Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAI.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAI.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAI.java	(revision 31634)
@@ -0,0 +1,153 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Greenstone Administrator Interface, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.admin;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.lang.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.plaf.*;
+import javax.swing.text.*;
+
+import org.greenstone.util.Configuration;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.admin.gui.SetServerPane;
+import org.greenstone.gsdl3.util.GSPath;
+
+/** Containing the top-level "core" for the GAI(Greenstone Administrator
+ * Interface) this class is the common core for the GAI application and 
+ * applet. It first parses the command line arguments, preparing to update 
+ * the configuration as required. Next it loads several important support 
+ * classes such as the Configuration and Dictionary. Finally it creates the 
+ * other important managers and sends them on their way.
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version ###
+ */
+public class GAI
+{
+    public static Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+    public static String gsdl3_src_home;
+    public static String gsdl3_web_home;
+    public static String tomcat_home;
+    
+    public static File build_properties_file;
+
+    public static String images_path;
+    /** A public reference to the GAIManager. */
+    static public GAIManager ga_man;
+
+    /** a public reference to the Dictionary */
+    static public Dictionary dictionary = null;
+    
+    public GAI(String gsdl3_src_home, String gsdl3_web_home)
+    {
+	this.gsdl3_src_home = gsdl3_src_home;
+	this.gsdl3_web_home = gsdl3_web_home;
+	
+	// set up the Configuration
+	new Configuration();
+	
+	// Read Dictionary
+	this.dictionary = new Dictionary("gai", Configuration.getLocale("general.locale"));
+	this.build_properties_file = new File(this.gsdl3_src_home, "build.properties");
+	// this may change if using preinstalled tomcat
+	this.tomcat_home = this.gsdl3_src_home+File.separatorChar+"packages"+File.separatorChar+"tomcat";
+	
+	// should come from classpath ??
+	this.images_path = this.gsdl3_src_home+File.separatorChar+"resources"+File.separatorChar+"images"+File.separatorChar;
+	// start the GAIManager
+	ga_man  = new GAIManager(screen_size);
+	ga_man.display();
+
+    }
+
+    public static String getGSDL3Home(){
+	
+	String gsdl3Home = gsdl3_web_home;
+	String os = "linux";
+	
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+	    gsdl3Home = gsdl3Home.replaceAll("\\\\", "/");
+	    os = "windows";
+	}
+		
+	gsdl3Home = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Home = gsdl3Home.replaceAll("/", "\\\\");
+	}
+	return gsdl3Home;
+    }
+
+    public static String getGSDL3ExtensionHome(){
+	String gsdl3ExtHome = gsdl3_web_home;
+	String os = "linux";
+
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+	    gsdl3ExtHome = gsdl3ExtHome.replaceAll("\\\\", "/");
+	    os = "windows";
+	}
+	
+	gsdl3ExtHome = GSPath.removeLastLink(gsdl3ExtHome);
+	gsdl3ExtHome += "/ext";
+	
+	if(os.equals("windows")){
+	    gsdl3ExtHome = gsdl3ExtHome.replaceAll("/", "\\\\");
+	}
+	return gsdl3ExtHome;
+    }
+    
+    public static void main (String[] args){
+	// A serious hack, but its good enough to stop crappy 'Could not
+        // lock user prefs' error messages.  Thanks to Walter Schatz from
+        // the java forums.
+        System.setProperty("java.util.prefs.syncInterval","2000000");
+
+	if (args.length != 2) {
+	    System.err.println("Usage: java org.greenstone.admin.GAI <gsdl3 src home> <gsdl3 web home>");
+	    System.exit(1);
+	}
+	
+	File gsdl3_src_dir = new File(args[0]);
+	File gsdl3_web_dir = new File(args[1]);
+	if (!gsdl3_src_dir.isDirectory() || !gsdl3_web_dir.isDirectory()) {
+	    System.err.println("Usage: java org.greenstone.admin.GAI <gsdl3 src home> <gsdl3 web home>");
+	    System.err.println("src or web directory does not exist!");
+	    System.exit(1);
+	}
+	
+	GAI gai = new GAI(args[0], args[1]);
+
+    }
+
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIFirstSettingFrame.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIFirstSettingFrame.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIFirstSettingFrame.java	(revision 31634)
@@ -0,0 +1,219 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * Date: 03.2005
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+/* This program is intended to confirm the initial user's setting about
+ * the Tomcat server and MySQL server they would like to run GSDL3 with
+ */
+package org.greenstone.admin;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.lang.Object;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+
+import org.greenstone.util.Configuration;
+import org.greenstone.admin.gui.ParsingProgress;
+import org.greenstone.admin.gui.SetServerPane;
+import org.greenstone.admin.GAI;
+
+/** The initial setting pane is to request the user to input the preferred 
+ *  Tomcat and Mysql server,they would like to run GSDL3 with
+ *  @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *  @version
+ */
+public class GAIFirstSettingFrame
+    extends JFrame
+    implements ActionListener {
+
+    public static Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+   
+    /* This pane is to inform the user to set up the tomcat and mysql server 
+     * if they are the first time run GAI, if the user doen't want to set up
+     * the installation will be based on the gsdl3 default value*/
+    protected JScrollPane setting_message_pane = null;
+    protected JPanel first_setting_pane = null;
+    protected JPanel main_pane = null;
+    protected JTextArea first_setting_message = null;
+    protected JButton fset_yes_button = null;
+    protected JButton fset_no_button = null;
+    protected JPanel button_pane = null;
+    protected JPanel control_pane = null;
+    
+    public boolean setting_confirm = false;
+    
+    /** The various sizes for the screen layout*/
+    static private Dimension MIN_SIZE = new Dimension( 90,  90);
+    static private Dimension LIST_SIZE  = new Dimension(200, 450);
+    static private Dimension DIALOG_SIZE = new Dimension (400,300);
+    static private Dimension TABLE_SIZE = new Dimension(500,200);
+    static final Dimension SIZE = new Dimension(400,75);						   
+    
+    //Constructor
+    public GAIFirstSettingFrame() {
+	super();
+
+	this.setDefaultCloseOperation(EXIT_ON_CLOSE);
+	this.setSize(DIALOG_SIZE);
+	this.setTitle("First time setting Tomcat/MYSQL Server");
+
+	// create all the control button panes
+	button_pane  = new JPanel();
+
+	//First time running button
+	FirstSetYesButtonListener fsybl = new FirstSetYesButtonListener();
+	FirstSetNoButtonListener fsnbl = new FirstSetNoButtonListener();
+	
+	// ImagesIcon for the buttons
+	ImageIcon setYesButtonIcon = new ImageIcon(GAI.images_path + "refresh.gif");
+	ImageIcon setNoButtonIcon = new ImageIcon(GAI.images_path + "exit.gif");
+	
+	fset_yes_button = new JButton("Yes", setYesButtonIcon);
+	fset_yes_button.addActionListener(fsybl);
+	fset_yes_button.setMnemonic(KeyEvent.VK_S);
+	fset_yes_button.setToolTipText("Click this button to Set up the Tomcat/MYSQL server");
+	fset_yes_button.setEnabled(false);
+	fset_no_button = new JButton("No", setNoButtonIcon);
+	fset_no_button.addActionListener(fsnbl);
+	fset_no_button.setEnabled(false);
+	fset_no_button.setMnemonic(KeyEvent.VK_N);
+	fset_no_button.setToolTipText("Click this button to skip the First time Tomcat/MYSQL server setting");
+	
+	//First time running setting buttons
+	fsybl = null;
+	fsnbl = null;
+	display();
+    }
+
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+    public void actionPerformed(ActionEvent event) {
+    }
+
+    /** This method is callsed to actually layout the components.*/
+    public void display() {
+	//KeyListenerImpl key_listener = new KeyListenerImpl();
+	//MouseListenerImpl mouse_listener = new MouseListenerImpl();
+	//this.addKeyListener(key_listener);
+
+	first_setting_message = new JTextArea();
+	first_setting_message.setEditable(false);
+	first_setting_message.setLineWrap(true);
+	
+	first_setting_message.setText("If you are about the first time to run GAI, you can set up " +
+				      "your own TOMCAT/MYSQL server. Otherwise, your GSDL3 " +
+				      "will be installed with the TOMCAT/MYSQL setting come " +
+				      "with the package. You may change the setting later on.");
+
+	first_setting_message.setFont(new Font("Arial", Font.BOLD, 14));
+	
+	setting_message_pane = new JScrollPane(first_setting_message);
+	
+	// The pane to store setting_message_pane
+	first_setting_pane = new JPanel();
+	first_setting_pane.setBorder(BorderFactory.createEmptyBorder(5,5,20,5));
+	first_setting_pane.setLayout (new BorderLayout());
+	first_setting_pane.add(setting_message_pane, BorderLayout.CENTER);
+
+	// Button Control Layout 
+	fset_yes_button.setEnabled(true);
+	fset_no_button.setEnabled(true);
+
+	button_pane.setLayout (new GridLayout(1,2));
+	button_pane.add(fset_yes_button);
+	button_pane.add(fset_no_button);
+
+	control_pane = new JPanel();
+	control_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	control_pane.setLayout(new BorderLayout());
+	control_pane.add(new JLabel("Do you want to set up your own running Tomcat/Mysql Servers?"), BorderLayout.CENTER);
+	control_pane.add(button_pane, BorderLayout.SOUTH);
+			 
+	main_pane = (JPanel) getContentPane();
+	//main_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	main_pane.setLayout (new BorderLayout());
+	main_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	main_pane.add(first_setting_pane,BorderLayout.CENTER);
+	main_pane.add(control_pane, BorderLayout.SOUTH);
+	main_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	main_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+
+	// Center and display
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - SIZE.width) / 2,
+		    (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+    
+    public boolean checkSettingConfirm(){
+	return setting_confirm;
+    }
+
+    public void destroy() {
+    }
+        
+    /** This class serves as the listener for actions on the build button.*/
+    private class FirstSetYesButtonListener
+	implements ActionListener {
+	/** If you want to give up the change you have made to the build properties 
+	 *  file before you save the change,  This button is to reload the log file 
+	 *  whenever user want to */
+	public void actionPerformed(ActionEvent event) {
+	    SetServerPane set_server_pane = new SetServerPane ();
+	    set_server_pane.display();
+	    //disable the GAIFirstSettingFrame
+	    setVisible (false);
+	    setting_confirm = true;
+	}
+    }
+    private class FirstSetNoButtonListener
+	implements ActionListener {
+	// Exit the Adminstration tool 
+	public void actionPerformed(ActionEvent event) {
+	    setting_confirm = false;
+	    setVisible (false);
+	    //return setting_confirm;
+	    if (!setting_confirm) {
+		GAIManager gai = new GAIManager(screen_size);
+	    }
+	}
+    }
+}
+       
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIManager.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIManager.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/GAIManager.java	(revision 31634)
@@ -0,0 +1,359 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Administration Tool (GAI) application, part of the 
+ * Greenstone digital library suite from the New Zealand Digital Library 
+ * Project at the University of Waikato, New Zealand.
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ *########################################################################
+ */
+
+package org.greenstone.admin;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.admin.gui.TabbedPane;
+import org.greenstone.admin.gui.MenuBar;
+import org.greenstone.admin.gui.LogPane;
+import org.greenstone.admin.gui.ConfPane;
+import org.greenstone.admin.guiext.ExtPane;
+
+import org.greenstone.util.Configuration;
+
+//import java AWT classes
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.plaf.*;
+
+public class GAIManager 
+    extends JFrame 
+    implements ActionListener, ChangeListener{
+    
+    private Dimension size = null;
+    // The ConfPane contains the configuration details of Greenstone III build and installation
+    public ConfPane conf_pane = null;
+    // The LogPane contains the log details for Tomcat Server used by gsdl3
+    public LogPane log_pane = null;
+    // Not designed yet
+    public ExtPane ext_pane = null;
+    // Not designed yet
+    public JPanel monitor_pane = null;
+       
+    static public ImageIcon CONF_ICON = null; 
+    static public ImageIcon EXT_ICON = null; 
+    static public ImageIcon MONITOR_ICON = null; 
+    static public ImageIcon LOG_ICON = null; 
+    
+    // Create a menu bar in the main Pane
+    public MenuBar menu_bar = null;
+
+    // HelpFrame
+    
+    private JPanel previous_pane = null;
+    private JPanel content_pane = null;
+    private JTabbedPane tab_pane = null;
+
+    /** A threaded tab changer to try and avoid NPE on exit. */
+    private TabUpdater tab_updater = null;
+    
+    //Contructor
+    public GAIManager(Dimension size){
+	super();
+	this.size = size;
+	this.setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+	this.setTitle("GreenStone 3 Administration Tool");
+	/* Add a focus listener to ourselves. Thus if we gain focus when a Modal Dialog should 
+	 * instead have it, we can try to bring the modal dialog to the front*/
+	//this.addFocusListener(new GAIGUIFocusListener());	
+	//display();
+	Image img = Toolkit.getDefaultToolkit().getImage("admin32.png");
+	this.setIconImage(img);
+    }
+
+    private class GAIGUIFocusListener
+	extends FocusAdapter {
+	public void focusGained(FocusEvent e) {
+	    /*  if (ModalDialog.current_modal != null) {
+		ModalDialog.current_modal.makeVisible();
+		ModalDialog.current_modal.toFront();
+		}*/
+	}
+    }
+
+   public void display(){
+	CONF_ICON = new ImageIcon (GAI.images_path + "create.gif");
+	EXT_ICON = new ImageIcon (GAI.images_path + "information.gif");	
+	MONITOR_ICON = new ImageIcon (GAI.images_path + "zoom.gif");
+	LOG_ICON = new ImageIcon (GAI.images_path + "history.gif");
+	content_pane = (JPanel) this.getContentPane();
+
+	try {
+	    this.setSize(size);
+	    //create a menuBar
+	    menu_bar = new MenuBar(new MenuListenerImpl());
+	    this.setJMenuBar(menu_bar);
+	    
+	    //Create the tabbed pane and put it in the center
+	    tab_pane = new JTabbedPane();
+	    tab_pane.addChangeListener(this);
+	    
+	    //set up the Configuration Pane
+	    if (Configuration.get("admin.conf")){
+		conf_pane = new ConfPane();
+		conf_pane.display();
+		tab_pane.addTab("Configuration", CONF_ICON, conf_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(conf_pane),Configuration.get("admin.conf"));
+	    }
+	    	    
+	    //set up the ext Pane
+	    if (Configuration.get("admin.ext")){
+		ext_pane = new ExtPane();
+
+		ext_pane.display();
+		tab_pane.addTab("Extensions", EXT_ICON, ext_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(ext_pane),Configuration.get("admin.ext"));
+	    }
+	    	    
+	    //set up the Monitor Pane
+	    if (Configuration.get("admin.monitor")){
+		monitor_pane = new JPanel();
+		//monitor_pane.display();
+		tab_pane.addTab("Monitor", MONITOR_ICON, monitor_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(monitor_pane),Configuration.get("admin.monitor"));
+	    }
+
+	    //set up the Log Pane
+	    if (Configuration.get("admin.log")){
+		log_pane = new LogPane();
+		log_pane.display();
+		tab_pane.addTab("Log", LOG_ICON, log_pane);
+		tab_pane.setEnabledAt(tab_pane.indexOfComponent(log_pane),Configuration.get("admin.log"));
+	    }   
+	    content_pane.setLayout(new BorderLayout());
+	    content_pane.add(tab_pane, BorderLayout.CENTER);
+	    pack();
+	    setVisible(true);
+	} catch (Exception e){
+	    e.printStackTrace();
+	    System.exit(1);
+	}
+    }
+
+    /** Any implementation of <i>ActionListener</i> must include this method so that we can be informed 
+     *  when an action has occured. In this case we are listening to actions from the menu-bar, and should 
+     *  react appropriately.
+     * @param event An <strong>ActionEvent</strong> containing information about the action that has occured.
+     */
+    
+    public void actionPerformed (ActionEvent event){
+	Object esrc = event.getSource();
+	if (esrc == menu_bar.file_exit){
+	    if (conf_pane.configurationChanged()) {
+		// prompt users for save
+		int result = JOptionPane.showConfirmDialog((Component) null, "The Configuration files have been changed, do you want to save the change", "Save Confirmation", JOptionPane.YES_NO_OPTION);
+		if ( result == JOptionPane.YES_OPTION) {
+		    conf_pane.save();	
+		} 
+	    }
+	    exit();
+	}
+	else if (esrc == menu_bar.file_save){
+	    /* sliently and Globally save Project and Site configuration*/
+	    conf_pane.save();
+	}
+	else if (esrc == menu_bar.help_conf){
+	}
+	else if (esrc == menu_bar.help_ext){
+	} 
+	else if (esrc == menu_bar.help_monitor){
+	}
+	else if (esrc == menu_bar.help_log){
+	}
+	else if (esrc == menu_bar.help_about) {
+	}
+    }
+    
+    
+    public void destory (){
+	// Destroying create pane ensures the latest log has been closed
+	//if (status_pane != null) {
+	//}
+    }
+    /**Overridden from JFrame so we can exit safely when window is closed 
+     *(or destroyed). @param event A <strong>WindowEvent</strong> containing 
+     *information about the event that fired this call.
+     */
+    protected void processWindowEvent(WindowEvent event) {
+	if(event.getID() == WindowEvent.WINDOW_CLOSING) {
+	    exit();
+	}
+    }
+
+
+    /** Listens to actions upon the menu bar, and if it detects a click over 
+     *the help menu brings the help window to the front if it has become hidden.
+     */
+    private class MenuListenerImpl
+	implements MenuListener {
+	public void menuCanceled (MenuEvent e) {
+	}
+	
+	public void menuDeselected (MenuEvent e){
+	}
+	
+	public void menuSelected (MenuEvent e){
+	    if (e.getSource() == menu_bar.file_exit) {
+		exit();
+	    }
+	  
+	    if (e.getSource() == menu_bar.help){
+		if (menu_bar.help.isSelected()){
+		    menu_bar.help.doClick(10);
+		}
+	    }
+	}
+    }
+
+
+    /** This method ensures that all the things needing saving 
+     *  are saved before GAIManager.exit() is called.
+     */
+    public void exit() {
+	System.exit(0);
+    }
+
+
+    /** Any implementation of ChangeListener must include this method 
+     **so we can be informed when the state of one of the registered 
+     **objects changes. In this case we are listening to view changes 
+     **within the tabbed pane.
+     * @param event A ChangeEvent containing information about the event that fired this call.
+     */
+    public void stateChanged(ChangeEvent event) {
+	if(previous_pane != null) {
+	}
+	
+	//menu_bar.tabSelected(tab_pane.getSelectedIndex());
+	int selected_index = tab_pane.getSelectedIndex();
+	if (selected_index == tab_pane.indexOfComponent(log_pane)) {
+	    log_pane.gainFocus();
+	}
+	else if (selected_index == tab_pane.indexOfComponent(conf_pane)) {
+	    conf_pane.gainFocus();
+	}
+	previous_pane = (JPanel) tab_pane.getSelectedComponent();
+    }
+    
+    public void modeChanged (int mode){
+	if (log_pane != null){
+	    log_pane.modeChanged(mode);
+	}
+	if (conf_pane !=null) {
+	    conf_pane.modeChanged(mode);
+	}
+    }
+
+    //   public void refresh (int refresh_reason, boolean collection_loaded)
+    public void refresh () {
+	if (log_pane != null ){
+	}
+    }
+
+    public void returnToInitialPane(){
+	if (log_pane != null) {
+	    tab_pane.setSelectedComponent(log_pane);
+	}
+    }
+    public void wait(boolean waiting) {
+	Component glass_pane = getGlassPane();
+	if(waiting) {
+	    // Show wait cursor.
+	    //glass_pane.addMouseListener(mouse_blocker_listener);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+	    glass_pane.setVisible(true);
+	}
+	else {
+	    // Hide wait cursor.
+	    glass_pane.setVisible(false);
+	    glass_pane.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
+	    //glass_pane.removeMouseListener(mouse_blocker_listener);
+	}
+	glass_pane = null;
+    }
+
+    private class TabUpdater
+	implements Runnable {
+	private boolean ready = false;
+	private int conf_pos = -1;
+	private int ext_pos = -1;
+	private int monitor_pos = -1;
+	private int log_pos = -1;
+	private JTabbedPane tab_pane = null;
+	
+	public TabUpdater (JTabbedPane tab_pane, boolean ready){
+	    this.ready = ready;
+	    this.tab_pane = tab_pane;
+	    conf_pos = tab_pane.indexOfComponent(conf_pane);
+	    ext_pos = tab_pane.indexOfComponent(ext_pane);
+	    monitor_pos = tab_pane.indexOfComponent(monitor_pane);  
+	    log_pos = tab_pane.indexOfComponent(log_pane);
+	}
+	
+	public void run() {
+	    if (conf_pos != -1){
+		if (ready) {
+		    tab_pane.setEnabledAt (conf_pos,Configuration.get("admin.conf"));
+		} else {
+		    tab_pane.setEnabledAt (conf_pos,false);
+		}
+	    }
+	    if (ext_pos != -1){
+		if (ready) {
+		    tab_pane.setEnabledAt (ext_pos,Configuration.get("admin.ext"));
+		} else {
+		    tab_pane.setEnabledAt (ext_pos,false);
+		}
+	    }
+	    if (monitor_pos != -1){
+		if (ready) {
+		    tab_pane.setEnabledAt (monitor_pos,Configuration.get("admin.monitor"));
+		} else {
+		    tab_pane.setEnabledAt (monitor_pos,false);
+		}
+	    }
+	    if (log_pos != -1){
+		if (ready) {
+		    tab_pane.setEnabledAt (log_pos,Configuration.get("admin.log"));
+		} else {
+		    tab_pane.setEnabledAt (log_pos,false);
+		}
+	    }
+	}
+	public void setReady (boolean ready){
+
+	    this.ready = ready;
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/LoggedMessageArea.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/LoggedMessageArea.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/LoggedMessageArea.java	(revision 31634)
@@ -0,0 +1,44 @@
+package org.greenstone.admin;
+
+import java.io.File;
+
+import javax.swing.JTextArea;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
+
+public class LoggedMessageArea extends JTextArea
+{
+    Logger _logger = null;
+    boolean _loggingLoaded = false;
+
+    public LoggedMessageArea(Class loggerClass)
+    {
+	if(!_loggingLoaded){
+	    try{
+		File logPath = new File(".." + File.separatorChar + "ext" + File.separatorChar + "logs");
+		File logFile = new File(".." + File.separatorChar + "ext" + File.separatorChar + "logs" + File.separatorChar + "ext.log");
+	
+		if(!logPath.exists()){
+		    logPath.mkdirs();
+		}
+
+		if(!logFile.exists()){
+		    logFile.createNewFile();
+		}
+	    }
+	    catch(Exception ex){
+		ex.printStackTrace();
+	    }
+	    PropertyConfigurator.configure("log4j.properties");
+	    _loggingLoaded = true;
+	}
+	_logger = Logger.getLogger(loggerClass);
+    }
+
+    public void append(String s)
+    {
+	super.append(s);
+	_logger.info(s);
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/config.xml
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/config.xml	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/config.xml	(revision 31634)
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE Configuration [
+  <!ELEMENT Configuration  (GathererConfig, Other, DirectoryMappings, Associations)>
+  <!ELEMENT GathererConfig (Argument*)>
+  <!ELEMENT Argument       (#PCDATA)>
+  <!ATTLIST Argument
+            name           CDATA #REQUIRED>
+  <!ELEMENT Other          (Info*)>
+  <!ELEMENT Info           (Name, Desc, Arguments)>
+  <!ELEMENT Arguments      (Option*)>
+  <!ELEMENT Option         (Name, Desc, Type, Required, Default?, List?)>
+  <!ELEMENT Type           (#PCDATA)>
+  <!ELEMENT Required       (#PCDATA)>
+  <!ELEMENT Default        (#PCDATA)>
+  <!ELEMENT List           (Value*)>
+  <!ELEMENT Value          (Name, Desc?)>
+  <!ELEMENT Name           (#PCDATA)>
+  <!ELEMENT Desc           (#PCDATA)>
+  <!ELEMENT DirectoryMappings (Mapping*)>
+  <!ELEMENT Mapping           (#PCDATA)>
+  <!ATTLIST Mapping 
+            name              CDATA #REQUIRED
+	    file              CDATA #REQUIRED>
+  <!ELEMENT Associations (Entry*)>
+  <!ELEMENT Entry (#PCDATA)>
+  <!ATTLIST Entry
+            extension CDATA #REQUIRED>
+]>
+
+<Configuration version="2.53a">
+  <GathererConfig>
+    <!-- ***** General Options: apply globally to the GAI execution. ***** -->
+    <!-- The location and size of the main Gatherer window. -->
+    <Argument name="general.bounds">0, 0, 800, 600</Argument>
+    <!-- Are we expected to produce a debug file. Can be overidden at start up. default = [false] -->
+    <Argument name="general.debug">false</Argument>
+    <!-- The standard font used in the Gatherer. [Verdana, BOLD, 12] -->
+    <Argument name="general.font">Verdana, BOLD, 12</Argument>
+    <Argument name="general.library_url"/>
+    <Argument name="general.locale">en_NZ</Argument>
+    <!-- How many folders deep should we recurse before suspecting a cyclic path? [40] -->
+    <Argument name="general.max_folder_depth">40</Argument>
+    <!-- The mode controls what level a detail is presented to the user - ranging from a basic, no regex, no perl experience up to complete details for all processes. The values, ranging from lowest to highest detail, are: 1 = Assistant, 2 = Librarian, 3 = Systems, 4 = Expert [default 2]-->
+    <Argument name="general.mode">2</Argument>
+    <Argument name="general.open_collection"></Argument>
+    <!-- The application launching command used when previewing the collection. [default -blank-] -->
+    <Argument name="general.preview_program"></Argument>
+    <Argument name="general.proxy_host"></Argument>
+    <Argument name="general.proxy_port"></Argument>
+    <Argument name="general.show_file_size">false</Argument>
+    <!-- Do we expect to connect to the internet via a proxy server. [false] -->
+    <Argument name="general.use_proxy">false</Argument>
+    <Argument name="general.view_extracted_metadata">true</Argument>
+
+    <!-- Colouring -->
+    <Argument name="coloring.button_background">218, 237, 252</Argument>
+    <Argument name="coloring.button_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.button_selected_background">128, 180, 216</Argument>
+    <Argument name="coloring.button_selected_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.collection_heading_background">176, 208, 176</Argument>  
+    <Argument name="coloring.collection_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_selection_background">128, 180, 216</Argument>
+    <Argument name="coloring.collection_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_tree_background">224, 240, 224</Argument>
+    <Argument name="coloring.collection_tree_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.collection_noneditable_background">192, 192, 192</Argument>
+    <Argument name="coloring.collection_noneditable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.disabled">206, 206, 206</Argument>
+    <Argument name="coloring.editable_background">255, 255, 255</Argument>
+    <Argument name="coloring.editable_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.error_background">255, 0, 0</Argument>
+    <Argument name="coloring.error_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.scrollbar_background">192, 192, 192</Argument>
+    <Argument name="coloring.scrollbar_foreground">255, 255, 255</Argument>
+    <Argument name="coloring.workspace_heading_background">128, 180, 216</Argument>
+    <Argument name="coloring.workspace_heading_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_selection_background">176, 208, 176</Argument>
+    <Argument name="coloring.workspace_selection_foreground">0, 0, 0</Argument>
+    <Argument name="coloring.workspace_tree_background">218, 237, 252</Argument>
+    <Argument name="coloring.workspace_tree_foreground">0, 0, 0</Argument>
+
+    <!-- Mirroring -->
+    <!-- Mirroring has many particular options. These show the default settings. -->
+   <!-- <Argument name="mirroring.depth">0</Argument>
+    <Argument name="mirroring.no_parents">true</Argument>
+    <Argument name="mirroring.other_hosts">false</Argument>
+    <Argument name="mirroring.overwrite">false</Argument>
+    <Argument name="mirroring.page_requisites">true</Argument>
+    <Argument name="mirroring.quiet">true</Argument>
+    <Argument name="mirroring.remove_failed">false</Argument>-->
+
+    <!-- Warnings -->
+    <!-- Warnings can be disabled by the user. -->
+    <Argument name="warning.CollectionBuilt">true</Argument>
+    <Argument name="warning.DirectoryLevelMetadata">true</Argument>
+    <Argument name="warning.ExtractedMetadata">true</Argument>
+    <Argument name="warning.InheritedMetadata">true</Argument>
+    <Argument name="warning.MissingEXEC">true</Argument>
+    <Argument name="warning.MissingGSDL">true</Argument>
+    <Argument name="warning.MissingPERL">true</Argument>
+    <Argument name="warning.MissingWGET">true</Argument>
+    <Argument name="warning.NoMetadataSetsSelected">true</Argument>
+    <Argument name="warning.NoPluginExpectedToProcessFile">true</Argument>
+    <Argument name="warning.OldWGET">true</Argument>
+    <Argument name="warning.TitleClashes">true</Argument>
+
+    <!-- Workflow -->
+    <!-- Disabling a view here will cause it to be hidden from the Gatherer display. -->
+    <Argument name="admin.conf">true</Argument>
+    <Argument name="admin.ext">true</Argument> 
+    <Argument name="admin.monitor">true</Argument>
+    <Argument name="admin.log">true</Argument> 
+
+    <Argument name="GEMS.Preferences.Selected_Languages">en</Argument>
+  </GathererConfig>
+
+  <Other/>
+  <DirectoryMappings/>
+  <Associations>
+    <Entry extension="doc"/>
+    <Entry extension="gif"/>
+    <Entry extension="htm"/>
+    <Entry extension="html"/>
+    <Entry extension="jpg"/>
+    <Entry extension="jpeg"/>
+    <Entry extension="pdf"/>
+    <Entry extension="png"/>
+    <Entry extension="ps"/>
+    <Entry extension="rtf"/>
+    <Entry extension="txt"/>
+    <Entry extension="text"/>
+  </Associations>
+</Configuration>
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ClearLogFilePrompt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ClearLogFilePrompt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ClearLogFilePrompt.java	(revision 31634)
@@ -0,0 +1,124 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ * Modified by: Chi-Yu Huang
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+/*import org.greenstone.gatherer.Dictionary;
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.file.WorkspaceTree;  // !!! Don't like this here
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;*/
+
+
+/** This class provides the functionality to delete current collections from the 
+ ** GSDLHOME/collect/ directory. The user chooses the collection from a list, where 
+ ** each entry also displays details about itself, confirms the delete of a collection 
+ ** by checking a checkbox then presses the ok button to actually delete the collection.
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version 1.0
+ */
+public class ClearLogFilePrompt 
+	extends JDialog {
+    
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private ClearLogFilePrompt prompt = null;
+    private File log_file = null;
+
+    //A flag showing the deleting status
+    private boolean current_logfile_cleared = false;
+
+    /** The size of the delete prompt screen. */
+    static final Dimension SIZE = new Dimension(500, 500);
+
+    // Constructor.
+    public ClearLogFilePrompt() {
+	super();
+	prompt = this;
+    }
+    
+    /** Destructor. */
+    public void destroy() {
+	prompt = null;
+    }
+
+    /** This method causes the Clear Confirmation Dialog to be displayed. 
+     * returns true if it has deleted the log file that is currently open */
+    public boolean display(File log_file) {
+	int result = JOptionPane.showConfirmDialog((Component) null, "Do you really want to clear this log file", "Clear Confirmation", JOptionPane.YES_NO_OPTION);
+	if ( result == JOptionPane.YES_OPTION) {
+	    log_file.delete();
+	    try {
+		log_file.createNewFile();
+	    } catch (Exception e) {
+		e.printStackTrace();
+	    }	     
+	    current_logfile_cleared = true;
+	} else if (result == JOptionPane.NO_OPTION) {
+	    current_logfile_cleared = false;
+	    JOptionPane.showMessageDialog(prompt, "Cancel clearing the " + log_file.getPath() +" log file");
+	    prompt.dispose();
+	}
+	return current_logfile_cleared;
+    }
+    
+    
+    /** Shows a delete complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully deleted.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    /*public void resultPrompt(boolean success) {
+	if (success) {
+	    JOptionPane.showMessageDialog(prompt,"Selected log file has been deleted. You have to restart the action to create the log file");
+	} else {
+	    JOptionPane.showMessageDialog(prompt, "Cancel deleting the log file");
+	}
+	}*/
+    
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+    public void actionPerformed(ActionEvent event) {
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ConfPane.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ConfPane.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ConfPane.java	(revision 31634)
@@ -0,0 +1,1150 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * Modified: 03.2005
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.BorderLayout;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.Font;
+import java.awt.GridLayout;
+import java.awt.Point;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.awt.event.KeyAdapter;
+import java.awt.event.KeyEvent;
+import java.awt.event.MouseAdapter;
+import java.awt.event.MouseEvent;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Hashtable;
+
+import javax.swing.BorderFactory;
+import javax.swing.ImageIcon;
+import javax.swing.JButton;
+import javax.swing.JLabel;
+import javax.swing.JList;
+import javax.swing.JOptionPane;
+import javax.swing.JPanel;
+import javax.swing.JScrollPane;
+import javax.swing.JSplitPane;
+import javax.swing.JTable;
+import javax.swing.JTree;
+import javax.swing.event.TreeSelectionEvent;
+import javax.swing.event.TreeSelectionListener;
+import javax.swing.table.AbstractTableModel;
+import javax.swing.table.JTableHeader;
+import javax.swing.table.TableCellRenderer;
+import javax.swing.tree.DefaultMutableTreeNode;
+import javax.swing.tree.TreeSelectionModel;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.util.Configuration;
+
+/**
+ * The Configuration pane is to view the status of relevant configure files in
+ * GSIII
+ * 
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version
+ */
+public class ConfPane extends JPanel implements ActionListener
+{
+
+	/*
+	 * The pane to demonstrate Configuration information, including the build
+	 * property file being monitored and their content, and control functions
+	 */
+	protected JSplitPane main_conf_pane = null;
+
+	protected JPanel main_contentPane = null;
+	public Hashtable<Object, Object> conf_table;
+
+	/** The panel that contains a Conf_list */
+	private JPanel confList_pane = null;
+
+	/** The panel that contains the conf content. */
+	private JPanel confContent_pane = null;
+	private JScrollPane conf_table_pane = null;
+
+	/** The List showing all the Configuration files concerned. */
+	private JList conf_list = null;
+	private JTree conf_tree = null;
+
+	/** The label at the top of the confList_pane. */
+	private JLabel confList_label = null;
+	/** The label shown at the top of the confContent Pane. */
+	private JLabel confContent_label = null;
+
+	// The control buttons used to manipulate Configuration Pane
+	protected JPanel inner_button_pane = null;
+	protected JPanel inner_control_pane = null;
+	protected JPanel tomcat_button_pane = null;
+	protected JPanel mysql_button_pane = null;
+	protected JPanel setting_shown_pane = null;
+
+	/** Buttons in inner button pane */
+	private JButton startup_tomcat_button = null;
+	private JButton shutdown_tomcat_button = null;
+	private JButton restart_tomcat_button = null;
+	private JButton startup_mysql_button = null;
+	private JButton shutdown_mysql_button = null;
+	private JButton restart_mysql_button = null;
+
+	private JButton save_button = null;
+
+	private JLabel conf_label = null;
+	private JLabel conf_setting_label = null;
+
+	/* ConfPane empty AREA to show XML configuration file */
+	//private JTextArea conf_xml_area = null;
+
+	//control tomcat and mysql server running status
+	private String tomcat_server;
+	private String tomcat_port;
+	private String mysql_adminuser;
+	private String mysql_server;
+	private String mysql_port;
+	private String conf_pane_name;
+
+	private boolean tomcat_server_up = false;
+	private boolean mysql_server_up = false;
+
+	/** The scrollable area into which the configuration content is placed. */
+	//private JScrollPane conf_xml_content = null;
+
+	public SiteConfSetting site_conf = null;
+	public SiteConfSetting interface_conf = null;
+
+	/** The various sizes for the screen layout */
+	static private Dimension MIN_SIZE = new Dimension(90, 90);
+	static private Dimension LIST_SIZE = new Dimension(200, 450);
+	static private Dimension CONTENT_SIZE = new Dimension(600, 450);
+	static private Dimension TABLE_SIZE = new Dimension(500, 200);
+	static private int MINIMUM_TABLE_HEADER_SIZE = 15;
+
+	private JTable conf_setting_table = null;
+
+	//The details display in the table
+	int display_row_count = 5;
+	//int rowLength = 20;
+	//int colLength = 3;
+	//private Object[][] conf_display = new Object[rowLength][colLength];
+	//private Object[][] conf_setting;
+
+	private final String ONE = "1";
+	private final String TWO = "2";
+	private final String THREE = "3";
+
+	private ArrayList<ArrayList<Object>> conf_display = new ArrayList<ArrayList<Object>>();
+	private ArrayList<HashMap<String, Object>> conf_setting = new ArrayList<HashMap<String, Object>>();
+
+	//An array to store all the details in the build.properties file
+	private ArrayList<String> conf_array;
+
+	//Site configuration
+	// TODO!! there are more than one site and interface - need to extend this
+	public File site_conf_file = new File(GSFile.siteConfigFile(GSFile.siteHome(GAI.gsdl3_web_home, "localsite")));
+	public File interface_conf_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GAI.gsdl3_web_home, "default")));
+	public boolean project_conf_changed = false;
+
+	ConfSettingTableModel conf_table_model = null;
+	DefaultMutableTreeNode top = new DefaultMutableTreeNode("Configuration Files");
+
+	//Constructor
+	public ConfPane()
+	{
+		conf_table = new Hashtable<Object, Object>();
+		// create all the control button panes
+		inner_control_pane = new JPanel();
+		inner_button_pane = new JPanel();
+		tomcat_button_pane = new JPanel();
+		mysql_button_pane = new JPanel();
+
+		/*
+		 * the class for showing the site configurations including
+		 * siteConfig.xml and interfaceConfig.xml
+		 */
+		site_conf = new SiteConfSetting("siteConfig", site_conf_file);
+		interface_conf = new SiteConfSetting("interfaceConfig", interface_conf_file);
+
+		// Main Configuration Pane
+		main_conf_pane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
+		// Main pane for Configuration Content Pane
+		main_contentPane = new JPanel();
+
+		//Create a tree for a list Configuration files we are interested
+		//DefaultMutableTreeNode child1 = new DefaultMutableTreeNode();
+		DefaultMutableTreeNode child1 = new DefaultMutableTreeNode("Project Configuration");
+
+		DefaultMutableTreeNode compile = new DefaultMutableTreeNode("Compile");
+		DefaultMutableTreeNode tomcat = new DefaultMutableTreeNode("Tomcat");
+		DefaultMutableTreeNode proxy = new DefaultMutableTreeNode("Proxy");
+		DefaultMutableTreeNode mysql = new DefaultMutableTreeNode("MySQL");
+		DefaultMutableTreeNode gsdl = new DefaultMutableTreeNode("GSDL");
+		child1.add(compile);
+		child1.add(tomcat);
+		child1.add(proxy);
+		child1.add(mysql);
+		child1.add(gsdl);
+
+		DefaultMutableTreeNode child2 = new DefaultMutableTreeNode("Site Configuration");
+		//DefaultMutableTreeNode child2 = new DefaultMutableTreeNode();	
+		DefaultMutableTreeNode siteConf = new DefaultMutableTreeNode("SiteConfig");
+		DefaultMutableTreeNode interfaceConf = new DefaultMutableTreeNode("InterfaceConfig");
+		child2.add(siteConf);
+		child2.add(interfaceConf);
+
+		//DefaultMutableTreeNode child3 = new DefaultMutableTreeNode("Conf 3");
+		top.add(child1);
+		top.add(child2);
+		//top.add(child3);
+		conf_tree = new JTree(top);
+		conf_tree.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+		conf_tree.addTreeSelectionListener(new ConfTreeListener());
+
+		// inner button_pane Buttons
+		StartupTomcatButtonListener stbl = new StartupTomcatButtonListener();
+		ShutdownTomcatButtonListener stdbl = new ShutdownTomcatButtonListener();
+		RestartTomcatButtonListener rtbl = new RestartTomcatButtonListener();
+
+		StartupMysqlButtonListener smbl = new StartupMysqlButtonListener();
+		ShutdownMysqlButtonListener smdbl = new ShutdownMysqlButtonListener();
+		RestartMysqlButtonListener rmbl = new RestartMysqlButtonListener();
+
+		ImageIcon startupTomcatButtonIcon = new ImageIcon(GAI.images_path + "redo.gif");
+		ImageIcon shutdownTomcatButtonIcon = new ImageIcon(GAI.images_path + "stop.gif");
+		ImageIcon restartTomcatButtonIcon = new ImageIcon(GAI.images_path + "stop.gif");
+
+		ImageIcon startupMysqlButtonIcon = new ImageIcon(GAI.images_path + "redo.gif");
+		ImageIcon shutdownMysqlButtonIcon = new ImageIcon(GAI.images_path + "stop.gif");
+		ImageIcon restartMysqlButtonIcon = new ImageIcon(GAI.images_path + "stop.gif");
+		ImageIcon saveButtonIcon = new ImageIcon(GAI.images_path + "save.gif");
+
+		startup_tomcat_button = new JButton();
+		startup_tomcat_button.addActionListener(stbl);
+		startup_tomcat_button.setMnemonic(KeyEvent.VK_S);
+		startup_tomcat_button.setEnabled(false);
+		startup_tomcat_button.setText(GAI.dictionary.get("ConfPane.Tomcat_Startup"));
+		startup_tomcat_button.setToolTipText(GAI.dictionary.get("ConfPane.Tomcat_Startup_Tooltip"));
+
+		shutdown_tomcat_button = new JButton();
+		shutdown_tomcat_button.addActionListener(stdbl);
+		shutdown_tomcat_button.setEnabled(false);
+		shutdown_tomcat_button.setText(GAI.dictionary.get("ConfPane.Tomcat_Shutdown"));
+		shutdown_tomcat_button.setToolTipText(GAI.dictionary.get("ConfPane.Tomcat_Shutdown_Tooltip"));
+
+		restart_tomcat_button = new JButton();
+		restart_tomcat_button.addActionListener(rtbl);
+		restart_tomcat_button.setEnabled(false);
+		restart_tomcat_button.setText(GAI.dictionary.get("ConfPane.Tomcat_Restart"));
+		restart_tomcat_button.setToolTipText(GAI.dictionary.get("ConfPane.Tomcat_Restart_Tooltip"));
+
+		startup_mysql_button = new JButton();
+		startup_mysql_button.addActionListener(smbl);
+		startup_mysql_button.setMnemonic(KeyEvent.VK_S);
+		startup_mysql_button.setEnabled(false);
+		startup_mysql_button.setText(GAI.dictionary.get("ConfPane.MySQL_Startup"));
+		startup_mysql_button.setToolTipText(GAI.dictionary.get("ConfPane.MySQL_Startup_Tooltip"));
+
+		shutdown_mysql_button = new JButton();
+		shutdown_mysql_button.addActionListener(smdbl);
+		shutdown_mysql_button.setEnabled(false);
+		shutdown_mysql_button.setText(GAI.dictionary.get("ConfPane.MySQL_Shutdown"));
+		shutdown_mysql_button.setToolTipText(GAI.dictionary.get("ConfPane.MySQL_Shutdown_Tooltip"));
+
+		restart_mysql_button = new JButton();
+		restart_mysql_button.addActionListener(rmbl);
+		restart_mysql_button.setEnabled(false);
+		restart_mysql_button.setText(GAI.dictionary.get("ConfPane.MySQL_Restart"));
+		restart_mysql_button.setToolTipText(GAI.dictionary.get("ConfPane.MySQL_Restart_Tooltip"));
+
+		//tomcat control buttons
+		stbl = null;
+		stdbl = null;
+		rtbl = null;
+
+		//mysql control button
+		smbl = null;
+		smdbl = null;
+		rmbl = null;
+
+		//read in build properties
+		getConfContent();
+	}
+
+	/**
+	 * Any implementation of ActionListener requires this method so that when an
+	 * action is performed the appropriate effect can occur.
+	 */
+	public void actionPerformed(ActionEvent event)
+	{
+	}
+
+	/** This method is called to actually layout the components. */
+	public void display()
+	{
+		//Create Components.
+		//KeyListenerImpl key_listener = new KeyListenerImpl();
+		//MouseListenerImpl mouse_listener = new MouseListenerImpl();
+		//this.addKeyListener(key_listener);
+
+		//confList_Pane
+		confList_pane = new JPanel();
+		confList_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+		confList_pane.setPreferredSize(LIST_SIZE);
+		confList_pane.setSize(LIST_SIZE);
+		confList_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+		confList_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+
+		confList_label = new JLabel();
+		confList_label.setOpaque(true);
+		confList_label.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+		confList_label.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+		confList_label.setText(GAI.dictionary.get("ConfPane.Conf_List"));
+
+		// confContent_Pane
+		confContent_pane = new JPanel();
+		confContent_pane.setBorder(BorderFactory.createEmptyBorder(10, 20, 10, 20));
+		confContent_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+		confContent_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+
+		confContent_label = new JLabel();
+		confContent_label.setOpaque(true);
+		confContent_label.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+		confContent_label.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+		confContent_label.setText(GAI.dictionary.get("ConfPane.Conf_Content"));
+
+		conf_table_model = new ConfSettingTableModel();
+		conf_setting_table = new JTable(conf_table_model)
+		{
+			public String getToolTipText(MouseEvent e)
+			{
+				String tip = null;
+				Point p = e.getPoint();
+				int rowIndex = rowAtPoint(p);
+				int colIndex = columnAtPoint(p);
+				int realColumnIndex = convertColumnIndexToModel(colIndex);
+				if (realColumnIndex == 0)
+				{
+					tip = conf_display.get(rowIndex).get(realColumnIndex).toString();
+				}
+				else
+				{
+					tip = super.getToolTipText(e);
+				}
+				return tip;
+			}
+
+			public Component prepareRenderer(TableCellRenderer renderer, int rowIndex, int colIndex)
+			{
+				Component c = super.prepareRenderer(renderer, rowIndex, colIndex);
+				if (colIndex == 0)
+				{
+					c.setBackground(Configuration.getColor("coloring.table_noneditable_background"));
+				}
+				else
+				{
+					c.setBackground(Configuration.getColor("coloring.table_editable_background"));
+				}
+				return c;
+			}
+		};
+
+		//When the MySQL and TOMCAT servers are still running and detect double-clicking occuring (editing mode)
+		conf_setting_table.addMouseListener(new MouseAdapter()
+		{
+			public void mouseClicked(MouseEvent e)
+			{
+				if (e.getClickCount() == 2)
+				{
+					if (conf_pane_name.matches("MYSQL"))
+					{
+						mysql_server_up = false;
+						mysql_server_up = checkMysqlServer();
+						if (mysql_server_up)
+						{
+							JOptionPane.showMessageDialog((Component) null, "MySQL is running, shutdown the MySQL server before making any changes");
+						}
+					}
+					if (conf_pane_name.matches("TOMCAT"))
+					{
+						tomcat_server_up = false;
+						tomcat_server_up = checkTomcatServer();
+						if (tomcat_server_up)
+						{
+							JOptionPane.showMessageDialog((Component) null, "TOMCAT is running, shutdown the TOMCAT server before making any changes");
+						}
+					}
+				}
+			}
+		});
+		conf_setting_table.addKeyListener(new KeyAdapter()
+		{
+			public void keyPressed(KeyEvent ke)
+			{
+				if (ke.getKeyCode() == KeyEvent.VK_F2)
+				{
+					if (conf_pane_name.matches("MYSQL"))
+					{
+						mysql_server_up = checkMysqlServer();
+						if (mysql_server_up)
+						{
+							JOptionPane.showMessageDialog((Component) null, "MySQL is running, shutdown the MySQL server before making any changes");
+						}
+					}
+					if (conf_pane_name.matches("TOMCAT"))
+					{
+						tomcat_server_up = checkTomcatServer();
+						if (tomcat_server_up)
+						{
+							JOptionPane.showMessageDialog((Component) null, "TOMCAT is running, shutdown the TOMCAT server before making any changes");
+						}
+					}
+				}
+			}
+		});
+
+		//Set up tableHeader
+		JTableHeader table_header = conf_setting_table.getTableHeader();
+		Dimension table_header_preferred_size = table_header.getPreferredSize();
+		if (table_header_preferred_size.height < MINIMUM_TABLE_HEADER_SIZE)
+		{
+			table_header_preferred_size.setSize(table_header_preferred_size.width, MINIMUM_TABLE_HEADER_SIZE);
+			table_header.setPreferredSize(table_header_preferred_size);
+		}
+
+		table_header.setFont(new Font("Arial", Font.BOLD, 13));
+		conf_setting_table.setRowHeight(30);
+		conf_table_pane = new JScrollPane(conf_setting_table);
+		conf_table_pane.setVisible(false);
+		conf_table_pane.getViewport().setBackground(Configuration.getColor("coloring.collection_tree_background"));
+
+		// Layout Components
+		confList_pane.setLayout(new BorderLayout());
+		JScrollPane conf_treeView = new JScrollPane(conf_tree);
+		confList_pane.add(conf_treeView, BorderLayout.CENTER);
+		confList_pane.add(confList_label, BorderLayout.NORTH);
+		confContent_pane.setLayout(new BorderLayout());
+		confContent_pane.add(conf_table_pane, BorderLayout.CENTER);
+
+		inner_button_pane.setLayout(new GridLayout(1, 2));
+		inner_control_pane.setLayout(new BorderLayout());
+		inner_control_pane.setBorder(BorderFactory.createEmptyBorder(5, 10, 5, 10));
+		inner_control_pane.setPreferredSize(new Dimension(50, 50));
+		inner_control_pane.setSize(new Dimension(50, 50));
+		inner_control_pane.add(inner_button_pane, BorderLayout.CENTER);
+
+		main_contentPane.setLayout(new BorderLayout());
+		main_contentPane.add(confContent_label, BorderLayout.NORTH);
+		main_contentPane.add(confContent_pane, BorderLayout.CENTER);
+		main_contentPane.add(inner_control_pane, BorderLayout.SOUTH);
+		main_conf_pane.add(confList_pane, JSplitPane.LEFT);
+		main_conf_pane.add(main_contentPane, JSplitPane.RIGHT);
+		main_conf_pane.setDividerLocation(LIST_SIZE.width - 10);
+
+		this.setLayout(new BorderLayout());
+		this.add(main_conf_pane, BorderLayout.CENTER);
+	}
+
+	public void save()
+	{
+		if (project_conf_changed)
+		{
+			saveProjectConf(GAI.build_properties_file);
+		}
+		if (site_conf.confChanged())
+		{
+			site_conf.saveFile(site_conf_file);
+		}
+		if (interface_conf.confChanged())
+		{
+			interface_conf.saveFile(interface_conf_file);
+		}
+
+	}
+
+	public boolean configurationChanged()
+	{
+		return (project_conf_changed || site_conf.confChanged() || interface_conf.confChanged());
+
+	}
+
+	private class StartupTomcatButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			/*
+			 * before startup Tomcat server, we want to make sure project and
+			 * Site configuration files are saved
+			 */
+			if (project_conf_changed)
+			{
+				saveProjectConf(GAI.build_properties_file);
+			}
+			/*
+			 * boolean site_conf_changed = site_conf.confChanged(); boolean
+			 * interface_conf_changed=interface_conf.confChanged(); if
+			 * (site_conf_changed) { site_conf.saveFile(site_conf_file); } if
+			 * (interface_conf_changed) {
+			 * interface_conf.saveFile(interface_conf_file); }
+			 */
+			tomcat_server_up = checkTomcatServer();
+			if (!tomcat_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable startupTomcatThread = threadControl.new startupTomcatServer("start-tomcat");
+				new Thread(startupTomcatThread).start();
+				tomcat_server_up = true;
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server has been Startup successfully!");
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server is running");
+			}
+			changeTomcatButtonPane1();
+		}
+	}
+
+	private class ShutdownTomcatButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			tomcat_server_up = checkTomcatServer();
+			if (tomcat_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable shutdownTomcatThread = threadControl.new shutdownTomcatServer("stop-tomcat");
+				new Thread(shutdownTomcatThread).start();
+				tomcat_server_up = false;
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server has been shutted down !");
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server was not running!");
+			}
+			changeTomcatButtonPane2();
+		}
+	}
+
+	private class RestartTomcatButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			//before we restart Tomcat server, we want to make sure the conf_pane settings are saved
+			if (project_conf_changed)
+			{
+				saveProjectConf(GAI.build_properties_file);
+			}
+			/*
+			 * boolean site_conf_changed = site_conf.confChanged(); boolean
+			 * interface_conf_changed=interface_conf.confChanged(); if
+			 * (site_conf_changed) { site_conf.saveFile(site_conf_file); } if
+			 * (interface_conf_changed) {
+			 * interface_conf.saveFile(interface_conf_file); }
+			 */
+			tomcat_server_up = checkTomcatServer();
+			if (tomcat_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable restartTomcatThread = threadControl.new restartTomcatServer("restart-tomcat");
+				new Thread(restartTomcatThread).start();
+				tomcat_server_up = true;
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server has been Restarted successfully!");
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "Tomcat server was not running!");
+			}
+		}
+	}
+
+	private class StartupMysqlButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			//before we startup the MySQL server, we want to make sure the Conf_pane settings are saved
+			saveProjectConf(GAI.build_properties_file);
+			mysql_server_up = checkMysqlServer();
+			if (!mysql_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable startupMysqlThread = threadControl.new startupMysqlServer("start-mysql");
+				new Thread(startupMysqlThread).start();
+				mysql_server_up = true;
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server has been Startup successfully!");
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server has been running");
+			}
+			changeMysqlButtonPane1();
+		}
+	}
+
+	private class ShutdownMysqlButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			mysql_server_up = checkMysqlServer();
+			if (mysql_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable shutdownMysqlThread = threadControl.new shutdownMysqlServer("stop-mysql");
+				new Thread(shutdownMysqlThread).start();
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server has been Shutdown successfully !");
+				mysql_server_up = false;
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server was not running!");
+			}
+			changeMysqlButtonPane2();
+		}
+	}
+
+	private class RestartMysqlButtonListener implements ActionListener
+	{
+		public void actionPerformed(ActionEvent event)
+		{
+			saveProjectConf(GAI.build_properties_file);
+			mysql_server_up = checkMysqlServer();
+			if (mysql_server_up)
+			{
+				ThreadControl threadControl = new ThreadControl();
+				Runnable restartMysqlThread = threadControl.new restartMysqlServer("restart-mysql");
+				new Thread(restartMysqlThread).start();
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server has been Restarted successfully!");
+				mysql_server_up = true;
+			}
+			else
+			{
+				JOptionPane.showMessageDialog((Component) null, "MYSQL server was not running!");
+			}
+		}
+	}
+
+	// Save the configuration file globally,when click File->Save or restart (or startup) Tomcat and MySQl server
+	public void saveProjectConf(File file)
+	{
+		String new_string;
+		Enumeration<Object> keys = conf_table.keys();
+		String key;
+		String value;
+		String filename = file.getPath();
+		while (keys.hasMoreElements())
+		{
+			key = (String) keys.nextElement();
+			value = (String) conf_table.get(key);
+			for (int j = 0; j < conf_array.size(); j++)
+			{
+				if (conf_array.get(j).startsWith(key.toLowerCase()))
+				{
+					new_string = key + "=" + value;
+					conf_array.set(j, new_string.toLowerCase());
+				}
+			}
+		}
+
+		try
+		{
+			BufferedWriter conf_out = new BufferedWriter(new FileWriter(filename));
+			for (int j = 0; j < conf_array.size(); j++)
+			{
+				conf_out.write(conf_array.get(j).toString());
+				conf_out.newLine();
+			}
+			conf_out.close();
+			getConfContent();
+			project_conf_changed = false;
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public boolean projectConfChanged()
+	{
+		return project_conf_changed;
+	}
+
+	public void getConfContent()
+	{
+		String filename = GAI.build_properties_file.getPath();
+		if (!GAI.build_properties_file.exists())
+		{
+			JOptionPane.showMessageDialog((Component) null, "Build property file does not exist");
+		}
+		else
+		{
+			readProjectConf(filename);
+		}
+	}
+
+	/*
+	 * Read build.properties file
+	 * 
+	 * @param
+	 */
+	public void readProjectConf(String filename)
+	{
+		System.err.println("FILENAME = " + filename);
+		String fileLine;
+		/*
+		 * inside the array will store, the conf_setting[i][0]:Comment,
+		 * conf_setting[i][1]:Para and conf_setting[i][2]:Value conf_array[]
+		 * store all the details from the build.properties
+		 */
+		conf_array = new ArrayList<String>();
+		conf_setting.clear();
+
+		try
+		{
+			BufferedReader conf_in = new BufferedReader(new FileReader(filename));
+			while ((fileLine = conf_in.readLine()) != null)
+			{
+				HashMap<String, Object> map = new HashMap<String, Object>();
+				// This is an empty line
+				if (fileLine.matches("^\\s*$"))
+				{
+					// Do Nothing
+				}
+				else if (fileLine.matches("##.+"))
+				{
+					//This line shows the specific service for these conf setting
+					conf_array.add(fileLine);
+				}
+				else if (fileLine.matches("#.+"))
+				{
+					//This line is Configuration Comment line
+					conf_array.add(fileLine);
+					map.put(ONE, fileLine.substring(1));
+					conf_setting.add(map);
+				}
+				else if (!fileLine.matches("#.+") && !fileLine.matches("^\\s*$"))
+				{
+					//This line is Setting line
+					int end_index = fileLine.indexOf("=");
+					if (end_index != -1)
+					{
+						map.put(TWO, fileLine.substring(0, end_index).toUpperCase());
+						map.put(THREE, fileLine.substring(end_index + 1));
+						conf_table.put(map.get(TWO), map.get(THREE));
+
+						conf_array.add(fileLine);
+
+						if (map.get(TWO).toString().matches("TOMCAT.SERVER"))
+						{
+							tomcat_server = map.get(THREE).toString();
+						}
+						if (map.get(TWO).toString().matches("TOMCAT.PORT"))
+						{
+							tomcat_port = map.get(THREE).toString();
+						}
+						if (map.get(TWO).toString().matches("MYSQL.ADMIN.USER"))
+						{
+							mysql_adminuser = map.get(THREE).toString();
+						}
+						if (map.get(TWO).toString().matches("MYSQL.SERVER"))
+						{
+							mysql_server = map.get(THREE).toString();
+						}
+						if (map.get(TWO).toString().matches("MYSQL.PORT"))
+						{
+							mysql_port = map.get(THREE).toString();
+						}
+
+						conf_setting.add(map);
+					}
+				}
+				else
+				{
+					// Wrong character in the line
+				}
+			}
+
+			conf_in.close();
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public void clearTable()
+	{
+		conf_display.clear();
+	}
+
+	private boolean checkTomcatServer()
+	{
+		try
+		{
+			if (tomcat_server != "" && tomcat_port != "")
+			{
+				String http_string = "http://" + tomcat_server + ":" + tomcat_port + "/";
+				URL tomcatURL = new URL(http_string);
+				HttpURLConnection tomcatConn = (HttpURLConnection) tomcatURL.openConnection();
+				tomcatConn.connect();
+				//necessary to get a Response, even if you don't care about contents
+				InputStream connIn = tomcatConn.getInputStream();
+				int pageStatus = tomcatConn.getResponseCode();
+				if (pageStatus == HttpURLConnection.HTTP_NOT_FOUND)
+				{//404 error
+					//Tomcat is not running
+					tomcat_server_up = false;
+				}
+				else
+				{
+					tomcat_server_up = true;
+				}
+			}
+			else
+			{
+				System.err.println("Either Tomcat server or Tomcat port was not set properly");
+			}
+		}
+		catch (Exception ex)
+		{
+			tomcat_server_up = false;
+			System.err.println(ex.getMessage());
+		}
+		return tomcat_server_up;
+	}
+
+	private boolean checkMysqlServer()
+	{
+		Connection conn = null;
+		try
+		{
+			String mysql_userName = "gsdl3reader";
+			String mysql_password = "";
+			String mysql_url = "jdbc:mysql://" + mysql_server + ":" + mysql_port + "/localsite_gs3mgdemo";
+			//System.err.println("what is Mysql_URL:" + mysql_url);
+			Class.forName("com.mysql.jdbc.Driver").newInstance();
+			conn = DriverManager.getConnection(mysql_url, mysql_userName, mysql_password);
+			if (conn == null)
+			{
+				mysql_server_up = false;
+			}
+			else
+			{
+				mysql_server_up = true;
+			}
+		}
+		catch (Exception e)
+		{
+			mysql_server_up = false;
+			System.err.println("Cannot connect to database server");
+		}
+		return mysql_server_up;
+	}
+
+	public class ConfSettingTableModel extends AbstractTableModel
+	{
+		String[] columnNames = { "Configuration Parameter", "Configuration Value" };
+
+		public int getColumnCount()
+		{
+			return columnNames.length;
+		}
+
+		public int getRowCount()
+		{
+			return display_row_count;
+		}
+
+		public String getColumnName(int col)
+		{
+
+			return columnNames[col];
+		}
+
+		public Object getValueAt(int row, int col)
+		{
+			try
+			{
+				return conf_display.get(row).get(col + 1);
+			}
+			catch (Exception ex)
+			{
+				return "";
+			}
+		}
+
+		public boolean isCellEditable(int row, int col)
+		{
+			if (col == 0)
+			{
+				return false;
+			}
+			else if (conf_display.get(row).get(col).toString().matches("MYSQL.+"))
+			{
+				mysql_server_up = checkMysqlServer();
+				if (mysql_server_up)
+				{
+					return false;
+				}
+			}
+			else if (conf_display.get(row).get(col).toString().matches("TOMCAT.+"))
+			{
+				tomcat_server_up = checkTomcatServer();
+				if (tomcat_server_up)
+				{
+					return false;
+				}
+			}
+			return true;
+		}
+
+		public void setValueAt(Object value, int row, int col)
+		{
+			if (!isCellEditable(row, col))
+				return;
+			if (conf_display.get(row) == null)
+			{
+				ArrayList<Object> list = new ArrayList<Object>();
+				list.add(col + 1, value);
+				conf_display.add(row, list);
+			}
+			else
+			{
+				conf_display.get(row).add(col + 1, value);
+			}
+			project_conf_changed = true;
+			//System.err.println("**ConfDisplay:" + conf_display[row][col].toString());
+			conf_table.put(conf_display.get(row).get(col).toString(), value);
+			fireTableCellUpdated(row, col + 1);
+			updateUI();
+		}
+	}
+
+	public int showProjectConf(String conf_param)
+	{
+		conf_table_pane.setVisible(true);
+		clearPrevConfPane();
+		confContent_pane.add(conf_table_pane, BorderLayout.CENTER);
+		confContent_pane.revalidate();
+		clearTable();
+
+		int row_count = 0;
+		for (int i = 0; i < conf_setting.size(); i++)
+		{
+			if (conf_setting.get(i).get(TWO) != null && conf_setting.get(i).get(TWO).toString().matches("^(" + conf_param + ").+"))
+			{
+				row_count = row_count + 1;
+
+				ArrayList<Object> list = new ArrayList<Object>();
+				list.add(conf_setting.get(i).get(ONE));
+				list.add(conf_setting.get(i).get(TWO));
+				list.add(conf_table.get(conf_setting.get(i).get(TWO)));
+				conf_display.add(list);
+			}
+		}
+		return row_count;
+	}
+
+	public void displaySiteConf(SiteConfSetting site_conf_tmp)
+	{
+		clearPrevConfPane();
+		confContent_pane.add(site_conf_tmp, BorderLayout.CENTER);
+		try
+		{
+			site_conf_tmp.showText();
+			site_conf_tmp.goTop();
+			confContent_pane.revalidate();
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public void clearPrevConfPane()
+	{
+		confContent_pane.remove(conf_table_pane);
+		confContent_pane.remove(site_conf);
+		confContent_pane.remove(interface_conf);
+	}
+
+	private class ConfTreeListener implements TreeSelectionListener
+	{
+		public void valueChanged(TreeSelectionEvent e)
+		{
+			String option = conf_tree.getLastSelectedPathComponent().toString();
+			conf_table_pane.setVisible(true);
+			if (option == "Compile")
+			{
+				conf_pane_name = "COMPILE";
+				display_row_count = showProjectConf(conf_pane_name);
+				inner_button_pane.removeAll();
+				inner_button_pane.setLayout(new GridLayout(1, 3));
+			}
+			else if (option == "Tomcat")
+			{
+				conf_pane_name = "TOMCAT";
+				display_row_count = showProjectConf(conf_pane_name);
+				//Setup Tomcat button control pane
+				inner_button_pane.removeAll();
+				inner_button_pane.setLayout(new GridLayout(1, 3));
+				if (checkTomcatServer())
+				{
+					changeTomcatButtonPane1();
+				}
+				else
+				{
+					changeTomcatButtonPane2();
+				}
+			}
+			else if (option == "Proxy")
+			{
+				conf_pane_name = "PROXY";
+				display_row_count = showProjectConf(conf_pane_name);
+				//Setup Proxy button control pane
+				inner_button_pane.removeAll();
+				inner_button_pane.setLayout(new GridLayout(1, 3));
+			}
+			else if (option == "MySQL")
+			{
+				conf_pane_name = "MYSQL";
+				display_row_count = showProjectConf(conf_pane_name);
+				// Setup MySQL button control pane
+				inner_button_pane.removeAll();
+				inner_button_pane.setLayout(new GridLayout(1, 3));
+				if (checkMysqlServer())
+				{
+					changeMysqlButtonPane1();
+				}
+				else
+				{
+					changeMysqlButtonPane2();
+				}
+			}
+			else if (option == "GSDL")
+			{
+				conf_pane_name = "GSDL";
+				display_row_count = showProjectConf(conf_pane_name);
+				inner_button_pane.removeAll();
+			}
+			else if (option == "SiteConfig")
+			{
+				conf_pane_name = "SiteConfig";
+				inner_button_pane.removeAll();
+				displaySiteConf(site_conf);
+			}
+			else if (option == "InterfaceConfig")
+			{
+				conf_pane_name = "InterfaceConfig";
+				inner_button_pane.removeAll();
+				displaySiteConf(interface_conf);
+			}
+			updateUI();
+		}
+	}
+
+	public void modeChanged(int mode)
+	{
+		return;
+	}
+
+	public void gainFocus()
+	{
+		return;
+	}
+
+	public void changeTomcatButtonPane1()
+	{
+		inner_button_pane.removeAll();
+		inner_button_pane.setLayout(new GridLayout(1, 2));
+		inner_button_pane.add(restart_tomcat_button);
+		inner_button_pane.add(shutdown_tomcat_button);
+		restart_tomcat_button.setEnabled(true);
+		shutdown_tomcat_button.setEnabled(true);
+		updateUI();
+	}
+
+	public void changeTomcatButtonPane2()
+	{
+		inner_button_pane.removeAll();
+		inner_button_pane.setLayout(new GridLayout(1, 2));
+		inner_button_pane.add(startup_tomcat_button);
+		inner_button_pane.add(shutdown_tomcat_button);
+		startup_tomcat_button.setEnabled(true);
+		shutdown_tomcat_button.setEnabled(false);
+		updateUI();
+	}
+
+	public void changeMysqlButtonPane1()
+	{
+		inner_button_pane.removeAll();
+		inner_button_pane.setLayout(new GridLayout(1, 2));
+		inner_button_pane.add(restart_mysql_button);
+		inner_button_pane.add(shutdown_mysql_button);
+		restart_mysql_button.setEnabled(true);
+		shutdown_mysql_button.setEnabled(true);
+		updateUI();
+	}
+
+	public void changeMysqlButtonPane2()
+	{
+		inner_button_pane.removeAll();
+		inner_button_pane.setLayout(new GridLayout(1, 2));
+		inner_button_pane.add(startup_mysql_button);
+		inner_button_pane.add(shutdown_mysql_button);
+		startup_mysql_button.setEnabled(true);
+		shutdown_mysql_button.setEnabled(false);
+		updateUI();
+	}
+
+	public void afterDisplay()
+	{
+		return;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/DeleteLogFilePrompt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/DeleteLogFilePrompt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/DeleteLogFilePrompt.java	(revision 31634)
@@ -0,0 +1,120 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ * Modified by: Chi-Yu Huang
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+/*
+import org.greenstone.gatherer.Gatherer;
+import org.greenstone.gatherer.LocalLibraryServer;
+import org.greenstone.gatherer.collection.BasicCollectionConfiguration;
+import org.greenstone.gatherer.file.WorkspaceTree;  // !!! Don't like this here
+import org.greenstone.gatherer.util.ArrayTools;
+import org.greenstone.gatherer.util.StaticStrings;*/
+
+
+/** This class provides the functionality to delete current collections from the 
+ ** GSDLHOME/collect/ directory. The user chooses the collection from a list, where 
+ ** each entry also displays details about itself, confirms the delete of a collection 
+ ** by checking a checkbox then presses the ok button to actually delete the collection.
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version 1.0
+ */
+public class DeleteLogFilePrompt 
+	extends JDialog {
+    
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private DeleteLogFilePrompt prompt = null;
+    private File log_file = null;
+
+    //A flag showing the deleting status
+    private boolean current_logfile_deleted = false;
+
+    /** The size of the delete prompt screen. */
+    static final Dimension SIZE = new Dimension(500, 500);
+
+    // Constructor.
+    public DeleteLogFilePrompt() {
+	super();
+	prompt = this;
+    }
+    
+    /** Destructor. */
+    public void destroy() {
+	prompt = null;
+    }
+
+    /** This method causes the Delete Confirmation Dialog to be displayed. 
+     * returns true if it has deleted the log file that is currently open */
+    public boolean display(File log_file) {
+	int result = JOptionPane.showConfirmDialog((Component) null, "Do you really want to delete this log file", "Delete Confirmation", JOptionPane.YES_NO_OPTION);
+	if ( result == JOptionPane.YES_OPTION) {
+	    log_file.delete();
+	    current_logfile_deleted = true;
+	    JOptionPane.showMessageDialog(prompt,log_file.getPath() + " log file has been deleted. You have to restart the action to create the log file");
+	} else if (result == JOptionPane.NO_OPTION) {
+	    current_logfile_deleted = false;
+	    JOptionPane.showMessageDialog(prompt, "Cancel deleting the " + log_file.getPath() +" log file");
+	    prompt.dispose();
+	}
+	return current_logfile_deleted;
+    }
+    
+    
+    /** Shows a delete complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully deleted.
+     * @see org.greenstone.gatherer.collection.Collection
+     */
+    /*public void resultPrompt(boolean success) {
+	if (success) {
+	    JOptionPane.showMessageDialog(prompt,"Selected log file has been deleted. You have to restart the action to create the log file");
+	} else {
+	    JOptionPane.showMessageDialog(prompt, "Cancel deleting the log file");
+	}
+	}*/
+    
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+    public void actionPerformed(ActionEvent event) {
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/LogPane.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/LogPane.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/LogPane.java	(revision 31634)
@@ -0,0 +1,417 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * Modified: 03.2005
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+
+import org.greenstone.admin.GAIManager;
+import org.greenstone.admin.GAI;
+import org.greenstone.util.Configuration;
+
+/** The Log pane is to view the status of relevant log files in GSIII
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version
+ */
+public class LogPane
+    extends JPanel
+    implements ActionListener {
+
+    /* The pane to demonstrate log information, including  the log files being 
+     *monitored and their content*/
+    protected JSplitPane main_log_pane = null;
+
+    /** The panel that contains a log_list */
+    private JPanel logList_pane =  null;
+
+    /** The panel that contains the log content. */
+    private JPanel logContent_pane =  null;
+
+    /** The List showing all the log files concerned. */
+    private JList log_list=null;
+
+    /** The scrollable area into which the log content is placed. */
+    private JScrollPane log_content = null;
+
+    /** The label at the top of the logList_pane. */
+    private JLabel logList_label =  null;
+
+    /** The label shown at the top of the logContent Pane. */
+    private JLabel logContent_label  =  null;
+
+    /* Log TEXT AREA*/
+    private JTextArea log_textarea = null;
+
+    // The control buttons used to manipulate log Pane
+    protected JPanel button_pane = null;
+    
+    /** Buttons */
+    private JButton reload_button = null;
+    private JButton clear_button = null;
+
+    /*The pane which contains the controls for log files */
+    private JPanel control_pane = null;
+
+    private ArrayList<File> logFiles = new ArrayList<File>();
+    private File currentlySelectedFile = null;
+      
+    /** The various sizes for the screen layout*/
+    static private Dimension MIN_SIZE = new Dimension( 90,  90);
+    static private Dimension LIST_SIZE  = new Dimension(200, 450);
+    static private Dimension LOGPANE_SIZE = new Dimension (800,450);
+
+    //Constructor
+    public LogPane() {
+
+	logFiles.add(new File (GAI.tomcat_home+File.separatorChar+"logs"+File.separatorChar+"catalina.out"));
+	logFiles.add(new File (GAI.getGSDL3ExtensionHome() + File.separatorChar + "logs" + File.separatorChar + "ext.log"));
+
+	// create all the necessary panes
+	control_pane = new JPanel();
+	button_pane = new JPanel();
+
+	// Log_Pane
+	main_log_pane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
+	main_log_pane.setPreferredSize(LOGPANE_SIZE);
+	//log_pane.setSize(LOGPANE_SIZE);
+	
+	// Log_list
+	String[] log_files = { "Tomcat log file", "Extension log file"};
+	log_list = new JList(log_files);
+	log_list.setBorder(BorderFactory.createLoweredBevelBorder());
+	log_list.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+	log_list.setVisibleRowCount(4);
+	log_list.setBackground (Configuration.getColor("coloring.workspace_tree_background"));
+	log_list.setForeground (Configuration.getColor("coloring.workspace_tree_foreground"));
+	log_list.addListSelectionListener(new LogListListener());
+	
+	// Log_TextArea
+	log_textarea = new JTextArea();
+	log_textarea.setEditable(false);
+		
+	// Buttons
+	ReloadButtonListener rbl = new ReloadButtonListener();
+	ClearButtonListener dbl = new ClearButtonListener();
+	ImageIcon reloadButtonIcon = new ImageIcon(GAI.images_path + "toolbarButtonGraphics/general/Refresh16.gif");
+	ImageIcon clearButtonIcon = new ImageIcon(GAI.images_path + "toolbarButtonGraphics/general/Delete16.gif");
+
+	//reload_button = new JButton("Reload", reloadButtonIcon);
+	reload_button = new JButton();
+	reload_button.addActionListener(rbl);
+	reload_button.setEnabled(false);
+	reload_button.setMnemonic(KeyEvent.VK_R);
+	reload_button.setText(GAI.dictionary.get("LogPane.Reload_Log"));
+	reload_button.setToolTipText(GAI.dictionary.get("LogPane.Reload_Log_Tooltip"));
+
+	clear_button = new JButton();
+	clear_button.addActionListener(dbl);
+	clear_button.setEnabled(false);
+	clear_button.setMnemonic(KeyEvent.VK_C);
+	clear_button.setText(GAI.dictionary.get("LogPane.Clear_Log"));
+	clear_button.setToolTipText(GAI.dictionary.get("LogPane.Clear_Log_Tooltip"));
+
+	rbl = null;
+	dbl = null;
+    }
+
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+
+    public void actionPerformed(ActionEvent event) {
+    }
+
+
+    /** This method is callsed to actually layout the components.*/
+    public void display() {
+	// Create Components.
+	//KeyListenerImpl key_listener = new KeyListenerImpl();
+	//MouseListenerImpl mouse_listener = new MouseListenerImpl();
+	//this.addKeyListener(key_listener);
+
+	// logList_Pane
+	logList_pane = new JPanel();
+	logList_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	logList_label = new JLabel();
+	logList_label.setOpaque(true);
+	logList_label.setBackground(Configuration.getColor("coloring.workspace_heading_background"));
+	logList_label.setForeground(Configuration.getColor("coloring.workspace_heading_foreground"));
+	logList_label.setText(GAI.dictionary.get("LogPane.Log_List"));
+
+	// logContent_Pane
+	logContent_pane = new JPanel();
+	logContent_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	logContent_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	logContent_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	logContent_label = new JLabel();
+	logContent_label.setOpaque(true);
+	logContent_label.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	logContent_label.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	logContent_label.setText(GAI.dictionary.get("LogPane.Log_Content"));
+	
+	// TEXTAREA Layout
+	log_content = new JScrollPane(log_textarea);
+
+	// Button Layout 
+	button_pane.setLayout (new GridLayout(1,3));
+	button_pane.add(reload_button);
+	button_pane.add(clear_button);
+	
+	control_pane.setLayout (new BorderLayout());
+	//control_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	control_pane.setBorder(BorderFactory.createEmptyBorder(05,10,5,10));
+	control_pane.setPreferredSize(new Dimension(50,50));
+	control_pane.setSize(new Dimension(50,50));
+	control_pane.add (button_pane, BorderLayout.CENTER);
+
+	// Layout Components
+	logList_pane.setLayout(new BorderLayout());
+	logList_pane.add(log_list, BorderLayout.CENTER);
+	logList_pane.add(logList_label, BorderLayout.NORTH);
+		
+	logContent_pane.setLayout(new BorderLayout());
+	logContent_pane.add(log_content,BorderLayout.CENTER);
+	logContent_pane.add(logContent_label, BorderLayout.NORTH);
+	logContent_pane.add(control_pane, BorderLayout.SOUTH);
+	
+	JPanel filterPanel = new JPanel();
+	filterPanel.setLayout(new BorderLayout());
+
+	JTextField filterField = new JTextField();
+	filterPanel.add(filterField, BorderLayout.CENTER);
+	
+	JButton filterButton = new JButton("Filter");
+	filterButton.addActionListener(new FilterButtonListener(filterField));
+	filterPanel.add(filterButton, BorderLayout.EAST);
+
+	JPanel contentFilterPanel = new JPanel();
+	contentFilterPanel.setLayout(new BorderLayout());
+	contentFilterPanel.add(logContent_pane, BorderLayout.CENTER);
+	contentFilterPanel.add(filterPanel, BorderLayout.NORTH);
+
+	main_log_pane.add(logList_pane, JSplitPane.LEFT);
+	main_log_pane.add(contentFilterPanel, JSplitPane.RIGHT);
+	main_log_pane.setDividerLocation(LIST_SIZE.width - 10);
+	
+	this.setLayout(new BorderLayout());
+	this.add(main_log_pane, BorderLayout.CENTER);
+	//this.add(control_pane, BorderLayout.SOUTH);
+    }
+
+    public class FilterButtonListener implements ActionListener
+    {
+	JTextField _filterField = null;
+
+	public FilterButtonListener(JTextField filterField)
+	{
+	    _filterField = filterField;
+	}
+	
+	public void actionPerformed(ActionEvent e)
+	{
+	    readFile(currentlySelectedFile.getPath(), _filterField.getText());
+	}
+    }
+
+    /** Called whenever this pane gains focus, this method ensures that the various 
+     **tree renderers are correctly colouring the tree (as these settings sometimes get lost).
+     * @param event A <strong>FocusEvent</strong> containing details about the focus action performed.
+     */
+    /*public void focusGained(FocusEvent event) {
+	DefaultTreeCellRenderer def = new DefaultTreeCellRenderer();
+	DefaultTreeCellRenderer w = (DefaultTreeCellRenderer)workspace_tree.getCellRenderer();
+	DefaultTreeCellRenderer c = (DefaultTreeCellRenderer)collection_tree.getCellRenderer();
+	if(event.getSource() == workspace_tree) {
+	    w.setBackgroundSelectionColor(def.getBackgroundSelectionColor());
+	    c.setBackgroundSelectionColor(Color.lightGray);
+	}
+	else if(event.getSource() == collection_tree) {
+	    c.setBackgroundSelectionColor(def.getBackgroundSelectionColor());
+	    w.setBackgroundSelectionColor(Color.lightGray);
+	}
+	repaint();
+	}*/
+    
+    /** Implementation side-effect, not used in any way.
+     * @param event A <strong>FocusEvent</strong> containing details about the focus action performed.
+     */
+    /*public void focusLost(FocusEvent event) {
+      }*/
+    
+    /** Called to inform this control panel that it has just gained focus as an effect of the user clicking on its tab.
+     */
+    public void gainFocus() {
+	// Update the meta-audit view to show the current selection, if any.
+	//Gatherer.g_man.meta_audit.setRecords(getCollectionTreeSelection());
+    }
+    
+    /** Called whenever the detail mode changes to ensure the filters are at an appropriate 
+     **level (ie only editable by those that understand regular expression matching)
+     * @param mode the mode level as an int
+     */
+    /*  public void modeChanged(int mode) {
+	collection_filter.setEditable(mode > Configuration.LIBRARIAN_MODE);
+	workspace_filter.setEditable(mode > Configuration.LIBRARIAN_MODE);
+	}*/
+    
+    private class KeyListenerImpl
+	extends KeyAdapter {
+	private boolean vk_left_pressed = false;
+	public void keyReleased(KeyEvent event) {
+	}
+	// we need to watch for left clicks on an unopened folder - should shift the focus to the
+	//parent folder. But because there is some other mysterious key listener that does opening and 
+	//closing folders on right and left clicks, we must detect the situation before the other handler 
+	//has done its job, and process it after.*/
+	public void keyPressed(KeyEvent event) {
+	    if (event.getKeyCode() == KeyEvent.VK_LEFT) {
+		
+	    }
+	}     
+    }
+
+    /** This class listens for mouse clicks and responds right mouse button clicks (popup menu)., left
+     **  mouse button:log file selected */
+    private class MouseListenerImpl
+	extends MouseAdapter {
+	/* Any subclass of MouseAdapter can override this method to respond to mouse click events. 
+	 *In this case we want to open a pop-up menu if we detect a right mouse click over one of our 
+	 *registered components, and start an external application if someone double clicks on a certain 
+	 *file record. */
+	public void mouseReleased(MouseEvent e) {}
+	public void mouseExited(MouseEvent e) {}
+	public void mousePressed(MouseEvent e) {}
+	public void mouseEntered(MouseEvent e) {}
+	
+    }
+    
+    /** This class serves as the listener for actions on the build button. */
+    private class ReloadButtonListener
+	implements ActionListener {
+	/** As the log files could be modified in the runtime environment
+	 *  This button is to reload the log file whenever user want to */
+	public void actionPerformed(ActionEvent event) {
+	    // Remember that for lower thresholds the above doesn't work, so try this instead
+	    reload_button.setEnabled(true);
+	    clear_button.setEnabled(true);
+	    updateLogsContent(currentlySelectedFile.getPath());
+	}
+    }
+    
+    private class ClearButtonListener
+	implements ActionListener {
+	public void actionPerformed(ActionEvent event) {
+	    if (!currentlySelectedFile.exists()){
+		JOptionPane.showMessageDialog((Component) null,currentlySelectedFile.getPath() + " log file does not exist");
+		return;
+	    } else {
+		ClearLogFilePrompt cfp  = new ClearLogFilePrompt();
+		boolean file_cleared = cfp.display(currentlySelectedFile);
+		if (file_cleared) {
+		    log_textarea.setText("");
+		}
+	    }
+	    clear_button.setEnabled(false);
+	}
+    }
+    
+    public void updateLogsContent(String filename){
+	if (!currentlySelectedFile.exists()){ 
+	    log_textarea.setText("");
+	    JOptionPane.showMessageDialog((Component) null, filename+" log file does not exist");  
+	    clear_button.setEnabled(false);
+	} else {
+	    readFile(filename, "");
+	}
+    }
+
+    public void readFile (String filename, String filter) {
+	log_textarea.setText("");
+	String fileLine;
+	try {
+	    BufferedReader in = new BufferedReader(new FileReader(filename));
+	    while ((fileLine = in.readLine()) != null) {
+		if(fileLine.contains(filter)){
+		    log_textarea.append(fileLine);
+		    log_textarea.append("\n");
+		}
+	    }
+	} catch (Exception e) {
+	    e.printStackTrace();
+	}	    
+    }
+   
+    private class LogListListener implements ListSelectionListener {
+	public void valueChanged (ListSelectionEvent e){
+	    if (e.getValueIsAdjusting() == false){
+		if (log_list.getSelectedIndex() == -1){
+		    //no selection
+		} else if (log_list.getSelectedIndex () == 0 ) {
+		    File log_file = logFiles.get(0);
+		    currentlySelectedFile = log_file;
+		    String filename = log_file.getPath();
+		    updateLogsContent(filename);
+		    reload_button.setEnabled(true);
+		    clear_button.setEnabled(true);
+		} else if (log_list.getSelectedIndex () == 1) {
+		    File log_file = logFiles.get(1);
+		    currentlySelectedFile = log_file;
+		    String filename = log_file.getPath();
+		    updateLogsContent(filename);
+		    clear_button.setEnabled(true);
+		    reload_button.setEnabled(true);
+		} else if (log_list.getSelectedIndex () == 2) {
+		    log_textarea.setText("");
+		    JOptionPane.showMessageDialog((Component) null,"This file has not been defined yet");	
+		    clear_button.setEnabled(false);
+		    reload_button.setEnabled(false);
+		}
+	    }
+	}
+    }
+    public void modeChanged (int mode){
+	return;
+    }
+    public void afterDisplay() {
+	return;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/MenuBar.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/MenuBar.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/MenuBar.java	(revision 31634)
@@ -0,0 +1,203 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.util.Configuration;
+
+/** The menu bar for the Administration tools main GUI. 
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version
+ */
+public class MenuBar
+    extends JMenuBar {
+    /** The icon to be displayed alongside the context choosen help file. */
+    private int current_tab = -1;
+    
+    private JMenu file = null;
+    private JMenu edit = null;
+    public JMenu help  = null;
+    // public JMenuItem file_cdimage    = null;
+    // public JMenuItem file_close      = null;
+    //public JMenuItem file_delete     = null;
+    public JMenuItem file_exit = null;
+    public JMenuItem file_save = null;
+    public JMenuItem help_general;
+    public JMenuItem help_conf;
+    public JMenuItem help_ext;
+    public JMenuItem help_monitor;
+    public JMenuItem help_log;
+    public JMenuItem help_about;
+
+    static public ImageIcon HELP_ICON = null; 
+    static public ImageIcon BLANK_ICON = null;
+    
+    //    public MenuBar(MenuListener Menu_listener)
+    public MenuBar(MenuListener menu_listener)
+    {
+	HELP_ICON = new ImageIcon(GAI.images_path +"help.gif");
+	BLANK_ICON = new ImageIcon(GAI.images_path+"blank.gif");
+
+	//file = new JMenu("File");
+	file = new JMenu();
+	file.setMnemonic(KeyEvent.VK_F);
+	file.setText(GAI.dictionary.get("Menu.File"));
+	
+	//file_save = new JMenuItem("Save");
+	file_save = new JMenuItem();
+	file_save.addActionListener(GAI.ga_man);
+	file_save.setMnemonic(KeyEvent.VK_S);
+	file_save.setText(GAI.dictionary.get("Menu.File_Save"));
+	
+	//file_exit = new JMenuItem("Exit");
+	file_exit = new JMenuItem();
+	file_exit.addActionListener(GAI.ga_man);
+	file_exit.setMnemonic(KeyEvent.VK_X);
+	file_exit.setText(GAI.dictionary.get("Menu.File_Exit"));
+
+	//Layout (File menu)
+	file.add(file_save);
+	file.add(new JSeparator());
+	file.add(file_exit);
+
+	// Edit menu
+	//edit = new JMenu("Edit");
+	edit = new JMenu();
+	edit.setMnemonic(KeyEvent.VK_E);
+	edit.setText(GAI.dictionary.get("Menu.Edit"));
+			  
+	// Help menu
+	//help = new JMenu("Help");
+	help = new JMenu();
+	help.setMnemonic(KeyEvent.VK_H);
+	help.setIcon(HELP_ICON);
+	help.setText(GAI.dictionary.get("Menu.Help"));
+	
+	help_general = new JMenuItem();
+	help_general.addActionListener(GAI.ga_man);
+	help_general.setText(GAI.dictionary.get("Source.General"));
+		
+	help_conf = new JMenuItem();
+	help_conf.addActionListener(GAI.ga_man);
+	help_conf.setText(GAI.dictionary.get("GAI.Configuration"));
+		
+	help_ext = new JMenuItem();
+	help_ext.addActionListener(GAI.ga_man);
+	help_ext.setText(GAI.dictionary.get("GAI.Ext"));
+	
+	help_monitor = new JMenuItem();
+	help_monitor.addActionListener(GAI.ga_man);
+	help_monitor.setText(GAI.dictionary.get("GAI.Monitor"));
+	
+	help_log = new JMenuItem();
+	help_log.addActionListener(GAI.ga_man);
+	help_log.setText(GAI.dictionary.get("GAI.Log"));
+	
+	help_about = new JMenuItem();
+	help_about.addActionListener(GAI.ga_man);
+	help_about.setText(GAI.dictionary.get("Menu.Help_About"));
+			
+	
+	// Layout (help menu)
+	help.add(help_general);
+	help.add(new JSeparator());
+	if (Configuration.get("admin.conf")) {
+	    help.add(help_conf);
+	}
+	if (Configuration.get("admin.ext")) {
+	    help.add(help_ext);
+	}
+	if (Configuration.get("admin.monitor")) {
+	    help.add(help_monitor);
+	}
+	if (Configuration.get("admin.log")) {
+	    help.add(help_log);
+	}
+	
+	help.add(new JSeparator());
+	help.add(help_about);
+
+	// Layout (menu bar)
+	add(file);
+	add(Box.createHorizontalStrut(15));
+	add(edit);
+	add(Box.createHorizontalGlue());
+	add(help);
+    }
+
+    /** Once a quit has been requested by the user, prevent any further menu selections. */
+    public void exit() {
+	file.setEnabled(false);
+	edit.setEnabled(false);
+	help.setEnabled(false);
+    }
+
+    /*public void refresh(int refresh_reason, boolean ready)
+    {
+	file_close.setEnabled(ready);
+	file_save.setEnabled(ready);
+	}*/
+
+
+    /** In order to provide context aware help advice we keep track of which
+     *  tab the user has open, and then highlight that help menu item with
+     *  separators.
+     *  @param tab_index The index of the selected tab (0-7).
+     */
+    public void tabSelected(int tab_index) {
+	JMenuItem selected;
+	if(current_tab != -1) {
+	    // Remove the image
+	    selected = help.getItem(current_tab);
+	    if(selected != null) {
+		selected.setIcon(BLANK_ICON);
+	    }
+	}
+	current_tab = tab_index + 2;
+	selected = help.getItem(current_tab);
+	if(selected != null) {
+	    selected.setIcon(HELP_ICON);
+	    
+	}
+	selected = null;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ParsingProgress.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ParsingProgress.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ParsingProgress.java	(revision 31634)
@@ -0,0 +1,123 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Core GUI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: John Thompson, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.lang.Object;
+import javax.swing.*;
+
+/** This class provides a progress bar to be displayed whenever the module 
+ * is running some background tasks
+ * @author John Thompson, Greenstone Digital Library, University of Waikato
+ * @version 2.3
+ */
+public class ParsingProgress
+    extends JDialog {
+    //implements ActionListener {
+    
+    public final static int ONE_SECOND = 1000;
+
+    /** The content pane within the dialog box. */
+    private JPanel content_pane = null;
+    private Timer timer;
+    /** The progress bar itself. */
+    private JProgressBar progressBar = null;
+    
+    /** The default size of the progress dialog. */
+    static final Dimension SIZE = new Dimension(400,75);
+    
+    
+    /** Constructor.
+     * @param title The title to show on the dialog, as a <strong>String</strong>.
+     * @param message The message to show on the dialog, as a <strong>String</strong>.
+     * @param max The total amount of the time the process is running before 
+     * the progress bar can be disposed of, as an <i>int</i>.
+     */
+    public ParsingProgress(String title, String message, int max) {
+	super();
+	this.setSize(SIZE);
+	this.setTitle(title);
+	//updateUI();
+
+	// Creation
+	this.content_pane = (JPanel) getContentPane();
+	this.content_pane = new JPanel();
+	
+	this.progressBar = new JProgressBar();
+	this.progressBar.setMaximum(max);
+	this.progressBar.setIndeterminate(true);
+       
+
+	// Layout
+	this.content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	this.content_pane.setLayout(new BorderLayout());
+	this.content_pane.add(new JLabel(message), BorderLayout.NORTH);
+	this.content_pane.add(progressBar, BorderLayout.CENTER);
+	
+	//Create a timer
+	//timer = new Timer (ONE_SECOND, new ActionListener() {
+	//	public void actionPerformed (ActionEvent evt) {
+		    progressBar.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+		    
+		    //container.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
+		    /*  progressBar.setValue(max);
+			Toolkit.getDefaultToolkit().beep();
+			timer.stop();
+			startButton.setEnabled(true);
+			setCursor(null); //turn off the wait cursor
+			progressBar.setValue(progressBar.getMinimum());*/
+		    //}
+		    //});
+	
+	// Center and display
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - SIZE.width) / 2,
+		    (screen_size.height - SIZE.height) / 2);
+	
+	setVisible(true);
+    }
+    
+    public void destroy() {
+    }
+    
+    /** Method which increments the progress count by one, which should be called 
+     ** after every successful parsing of a classifier or plugin.
+     */
+    public void inc() {
+	this.progressBar.setValue(progressBar.getValue() + 1);
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SaveChangePrompt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SaveChangePrompt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SaveChangePrompt.java	(revision 31634)
@@ -0,0 +1,108 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+import org.greenstone.admin.gui.ConfPane;
+
+/** This class provides the functionality to show a prompt to save changed file  
+ * @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * @version 1.0
+ */
+public class SaveChangePrompt
+    extends JDialog {
+    
+    /** A reference to ourself so any inner-classes can dispose of us. */
+    private SaveChangePrompt prompt = null;
+    private File file = null;
+        
+    //A flag showing the saving change status
+    private boolean current_change_saved = false;
+    
+    /** The size of the save prompt screen. */
+    static final Dimension SIZE = new Dimension(500, 500);
+    
+    // Constructor.
+    public SaveChangePrompt() {
+	super();
+	prompt = this;
+    }
+    
+    /** Destructor. */
+    public void destroy() {
+	prompt = null;
+    }
+    
+    /** This method causes the Save Change Confirmation Dialog to be displayed.
+     * returns true if it has save the change file that is currently open */
+    // public boolean display(File file) {
+	/*int result = JOptionPane.showConfirmDialog((Component) null, "Do you really want to save the change", "Save Confirmation", JOptionPane.YES_NO_OPTION);
+	  if ( result == JOptionPane.YES_OPTION) {
+	  ConfPane confPane = new ConfPane();
+	  confPane.saveNewConfSetting();
+	  confPane.writeConfFile(file);
+	  current_change_saved = true;
+	  } else if (result == JOptionPane.NO_OPTION) {
+	    current_change_saved = false;
+	    prompt.dispose();
+	    }
+	    resultPrompt(current_change_saved);
+	    return current_change_saved;*/
+    //}
+    
+    /** Shows a save complete prompt. 
+     * @param success A <strong>boolean</strong> indicating if the collection was successfully saved
+     */
+    public void resultPrompt(boolean success) {
+	if (success) {
+	    JOptionPane.showMessageDialog(prompt,"The build property file has been saved.");
+	} else {
+	    JOptionPane.showMessageDialog(prompt, "Cancel saving the change to build property file");
+	}
+    }
+    
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+    public void actionPerformed(ActionEvent event) {
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SetServerPane.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SetServerPane.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SetServerPane.java	(revision 31634)
@@ -0,0 +1,371 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * Date: 05.2005
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+/* This program is intended to confirm the initial user's setting about
+ * the Tomcat server and MySQL server they would like to run GSDL3 with
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.lang.Object;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import javax.swing.table.*;
+import java.awt.event.MouseEvent;
+import java.net.*;
+import java.sql.*;
+
+import org.greenstone.util.Configuration;
+import org.greenstone.admin.gui.ParsingProgress;
+import org.greenstone.admin.GAI;
+
+
+/** The initial setting pane is to request the user to input the preferred 
+ * Tomcat and Mysql server,they would like to run GSDL3 with
+ *  @author Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *  @version
+ */
+public class SetServerPane
+    extends JFrame
+    implements ActionListener {
+
+    /* This pane is to allow the user to set up their own running Tomcat/Mysql server 
+     * before the installation of the GSDL3*/
+    // protected JScrollPane setting_message_pane = null;
+    protected JPanel main_pane = null;
+    protected JPanel button_pane = null;
+    protected JPanel outter_control_pane = null;
+    protected JScrollPane setting_table_pane = null;
+    protected JTable server_setting_table = null;
+    
+    private JButton save_button = null;
+    private JButton install_button = null;
+    private JButton exit_button = null;
+
+    ServerSettingTableModel server_setting_table_model = null;
+    private boolean setting_confirm = false;
+    private ArrayList<String> conf_array;
+    private boolean success = false;
+    private boolean file_saved = true;
+    
+    /** The various sizes for the screen layout*/
+    static private Dimension MIN_SIZE = new Dimension( 90,  90);
+    static private Dimension LIST_SIZE  = new Dimension(200, 450);
+    static private Dimension DIALOG_SIZE = new Dimension (400,300);
+    static private Dimension FRAME_SIZE = new Dimension (500, 450);
+    static private Dimension TABLE_SIZE = new Dimension(500,200);
+    static final Dimension SIZE = new Dimension(400,400);
+    
+    //Constructor
+    public SetServerPane() {
+	super();
+	this.setDefaultCloseOperation(EXIT_ON_CLOSE);
+	this.setSize(FRAME_SIZE);
+	this.setTitle("Setting Tomcat/MYSQL Servers");
+	
+	// create all the control button panes
+	button_pane  = new JPanel();
+
+	//First time running button
+	SaveButtonListener ybl = new SaveButtonListener();
+	InstallButtonListener nbl = new InstallButtonListener();
+	ExitButtonListener ebl = new ExitButtonListener();
+	
+	// ImagesIcon for the buttons
+	ImageIcon SaveButtonIcon = new ImageIcon(GAI.images_path + "refresh.gif");
+	ImageIcon InstallButtonIcon = new ImageIcon(GAI.images_path + "exit.gif");
+	ImageIcon ExitButtonIcon = new ImageIcon(GAI.images_path + "exit.gif");
+
+	save_button = new JButton("Save Setting", SaveButtonIcon);
+	save_button.addActionListener(ybl);
+	save_button.setMnemonic(KeyEvent.VK_S);
+	save_button.setToolTipText("Click this button to save up the setting");
+	save_button.setEnabled(false);
+
+	install_button = new JButton("Install GSDL3", InstallButtonIcon);
+	install_button.addActionListener(nbl);
+	install_button.setEnabled(false);
+	install_button.setMnemonic(KeyEvent.VK_I);
+	install_button.setToolTipText("Click this button to Install Greenstone III");
+
+	exit_button = new JButton("Exit Setting and Installation", ExitButtonIcon);
+	exit_button.addActionListener(ebl);
+	exit_button.setEnabled(false);
+	install_button.setMnemonic(KeyEvent.VK_I);
+	install_button.setToolTipText("Click this button to Exit the Setting and Installation of Greenstone III");
+
+	//Setting&Installation buttons
+	ybl = null;
+	nbl = null;
+	ebl = null;
+    }
+
+    /** Any implementation of ActionListener requires this method so that when an 
+     **action is performed the appropriate effect can occur.*/
+    public void actionPerformed(ActionEvent event) {
+    }
+
+    /** This method is callsed to actually layout the components.*/
+    public void display() {
+	//KeyListenerImpl key_listener = new KeyListenerImpl();
+	//MouseListenerImpl mouse_listener = new MouseListenerImpl();
+	//this.addKeyListener(key_listener);
+	
+	server_setting_table_model = new ServerSettingTableModel();
+	server_setting_table = new JTable(server_setting_table_model) {
+		protected String[] rowToolTips = {
+		    "The name of the machine that Tomcat is/will be run on",
+		    "The port number that Tomcat is/will be run on",
+		    "The base directory for the existing Tomcat installation",
+		    "The name of the machine that Mysql is/will be run on",
+		    "The port number that Mysql is/will be run on",
+		    "The base directory for the existing MYSQL installations",
+		    "The user name for the administrator of using MySQL",
+		    "The socket that Mysql is/will be run on"};
+		
+		public String getToolTipText(MouseEvent e){
+		    String tip = null;
+		    Point p = e.getPoint();
+		    int rowIndex = rowAtPoint(p);
+		    int colIndex = columnAtPoint(p);
+		    int realColumnIndex = convertColumnIndexToModel(colIndex);
+		    if (realColumnIndex == 0){
+			tip = rowToolTips[rowIndex];
+		    } else {
+			tip = super.getToolTipText(e);
+		    }
+		    return tip;
+		}
+	    };
+	
+	//Set up tableHeader
+	JTableHeader header = server_setting_table.getTableHeader();
+	header.setFont(new Font("Arial", Font.BOLD, 14));
+	
+	server_setting_table.setRowHeight(30);
+	setting_table_pane = new JScrollPane(server_setting_table);
+	
+	// Button Control Layout 
+	save_button.setEnabled(true);
+	install_button.setEnabled(true);
+	exit_button.setEnabled(true);
+	
+	button_pane.setLayout (new GridLayout(1,3));
+	button_pane.add(save_button);
+	button_pane.add(install_button);
+	button_pane.add(exit_button);
+
+	outter_control_pane = new JPanel();
+	outter_control_pane.setLayout (new BorderLayout());
+	outter_control_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	outter_control_pane.setPreferredSize(new Dimension(50,50));
+	outter_control_pane.setSize(new Dimension(50,50));
+	outter_control_pane.add (button_pane, BorderLayout.CENTER);
+	
+	main_pane = (JPanel) getContentPane();
+	main_pane.setLayout (new BorderLayout());
+	main_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
+	main_pane.add(setting_table_pane, BorderLayout.CENTER);
+	main_pane.add(outter_control_pane, BorderLayout.SOUTH);
+	main_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	main_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	
+	// Center and display
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - SIZE.width) / 2,
+		    (screen_size.height - SIZE.height) / 2);
+	setVisible(true);
+    }
+    
+    public void destroy() {
+    }
+        
+    /** This class serves as the listener for actions on the build button.*/
+    private class SaveButtonListener
+	implements ActionListener {
+	/** This button is to save any change you have added
+	 ** to the Tomcat/Mysql servers setting*/
+	public void actionPerformed(ActionEvent event) {
+	    setting_confirm = true;
+	    int result = JOptionPane.showConfirmDialog((Component) null, "Do you really want to save the setting?", "Save Confirmation", JOptionPane.YES_NO_OPTION);
+	    if ( result == JOptionPane.YES_OPTION) {
+		updateSetting (GAI.build_properties_file);
+		file_saved = writeFile (GAI.build_properties_file);
+		if (file_saved) {
+		    JOptionPane.showMessageDialog((Component) null,"Change has been saved succefully!");
+		} else {
+		    JOptionPane.showMessageDialog((Component) null,"Change has not been saved succefully!");
+		    file_saved = false;
+		} 
+	    } else if (result == JOptionPane.NO_OPTION) {
+		JOptionPane.showMessageDialog((Component) null,"Change has not been saved!");
+		file_saved = false;
+	    }
+	}
+    }
+
+    public void updateSetting(File build_properties_file) {
+	readFile (GAI.build_properties_file);
+	String new_string;
+	for (int i=0; i < server_setting_table_model.getRowCount(); i++){
+	    //System.err.println("What is the value here:" + server_setting_table_model.getValueAt(i,0).toString());
+	    if (!server_setting_table_model.getValueAt(i,1).toString().matches("^\\s*$")){
+		for (int j=0; j < conf_array.size(); j++){
+		    if (conf_array.get(j).startsWith(server_setting_table_model.getValueAt(i,0).toString().toLowerCase())){
+			//System.err.println("What is the value in conf_array:" + conf_array.get(j));
+			new_string = server_setting_table_model.getValueAt(i,0).toString()+"="+server_setting_table_model.getValueAt(i,1).toString();
+			conf_array.set(j, new_string.toLowerCase());
+		    }
+		}
+	    }
+	}
+	
+    }
+    public void readFile (File build_properties_file) {
+	String filename = build_properties_file.getPath();
+	String fileLine;
+	/*conf_array[] store all the details from the build.properties*/
+	conf_array = new ArrayList<String>();
+	try {
+	    BufferedReader conf_in = new BufferedReader(new FileReader(filename));
+	    while ((fileLine = conf_in.readLine()) != null) {
+		// Besides an empty line, all the other lines will be stored in the conf_array
+		if (!fileLine.matches("^\\s*$")) {
+		    conf_array.add(fileLine);
+		} 
+	    }
+	    conf_in.close();
+	} catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+    
+    public boolean writeFile(File build_properties_file){
+	String filename = build_properties_file.getPath();
+	try {
+	    BufferedWriter conf_out = new BufferedWriter(new FileWriter(filename));
+	    for (int j=0 ; j < conf_array.size(); j++){
+		conf_out.write(conf_array.get(j).toString());
+		conf_out.newLine();
+	    }
+	    success = true;
+	    conf_out.close();
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    success = false;
+	}
+	return success;
+    }
+
+    private class InstallButtonListener
+	implements ActionListener {
+	// Exit the Adminstration tool 
+	public void actionPerformed(ActionEvent event) {
+	    setting_confirm = false;
+	    /*if (!setting_confirm) {
+	      GAIManager gai = new GAIManager("/research/chi/gsdl3-test/gsdl3", screen_size);
+	      }*/
+	}
+    }
+    
+    private class ExitButtonListener
+	implements ActionListener {
+	// Exit the Adminstration tool 
+	public void actionPerformed(ActionEvent event) {
+	    if (!file_saved) {
+		int result = JOptionPane.showConfirmDialog((Component) null, "The value of configuration has been changed,do you really want to exit?", "Exit Confirmation", JOptionPane.YES_NO_OPTION);
+		if ( result == JOptionPane.YES_OPTION) {
+		    System.exit(1);
+		} else if (result == JOptionPane.NO_OPTION) {
+		    JOptionPane.showMessageDialog((Component) null,"Press Save button to save the change!");
+		}
+	    } else {
+		System.exit(1);
+	    }	    
+	}
+    }
+    
+    class ServerSettingTableModel extends AbstractTableModel {
+	String[] columnNames = {"Configuration Parameter",
+				"Configuration Value"};
+	Object [][] conf_display = {{"TOMCAT.SERVER", ""},
+				    {"TOMCAT.PORT", ""},
+				    {"TOMCAT.INSTALLED.PATH",""},
+				    {"MYSQL.SERVER",""},
+				    {"MYSQL.PORT",""},
+				    {"MYSQL.INSTALLED.PATH",""},
+				    {"MYSQL.ADMIN.USER",""},
+				    {"MYSQL.SOCKET",""}};
+	
+	public int getColumnCount(){
+	    return columnNames.length;
+	}
+	
+	public int getRowCount() {
+	    return conf_display.length;
+	}
+	
+	public String getColumnName(int col){
+	    return columnNames[col];
+	}
+	
+	public Object getValueAt(int row,int col){
+	    return conf_display[row][col];
+	}
+	
+	public boolean isCellEditable(int row, int col){
+	    if (col == 0){
+		return false;
+	    } else {
+		return true;
+	    }
+	}
+
+	public void setValueAt(Object value, int row, int col){
+	    conf_display[row][col] = value;
+	    fireTableCellUpdated(row,col);
+	}
+    }
+
+}
+       
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SiteConfSetting.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SiteConfSetting.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/SiteConfSetting.java	(revision 31634)
@@ -0,0 +1,151 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ * Modified: 03.2005
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.util.*;
+import java.lang.Object;
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import javax.swing.table.*;
+
+import org.greenstone.admin.GAI;
+
+/*Site configuration demonstrated in XML format
+ *@author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *@version:
+ */
+public class SiteConfSetting
+    extends JPanel {
+    // implements TextListener {
+
+    //The Text area to demonstarte configuration content
+    private JTextArea site_conf_content = null;
+
+    //The scrollable area into which the configuration content is placed
+    private JScrollPane site_conf_area = null;
+    
+    private String site_conf_name;
+    private boolean conf_changed = false;
+    private StringBuffer site_conf_in;
+    
+    public SiteConfSetting(String conf_name, File conf_file) {
+	super();
+	this.site_conf_name = conf_name;
+	/*A StringBuffer to store the content of Site Configuration files such as 
+	 *SiteConfig.xml and InterfaceConfig.xml files*/
+	this.site_conf_in = new StringBuffer();
+	this.site_conf_in = readConfFile(conf_file);
+	this.site_conf_content = new JTextArea();
+	this.site_conf_content.setEditable(true);
+	this.site_conf_content.append(this.site_conf_in.toString());
+	this.site_conf_area = new JScrollPane(site_conf_content);  
+	this.site_conf_content.getDocument().addDocumentListener(new ConfDocumentListener());
+	setLayout(new BorderLayout());
+	add(site_conf_area,BorderLayout.CENTER);
+    }
+    
+    private class ConfDocumentListener implements DocumentListener {
+        final String newline = "\n";
+	
+        public void insertUpdate(DocumentEvent e) {
+	    conf_changed = true;
+        }
+        
+	public void removeUpdate(DocumentEvent e) {
+	    conf_changed = true;
+	}
+	
+        public void changedUpdate(DocumentEvent e) {
+	    conf_changed = true;
+	}
+	
+        public void updateLog(DocumentEvent e, String action) {
+	}
+    }
+    
+    public boolean confChanged ()
+    {
+	return this.conf_changed;
+    }
+    
+    public void showText()
+    {
+	site_conf_content.getText();
+    }
+    
+    public void goTop()
+    {
+	//Go to the top of ScrollPane
+	site_conf_content.setCaretPosition(0);
+    }
+    
+    public void saveFile(File filename) 
+    {
+	try {
+	    BufferedWriter site_out = new BufferedWriter(new FileWriter(filename.getPath()));
+	    String text = site_conf_content.getText();
+	    site_out.write(text);
+	    site_out.close();
+	    conf_changed = false;
+	} catch (Exception e) {
+	    e.printStackTrace();
+	}
+    }
+    
+    private StringBuffer readConfFile(File filename)
+    {
+	StringBuffer file_buffer = new StringBuffer();
+	String fileLine;
+	try {
+	    BufferedReader in = new BufferedReader(new FileReader(filename));
+	    while ((fileLine = in.readLine())!= null) {
+		file_buffer.append(fileLine);
+		file_buffer.append("\n");
+	    }
+   	} catch (Exception e) {
+	    e.printStackTrace();
+	}	    
+	return file_buffer;
+    }
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/TabbedPane.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/TabbedPane.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/TabbedPane.java	(revision 31634)
@@ -0,0 +1,90 @@
+/**
+ *#########################################################################
+ *
+ * A component of the Gatherer application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+
+package org.greenstone.admin.gui;
+
+
+import javax.swing.JTabbedPane;
+import javax.swing.ImageIcon;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.JFrame;
+import javax.swing.JComponent;
+import java.awt.*;
+import java.awt.event.KeyEvent;
+
+public class TabbedPane extends JPanel {
+
+    private static final Dimension SIZE = new Dimension(800, 540);
+
+    public TabbedPane() {
+	super(new GridLayout(1, 1));
+	setBackground(new Color(176,208,176));
+	
+        JTabbedPane tabbedPane = new JTabbedPane();
+        JComponent panel1 = makeTextPanel("Log information Pane");
+	/*   tabbedPane.addTab("Log Pane", icon, panel1,
+	     "Log Pane");*/
+	panel1.setPreferredSize(SIZE);
+	panel1.setBackground(new Color(176,208,176));
+	tabbedPane.addTab("Log Pane", panel1);
+	tabbedPane.setMnemonicAt(0, KeyEvent.VK_L);
+      
+
+        JComponent panel2 = makeTextPanel("Runtime Status");
+        /*tabbedPane.addTab("Status Pane", icon, panel2,
+	  "Status Pane");*/
+
+	panel2.setPreferredSize(SIZE);
+	panel2.setBackground(new Color(176,208,176));
+	tabbedPane.addTab("Status Pane", panel2);
+        tabbedPane.setMnemonicAt(1, KeyEvent.VK_R);
+
+        //Add the tabbed pane to this panel.
+        add(tabbedPane);
+    }
+
+    protected JComponent makeTextPanel(String text) {
+        JPanel panel = new JPanel(false);
+        JLabel filler = new JLabel(text);
+        filler.setHorizontalAlignment(JLabel.CENTER);
+        panel.setLayout(new GridLayout(1, 1));
+        panel.add(filler);
+        return panel;
+    }
+
+    /** Returns an ImageIcon, or null if the path was invalid. */
+    protected static ImageIcon createImageIcon(String path) {
+        java.net.URL imgURL = TabbedPane.class.getResource(path);
+        if (imgURL != null) {
+            return new ImageIcon(imgURL);
+        } else {
+            System.err.println("Couldn't find file: " + path);
+            return null;
+        }
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ThreadControl.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ThreadControl.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/gui/ThreadControl.java	(revision 31634)
@@ -0,0 +1,388 @@
+/**
+ *#########################################################################
+ *
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Author: Chi-Yu Huang, Greenstone Digital Library, University of Waikato
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.admin.gui;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.net.*;
+import javax.swing.*;
+import javax.swing.event.*;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.admin.GAIManager;
+import org.greenstone.admin.gui.LogPane;
+import org.greenstone.admin.gui.ParsingProgress;
+import org.apache.tools.ant.*;
+import org.apache.tools.ant.helper.ProjectHelperImpl;
+
+
+
+public class ThreadControl 
+    extends Thread {
+    
+    private GAIManager gai_man;
+
+    public ThreadControl(){
+    }
+
+    public void run(){
+    }   
+
+    
+    public void executeAntTarget(String command_string) throws IOException  {
+	Project ant = new Project();
+	File buildFile = new File (GAI.gsdl3_src_home + File.separator + "build.xml");
+	
+	//Set up a PrintStream pointing at a file
+	//FileOutputStream ant_out =
+	//   new FileOutputStream(GAI.gsdl3_src_home + File.separator + "ant_out.log");
+	//FileOutputStream ant_err = 
+	//   new FileOutputStream(GAI.gsdl3_src_home + File.separator + "ant_err.log");
+	
+	//	BufferedOutputStream ant_out_buffer = 
+	//   new BufferedOutputStream(ant_out, 1024);
+	//BufferedOutputStream ant_err_buffer = 
+	///   new BufferedOutputStream(ant_err, 1024);
+	
+	//PrintStream ant_out_ps =
+	//   new PrintStream(ant_out_buffer, false);
+	//PrintStream ant_err_ps =
+	//   new PrintStream(ant_err_buffer, false);
+	
+	DefaultLogger log = new DefaultLogger();
+        //log.setErrorPrintStream(ant_err_ps);
+	//log.setOutputPrintStream(ant_out_ps);
+
+	log.setErrorPrintStream(System.err);
+	log.setOutputPrintStream(System.out);
+        log.setMessageOutputLevel(Project.MSG_INFO);
+	ant.addBuildListener(log);
+	try {
+	    ant.init();
+	    ant.fireBuildStarted();
+	    ProjectHelper helper = ProjectHelper.getProjectHelper();
+	    helper.parse(ant, buildFile);
+	    ant.executeTarget(command_string);
+	    ant.fireBuildFinished(null);
+	}
+	catch (BuildException e) {
+	    e.printStackTrace(System.err); 
+	    ant.fireBuildFinished(e);
+	}
+	/*finally {
+	    System.setOut(ant_out_ps);
+	    System.setErr(ant_err_ps);
+	    }*/
+
+	//boolean ant_success = checkAntResult();
+	//ant_out_ps.close();
+	//ant_err_ps.close();
+	/*if (!ant_success){
+	    int result = JOptionPane.showConfirmDialog((Component) null, "Ant building call was failed, do you want to see the log file?", "Exit Confirmation", JOptionPane.YES_NO_OPTION);
+	    if ( result == JOptionPane.YES_OPTION) {
+		//System.exit(1);
+		// Showing ant_out.log content
+	    } else if (result == JOptionPane.NO_OPTION) {
+		JOptionPane.showMessageDialog((Component) null,"You can see the log file from Log Pane!");
+	    } 
+	} else {
+	    //System.err.println("****Ant build successed");
+	    //JOptionPane.showMessageDialog((Component) null,"You can see the log file from Log Pane!");
+	    
+	    }*/
+    }
+
+    /*public boolean checkAntResult(){
+	try {
+	    String filename = GAI.gsdl3_src_home + File.separator +"ant_out.log";
+	    String fileLine;
+	    
+	    BufferedReader ant_result = new BufferedReader(new FileReader(filename));
+	    
+	    while ((fileLine = ant_result.readLine()) != null) {
+		if (fileLine.matches("BUILD SUCCESSFUL")){
+		    //return true;
+		    return false;
+		} else if (fileLine.matches("BUILD FAILED")){
+		    //System.err.println("***Ant result"+fileLine);
+		    return false;
+		}
+	    }
+	} catch (Exception e) {
+	    e.printStackTrace();
+	   
+	}	
+	return false;
+	}*/
+    
+
+    public class restartTomcatServer 
+	implements Runnable {
+	private String command_string = null;
+	/* Construct a restartTomcatServer Object*/
+	public restartTomcatServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	
+	public void run() {
+	    try {
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class startupTomcatServer 
+	implements Runnable {
+	private String command_string = null;
+	/* Construct a startupTomcatServer Object*/
+	public startupTomcatServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try { 
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class shutdownTomcatServer 
+	implements Runnable {
+	private String command_string = null;
+	/* Construct a shutdownTomcatServer Object*/
+	public shutdownTomcatServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try { 
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class startupMysqlServer 
+	implements Runnable {
+	private String command_string = null;
+
+	/* Construct a startupMysqlServer Object*/
+	public startupMysqlServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try { 
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class shutdownMysqlServer 
+	implements Runnable {
+	private String command_string = null;
+	/* Construct a shutdownMysqlServer Object*/
+	public shutdownMysqlServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try { 
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class restartMysqlServer 
+	implements Runnable {
+	private String command_string = null;
+	/* Construct a restartMysqlServer Object*/
+	public restartMysqlServer(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try { 
+		executeAntTarget(this.command_string);
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    public class installGS3 
+	implements Runnable {
+	private String command_string = null;
+
+	/* Install GSDL3 through ant program*/
+	public installGS3(String command_string) {
+	    this.command_string = command_string;
+	}
+	public void run() {
+	    try {
+     
+		executeAntTarget(this.command_string);
+		// ant_success;
+	    } catch(Exception e) { 
+		System.out.println(e.getMessage()); 
+	    }
+	}
+    }
+
+    
+}
+
+
+/***
+ * This program shows how you can time a set of threads.  The main()
+ * program demonstrates the timing classes by spawning and timing a
+ * set of threads that just execute a dummy loop.
+ ***/
+/*public class ThreadTiming {
+
+    /** Times a group of threads.  Works with TimingWrapper. */
+//    public static class ThreadTimer {
+//	private boolean __resultReady;
+//	private int __monitor[], __numThreads;
+//	private long __begin, __end;
+	
+	/** Creates a ThreadTimer that can time a given number of threads. */
+//	public ThreadTimer(int numThreads) {
+//	    __monitor     = new int[1];
+//	    __monitor[0]  = 0;
+//	    __numThreads  = numThreads;
+//	    __resultReady = false;
+//	}
+	
+	/** Starts the timer when all the threads are ready to start. */
+//	public void start() {
+//    synchronized(__monitor) {
+//		if(++__monitor[0] >= __numThreads) {
+//		    __begin = System.currentTimeMillis();
+//		    __monitor.notifyAll();
+//		} else {
+//		    while(true) {
+//			try {
+//			    __monitor.wait();
+//			} catch(InterruptedException e) {
+//			    continue;
+//			}
+//			break;
+//		    }
+//		}
+//	    }
+//	}
+	
+	/** Stops the timer when the last thread is done. */
+//	public void stop() {
+//	    synchronized(__monitor) {
+//	if(--__monitor[0] <= 0) {
+//		    __end   = System.currentTimeMillis();
+//		    synchronized(this) {
+//		__resultReady = true;
+//		notify();
+//	    }
+//	}
+//   }
+//}/
+	
+	/** Calling thread waits until timing result is ready. */
+//	public synchronized void waitForResult() {
+//	    while(!__resultReady) {
+//		try {
+///	    wait();
+//	} catch(InterruptedException e) {
+//	}
+//    }
+//    __resultReady = false;
+//}
+//
+///** Returns elapsed time in milliseconds. */
+//public long getElapsedTime() {
+//	    return (__end - __begin);
+//}
+//  }
+//  
+    /** Wraps a thread so it can be timed with a group. */
+//    public static class TimingWrapper implements Runnable {
+//	private Runnable __runMe;
+//private ThreadTimer __timer;
+
+//	public TimingWrapper(Runnable runMe, ThreadTimer timer) {
+//    __runMe   = runMe;
+//    __timer   = timer;
+//}
+	
+//public void run() {
+//    // Note, this timing approach does not measure thread creation
+//    // and thread joining overhead.
+//    __timer.start();
+//    __runMe.run();
+//    __timer.stop();
+//}
+//  }
+
+
+//public static final int NUM_THREADS = 10;
+//public static final int NUM_LOOPS   = 10000000;
+    
+//   public static void main(String[] args) {
+//int thread;
+//ThreadTimer timer;
+//
+//	timer = new ThreadTimer(NUM_THREADS);
+
+	// Start a number of threads that iterate a dummy loop
+//	for(thread = 0; thread < NUM_THREADS; thread++)
+//    new Thread(new TimingWrapper(new Runnable() {
+//	    public void run() {
+//		for(int i=0; i < NUM_LOOPS; i++);
+//	    }
+//	}, timer)).start();
+//
+//timer.waitForResult();
+	
+//	System.out.println("Elapsed Seconds: " +
+//			   ((double)timer.getElapsedTime())/1000.0);
+//   }
+    
+//}*/
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BaseExt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BaseExt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BaseExt.java	(revision 31634)
@@ -0,0 +1,78 @@
+package org.greenstone.admin.guiext;
+
+import java.awt.Insets;
+import java.io.File;
+
+import javax.swing.JPanel;
+import javax.swing.JScrollPane;
+import javax.swing.JTextArea;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.*;
+
+import org.greenstone.admin.GAI;
+
+public abstract class BaseExt {
+    
+    static protected JScrollPane description_pane;
+    static protected JPanel main_pane;
+    static protected JPanel control_pane;
+    static protected JTextArea descriptionTextArea;
+    static protected String description;
+	
+    public JPanel button_pane;
+    public String extension_name;
+    public String destination_folder;
+    public String url;
+    public String group;
+    public String download_type;
+    public Boolean configurable;
+    
+    GlobalProperties globalProperty = null;
+    static String fileSeparator = File.separator;
+    GSPath gspath = null;
+    
+    public BaseExt(){
+    }
+    
+    protected abstract JPanel getControlPane();
+    
+    protected abstract JPanel getDescriptionPane();    
+    
+    protected abstract boolean getCommandStatus(String Command);
+    
+    public boolean doCallback(String methodName)
+    {
+	return false;
+    }
+
+    public boolean doCommand(String cmd)
+    {
+	return false;
+    }
+    
+    public static Element getRootElement(String url){
+    	
+	Element rootNode = null;
+    	try{
+    	    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+    	    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+    	    Document doc = docBuilder.newDocument();
+    	    doc = docBuilder.parse (new File(url));
+    	    rootNode = doc.getDocumentElement();  
+    	    return rootNode;
+    	}catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+    	}
+    }
+    
+    protected abstract Object[][] getConfigureContent();
+    
+    //protected abstract Boolean isConfigurable();
+	
+    protected abstract JPanel getButtonPane();
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BasePanel.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BasePanel.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/BasePanel.java	(revision 31634)
@@ -0,0 +1,8 @@
+package org.greenstone.admin.guiext;
+
+import javax.swing.JPanel;
+
+public interface BasePanel
+{
+    public JPanel getPanel();
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Callback.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Callback.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Callback.java	(revision 31634)
@@ -0,0 +1,96 @@
+package org.greenstone.admin.guiext;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import javax.swing.JTextArea;
+
+import org.w3c.dom.Element;
+
+public class Callback implements Runnable
+{
+    String _class = null;
+    String _param = null;
+    CommandStep _parent = null;
+    
+    public Callback(Element callbackElement, CommandStep parent)
+    {
+	_parent = parent;
+
+	if(callbackElement != null){
+	    _class = ExtXMLHelper.getValueFromSingleElement(callbackElement, true);
+	    _param = callbackElement.getAttribute("type");
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.CALLBACK + "> element is null");
+	}
+    }
+
+    public void run()
+    {
+	_parent.getParent().getParent().loadGuiExtFile();
+	JTextArea messageArea = _parent.getMessageArea();
+
+	Class extClass = null;
+	try{
+	    extClass = Class.forName(_class);
+	}
+	catch(Exception ex){
+	    System.err.println("Could not create the extension class used for callback methods, either the class name is incorrect or the class does not exist inside the guiext.jar file");
+	}
+	Constructor classConstructor = null;
+	Object extObj = null;
+	
+	try{
+	    classConstructor = extClass.getConstructor(new Class[0]);
+	    extObj = classConstructor.newInstance(new Object[0]);
+	}
+	catch(Exception ex){
+	    ex.printStackTrace();
+	    System.err.println("Could not create the extension class used for callback methods, either the class name is incorrect or the class does not exist inside the guiext.jar file");
+	    return;
+	}
+
+	Class[] params = new Class[]{String.class};
+	
+	Method callbackMethod = null;
+	try{
+	    callbackMethod = extObj.getClass().getDeclaredMethod("doCallback", params);
+	}
+	catch(Exception ex){
+	    System.err.println("Specified class does not have a doCallback(String) method");
+	    _parent.threadError();
+	    return;
+	}
+
+	messageArea.append("\nCalling the callback method with \"" + _param +"\" as an argument");
+	Boolean success = null;
+	try{
+	    success = (Boolean)callbackMethod.invoke(extObj, (Object)_param);
+	}
+	catch(Exception ex){
+	    System.err.println("Error while attempting to invoke specified method -> ");
+	    ex.printStackTrace();
+	    _parent.threadError();
+	    return;
+	}
+	
+	if(!success){
+	    System.err.println("Call to doCallback method with the value \"" + _param + "\" returned unsuccessfully");
+	    _parent.threadError();
+	    return;
+	}
+
+	_parent.startNextThread();
+    }
+    
+    public String getParam()
+    {
+	return _param;
+    }
+    
+    public CommandStep getParent()
+    {
+	return _parent;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Command.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Command.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Command.java	(revision 31634)
@@ -0,0 +1,176 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+
+import java.util.HashMap;
+
+import javax.swing.JFrame;
+import javax.swing.JTextArea;
+import javax.swing.JOptionPane;
+
+public class Command implements Runnable
+{
+    HashMap<String, String> _osCommands = new HashMap<String, String>();
+    CommandStep _parent = null;
+
+    public Command(Element commandElement, CommandStep parent) 
+    {
+	_parent = parent;						
+
+	if(commandElement != null){
+	    Element[] commands = ExtXMLHelper.getMultipleChildElements(commandElement, "os", true);
+	    for(int i = 0; i < commands.length; i++){
+		Element currentCommand = commands[i];
+		String os = currentCommand.getAttribute("name");
+
+		if(os.equals("")){
+		    System.err.println("A <" + ExtXMLHelper.COMMAND + "> element does not have an \"os\" attribute");
+		}
+		else{
+		    _osCommands.put(os, ExtXMLHelper.getValueFromSingleElement(currentCommand, false));
+		}
+	    }
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.COMMAND + "> element is null");
+	}
+    }
+
+    public void run()
+    {
+	JTextArea messageArea = _parent.getMessageArea();
+	String command = getCommandForCurrentOS();
+	
+	if(command == null){
+	    System.err.println("No commands exist for your current operating system, to avoid seeing this warning please place <os name=\"default\"/> into the set of commands for this step in the extension_project_list.xml file located in your extension directory");
+	    _parent.threadError();
+	    return;
+	}
+	else if(command == ""){
+	    _parent.threadError();
+	    return;
+	}
+		
+	File workingDirectory = new File(_parent.getParent().getParent().getExtensionDirectory());
+	Process commandLineProc = null;
+	try{
+	    commandLineProc = null;
+	    
+	    if(System.getProperty("os.name").contains("Windows")){
+		String[] args = new String[3];
+		args[0] = "cmd.exe";
+		args[1] = "/C";
+		args[2] = command;
+
+		String allArgs = new String();
+		for(int i = 0; i < args.length; i++){
+		    if(i != 0){allArgs += " ";}
+		    allArgs += args[i];
+		}
+
+		messageArea.append("\nExecuting \"" + allArgs + "\" on the command line\n");
+
+		commandLineProc = Runtime.getRuntime().exec(args, null, workingDirectory);
+	    }
+	    else{
+		String[] args = new String[3];
+		args[0] = "sh";
+		args[1] = "-c";
+		args[2] = command;
+
+		String allArgs = new String();
+		for(int i = 0; i < args.length; i++){
+		    if(i != 0){allArgs += " ";}
+		    allArgs += args[i];
+		}
+
+		messageArea.append("\nExecuting \"" + allArgs + "\" on the command line\n");
+		commandLineProc = Runtime.getRuntime().exec(args, null, workingDirectory);
+	    }
+	    
+	    BufferedReader stdInput = new BufferedReader(new InputStreamReader(commandLineProc.getInputStream()));
+
+	    Thread stdPrinter = new PrinterThread(messageArea, stdInput);
+	    stdPrinter.start();
+
+	    BufferedReader stdError = new BufferedReader(new InputStreamReader(commandLineProc.getErrorStream()));
+
+	    Thread errPrinter = new PrinterThread(messageArea, stdError);
+	    errPrinter.start();
+	    
+	    int success = commandLineProc.waitFor();
+	    
+	    if(success != 0){
+		System.err.println("Command line process \"" + command  + "\" returned unsuccessfully with the value \"" + success + "\"");
+		_parent.threadError();
+		return;
+	    }
+	}
+	catch(Exception ex){
+	    ex.printStackTrace();
+	    _parent.threadError();
+	    return;
+	} 
+	_parent.startNextThread();
+    }
+
+    public boolean executeCommand()
+    {
+	return false;
+    }
+
+    public CommandStep getParent()
+    {
+	return _parent;
+    }
+
+    public String getCommandForCurrentOS()
+    {
+	String currentos = System.getProperty("os.name");
+
+	String command = _osCommands.get(currentos);
+	if(command != null){
+	    return command;
+	}
+	
+	if(currentos.contains("Windows")){
+	    command = _osCommands.get("Windows");
+	    
+	    if(command != null){
+		return command;
+	    }   
+	}
+	return _osCommands.get("default");
+    }
+
+    public class PrinterThread extends Thread
+    {
+	JTextArea _messageArea = null;
+	BufferedReader _output = null;
+
+	public PrinterThread(JTextArea messageArea, BufferedReader output)
+	{
+	    _messageArea = messageArea;
+	    _output = output;
+	}
+	
+	public void run()
+	{
+	    String s = "";
+
+	    try{
+		while ((s = _output.readLine()) != null) {
+		    _messageArea.append(s + "\n");
+		    _messageArea.setSelectionEnd(_messageArea.getDocument().getLength());
+		}  
+	    }
+	    catch(Exception ex){
+		ex.printStackTrace();
+	    }
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/CommandStep.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/CommandStep.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/CommandStep.java	(revision 31634)
@@ -0,0 +1,117 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+
+import java.awt.BorderLayout;
+
+import java.awt.event.ActionListener;
+import java.awt.event.ActionEvent;
+
+import javax.swing.JTextArea;
+import javax.swing.JButton;
+import javax.swing.JPanel;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.admin.LoggedMessageArea;
+
+public class CommandStep extends Step
+{
+    protected ArrayList<Runnable> _commandsAndCallbacks = new ArrayList<Runnable>();
+    protected LoggedMessageArea _messageArea = new LoggedMessageArea(this.getClass());
+    protected int _threadCount = -1;
+
+    /*
+    Command[] _commands = null;
+    Callback[] _callbacks = null;
+    */
+
+    public CommandStep(Element commandStepElement, SequenceList parent)
+    {
+	super(commandStepElement, parent);
+	
+	if(commandStepElement != null){
+	    NodeList commandAndCallbackElements = commandStepElement.getElementsByTagName("*");
+
+	    for(int i = 0; i < commandAndCallbackElements.getLength(); i++){
+		Element currentChild = (Element)commandAndCallbackElements.item(i);
+		if(currentChild.getTagName().equalsIgnoreCase(ExtXMLHelper.COMMAND)){
+		    _commandsAndCallbacks.add(new Command(currentChild, this));
+		}
+		else if (currentChild.getTagName().equalsIgnoreCase(ExtXMLHelper.CALLBACK)){
+		    _commandsAndCallbacks.add(new Callback(currentChild, this));
+		}
+		else if (currentChild.getTagName().equalsIgnoreCase(ExtXMLHelper.OS)){
+		}
+		else{
+		    System.err.println(currentChild.getTagName() + "A child of this command <" + ExtXMLHelper.STEP + "> is not either a <" + ExtXMLHelper.CALLBACK + "> element or a <" + ExtXMLHelper.COMMAND  + "> element");
+		}
+	    }
+
+	    if(_commandsAndCallbacks.size() == 0){
+		System.err.println("This command step does not have any <" + ExtXMLHelper.COMMAND  + "> or <" + ExtXMLHelper.CALLBACK  +"> elements");
+	    }
+
+	    _button.addActionListener(new CommandButtonListener());
+
+	}
+	else{
+	    System.err.println("This command <" + ExtXMLHelper.STEP + "> element is null");
+	}
+    }
+
+    public class CommandButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    ExtensionInformation info = _parent.getParent();
+
+	    JPanel panel = new JPanel();
+	    panel.setLayout(new BorderLayout());
+	    panel.add(_messageArea, BorderLayout.CENTER);
+	    info.changeExtPane(panel);
+
+	    _button.setEnabled(false);
+	    _button.setText("Running...");
+
+	    _threadCount = 0;
+	    Thread newThread = new Thread(_commandsAndCallbacks.get(0));
+	    newThread.start();
+	}
+    }
+
+    public JTextArea getMessageArea()
+    {
+	return _messageArea;
+    }
+
+    public void threadError()
+    {
+	_button.setText(_label);
+	_button.setEnabled(true);
+    }
+
+    public void startNextThread()
+    {
+	_threadCount++;
+
+	if(_threadCount < _commandsAndCallbacks.size()){
+	    Thread newThread = new Thread(_commandsAndCallbacks.get(_threadCount));
+	    newThread.start();
+	}
+	else{
+	    _button.setText(_label);
+	    _button.setEnabled(true);
+	    _parent.registerStepCompletion(CommandStep.this);
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/DownloadStep.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/DownloadStep.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/DownloadStep.java	(revision 31634)
@@ -0,0 +1,345 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+
+import java.io.File;
+
+import javax.swing.JButton;
+import javax.swing.JFrame;
+import javax.swing.JOptionPane;
+import javax.swing.JTextArea;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+
+import java.awt.BorderLayout;
+
+import java.awt.event.ActionListener;
+import java.awt.event.ActionEvent;
+
+import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
+import org.tmatesoft.svn.core.internal.io.fs.FSRepositoryFactory;
+import org.tmatesoft.svn.core.internal.io.svn.SVNRepositoryFactoryImpl;
+import org.tmatesoft.svn.core.wc.SVNClientManager;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
+import org.tmatesoft.svn.core.wc.SVNUpdateClient;
+import org.tmatesoft.svn.core.wc.ISVNEventHandler;
+import org.tmatesoft.svn.core.wc.SVNEvent;
+import org.tmatesoft.svn.core.wc.SVNEventAction;
+import org.tmatesoft.svn.core.wc.SVNRevision;
+import org.tmatesoft.svn.core.wc.SVNStatusType;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNDepth;
+import org.tmatesoft.svn.core.SVNCancelException;
+
+import org.greenstone.admin.GAI;
+import org.greenstone.admin.LoggedMessageArea;
+
+public class DownloadStep extends Step
+{
+    Source _mainSource = null;
+    Source[] _auxSources = null;
+    LoggedMessageArea _messageArea = new LoggedMessageArea(this.getClass());
+
+    public DownloadStep(SequenceList parent)
+    {
+	super("AUTOMATIC_DOWNLOAD", "Download Extension", "", "", parent);
+
+	ExtensionInformation info = _parent.getParent();
+	
+	_mainSource = new Source("svn", info.getBaseURL() + info.getFileStem() + "/trunk/src", "", this);
+
+	_button.addActionListener(new DownloadButtonListener());
+    }
+
+    public DownloadStep(Element downloadStepElement, SequenceList parent)
+    {
+	super(downloadStepElement, parent);
+	
+	if(downloadStepElement != null){    
+	    Element mainSourceElement = ExtXMLHelper.getSingleChildElement(downloadStepElement, ExtXMLHelper.MAIN_SOURCE, true);
+	    if(mainSourceElement == null){
+		System.err.println("This download <" + ExtXMLHelper.STEP + "> element has no <" + ExtXMLHelper.MAIN_SOURCE + "> element");
+	    }
+	    else{
+		_mainSource = new Source(mainSourceElement, this);
+	    }
+	
+	    Element[] auxSourceElements = ExtXMLHelper.getMultipleChildElements(downloadStepElement, ExtXMLHelper.AUX_SOURCE, false);
+	    if(auxSourceElements != null){
+		_auxSources = new Source[auxSourceElements.length];
+		for(int i = 0; i < auxSourceElements.length; i++){
+		    _auxSources[i] = new Source(auxSourceElements[i], this);
+		}
+	    }
+	    
+	    _button = new JButton(_label);
+	    _button.setEnabled(false);
+	    _button.addActionListener(new DownloadButtonListener());
+	}
+	else{
+	    System.err.println("This download <" + ExtXMLHelper.STEP + "> element is null");
+	}
+    }
+
+    public class DownloadButtonListener implements ActionListener
+    {
+	boolean _svnInitialised = false;
+	public DownloadButtonListener()
+	{
+	    if(!_svnInitialised){
+		//Setup the SVN client system
+		DAVRepositoryFactory.setup();
+		SVNRepositoryFactoryImpl.setup();
+		FSRepositoryFactory.setup();
+	    }
+	    _svnInitialised = true;
+	}
+	
+	public void actionPerformed(ActionEvent e)
+	{
+	    _button.setEnabled(false);
+	    _button.setText("Downloading...");
+
+	    DownloadThread downloadThread = new DownloadThread();
+	    downloadThread.start();
+	}
+    }
+
+    public class DownloadThread extends Thread
+    {
+	public void run()
+	{
+	    ExtensionInformation info = _parent.getParent();
+
+	    JPanel panel = new JPanel();
+	    panel.setLayout(new BorderLayout());
+	    panel.add(_messageArea, BorderLayout.CENTER);
+	    info.changeExtPane(panel);
+
+	    String fileStem = info.getFileStem();
+	    String defaultDownloadLocation = GAI.getGSDL3ExtensionHome() + System.getProperty("file.separator") + fileStem;
+
+	    ArrayList<Source> sourceElements = new ArrayList<Source>();
+	    sourceElements.add(_mainSource);
+
+	    if(_auxSources != null){
+		for(int i = 0; i < _auxSources.length; i++){
+		    sourceElements.add(_auxSources[i]);
+		}
+	    }
+
+	    for(int i = 0; i < sourceElements.size(); i++){
+		
+		Source currentSource = sourceElements.get(i);
+		String destinationFolder = null;
+
+		if(currentSource.getMethod().equals("svn")){
+		    String name = "";
+		    String password = "";
+
+		    //Setup the SVN client
+		    SVNClientManager svnManager = SVNClientManager.newInstance(SVNWCUtil.createDefaultOptions(true), name, password);
+		    SVNUpdateClient svnCheckout = svnManager.getUpdateClient();
+		    svnCheckout.setEventHandler(new SVNCheckoutEventHandler());
+		    svnCheckout.setIgnoreExternals(false);
+
+		    //The i != 0 enforces that the mainSource should always go to the default download location
+		    if(i != 0 && !currentSource.getFolder().equals("")){
+			destinationFolder = currentSource.getFolder();
+		    }
+		    else{
+			destinationFolder = defaultDownloadLocation;
+		    }
+		    
+		    //Check if the directory already exists. If it does exist then as the use if they wish to delete it
+		    File wcDir = new File(destinationFolder);
+		    if (wcDir.exists()) {
+			
+			Object[] option = {"Continue","Cancel"};
+			int n = JOptionPane.showOptionDialog(new JFrame(),"The folder for this extension already exists, continuing will remove all the files within this folder before proceeding.","Attention", 
+							     JOptionPane.YES_NO_CANCEL_OPTION,
+							     JOptionPane.QUESTION_MESSAGE,
+							     null,
+							     option,
+							     option[0]);
+			if(n == 0){
+			    ExtPane.deleteDir(wcDir);
+			}
+			else{
+			    _button.setEnabled(true);
+			    _button.setText(_label);
+			    return;
+			}
+		    }
+		    wcDir.mkdirs();
+		    
+		    //Perform the specified checkout
+		    String url = currentSource.getURL();
+		    try{
+			SVNURL repositoryURL = SVNURL.parseURIEncoded(url);
+			long x = svnCheckout.doCheckout(repositoryURL, wcDir, SVNRevision.HEAD, SVNRevision.HEAD, SVNDepth.INFINITY, true);	
+		    }
+		    catch(SVNException ex){
+			ex.printStackTrace();
+			_messageArea.append("Error retrieving file from SVN");
+			_button.setEnabled(true);
+			_button.setText(_label);
+			return;
+		    }
+		}
+	    }
+	    
+	    _button.setEnabled(true);
+	    _button.setText(_label);
+	    _parent.registerStepCompletion(DownloadStep.this);
+	}
+    }
+
+    public class SVNCheckoutEventHandler implements ISVNEventHandler
+    {	
+	public void handleEvent(SVNEvent event, double progress)
+	{
+	    /*
+	     * Gets the current action. An action is represented by SVNEventAction.
+	     * In case of an update an  action  can  be  determined  via  comparing 
+	     * SVNEvent.getAction() and SVNEventAction.UPDATE_-like constants. 
+	     */
+		
+	    //_messageArea.append(" event occurs");
+	    SVNEventAction action = event.getAction();
+	    String pathChangeType = " ";
+	    if (action == SVNEventAction.UPDATE_ADD) {
+		/*
+		 * the item was added
+		 */
+		pathChangeType = "A";
+	    } else if (action == SVNEventAction.UPDATE_DELETE) {
+		/*
+		 * the item was deleted
+		 */
+		pathChangeType = "D";
+	    } else if (action == SVNEventAction.UPDATE_UPDATE) {
+		/*
+		 * Find out in details what  state the item is (after  having  been 
+		 * updated).
+		 * 
+		 * Gets  the  status  of  file/directory  item   contents.  It   is 
+		 * SVNStatusType  who contains information on the state of an item.
+		 */
+		SVNStatusType contentsStatus = event.getContentsStatus();
+		if (contentsStatus == SVNStatusType.CHANGED) {
+		    /*
+		     * the  item  was  modified in the repository (got  the changes 
+		     * from the repository
+		     */
+		    pathChangeType = "U";
+		}else if (contentsStatus == SVNStatusType.CONFLICTED) {
+		    /*
+		     * The file item is in  a  state  of Conflict. That is, changes
+		     * received from the repository during an update, overlap  with 
+		     * local changes the user has in his working copy.
+		     */
+		    pathChangeType = "C";
+		} else if (contentsStatus == SVNStatusType.MERGED) {
+		    /*
+		     * The file item was merGed (those  changes that came from  the 
+		     * repository  did  not  overlap local changes and were  merged 
+		     * into the file).
+		     */
+		    pathChangeType = "G";
+		}
+	    } else if (action == SVNEventAction.UPDATE_EXTERNAL) {
+		/*for externals definitions*/
+		_messageArea.append("Fetching external item into '"
+				   + event.getFile().getAbsolutePath() + "'");
+		_messageArea.append("External at revision " + event.getRevision());
+		return;
+	    } else if (action == SVNEventAction.UPDATE_COMPLETED) {
+		/*
+		 * Updating the working copy is completed. Prints out the revision.
+		 */
+		//_messageArea.append("At revision " + event.getRevision());
+        	_messageArea.append("At revision " + event.getRevision()+"\n");
+        	//_messageArea.append("The extension ("+  _extensionName +") has been downloaded to the local folder: \n"+ _destination +"\n"); //xxxxx
+        	_messageArea.setSelectionEnd(_messageArea.getDocument().getLength());
+		return;
+	    } else if (action == SVNEventAction.ADD){
+		_messageArea.append("A     " +  event.getURL().getPath());
+		return;
+	    } else if (action == SVNEventAction.DELETE){
+		_messageArea.append("D     " +  event.getURL().getPath());
+		return;
+	    } else if (action == SVNEventAction.LOCKED){
+		_messageArea.append("L     " +  event.getURL().getPath());
+		return;
+	    } else if (action == SVNEventAction.LOCK_FAILED){
+		_messageArea.append("failed to lock    " +  event.getURL().getPath());
+		return;
+	    }
+	    
+	    /*
+	     * Now getting the status of properties of an item. SVNStatusType  also
+	     * contains information on the properties state.
+	     */
+	    SVNStatusType propertiesStatus = event.getPropertiesStatus();
+	    /*
+	     * At first consider properties are normal (unchanged).
+	     */
+	    String propertiesChangeType = " ";
+	    if (propertiesStatus == SVNStatusType.CHANGED) {
+		/*
+		 * Properties were updated.
+		 */
+		propertiesChangeType = "U";
+	    } else if (propertiesStatus == SVNStatusType.CONFLICTED) {
+		/*
+		 * Properties are in conflict with the repository.
+		 */
+		propertiesChangeType = "C";
+	    } else if (propertiesStatus == SVNStatusType.MERGED) {
+		/*
+		 * Properties that came from the repository were  merged  with  the
+		 * local ones.
+		 */
+		propertiesChangeType = "G";
+	    }
+	    
+	    /*
+	     * Gets the status of the lock.
+	     */
+	    String lockLabel = " ";
+	    SVNStatusType lockType = event.getLockStatus();
+	    
+	    if (lockType == SVNStatusType.LOCK_UNLOCKED) {
+		/*
+		 * The lock is broken by someone.
+		 */
+		lockLabel = "B";
+	    }
+
+	    /*
+	    System.err.println("pathChangeType = " + pathChangeType);
+	    System.err.println("propertiesChangeType = " + propertiesChangeType);
+	    System.err.println("lockLabel = " + lockLabel);
+	    System.err.println("event = " + event);
+	    System.err.println("event.getURL = " + event.getURL());
+	    */
+
+	    final String content = pathChangeType
+		+ propertiesChangeType
+		+ lockLabel
+		+ "       "
+		+ (event.getURL() == null ? "" : event.getURL().getPath()) + "\n";
+
+	    _messageArea.append(content);
+	    _messageArea.setSelectionEnd(_messageArea.getDocument().getLength());
+	}
+
+	public void checkCancelled() throws SVNCancelException
+	{
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtPane.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtPane.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtPane.java	(revision 31634)
@@ -0,0 +1,508 @@
+package org.greenstone.admin.guiext;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.*;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.*;
+
+import java.net.URLClassLoader;
+import java.net.URL;
+
+import javax.swing.*;
+import javax.swing.event.*;
+import javax.swing.tree.*;
+import javax.swing.table.TableCellRenderer;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.greenstone.admin.GAIManager;
+import org.greenstone.admin.GAI;
+
+import org.greenstone.util.Configuration;
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSPath;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+
+public class ExtPane extends JPanel  {
+
+    /* The pane to demonstrate extension information*/
+
+    protected JSplitPane main_ext_pane = null;
+
+    /** The panel that contains a log_list */
+    private JPanel extensionList_pane =  null;
+
+    /** The panel that contains the log content. */
+    protected static JPanel extensionContent_pane =  null;
+
+    /** The label at the top of the logList_pane. */
+    protected JLabel extensionList_label =  null;
+      
+    /** The label shown at the top of the logContent Pane. */
+    protected JLabel extensionContent_label  =  null;
+    
+    /** The scrollable area into which the log content is placed. */
+    protected JScrollPane description_content = null;
+    
+    /* Log TEXT AREA*/
+    protected JTextArea description_textarea = null;
+   
+    protected JScrollPane message_content = null;
+    
+    /* Log TEXT AREA*/
+    public static JTextArea message_textarea = null;
+
+    protected static JPanel current_description_pane = null;
+
+    protected JPanel button_pane = null;
+    
+    protected static JPanel control_pane = null;
+
+    protected JPanel main_content_pane = null;
+    
+    protected JTree extensionTreeList = null;
+      
+    protected DefaultMutableTreeNode top = new DefaultMutableTreeNode("Extension List");
+    
+    protected HashMap<String, ArrayList> extensionInformation = null;
+      
+    protected JPanel extensionContentHeaderPane = null;
+    
+    protected String file_location;
+    
+    protected final String extension_status_root = "Configuration";
+    protected File extension_status_file = null;
+    protected String extension_file = null;
+    
+    /** The various sizes for the screen layout*/
+    static protected Dimension MIN_SIZE = new Dimension( 90,  90);
+    static protected Dimension LIST_SIZE  = new Dimension(200, 450);
+    static protected Dimension LOGPANE_SIZE = new Dimension (800,450);
+    static public int left_padding = 5;
+    public String fileSeparator = File.separator;
+    
+    protected Element extFileRootElement = null;
+    protected HashMap<String, ExtensionInformation> extensions = null;
+    protected HashMap extClasses = null;
+    protected Properties dependencies = new Properties();
+    protected JPanel extPanel = new JPanel();
+    protected JSplitPane extensionListContainer = new JSplitPane(JSplitPane.VERTICAL_SPLIT);
+       
+    public ExtPane() {
+  
+	String gsdl3Home = GAI.getGSDL3Home();
+
+	file_location = gsdl3Home+fileSeparator+"ext"+fileSeparator+"extension_project_list.xml";
+	
+	extFileRootElement = getRootElement();
+
+	//Load the steps that have already been performed (used for dependencies)
+	loadExtensionInformation();
+	    
+	// create all the necessary panes
+	control_pane = new JPanel();
+	button_pane = new JPanel();
+	current_description_pane = new JPanel();
+	extensionContentHeaderPane = new JPanel();
+	extensionInformation = new HashMap<String, ArrayList>();
+
+	// Extension_Pane
+	main_ext_pane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
+	main_ext_pane.setPreferredSize(LOGPANE_SIZE);
+
+	// Extension_TextArea
+	description_textarea = new JTextArea();
+	description_textarea.setEditable(false);
+	description_textarea.setLineWrap(true);
+	description_textarea.setWrapStyleWord(true);
+	description_textarea.setMargin(new Insets(0,left_padding,0,0));
+    }
+
+    public void loadExtensionInformation()
+    {
+	Element[] extensionElements = ExtXMLHelper.getMultipleChildElements(extFileRootElement, ExtXMLHelper.EXTENSION, true);
+	if(extensionElements != null){
+	    extensions = new HashMap<String, ExtensionInformation>();
+
+	    for(int i = 0; i < extensionElements.length; i++){
+		ExtensionInformation ei = new ExtensionInformation(extensionElements[i], extFileRootElement.getAttribute("baseURL"));
+		extensions.put(ei.getName(), ei);
+	    }
+	}
+	else{
+	    System.err.println("There are no <" + ExtXMLHelper.EXTENSION  + "> elements in the extension_project_list.xml file in the extension directory");
+	}
+    }
+
+    /** This method is callsed to actually layout the components.*/
+    public void display() {
+         	
+	load_Admin_UI();
+	HashMap<String, ArrayList<String>> projectNameMap = getProjectList();
+    	Set<String> s = projectNameMap.keySet();
+    	Iterator<String> it = s.iterator();
+    	int i = 0;
+    	
+	while(it.hasNext()){
+	    String projectNameGroup = it.next(); 
+	    ArrayList alist = projectNameMap.get(projectNameGroup);
+	    DefaultMutableTreeNode projecti = new DefaultMutableTreeNode (projectNameGroup);
+	    
+	    for (int j = 0; j< alist.size(); j++){
+		String projectName = (String)alist.get(j);
+		DefaultMutableTreeNode projectNameGroupj = new DefaultMutableTreeNode (projectName);
+		projecti.add(projectNameGroupj);
+	    }
+	    top.add(projecti);
+	    i++;
+    	}
+
+    	extensionTreeList  = new JTree(top); 
+    	extensionTreeList.getSelectionModel().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION);
+    	extensionTreeList.addTreeSelectionListener(new ConfTreeListener());
+    	extensionTreeList.addMouseListener(new ConfTreeMouseListener());
+    	
+	// extensionList_Pane
+	extensionList_pane = new JPanel();
+	extensionList_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	extensionList_label = new JLabel();
+	extensionList_label.setOpaque(true);
+	extensionList_label.setBackground(Configuration.getColor("coloring.workspace_heading_background"));
+	extensionList_label.setForeground(Configuration.getColor("coloring.workspace_heading_foreground"));
+	extensionList_label.setText("Available Extensions");
+	extensionList_label.setBorder(BorderFactory.createEmptyBorder(0, left_padding, 0, 0));
+	// extensionContent_Pane
+	extensionContent_pane = new JPanel();
+	extensionContent_pane.setBorder(BorderFactory.createLoweredBevelBorder());
+	extensionContent_pane.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	extensionContent_pane.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	
+	extensionContent_label = new JLabel();
+	extensionContent_label.setBorder(BorderFactory.createEmptyBorder(0, left_padding, 0, 0));
+	extensionContent_label.setOpaque(false);
+	extensionContent_label.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	extensionContent_label.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	extensionContent_label.setText("Extension Content");
+
+	// Layout Components
+	extensionList_pane.setLayout(new BorderLayout());
+	extensionList_pane.add(extensionTreeList, BorderLayout.CENTER);
+	extensionList_pane.add(extensionList_label, BorderLayout.NORTH);
+	extensionListContainer.add(extensionList_pane, JSplitPane.TOP);
+	extensionListContainer.setDividerLocation(600);
+	
+	extPanel.setLayout(new BorderLayout());
+	ExtensionInformation.setExtPane(extPanel);
+
+	JScrollPane extScrollPane = new JScrollPane(extPanel);
+
+	extensionContent_pane.setLayout(new BorderLayout());
+	//extensionContent_pane.add(main_content_pane,BorderLayout.SOUTH);
+	extensionContent_pane.add(extensionContent_label, BorderLayout.NORTH);
+	extensionContent_pane.add(extScrollPane, BorderLayout.CENTER);
+	
+	main_ext_pane.add(extensionListContainer, JSplitPane.LEFT);
+	main_ext_pane.add(extensionContent_pane, JSplitPane.RIGHT);
+	main_ext_pane.setDividerLocation(LIST_SIZE.width);
+
+	this.setLayout(new BorderLayout());
+	this.add(main_ext_pane, BorderLayout.CENTER);
+    }
+    
+    
+    protected Element getRootElement(){
+    	
+	Element rootNode = null;
+    	
+	try{
+    	    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+    	    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+    	    Document doc = docBuilder.newDocument();
+    	    doc = docBuilder.parse (new File(file_location));
+    	    rootNode = doc.getDocumentElement();  
+    	    return rootNode;
+    	
+	}catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+    	}
+    }
+    
+    protected HashMap<String, ArrayList<String>> getProjectList(){
+    	
+	HashMap<String, ArrayList<String>> projectGroupMap = new HashMap<String, ArrayList<String>>();
+	Element rootNode = extFileRootElement;
+	NodeList projectList =  rootNode.getElementsByTagName(ExtXMLHelper.EXTENSION);
+
+    	for(int i = 0; i<projectList.getLength(); i++){
+	    
+	    Element projectNode = (Element)projectList.item(i);
+	    NodeList nameList = projectNode.getElementsByTagName(ExtXMLHelper.NAME);
+	    String name = nameList.item(0).getChildNodes().item(0).getNodeValue();
+    		
+	    NodeList groupList = projectNode.getElementsByTagName(ExtXMLHelper.GROUP);
+	    String group = groupList.item(0).getChildNodes().item(0).getNodeValue();
+
+	    if(projectGroupMap.containsKey(group)){
+		ArrayList<String> alist = projectGroupMap.get(group);
+		alist.add(name);
+		projectGroupMap.put(group, alist);
+	    }
+	    else{
+		ArrayList<String> alist = new ArrayList<String>();
+		alist.add(name);
+		projectGroupMap.put(group, alist);
+	    }
+    	}
+	return projectGroupMap;
+    }
+
+    protected class ConfTreeMouseListener implements MouseListener
+    {
+	public void mouseClicked(MouseEvent e)
+	{
+	    Object comp = extensionTreeList.getLastSelectedPathComponent();
+	    if(comp == null){
+		return;
+	    }
+
+	    String option = comp.toString();
+	    if(option == null){
+		return;
+	    }
+
+	    ExtensionInformation info = extensions.get(option);
+
+	    if(info != null){
+		extPanel.removeAll();
+		
+		JTextArea descArea = new JTextArea();
+		descArea.setText(info.getDescription());
+		descArea.setEditable(false);
+		descArea.setLineWrap(true);
+		descArea.setWrapStyleWord(true);
+		
+		//JEditorPane descArea = new JEditorPane("text/plain" info.getDescription());
+		//descArea.setEditable(false);
+
+		extPanel.add(descArea, BorderLayout.CENTER);
+		extPanel.revalidate();
+		extPanel.repaint();
+	    }
+	}
+         
+	public void mouseEntered(MouseEvent e){}
+	public void mouseExited(MouseEvent e){}
+	public void mousePressed(MouseEvent e){}
+	public void mouseReleased(MouseEvent e){} 
+    }
+     
+    protected class ConfTreeListener implements TreeSelectionListener{
+	
+	public void valueChanged (TreeSelectionEvent e){
+
+	    String option = new String();
+	    
+	    try{
+		option = extensionTreeList.getLastSelectedPathComponent().toString();
+	    }catch(Exception ex){
+	    	
+	    }
+	    
+	    if(extensionInformation.containsKey(option)){
+		
+		try{
+		    extensionListContainer.setBottomComponent(getProjectButtonPane(option));
+		    extensionListContainer.setDividerLocation(250);
+
+		    extensionContent_label.setText(option);    
+		}
+		catch(Exception ex){
+		    ex.printStackTrace();
+		    description_textarea.setText("");
+		    description_textarea.append("Sorry, this extension is not available!");
+		    extensionContent_pane.add(description_content, BorderLayout.CENTER);
+		    extensionContent_pane.validate();
+		    extensionContent_pane.repaint();
+		}
+	    }
+	    else{
+	    	if(control_pane.getComponentCount()>0){
+		    //System.out.println("remove");
+		    control_pane.remove(0);
+		    control_pane.revalidate();
+		    control_pane.repaint();
+		    control_pane.add(new JPanel());
+		    control_pane.revalidate();
+		    control_pane.repaint();
+	    	}
+		
+	    	if(extensionContent_pane.getComponentCount()>0){
+		    
+		    extensionContent_label.setText("Extension Content");
+		    //extensionContent_pane.remove(2);
+		    //extensionContent_pane.revalidate();
+		    //extensionContent_pane.repaint();
+		    
+		    
+		    //JTextArea ja= new JTextArea();
+		    //JScrollPane jp = new JScrollPane(ja);
+		    //extensionContent_pane.add(jp);
+		    //extensionContent_pane.revalidate();
+		    //extensionContent_pane.repaint();
+		    //xxxxx
+	    	}
+	    }
+	}
+    }
+
+    public String loadDescription(String className){
+	
+	String description = "";
+    	Element rootNode = extFileRootElement;
+    	NodeList projectNodeList = rootNode.getElementsByTagName(ExtXMLHelper.EXTENSION);
+    	
+    	for(int i = 0; i < projectNodeList.getLength(); i++){
+	    
+	    Element projectNode = (Element)projectNodeList.item(i);
+	    NodeList nameNodeList = projectNode.getElementsByTagName(ExtXMLHelper.NAME);
+	    String name = nameNodeList.item(0).getChildNodes().item(0).getNodeValue();
+	    
+	    if(name.equals(className)){
+		NodeList descriptionNodeList = projectNode.getElementsByTagName(ExtXMLHelper.DESCRIPTION);
+		description = descriptionNodeList.item(0).getChildNodes().item(0).getNodeValue();
+	    }
+    	}    	
+	return description;
+    }
+
+    public void load_Admin_UI(){
+    	
+	Element rootNode = extFileRootElement;
+    
+	NodeList projectList =  rootNode.getElementsByTagName(ExtXMLHelper.EXTENSION);
+
+	for(int i = 0; i < projectList.getLength(); i++){
+        	
+	    Element projectNode = (Element)projectList.item(i);
+	    ArrayList alist = new ArrayList();    		
+	    NodeList nameList = projectNode.getElementsByTagName(ExtXMLHelper.NAME);
+	    String name = nameList.item(0).getChildNodes().item(0).getNodeValue();
+	    
+	    //Maybe recode this? //xxxxx
+	    try{
+		extensionInformation.put(name, alist);
+	    }catch (Exception ex){
+		ex.printStackTrace();
+	    }
+	}
+    }
+    
+    public static void updateButtonPane(JPanel pane){
+	    control_pane.remove(0);
+	    control_pane.revalidate();
+	    control_pane.repaint();
+	    control_pane.add(pane);
+    }
+    
+    public static void updateExtensionContentPane(){
+	     CardLayout cl = (CardLayout)( current_description_pane.getLayout());
+	     cl.next(current_description_pane);
+
+    }
+
+    public static boolean deleteDir(File dir) {
+        
+    	if (dir.isDirectory()) {
+	    String[] children = dir.list();
+	    for (int i=0; i<children.length; i++) {
+		boolean success = deleteDir(new File(dir, children[i]));
+		if (!success) {
+		    return false;
+		}
+	    }
+	}
+    	return dir.delete();
+    }
+
+   
+    
+  /*  
+    private void updateStatus(String project_name, String action){
+	
+	try{
+	    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+	    Document doc = docBuilder.newDocument();
+	    Element root = doc.createElement(extension_status_root);
+			
+	    boolean fileExist = extension_status_file.exists();
+	    
+	    if(!fileExist){
+		extension_status_file = new File (extension_file);
+		root = doc.createElement(extension_status_root);
+	    }
+	    else{
+		doc = docBuilder.parse (extension_status_file);
+		root = doc.getDocumentElement();
+	    }
+	    
+	    
+	    TransformerFactory tf = TransformerFactory.newInstance();     
+	    Transformer transformer= tf.newTransformer();     
+	    DOMSource source= new DOMSource(root);     
+	    transformer.setOutputProperty(OutputKeys.INDENT,"yes");     
+			
+	    Writer pwx= new  BufferedWriter(new OutputStreamWriter(new FileOutputStream(extension_status_file),"UTF-8"));     
+	    StreamResult result= new StreamResult(pwx);     
+	    transformer.transform(source,result);  
+	    pwx.close();
+	    
+	    root = null;
+	    docBuilderFactory = null;
+	    docBuilder = null;
+	    doc = null;
+	    
+	}catch (Exception e) {
+	    System.out.println(e);
+	}	
+    }*/
+
+    public JPanel getProjectButtonPane(String projectName){
+
+    	String option = extensionTreeList.getLastSelectedPathComponent().toString();
+    	JPanel buttonPane = new JPanel();
+
+	ExtensionInformation extension = extensions.get(projectName);
+
+	String gsdl3ExtHome = GAI.getGSDL3ExtensionHome();
+
+	JButton[] buttons = extension.getSequenceList().getButtons();
+
+	GridLayout gl = new GridLayout(buttons.length, 1);
+	gl.setVgap(5);
+	buttonPane.setLayout(gl);
+	
+	for(int j = 0; j < buttons.length; j++){
+	    buttonPane.add(buttons[j]);
+	}
+	
+	extension.getSequenceList().updateButtons();
+	
+	return buttonPane;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtXMLHelper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtXMLHelper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtXMLHelper.java	(revision 31634)
@@ -0,0 +1,142 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Node;
+
+public class ExtXMLHelper
+{
+    static public final String EXTENSION = "extension";
+    static public final String GROUP = "group";
+    static public final String FILE_STEM = "fileStem";
+    static public final String NAME = "name";
+    static public final String DOWNLOAD = "download";
+    static public final String MAIN_SOURCE = "mainSource";
+    static public final String AUX_SOURCE = "auxSource";
+    static public final String SOURCE = "source";
+    static public final String INSTALL = "install";
+    static public final String UNINSTALL = "uninstall";
+    static public final String ENABLE = "enable";
+    static public final String DISABLE = "disable";
+    static public final String DESTINATION = "destination";
+    static public final String DESCRIPTION = "description";
+    static public final String PROPERTY = "property";
+    static public final String SEQUENCE_LIST = "sequenceList";
+    static public final String STEP ="step";
+    static public final String COMMAND ="command";
+    static public final String CALLBACK ="callback";
+    static public final String OPTION_LIST = "optionList";
+    static public final String OPTION = "option";
+    static public final String CLASS = "class";
+    static public final String OS = "os";
+
+    private ExtXMLHelper(){}
+    
+    public static String getValueFromSingleElement(Element parent, String valueName, boolean strict)
+    {
+	if(parent == null){
+	    System.err.println("Parent element is null");
+	    return null;
+	}
+	else if(valueName == null){
+	    System.err.println("Value name element is null");
+	    return null;
+	}
+
+	Element e = getSingleChildElement(parent, valueName, strict);
+	if(e == null){
+	    System.err.println("Cannot get a value from element \"" + valueName +"\" as it does not exist in this parent");
+	    return null;
+	}
+
+	String s = getValueFromSingleElement(e, strict);
+	if(s == null){
+	    if(strict){
+		System.err.println("Cannot get a value from the element \"" + valueName + "\" as it is empty");
+	    }
+	    
+	    return null;
+	}
+
+	return s;
+    }
+
+    public static String getValueFromSingleElement(Element e, boolean strict)
+    {
+	if(e == null){
+	    System.err.println("Element is null");
+	    return null;
+	}
+
+	Node textChild = e.getFirstChild();
+	if(textChild == null){
+	    if(strict){
+		System.err.println("This element has no value");
+	    }
+	    return null;
+	}
+
+	String value = textChild.getNodeValue();
+	if(value.equals("")){
+	    if(strict){
+		System.err.println("The value for this element is empty");
+	    }
+	    return null;
+	}
+	return value;
+    }
+
+    public static Element getSingleChildElement(Element parent, String elementName, boolean strict)
+    {
+	if(parent == null){
+	    System.err.println("Parent element is null");
+	    return null;
+	}
+	else if(elementName == null){
+	    System.err.println("Element name is null");
+	    return null;
+	}
+
+	NodeList nodeList = parent.getElementsByTagName(elementName);
+	if(nodeList == null || nodeList.getLength() == 0){
+	    if(strict){
+		System.err.println("The element " + elementName + " does not exist in this parent element");
+	    }
+	    return null;
+	}
+	else if(nodeList.getLength() > 1){
+	    System.err.println("The element " + elementName + " has more than one element in this parent element");
+	    return null;
+	}
+
+	return (Element)nodeList.item(0);
+    }
+    
+    public static Element[] getMultipleChildElements(Element parent, String elementName, boolean strict)
+    {
+	if(parent == null){
+	    System.err.println("Parent element is null");
+	    return null;
+	}
+	else if(elementName == null){
+	    System.err.println("Element name is null");
+	    return null;
+	}
+
+	NodeList nodeList = parent.getElementsByTagName(elementName);
+	if(nodeList == null || nodeList.getLength() == 0){
+	    if(strict){
+		System.err.println("The element " + elementName + " does not exist in this parent element");
+	    }
+	    return null;
+	}
+	
+	Element[] elements = new Element[nodeList.getLength()];
+	
+	for(int i = 0; i < nodeList.getLength(); i++){
+	    elements[i] = (Element)nodeList.item(i);
+	}
+
+	return elements;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtensionInformation.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtensionInformation.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/ExtensionInformation.java	(revision 31634)
@@ -0,0 +1,141 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.net.URLClassLoader;
+import java.net.URL;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Constructor;
+
+import java.io.File;
+
+import java.awt.BorderLayout;
+
+import javax.swing.JTextArea;
+import javax.swing.JPanel;
+
+import org.greenstone.admin.GAI;
+
+public class ExtensionInformation
+{
+    protected String _name = null;
+    protected String _group = null;
+    protected String _fileStem = null;
+    protected String _description = null;
+    protected String _baseURL = null;
+    protected SequenceList _sequenceList= null;
+    protected boolean _guiExtLoaded = false;
+
+    protected static JPanel _extPane = new JPanel();
+
+    public ExtensionInformation(Element extensionElement, String baseURL){
+	if(extensionElement != null){
+	    _name = ExtXMLHelper.getValueFromSingleElement(extensionElement, ExtXMLHelper.NAME, true);
+	    _group = ExtXMLHelper.getValueFromSingleElement(extensionElement, ExtXMLHelper.GROUP, true);
+	    _fileStem = ExtXMLHelper.getValueFromSingleElement(extensionElement, ExtXMLHelper.FILE_STEM, true);
+	    _description = ExtXMLHelper.getValueFromSingleElement(extensionElement, ExtXMLHelper.DESCRIPTION, true);
+	    _baseURL = baseURL;
+	    _sequenceList = new SequenceList(ExtXMLHelper.getSingleChildElement(extensionElement, ExtXMLHelper.SEQUENCE_LIST, true), this);
+	}
+	else{
+	    System.err.println("<" + ExtXMLHelper.EXTENSION + "> element is null");
+	}
+    }
+
+    public String getBaseURL()
+    {
+	return _baseURL;
+    }
+
+    public String getName()
+    {
+	return _name;
+    }
+
+    public String getGroup()
+    {
+	return _group;
+    }
+    
+    public String getFileStem()
+    {
+	return _fileStem;
+    }
+
+    public String getDescription()
+    {
+	return _description;
+    }
+
+    public SequenceList getSequenceList()
+    {
+	return _sequenceList;
+    }
+
+    public void loadExternalJar(String filename){
+	//Load the dynamic jar loader
+	URLClassLoader sysloader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+	Class<URLClassLoader> sysclass = URLClassLoader.class;
+	final Class[] parameters = new Class[]{URL.class};
+	Method method = null;
+	try{
+	    method = sysclass.getDeclaredMethod("addURL", parameters);
+	}
+	catch(Exception ex){
+	    ex.printStackTrace();
+	}
+	method.setAccessible(true);
+
+	File externalFile = new File(filename);
+	if(!externalFile.exists()){
+	    System.err.println("Cannot load the external jar file at " + filename + " as it does not exist");
+	    return;
+	}
+
+	try {
+	    URL u = new URL(externalFile.toURI().toURL().toString());
+	    method.invoke(sysloader, new Object[]{u});
+	} catch (Throwable t) {
+	    t.printStackTrace();
+	}
+
+	_guiExtLoaded = true;
+    }
+
+    public void loadGuiExtFile()
+    {
+	if(!_guiExtLoaded){
+	    loadExternalJar(getExtensionDirectory() + System.getProperty("file.separator") + "guiext.jar");		
+	}
+    }    
+
+    public boolean isGuiExtLoaded()
+    {
+	return _guiExtLoaded;
+    }
+
+    public void changeExtPane(JPanel extPane)
+    {
+	_extPane.removeAll();
+	_extPane.add(extPane, BorderLayout.CENTER);
+	_extPane.revalidate();
+	_extPane.repaint();
+    }
+
+    public static void setExtPane(JPanel extPane)
+    {	
+	_extPane = extPane;
+    }
+
+    public JPanel getExtPane()
+    {
+	return _extPane;
+    }
+
+    public String getExtensionDirectory()
+    {
+	return GAI.getGSDL3ExtensionHome() + System.getProperty("file.separator") + _fileStem;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Option.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Option.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Option.java	(revision 31634)
@@ -0,0 +1,201 @@
+package org.greenstone.admin.guiext;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import javax.swing.ImageIcon;
+import javax.swing.JTable;
+
+import org.greenstone.admin.GAI;
+
+import org.w3c.dom.Element;
+
+public class Option
+{
+    protected String _type = null;
+    protected String _name = null;
+    protected String _id = null;
+    protected String _value = null;    
+    protected OptionList _parent = null;
+    
+    protected Object _classObject = null;
+    protected Method _checkMethod = null;
+
+    protected ImageIcon _image = null;
+
+    private boolean _canCheck = false;
+    private int _checkResult = -1;
+
+    private final String GOOD_ICON = GAI.getGSDL3Home() + System.getProperty("file.separator") + "resources" + System.getProperty("file.separator") + "images" + System.getProperty("file.separator") + "icongreentick.png";
+    private final String BAD_ICON = GAI.getGSDL3Home() + System.getProperty("file.separator") + "resources" + System.getProperty("file.separator") + "images" + System.getProperty("file.separator") + "iconredexclamation.png";
+    private final String ERROR_ICON = GAI.getGSDL3Home() + System.getProperty("file.separator") + "resources" + System.getProperty("file.separator") + "images" + System.getProperty("file.separator") + "iconorangetriange.png";
+
+    public Option(Element optionElement, OptionList parent)
+    {
+	_parent = parent;
+
+	if(optionElement != null){
+	    _id = optionElement.getAttribute("id");
+	    if(_id == null || _id.equals("")){
+		System.err.println("This option element does not have an id");
+	    }
+
+	    _name = optionElement.getAttribute("label");
+	    if(_name == null || _name.equals("")){
+		_name = _id;
+	    }
+	    
+	    _value = ExtXMLHelper.getValueFromSingleElement(optionElement, false);
+	    
+	    if(_value == null){
+		_value = "";
+	    }
+
+	    String checkClass = optionElement.getAttribute("checkClass");
+	    String checkMethod = optionElement.getAttribute("checkMethod");
+
+	    if(!checkClass.equals("") && !checkMethod.equals("")){
+		loadCheckMethod(checkClass, checkMethod);
+	    }
+
+	    performCheck();
+
+	    _type = optionElement.getAttribute("type");
+	    
+	    if(!(_type.equals("") || _type.equals("password") || _type.equals("filebrowse") || _type.equals("folderbrowse"))){
+		System.err.println("This option element specifies a type that is not \"password\", \"filebrowse\" or \"folderbrowse\"");	    
+	    }
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.OPTION + "> element is null");
+	}
+    }
+
+    private void loadCheckMethod(String checkClassString, String checkMethodString)
+    {
+	_parent.getParent().getParent().getParent().loadGuiExtFile();
+	
+	Class checkClass = null;
+	try{
+	    checkClass = Class.forName(checkClassString);
+	}
+	catch(Exception ex){
+	    System.err.println("Could not get an instance of the check class, either the class name is incorrect or the class does not exist inside the guiext.jar file");
+	    return;
+	}
+	
+	try{
+	    Constructor classConstructor = checkClass.getConstructor(new Class[0]);
+	    _classObject = classConstructor.newInstance(new Object[0]);
+	}
+	catch(Exception ex){
+	    System.err.println("Could not instantiate the check class, either the class name is incorrect or the class does not exist inside the guiext.jar file");
+	    return;
+	}
+
+	try{
+	    _checkMethod = checkClass.getDeclaredMethod(checkMethodString, new Class[]{Object.class});
+	}
+	catch(Exception ex){
+	    System.err.println("Could not get the check method, either the method name is incorrect or the method does not take an Object as a parameter");
+	    return;
+	}
+	   
+	if(!(_checkMethod.getReturnType().equals(Boolean.TYPE))){
+	    System.err.println("The check method does not return a boolean value as is required");
+	    return;
+	}
+	_canCheck = true;
+    }
+
+    private void setImageFromCheck()
+    {
+	if(_canCheck){
+	    if(_checkResult == 0){
+		_image = new ImageIcon(GOOD_ICON);
+	    }
+	    else if(_checkResult == 1){
+		_image = new ImageIcon(BAD_ICON);
+	    }
+	    else{
+		_image = new ImageIcon(ERROR_ICON);
+	    }
+	}
+	else{
+	    _image = new ImageIcon(GOOD_ICON);
+	}
+    }
+    
+    private void performCheck()
+    {
+	if(_canCheck){
+	    Boolean result = null;
+	    try{
+		result = ((Boolean)_checkMethod.invoke(_classObject, new Object[]{_value}));
+	    }
+	    catch(Exception ex){
+		_checkResult = -1;
+	    }
+	    if(result){
+		_checkResult = 0;
+	    }
+	    else{
+		_checkResult = 1;
+	    }
+	}
+	setImageFromCheck();
+    }
+
+    public boolean isCheckable()
+    {
+	return _canCheck;
+    }
+
+    public int getCheckResult()
+    {
+	return _checkResult;
+    }
+    
+    public String getName()
+    {
+	return _name;
+    }
+
+    public String getValue()
+    {
+	return _value;
+    }
+
+    public void setValue(String value)
+    {
+	_value = value;
+	performCheck();
+
+	JTable table = _parent.getParent().getTableFromOptionList(_parent);
+	
+	if(table != null){
+	    table.repaint();
+	    table.revalidate();
+	}
+    }
+
+    public String getId()
+    {
+	return _id;
+    }
+    
+    public OptionList getParent()
+    {
+	return _parent;
+    }
+    
+    public String getType()
+    {
+	return _type;
+    }
+
+    public ImageIcon getImage()
+    {
+	return _image;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/OptionList.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/OptionList.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/OptionList.java	(revision 31634)
@@ -0,0 +1,103 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+import java.util.Properties;
+
+public class OptionList
+{
+    Option[] _options = null;
+    String _label = null;
+    String _id = null;
+    String _file = null;
+    PropertiesStep _parent = null;
+    
+    public OptionList(Element optionListElement, PropertiesStep parent, boolean fromFile)
+    {
+	_parent = parent;
+
+	if(optionListElement != null){
+	    _label = optionListElement.getAttribute("label");
+	    _id = optionListElement.getAttribute("id");
+	    _file = optionListElement.getAttribute("file");
+	    
+	    Element[] optionElements = ExtXMLHelper.getMultipleChildElements(optionListElement, ExtXMLHelper.OPTION, true);
+	    if(optionElements == null){
+		System.err.println("This optionList element does not contain any option elements");
+	    }
+
+	    Properties properties = null;
+	    if(fromFile){
+		File propertiesFile = null;
+		if(_file.equals("")){
+		    propertiesFile = new File(_parent.getParent().getParent().getExtensionDirectory() + System.getProperty("file.separator") + "build.properties");
+		}
+		else{
+		    propertiesFile = new File(_parent.getParent().getParent().getExtensionDirectory() + System.getProperty("file.separator") + _file);
+		}
+
+		if(propertiesFile.exists()){
+		    properties = new Properties();
+		    
+		    try{
+			properties.load(new FileInputStream(propertiesFile));
+		    }
+		    catch(Exception ex){
+			System.err.println("Error loading previous properties values from the properties file, using default values");
+			//ex.printStackTrace();
+		    }
+		}
+	    }
+	    
+	    _options = new Option[optionElements.length];
+	    for(int i = 0; i < optionElements.length; i++){
+		_options[i] = new Option(optionElements[i], this);
+
+		if(fromFile && properties != null){
+		    String value = null;
+		    if(_id.equals("")){
+			value = properties.getProperty(_options[i].getId());
+		    }
+		    else{
+			value = properties.getProperty(_id + "." + _options[i].getId());
+		    }
+
+		    if(value != null){
+			_options[i].setValue(value);
+		    }
+		}
+	    }
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.OPTION_LIST  + "> element is null");
+	}
+    }
+
+    public Option[] getOptions()
+    {
+	return _options;
+    }
+
+    public String getLabel()
+    {
+	return _label;
+    }
+
+    public String getId()
+    {
+	return _id;
+    }
+
+    public String getFilename()
+    {
+	return _file;
+    }
+    
+    public PropertiesStep getParent()
+    {
+	return _parent;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PanelStep.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PanelStep.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PanelStep.java	(revision 31634)
@@ -0,0 +1,66 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import javax.swing.JButton;
+import javax.swing.JPanel;
+
+import java.awt.event.ActionListener;
+import java.awt.event.ActionEvent;
+
+import java.lang.reflect.Constructor;
+
+public class PanelStep extends Step
+{
+    String _controlPanelClass = null;
+
+    public PanelStep(Element panelStepElement, SequenceList parent)
+    {
+	super(panelStepElement, parent);
+
+	if(panelStepElement != null){
+
+	    _controlPanelClass = ExtXMLHelper.getValueFromSingleElement(panelStepElement, ExtXMLHelper.CLASS, true);
+
+	    if(_controlPanelClass == null || _controlPanelClass.equals("")){
+		System.err.println("The <" + ExtXMLHelper.CLASS + "> element inside this panel <" + ExtXMLHelper.STEP + "> either does not exist or is empty");
+	    }
+	    
+	    _button.addActionListener(new PanelButtonListener());   
+	}
+	else{
+	     System.err.println("This panel <" + ExtXMLHelper.STEP + "> element is null");
+	}
+    }
+
+    public class PanelButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    _parent.getParent().loadGuiExtFile();
+
+	    Class controlPanelClass = null;
+	    try{
+		controlPanelClass = Class.forName(_controlPanelClass);
+	    }
+	    catch(Exception ex){
+		ex.printStackTrace();
+		return;
+	    }
+
+	    Object classObj = null;
+	    try{
+		Constructor classConstructor = controlPanelClass.getConstructor(new Class[0]);
+		classObj = classConstructor.newInstance(new Object[0]);
+	    }
+	    catch(Exception ex){
+		System.err.println("Could not create the control panel class, either the class name is incorrect or the class does not exist inside the guiext.jar file");
+		return;
+	    }
+		
+	    _parent.getParent().changeExtPane(((BasePanel)(classObj)).getPanel());
+
+	    _parent.registerStepCompletion(PanelStep.this);
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PropertiesStep.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PropertiesStep.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/PropertiesStep.java	(revision 31634)
@@ -0,0 +1,591 @@
+package org.greenstone.admin.guiext;
+
+import javax.crypto.Cipher;
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import javax.swing.AbstractCellEditor;
+import javax.swing.BorderFactory;
+import javax.swing.ImageIcon;
+import javax.swing.JButton;
+import javax.swing.JFileChooser;
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JOptionPane;
+import javax.swing.JPanel;
+import javax.swing.JPasswordField;
+import javax.swing.JScrollPane;
+import javax.swing.JTable;
+import javax.swing.JTextField;
+import javax.swing.SwingConstants;
+
+import javax.swing.event.DocumentListener;
+import javax.swing.event.DocumentEvent;
+
+import javax.swing.table.AbstractTableModel;
+import javax.swing.table.TableCellEditor;
+import javax.swing.table.TableColumn;
+import javax.swing.table.TableModel;
+
+import java.awt.event.ActionListener;
+import java.awt.event.ActionEvent;
+import java.awt.event.KeyListener;
+import java.awt.event.KeyEvent;
+
+import java.awt.BorderLayout;
+import java.awt.Color;
+import java.awt.Component;
+import java.awt.Font;
+import java.awt.GridLayout;
+
+import java.util.Properties;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.util.Configuration;
+import org.greenstone.admin.LoggedMessageArea;
+
+public class PropertiesStep extends Step
+{
+    OptionList[] _optionLists = null;
+    JTable[] _tables = null;
+    OptionList[] _modifiedOptionLists = null;
+    
+    HashMap<OptionList, PropertyTable> _optionListTableMap = new HashMap<OptionList, PropertyTable>();
+
+    public PropertiesStep(Element propertiesStepElement, SequenceList parent)
+    {
+	super(propertiesStepElement, parent);
+	
+	if(propertiesStepElement != null){
+	    ExtensionInformation info = _parent.getParent();
+
+	    Element[] optionListElements = ExtXMLHelper.getMultipleChildElements(propertiesStepElement, ExtXMLHelper.OPTION_LIST, true);
+
+	    if(optionListElements != null){
+		
+		_optionLists = new OptionList[optionListElements.length];
+		_modifiedOptionLists = new OptionList[optionListElements.length];
+
+		for(int i = 0; i < optionListElements.length; i++){
+		    _optionLists[i] = new OptionList(optionListElements[i], this, false);
+		    _modifiedOptionLists[i] = new OptionList(optionListElements[i], this, true);
+		}
+	    }
+	    else{
+		System.err.println("This properties <" + ExtXMLHelper.STEP + "> element has no <" + ExtXMLHelper.OPTION_LIST + "> elements");
+	    }
+	}
+	else{
+	    System.err.println("This properties <" + ExtXMLHelper.STEP + "> element is null" );
+	}
+	
+	_button.addActionListener(new PropertiesButtonListener());
+    }
+
+    public void setPropertiesToDefaults()
+    {
+	for(int i = 0; i < _optionLists.length; i++){
+	    Option[] _modifiedOptions = _modifiedOptionLists[i].getOptions();
+	    Option[] _defaultOptions = _optionLists[i].getOptions();
+	    
+	    for(int j = 0; j < _defaultOptions.length; j++){
+		_modifiedOptions[j].setValue(_defaultOptions[j].getValue());
+	    }
+	}
+    }
+
+    public JTable getTableFromOptionList(OptionList list)
+    {
+	return _optionListTableMap.get(list);
+    }
+
+    public class PropertiesButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    ExtensionInformation extension = _parent.getParent();
+	    
+	    String fileStem = extension.getFileStem();
+	    
+	    JPanel mainPanel = new JPanel();
+	    mainPanel.setBorder(BorderFactory.createLoweredBevelBorder());
+	    mainPanel.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	    mainPanel.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	    mainPanel.setLayout(new BorderLayout());
+	    
+	    JPanel tablePanel = new JPanel(new GridLayout(_optionLists.length, 1));
+	    
+	    if(_tables == null){
+		_tables = new JTable[_optionLists.length];
+	    }
+	    for(int i = 0; i < _optionLists.length; i++){
+		OptionList currentList = _modifiedOptionLists[i];
+	    
+		JLabel singleTableLabel = new JLabel(currentList.getLabel()); 
+		singleTableLabel.setHorizontalAlignment(SwingConstants.CENTER);
+		singleTableLabel.setFont(new Font("Arial", Font.BOLD, 13));
+
+		JPanel singleTablePanel = new JPanel();
+		singleTablePanel.setLayout(new BorderLayout());
+		singleTablePanel.add(singleTableLabel, BorderLayout.NORTH);
+		
+		Option[] options = currentList.getOptions();		
+
+		CustomTableModel tableModel = new CustomTableModel(new String[]{"Setting", "Value", "Check"}, options);
+		PropertyTable propertiesTable = new PropertyTable(tableModel, currentList);
+		TableColumn column = propertiesTable.getColumnModel().getColumn(2);
+		column.setPreferredWidth(32);
+		column.setMinWidth(32);
+		column.setMaxWidth(32);
+		column.setResizable(false);
+
+		_optionListTableMap.put(currentList, propertiesTable);
+
+		//The line below is necessary as the default grid colour on mac is white, which makes the lines invisible.
+		propertiesTable.setGridColor(Color.BLACK);
+
+		if(_tables[i] == null){
+		    _tables[i] = propertiesTable;
+		}
+
+		//The line below is a workaround for a bug in the JTable class that does not store properties if focus is given to another component.
+		propertiesTable.putClientProperty("terminateEditOnFocusLost", Boolean.TRUE);
+
+		singleTablePanel.add(propertiesTable, BorderLayout.CENTER);
+		
+		tablePanel.add(singleTablePanel);
+	    }
+
+	    JButton defaultsButton = new JButton("Restore default settings");
+	    defaultsButton.addActionListener(new DefaultSettingsButtonListener());
+	    
+	    JButton saveButton = new JButton("Save");
+	    saveButton.addActionListener(new SaveButtonListener());
+	    
+	    JPanel buttonPanel = new JPanel();
+	    buttonPanel.setLayout(new GridLayout(1, 2));
+	    buttonPanel.add(saveButton);
+	    buttonPanel.add(defaultsButton);
+	    
+	    mainPanel.setBorder(BorderFactory.createEmptyBorder(10,20,10,20));
+	    mainPanel.setBackground(Configuration.getColor("coloring.workspace_selection_background"));
+	    mainPanel.setForeground(Configuration.getColor("coloring.workspace_selection_foreground"));
+	    mainPanel.add(tablePanel, BorderLayout.CENTER);
+	    mainPanel.add(buttonPanel, BorderLayout.SOUTH);
+	    
+	    _parent.getParent().changeExtPane(mainPanel);
+	}
+    }
+
+    public class BrowseButtonListener implements ActionListener
+    {
+	Option _option = null;
+	JTextField _path = null;
+	JFileChooser _browser = null;
+	
+	public BrowseButtonListener(Option option, JTextField path)
+	{
+	    _option = option;
+	    _path = path;
+	    _browser = new JFileChooser();
+	}
+	
+	public void actionPerformed(ActionEvent e)
+	{
+	    if(e.getActionCommand().equals("edit")){
+		if(_option.getType().equalsIgnoreCase("folderbrowse")){
+		    _browser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
+		}
+		int returnValue = _browser.showOpenDialog(null);
+
+		if(returnValue == JFileChooser.APPROVE_OPTION){
+		    _option.setValue(_browser.getSelectedFile().getAbsolutePath());
+		    _path.setText(_option.getValue());
+		}
+	    }
+	}
+    }
+
+    public class PathBoxListener implements DocumentListener
+    {
+	Option _option = null;
+	JTextField _path = null;
+
+	public PathBoxListener(Option option, JTextField path)
+	{
+	    _option = option;
+	    _path = path;
+	}
+	
+	public void changedUpdate(DocumentEvent e)
+	{
+	    _option.setValue(_path.getText());
+	}
+	
+	public void insertUpdate(DocumentEvent e)
+	{
+	    _option.setValue(_path.getText());
+	}
+
+	public void removeUpdate(DocumentEvent e)
+	{
+	    _option.setValue(_path.getText());
+	}
+    }
+
+    public class PasswordOKButtonListener implements ActionListener
+    {
+	JPasswordField _password = null;
+	JPasswordField _confirm = null;
+	Option _option = null;
+	JFrame _passwordFrame = null;
+
+	public PasswordOKButtonListener(JPasswordField password, JPasswordField confirm, Option option, JFrame passwordFrame)
+	{
+	    _password = password;
+	    _confirm = confirm;
+	    _option = option;
+	    _passwordFrame = passwordFrame;
+	}
+
+	public void actionPerformed(ActionEvent e)
+	{
+	    if(Arrays.equals(_password.getPassword(), _confirm.getPassword())){
+		try{
+		    _option.setValue(new String(encrypt(_password.getPassword())));
+		}
+		catch(Exception ex){
+		    System.err.println("Error encrypting password");
+		}
+		_passwordFrame.dispose();
+	    }
+	    else{
+		JOptionPane.showMessageDialog(null, "The passwords you entered do not match.");
+		_password.setText("");
+		_confirm.setText("");
+	    }
+	}
+    }  
+
+    public static char[] encrypt(char[] value) throws Exception
+    {
+	byte[] salt = {'T', 'p'};
+	int count = 20;
+
+	KeyGenerator keygen = KeyGenerator.getInstance("AES");
+        SecretKey secretKey = keygen.generateKey();
+        byte[] raw = secretKey.getEncoded();
+        SecretKeySpec secretKeySpec = new SecretKeySpec(raw, "AES");
+
+	Cipher cipher = Cipher.getInstance("AES");
+	cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec);
+	
+	byte[] encryptedPassword = cipher.doFinal((new String(value)).getBytes("UTF-8"));
+
+	return new String(encryptedPassword).toCharArray();
+    }
+
+    public class PasswordCancelButtonListener implements ActionListener
+    {
+	JFrame _passwordFrame = new JFrame();
+
+	public PasswordCancelButtonListener(JFrame passwordFrame)
+	{
+	    _passwordFrame = passwordFrame;
+	}
+
+	public void actionPerformed(ActionEvent e)
+	{
+	    _passwordFrame.dispose();
+	}
+    }
+
+    public class PasswordFieldListener implements ActionListener
+    {
+	JPasswordField _password = new JPasswordField(10);
+	JPasswordField _confirm = new JPasswordField(10);
+	JFrame _passwordFrame = new JFrame("Enter password");
+
+	public PasswordFieldListener(Option option)
+	{
+	    JButton okButton = new JButton("Change");
+	    JButton cancelButton = new JButton("Cancel");
+	    
+	    okButton.addActionListener(new PasswordOKButtonListener(_password, _confirm, option, _passwordFrame));
+	    cancelButton.addActionListener(new PasswordCancelButtonListener(_passwordFrame));
+
+	    _passwordFrame.getContentPane().setLayout(new GridLayout(3,2));
+	    _passwordFrame.getContentPane().add(new JLabel("Password:"));
+	    _passwordFrame.getContentPane().add(_password);
+	    _passwordFrame.getContentPane().add(new JLabel("Confirm:"));
+	    _passwordFrame.getContentPane().add(_confirm);
+	    _passwordFrame.getContentPane().add(okButton);
+	    _passwordFrame.getContentPane().add(cancelButton);
+	    _passwordFrame.pack();
+	}
+	
+	public void actionPerformed(ActionEvent e)
+	{
+	    _password.setText("");
+	    _confirm.setText("");
+	    _passwordFrame.setLocationRelativeTo(null);
+	    _passwordFrame.setVisible(true);
+	}
+    }
+
+    public class PasswordEditor extends AbstractCellEditor implements TableCellEditor
+    {
+	JButton _onClickButton = new JButton("Change Password?");
+
+	public PasswordEditor(Option option)
+	{
+	    _onClickButton.addActionListener(new PasswordFieldListener(option));
+	}
+
+	public Object getCellEditorValue()
+	{
+	    return "";
+	}
+
+	public Component getTableCellEditorComponent(JTable table, Object path, boolean isSelected, int row, int column)
+	{
+	    return _onClickButton;
+	}
+    }
+
+    public class BrowseEditor extends AbstractCellEditor implements TableCellEditor
+    {
+	JPanel _browserPanel = null;
+	JTextField _path = null;
+	JButton _browserButton = null;
+	Option _option = null;
+	String _type = null;
+
+	public BrowseEditor(Option option, String type)
+	{
+	    _option = option;
+	    _type = type;
+	    
+	    _path = new JTextField(_option.getValue());
+	    _path.getDocument().addDocumentListener(new PathBoxListener(_option, _path));
+
+	    _browserButton = new JButton("Browse");
+	    _browserButton.setActionCommand("edit");
+	    _browserButton.addActionListener(new BrowseButtonListener(_option, _path));
+	    _browserButton.setBorderPainted(false);
+
+	    _browserPanel = new JPanel();
+	    _browserPanel.setLayout(new BorderLayout());
+	    _browserPanel.add(_path, BorderLayout.CENTER);
+	    _browserPanel.add(_browserButton, BorderLayout.EAST);
+	}
+
+	public Object getCellEditorValue()
+	{
+	    _path.setText(_option.getValue());
+	    return _option.getValue();
+	}
+
+	public Component getTableCellEditorComponent(JTable table, Object path, boolean isSelected, int row, int column)
+	{
+	    return _browserPanel;
+	}
+    }
+
+    public class PropertyTable extends JTable 
+    {
+	OptionList _properties = null;
+	TableCellEditor[] _editors = null;
+
+	public PropertyTable(TableModel tm, OptionList properties)
+	{
+	    super(tm);
+	    _properties = properties;
+	    
+	    Option[] options = _properties.getOptions();
+	    _editors = new TableCellEditor[options.length];
+
+	    for(int i = 0; i < options.length; i++){
+		Option currentOption = options[i];
+
+		if(currentOption.getType().equals("filebrowse")){
+		    _editors[i] = new BrowseEditor(currentOption, "file");
+		}
+		else if(currentOption.getType().equals("folderbrowse")){
+		    _editors[i] = new BrowseEditor(currentOption, "folder");
+		}
+		else if(currentOption.getType().equals("password")){
+		    _editors[i] = new PasswordEditor(currentOption);
+		}
+		else{
+		    _editors[i] = null;
+		}
+	    }
+	}
+
+	public TableCellEditor getCellEditor(int row, int column)
+	{
+	    if(column == 1 && row < _editors.length && _editors[row] != null){
+		return _editors[row];
+	    }
+	    return super.getCellEditor(row, column);
+	}
+    }
+    
+    public class SaveButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    for(int i = 0; i < _optionLists.length; i++){
+		OptionList currentOptionList = _modifiedOptionLists[i];
+		Option[] currentOptions = currentOptionList.getOptions();
+
+		Properties newProperties = new Properties();
+		File propertiesFile = null;
+		if(currentOptionList.getFilename().equals("")){
+		    propertiesFile = new File(_parent.getParent().getExtensionDirectory() + System.getProperty("file.separator") + "build.properties");
+		}
+		else{
+		    propertiesFile = new File(_parent.getParent().getExtensionDirectory() + System.getProperty("file.separator") + currentOptionList.getFilename());
+		}
+
+		if(propertiesFile.exists()){
+		    try{
+			newProperties.load(new FileInputStream(propertiesFile));
+		    }
+		    catch(Exception ex){
+			System.err.println("Could not load properties file before saving to it, existing values will be removed. Possible reasons for this include: \n - Invalid properties file format\n - File is write protected");
+		    }
+		}
+
+		for(int j = 0; j < currentOptions.length; j++){
+		    Option currentOption = currentOptions[j];
+		    if(currentOptionList.getId().equals("")){
+			newProperties.setProperty(currentOption.getId(), currentOption.getValue());
+		    }
+		    else{
+			newProperties.setProperty(currentOptionList.getId() + "." + currentOption.getId(), currentOption.getValue());
+		    }
+		}
+
+		try{
+		    if(currentOptionList.getFilename().equals("")){
+			newProperties.store(new FileOutputStream(_parent.getParent().getExtensionDirectory() + System.getProperty("file.separator") + "build.properties"), null);
+		    }
+		    else{
+			newProperties.store(new FileOutputStream(_parent.getParent().getExtensionDirectory() + System.getProperty("file.separator") + currentOptionList.getFilename()), null);
+		    }
+		}
+		catch(Exception ex){
+		    System.err.println("Could not save to properties file.");
+		}
+	    }
+
+	    _parent.registerStepCompletion(PropertiesStep.this);
+	}
+    }
+
+    public class DefaultSettingsButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    for(int i = 0; i < _tables.length; i++){
+		JTable currentTable = _tables[i]; 
+		Option[] currentDefaultOptions = _optionLists[i].getOptions();
+
+		for(int j = 0; j < currentDefaultOptions.length; j++){
+		    Option currentOption = currentDefaultOptions[j];
+		    currentTable.setValueAt(currentOption.getValue(), j, 1);
+		}
+		
+		currentTable.revalidate();
+		currentTable.repaint();
+	    }
+	}
+    }
+
+    public class CustomTableModel extends AbstractTableModel
+    {
+	private String[] _columnNames = null;
+    	private Option[] _data = null;
+	private ImageIcon[] _images = null;
+	
+    	public CustomTableModel(String[] columnNames, Option[] data){
+	    _columnNames = columnNames;
+	    _data = data;
+	    
+	    _images = new ImageIcon[_data.length];
+	    for(int i = 0; i < _data.length; i++){
+		_images[i] = _data[i].getImage();
+	    }
+    	}
+	
+        public int getColumnCount() {
+            return _columnNames.length;
+        }
+	
+        public int getRowCount() {
+            return _data.length;
+        }
+
+        public String getColumnName(int col) {
+            return _columnNames[col];
+        }
+
+        public Object getValueAt(int row, int col) {
+	    if(col == 0){
+		return _data[row].getName();
+	    }
+	    else if(col == 1){
+		if(_data[row].getType().equals("password") && !_data[row].getValue().equals("")){
+		    return "Password Set";
+		}
+		else if(_data[row].getType().equals("password") && _data[row].getValue().equals("")){
+		    return "No Password Set";
+		}
+		
+		return _data[row].getValue();
+	    }
+	    else if(col == 2){
+		return _data[row].getImage();
+	    }
+            else{
+		return null;
+	    }
+        }
+
+
+        public Class<? extends Object> getColumnClass(int c) {
+            return getValueAt(0, c).getClass();
+        }
+
+	public boolean isCellEditable(int row, int col) {
+	    if (col == 1) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+	
+        public void setValueAt(Object value, int row, int col) {
+	    Option o = _data[row];
+	    if(isCellEditable(row, col) && !o.getType().equals("password")){
+		o.setValue((String)value);
+		
+		if(o.isCheckable()){
+		    _images[row] = o.getImage();
+		}
+	    }
+	    fireTableDataChanged();
+        }
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/RemoveStep.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/RemoveStep.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/RemoveStep.java	(revision 31634)
@@ -0,0 +1,67 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import javax.swing.JButton;
+import javax.swing.JPanel;
+import javax.swing.JTextArea;
+
+import java.awt.BorderLayout;
+import java.awt.event.ActionListener;
+import java.awt.event.ActionEvent;
+
+import java.io.File;
+
+import java.lang.reflect.Constructor;
+
+public class RemoveStep extends Step
+{
+    public RemoveStep(DownloadStep dls, SequenceList parent)
+    {
+	super("AUTOMATIC_REMOVE", "Remove Extension", dls.getId(), dls.getId(), parent);
+
+	_button.addActionListener(new RemoveButtonListener());   
+    }
+
+    public RemoveStep(Element removeStepElement, SequenceList parent)
+    {
+	super(removeStepElement, parent);
+
+	if(removeStepElement != null){	    
+	    _button.addActionListener(new RemoveButtonListener());   
+	}
+	else{
+	     System.err.println("This remove <" + ExtXMLHelper.STEP + "> element is null");
+	}
+    }
+
+    public class RemoveButtonListener implements ActionListener
+    {
+	public void actionPerformed(ActionEvent e)
+	{
+	    _button.setEnabled(false);
+	    _button.setText("Removing...");
+	    ExtensionInformation info = _parent.getParent();
+	    JPanel descPanel = new JPanel();
+	    descPanel.setLayout(new BorderLayout());
+
+	    JTextArea descArea = new JTextArea();
+	    descArea.setEditable(false);
+	    descArea.setText(info.getDescription());
+	    descArea.setLineWrap(true);
+	    descArea.setWrapStyleWord(true);
+
+	    descPanel.add(descArea);
+
+	    _parent.getParent().changeExtPane(descPanel);
+
+	    ExtPane.deleteDir(new File(info.getExtensionDirectory()));
+
+	    _button.setEnabled(true);
+	    _button.setText(_label);
+
+	    _parent.clearPropertySteps();
+	    _parent.registerStepCompletion(RemoveStep.this);
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/SequenceList.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/SequenceList.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/SequenceList.java	(revision 31634)
@@ -0,0 +1,235 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import org.greenstone.admin.GAI;
+
+import java.util.Properties;
+import java.util.ArrayList;
+
+import javax.swing.JButton;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+
+public class SequenceList
+{
+    protected ArrayList<Step> _steps = new ArrayList<Step>();
+    protected ExtensionInformation _parent = null;
+    protected Properties _dependencies = null;
+
+    public SequenceList(Element sequenceElement, ExtensionInformation parent)
+    {
+	_parent = parent;
+
+	loadExtensionStatesFromFile();
+
+	if(sequenceElement != null){
+	    Element[] stepElements = ExtXMLHelper.getMultipleChildElements(sequenceElement, ExtXMLHelper.STEP, true);
+	    if(stepElements == null || stepElements.length == 0){
+		System.err.println("This <" + ExtXMLHelper.SEQUENCE_LIST + "> element does not have any valid <" + ExtXMLHelper.STEP + "> elements");
+	    }
+	    else{
+		for(int i = 0; i < stepElements.length; i++){
+		    Element currentStep = stepElements[i];
+		    String action = currentStep.getAttribute("action");
+		    
+		    if(i == 0 && !action.equals("download")){
+			_steps.add(new DownloadStep(this));
+		    }
+
+		    if(action.equals("download")){
+			_steps.add(new DownloadStep(currentStep, this));
+		    }
+		    else if(action.equals("properties")){
+			_steps.add(new PropertiesStep(currentStep, this));
+		    }
+		    else if(action.equals("button")){
+			_steps.add(new CommandStep(currentStep, this));
+		    }
+		    else if(action.equals("panel")){
+			_steps.add(new PanelStep(currentStep, this));
+		    }
+		    else if(action.equals("remove")){
+			_steps.add(new RemoveStep(currentStep, this));
+		    }
+		    else{
+			System.err.println("\"" + action + "\" is not a valid action attribute for <" + ExtXMLHelper.STEP + "> elements (valid options are \"download\", \"properties\", \"panel\", \"button\") or \"remove\"");
+		    }
+
+		    if((i == stepElements.length-1) && !action.equals("remove")){
+			_steps.add(new RemoveStep((DownloadStep)_steps.get(0), this));
+		    }
+		}
+	    }
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.SEQUENCE_LIST + "> element is null");
+	}
+    }
+    
+    public ExtensionInformation getParent()
+    {
+	return _parent;
+    }
+
+    public Step[] getSteps()
+    {
+	return (_steps.toArray(new Step[0]));
+    }
+
+    public JButton[] getButtons()
+    {
+	if(_steps == null){
+	    System.err.println("Cannot get buttons for this sequence list as it has no steps");
+	    return null;
+	}
+	
+	JButton[] buttons = new JButton[_steps.size()];
+	for(int i = 0; i < _steps.size(); i++){
+	    buttons[i] = _steps.get(i).getButton();
+	}
+	
+	return buttons;
+    }
+
+    private void loadExtensionStatesFromFile()
+    {
+	if(_dependencies == null){
+	    _dependencies = new Properties();
+	}
+
+	String fileName = GAI.getGSDL3ExtensionHome() + System.getProperty("file.separator") + "extensionStates.properties";
+	try{
+	    File stateFile = new File(fileName);
+	    
+	    if(stateFile.exists()){
+		_dependencies.load(new FileInputStream(new File(fileName)));
+	    }
+	}
+	catch(Exception ex){
+	    ex.printStackTrace();
+	}
+    }
+
+    private void saveExtensionStatesToFile()
+    {
+	String fileName = GAI.getGSDL3ExtensionHome() + System.getProperty("file.separator") + "extensionStates.properties";
+	try{
+	    _dependencies.store(new FileOutputStream(new File(fileName)), null);
+	}
+	catch(Exception ex){
+	    ex.printStackTrace();
+	}
+    }
+
+    public boolean addDependency(String id)
+    {
+	if(_dependencies != null){
+	    _dependencies.setProperty(_parent.getFileStem() + "." + id, "true");
+	    saveExtensionStatesToFile();
+	    return true;
+	}
+	else{
+	    System.err.println("Cannot add dependency as it has not yet been loaded");
+	    return false;
+	}
+    }
+
+    public void rollbackTo(String id)
+    {
+	if(_dependencies != null){
+	    rollbackToRecursive(id);
+	    saveExtensionStatesToFile();
+	}
+	else{
+	    System.err.println("Cannot remove dependency as it has not yet been loaded");
+	}
+    }
+
+    private void rollbackToRecursive(String id)
+    {
+	Step rollbackStep = null;
+	ArrayList<String> dependsSteps = new ArrayList<String>();
+	for(int i = 0; i < _steps.size(); i++){
+	    if((_steps.get(i)).getId().equals(id)){
+		rollbackStep = _steps.get(i);
+	    }
+	    
+	    String[] depends = _steps.get(i).getDependsOn().split(",");
+
+	    for(int j = 0; j < depends.length; j++){
+		if(depends[j].equals(id)){
+		    dependsSteps.add(_steps.get(i).getId());
+		}
+	    }
+	}
+	
+	_dependencies.setProperty(_parent.getFileStem() + "." + id, "false");
+
+	for(int i = 0; i < dependsSteps.size(); i++){
+	    rollbackToRecursive(dependsSteps.get(i));
+	}
+    }
+
+    public void clearPropertySteps()
+    {
+	for(int i = 0; i < _steps.size(); i++){
+	    Step currentStep = _steps.get(i);
+	    if(currentStep instanceof PropertiesStep){
+		((PropertiesStep)currentStep).setPropertiesToDefaults();
+	    }
+	}
+    }
+
+    public void registerStepCompletion(Step step)
+    {
+	addDependency(step.getId());
+	   
+	if(!step.getRollbacks().equals("")){
+	    String[] rollbacks = step.getRollbacks().split(",");
+	    for(int k = 0; k < rollbacks.length; k++){
+		rollbackTo(rollbacks[k]);
+	    }
+	}
+
+	updateButtons();
+    }
+
+    public void updateButtons()
+    {
+	loadExtensionStatesFromFile();
+
+	ExtensionInformation currentExtension = _parent;
+	String fileStem = currentExtension.getFileStem();
+
+	File extDir = new File(currentExtension.getExtensionDirectory());
+	if(!extDir.exists()){
+	    rollbackTo("");
+	}
+	
+	for(int i = 0; i < _steps.size(); i++){
+	    Step currentStep = _steps.get(i);
+	    JButton currentButton = currentStep.getButton();
+
+	    //If this step button depends on another button then disabled it
+	    String dependsOn = currentStep.getDependsOn();
+	    if(dependsOn.equals("")){
+		currentButton.setEnabled(true);
+	    }
+	    else{
+		String[] dependsList = dependsOn.split(",");
+		boolean enable = true;		
+		for(int j = 0; j < dependsList.length; j++){
+		    String propertyName = fileStem + "." + dependsList[j];
+
+		    if(_dependencies.getProperty(propertyName) == null || _dependencies.getProperty(propertyName).equals("false")){
+			enable = false;
+		    }
+		}
+		currentButton.setEnabled(enable);
+	    }
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Source.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Source.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Source.java	(revision 31634)
@@ -0,0 +1,61 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+public class Source
+{
+    String _method = null;
+    String _url = null;
+    String _folder = null;
+    DownloadStep _parent = null;
+
+    public Source(String method, String url, String folder, DownloadStep parent)
+    {
+	_parent = parent;
+	_method = method;
+	_url = url;
+	_folder = folder;
+    }
+
+    public Source(Element sourceElement, DownloadStep parent)
+    {
+	_parent = parent;
+
+	if(sourceElement != null){
+	    _method = sourceElement.getAttribute("method");
+	    if(_method.equals("")){
+		System.err.println("This <" + ExtXMLHelper.SOURCE + "> element does not contain a method attribute");
+	    }
+
+	    _folder = sourceElement.getAttribute("folder");
+	    
+	    _url = ExtXMLHelper.getValueFromSingleElement(sourceElement, true);
+	    if(_url == null || _url.equals("")){
+		System.err.println("This <" + ExtXMLHelper.SOURCE + "> element does not contain a url");
+	    }
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.SOURCE + "> element is null");
+	}
+    }
+
+    public DownloadStep getParent()
+    {
+	return _parent;
+    }
+
+    public String getMethod()
+    {
+	return _method;
+    }
+
+    public String getURL()
+    {
+	return _url;
+    }
+
+    public String getFolder()
+    {
+	return _folder;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Step.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Step.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/admin/guiext/Step.java	(revision 31634)
@@ -0,0 +1,112 @@
+package org.greenstone.admin.guiext;
+
+import org.w3c.dom.Element;
+
+import java.awt.BorderLayout;
+import java.awt.Insets;
+
+import javax.swing.JButton;
+import javax.swing.JTextArea;
+import javax.swing.JPanel;
+
+public class Step
+{
+    protected SequenceList _parent = null;
+    protected String _id = null;
+    protected String _action = null;
+    protected String _label = null;
+    protected String _dependsOn = null;
+    protected String _rollbacks = null;
+    protected JButton _button = null;
+    protected static JTextArea _commonMessageArea = new JTextArea();
+
+    public Step(String id, String label, String dependsOn, String rollbacks, SequenceList parent)
+    {
+	_parent = parent;
+	_id = id;
+	_label = label;
+	_dependsOn = dependsOn;
+	_rollbacks = rollbacks;
+
+	_button = new JButton(_label);
+	_button.setEnabled(false);
+    }
+
+    public Step(Element stepElement, SequenceList parent)
+    {
+	_parent = parent;
+	if(stepElement != null){
+	    _id = stepElement.getAttribute("id");
+	    if(_id.equals("")){
+		System.err.println("This <" + ExtXMLHelper.STEP + "> element has no id attribute or the id attribute is empty");
+	    }
+	    
+	    _label = stepElement.getAttribute("label");
+	    if(_label.equals("")){
+		System.err.println("This <" + ExtXMLHelper.STEP + "> element has no label attribute or the label attribute is empty");
+	    }
+	    
+	    _action = stepElement.getAttribute("action");
+	    if(_action.equals("")){
+		System.err.println("This <" + ExtXMLHelper.STEP + "> element has no action attribute or the action attribute is empty");
+	    }
+	    
+	    _dependsOn = stepElement.getAttribute("dependsOn");
+	    String[] dependencies = _dependsOn.split(",");
+	    for(int i = 0; i < dependencies.length; i++){
+		if(dependencies[i].equals(_id)){
+		    System.err.println("The " + _id + " step depends on itself which is not allowed");
+		    _dependsOn = "";
+		}
+	    }
+
+	    _rollbacks = stepElement.getAttribute("rollbackTo");
+
+	    _button = new JButton(_label);
+	    _button.setEnabled(false);
+	}
+	else{
+	    System.err.println("This <" + ExtXMLHelper.STEP + "> element is null");
+	}
+    }
+
+    public JButton getButton()
+    {
+	return _button;
+    }
+
+    public String getId()
+    {
+	return _id;
+    }
+
+    public String getAction()
+    {
+	return _action;
+    }
+
+    public String getLabel()
+    {
+	return _label;
+    }
+    
+    public String getDependsOn()
+    {
+	return _dependsOn;
+    }
+
+    public String getRollbacks()
+    {
+	return _rollbacks;
+    }
+
+    public SequenceList getParent()
+    {
+	return _parent;
+    }
+
+    public JTextArea getCommonMessageArea()
+    {
+	return _commonMessageArea;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/anttasks/README
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/anttasks/README	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/anttasks/README	(revision 31634)
@@ -0,0 +1,3 @@
+The ant tasks that were in here are useful for more than just Greenstone3, so I
+have moved them to the other-projects directory
+(http://svn.greenstone.org/other-projects/trunk/anttasks).
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/Phind.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/Phind.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/Phind.java	(revision 31634)
@@ -0,0 +1,945 @@
+/**********************************************************************
+ *
+ * Phind.java -- the Phind java applet - modified to work with gsdl3 kjdon
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+/*********************************************************************
+
+To use the applet, you'll need to embed it in a web page like this:
+
+<APPLET CODE="Phind.class" WIDTH=500 HEIGHT=500>
+
+  <PARAM NAME=collection  VALUE="fao.org">
+  <PARAM NAME=classifier  VALUE="1">
+  <PARAM NAME=phindcgi    VALUE="http://kowhai/cgi-bin/phindcgi">
+  <PARAM NAME=library     VALUE="http://kowhai/cgi-bin/library">
+  <PARAM NAME=backdrop    VALUE="http://kowhai/~paynter/transfer/phindtest/green1.jpg">
+  The Phind java applet.
+</APPLET>
+
+There are a bunch of other parameters; these are described in the 
+getParameters method below.  It is all done for you in Greenstone
+in the document.dm macro file (the _phindapplet_ macro).
+
+You may have problems with Java applet security.  Java applet's can only
+open socket connections (including the HTTP connections the applet uses 
+to get data) to the same server the applet was loaded from.  This means
+that your phindcgi, library, and (optional) backdrop URLs must be on the
+same machine that your web page was loaded from.
+
+**********************************************************************
+
+The applet comprises several classes:
+
+1. Phind (this file) is the applet class, loaded by the browser.
+   It also handles network connections.
+2. ResultDisplay is a Panel that sits in the applet and displays 
+   things; displays are connected in a doubly-linked list.
+3. ResultBox holds the results of a query.  Result boxes are shown
+   to the user through ResultDisplays.  Another doubly-linked list.
+4. ResultTitle acts as a caption to a ResultBox describing its contents.
+5. ResultCanvas is what the ResultBox data is drawn on.  It draws the
+   words on the screen, and handles user input from the mouse.
+6. ResultItem represents a single result object (a phrase or document).
+7. PhindTitle is for drawing backdrops in ResultDisplays.
+
+**********************************************************************/
+
+package org.greenstone.applet.phind;
+
+import java.awt.BorderLayout;
+import java.awt.Button;
+import java.awt.Choice;
+import java.awt.Color;
+import java.awt.Component;
+import java.awt.Dimension;
+import java.awt.event.ActionEvent;
+import java.awt.FlowLayout;
+import java.awt.Font;
+import java.awt.GridLayout;
+import java.awt.Image;
+import java.awt.Label;
+import java.awt.Panel;
+import java.awt.TextField;
+import java.awt.event.ActionListener;
+import java.net.URL;
+import java.io.DataInputStream;
+
+import java.net.Socket;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.io.IOException;
+
+import java.util.Vector;
+import java.util.Date;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+//import javax.xml.parsers.*;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+
+
+public class Phind extends java.applet.Applet 
+    implements ActionListener {
+    
+    // What is the collection called?
+    public String collection;
+
+    // Which phind classifier are we using? (There may be more than one.)
+    public String classifier;
+
+    // Internet address of phind resources
+    public String library_address, phindcgi_address;
+
+    // Initial search term
+    public String initialSearch;
+
+    // Number of phrases to retrieve at any one time
+    public int phraseBlockSize;
+
+    // Appearance parameters
+    public boolean vertical;
+    public int depth;
+
+    // Font
+    public int fontSize;
+    public String fontName;
+    public Font plainFont, boldFont;
+
+    // Do we want a background image in the applet?
+    public boolean showImage;
+    public String backdrop_address;
+    public Image backgroundImage;
+    public boolean showBorder;
+
+    // Colours
+    public Color panel_fg, panel_bg,
+	column_1_fg, column_1_bg, 
+	column_2_fg, column_2_bg,
+	highlight_fg, highlight_bg,
+	thesaurus_fg, thesaurus_bg, thesaurus_bar_fg, thesaurus_bar_bg,
+	expansion_fg, expansion_bg, expansion_bar_fg, expansion_bar_bg,
+	document_fg, document_bg, document_bar_fg, document_bar_bg,
+	message_fg, message_bg;
+    
+    // Column dimensions
+    int column_1_width, column_2_width;
+
+    // Where do we open new windows
+    String searchWindowName, documentWindowName;
+
+    // the mode of operation
+    int mode;
+    final int initMode = 0;
+    final int idleMode = 1;
+    final int searchMode = 2;
+
+    // Elements of the control panel
+    boolean showControlPanel;
+    Label titleLabel;
+    TextField wordField;
+    Button searchButton, prevButton, nextButton;
+
+    // Holders for the downloaded data
+    Panel resultPanel;
+    ResultDisplay firstDisplay, lastDisplay;
+
+    // The time at which the last query finished
+    Date lastQueryEndTime;
+
+    // lastQueryEndTime is stored to ensure a 1 second gap between a query
+    // returning and a new one beginning.  It is needed because the FAO
+    // folks in Rome have a huge lag, and frquently click several times
+    // while they wait; these clicks are turned into new queries, which
+    // they await again.  It is no elegant solution, but it seems like the
+    // easiest, given that I don't know threads.
+    // 1. The search button is easy to disable, and is disabled when a
+    //    socket connection is in progress.
+    // 2. ResutCanvas widgets can'r be similarly disabled because the
+    //    browser hides or wipes them, which looks very bad.
+    // 3. I cannot just ignore clicks on canvasses because the browser
+    //    caches the clicks while the socket connection is going on, and then
+    //    sends them through afterwards, when the canvas is accepting clicks
+    //    again.
+    // 4. Current sequence of events is to record the time the last query
+    //    ends, then whenever a click happens make sure a second has past.  if
+    //    you double-click the the first query is sent, returns, end-tie is
+    //    set, and the second (and any others made during query time) is
+    //    *immediately* processed, but since 1 second hasn't past it is
+    //    ignored.
+
+  
+    public String getAppletInfo() {
+	return "Phind by Gordon Paynter (paynter@cs.waikato.ac.nz). Copyright 1997-2000.";
+    }
+
+
+    public void init() {
+
+	mode = initMode;
+	
+	// Read applet parameters
+	getParameters();
+
+	// Initialise the user interface
+	setBackground(panel_bg);
+	lastQueryEndTime = new Date();
+
+	// fonts used to output text
+	plainFont = new Font(fontName, Font.PLAIN, fontSize);
+	boldFont  = new Font(fontName, Font.BOLD, fontSize);
+
+	// The phind applet layout manager
+	setLayout(new BorderLayout());
+
+	// Panel containing the displays is in the center of the display
+	resultPanel = new Panel();
+	if (vertical) {
+	    resultPanel.setLayout(new GridLayout(depth,1,0,2));
+	} else {
+	    System.out.println("horizontal");
+	    resultPanel.setLayout(new GridLayout(1,depth,2,0));
+	}
+	add("Center", resultPanel);
+
+	// Create ResultDisplays and place into the interface
+	ResultDisplay d1, d2 = null;
+	firstDisplay = new ResultDisplay(this, null);
+	resultPanel.add(firstDisplay);
+
+	if (depth == 1) {
+	    lastDisplay = firstDisplay;
+	} else {
+	    d1 = firstDisplay;
+	    for (int i = 2; i <= depth; i++) {
+		d2 = new ResultDisplay(this, d1);
+		resultPanel.add(d2);
+		d1 = d2;
+	    }
+	    lastDisplay = d2;
+	}
+
+	// The control panel
+	initialiseControlPanel();
+
+	// lets get started then
+	setStatus("Welcome to Phind.");
+	mode = idleMode;
+
+	// Perform initial search, if requested
+	if (initialSearch.length() > 0) {
+	    searchForWord(initialSearch);
+	}
+    }
+
+
+    // Display a message in the status bar
+    void setStatus(String status) {
+	showStatus(status);
+    }
+
+    // The user performs an action in the interface
+    /*   public boolean action(Event evt, Object arg) {
+
+	if (evt.target == searchButton) {
+	    System.out.println("evt.target==searchButton");
+	    searchForWord(getSearchTerm());
+	} else if (evt.target == wordField) {
+	    System.out.println("evt.target==wordField");
+	    searchForWord(getSearchTerm());
+	} else if (evt.target == prevButton) {
+	    shiftPrevious();
+	} else if (evt.target == nextButton) {
+	    shiftNext();
+	} else {
+	    System.out.println("unknown action: " + evt.toString() 
+			       + ", object: " + arg.toString());
+	}
+	return true;
+    }
+
+    */
+
+    public void actionPerformed(ActionEvent evt) {
+	
+	Component target = (Component)evt.getSource();
+	if (target==searchButton) {
+	    System.out.println("search button pressed");
+	    searchForWord(getSearchTerm());
+	} else if (target == wordField)  {
+	    System.out.println("word field entered");
+	    searchForWord(getSearchTerm());
+	} else if (target == prevButton) {
+	    System.out.println("prev button pressed");
+	    shiftPrevious();
+	}else if (target == nextButton) {
+	    System.out.println("prev button pressed");
+	    shiftNext();
+	} else {
+	    System.out.println("unknown action: " + evt.toString() );
+	    
+	}
+    }
+
+    // Search for a word 
+    //
+    // Called on two occasions:
+    //   when the "Search" Button is pressed, or
+    //   to perform an "initial search"
+    void searchForWord(String searchWord) {
+
+	System.err.println("in searchforword!!");
+	if (mode == idleMode) {
+	    
+	    setSearchTerm(searchWord);
+
+	    // Convert the String from UTF8 charaters into 
+	    // an encoding that is okay for a URL.
+	    searchWord = URLUTF8Encoder.encode(searchWord);
+
+	    // Look up the word
+	    if (searchWord.length() > 1) {
+		setStatus("searching for \"" + searchWord + "\""); 
+		firstDisplay.emptyContents();
+		ResultBox result = lookupPhraseOnServer(null, false, searchWord, searchWord, 2);
+
+		// if there is an error, return
+		if (result == null) {
+		    setStatus("No results for \"" + searchWord + "\""); 
+		    return;
+		}
+
+		// display the result
+		result.display = firstDisplay.display(result);
+		result.setSize(result.display.getSize());
+		result.paintAll(result.getGraphics());
+	    }
+
+	    enablePreviousAndNext();
+	}
+    }
+  
+
+    // Search for a phrase
+    //
+    // If querymode is 2, the user has clicked on a phrase.
+    // If querymode is 3, the user has requested more phrases.
+    // If querymode is 4, the user has requested more documents.
+    void searchForPhrase(ResultBox source, String key, String phrase, int queryMode) {
+
+	// System.out.println("searchForPhrase: " + key + " " + phrase + " " + queryMode);
+
+	if (mode == idleMode) {
+
+	    // If we are going to replace the first ResultDisplay, then empty it
+	    if (queryMode <= 2) {
+		if (source.display.next != null) source.display.next.emptyContents();
+	    }
+
+	    // look up the word
+	    setStatus("Searching for \"" + phrase + "\""); 
+	    ResultBox result = lookupPhraseOnServer(source, true, key, phrase, queryMode);
+	    if (result == null) {
+		setStatus("No result for \"" + phrase + "\""); 
+		return;
+	    }
+
+	    // If this is not already displayed, display it in the last free spot
+	    if (queryMode <= 2) {
+		result.display = lastDisplay.display(result);
+		result.setSize(result.display.getSize());
+		result.paintAll(result.getGraphics());
+	    }
+
+	    enablePreviousAndNext();
+	}
+    }
+
+
+    // Look up a phrase (or symbol) on the server
+    //
+    // Arguments are the source of the query (a ResultBox, or null if the
+    // query comes from hitting the search button), the key to search for
+    // (the text of a phrase or a symbol number), the phrase as a string, 
+    // and the query mode.
+    // Query modes are: 
+    //   0 = obsolete
+    //   1 = obsolete
+    //   2 = get first N phrases and URLs,
+    //   3 = get another N phrases into same window
+    //   4 = get another N documents into same window
+    //   5 = get another N thesaurus links into same window
+
+    ResultBox lookupPhraseOnServer(ResultBox source, 
+				   boolean keyKnown, String key, String phrase, 
+				   int queryMode) { 
+    	disableSearchButton();
+	mode = searchMode;
+	ResultBox r = null;
+
+	if (queryMode <= 2) {
+	    r = new ResultBox(this, collection, key, phrase, source);
+	} else if ((queryMode == 3) || (queryMode == 4) || (queryMode == 5)) {
+	    r = source;
+	}
+
+	try { 
+	    queryServer(keyKnown, key, queryMode, r);
+	} catch (Exception e) { 
+	    System.out.println("Phind query error: " + e.toString());
+	    setStatus("Query error: " + e.toString());
+	    mode = idleMode;
+	    enableSearchButton();
+	    return null;
+	}
+      
+	// The query is finished
+	setStatus(r.c.numberOfItems + " results for \"" + phrase + "\""); 
+	mode = idleMode;
+	enableSearchButton();
+	lastQueryEndTime = new Date();
+
+	return r;
+    }
+
+
+    // Query the phindcgi program
+    //
+    // Send a query (a word or symbol number) to the server
+    // and pass the response to a ResultBox.
+
+    void queryServer(boolean keyKnown, String word, int queryMode, ResultBox area) 
+	throws IOException {
+
+	// Build the query
+	String query = phindcgi_address + "c=" + collection + "&pc=" + classifier;
+
+	if (keyKnown) {
+	  query = query + "&ppnum=" + word;
+	} else {
+	  query = query + "&ppnum=0" + "&pptext=" + word;
+	}
+
+	
+	// Specify the set of results to return 
+	int first_e = 0;
+	int last_e = 0;
+	int first_d = 0;
+	int last_d = 0;
+	int first_l = 0;
+	int last_l = 0;
+
+	// the initial query
+	if (queryMode <= 2) {
+	  last_e = phraseBlockSize;
+	  last_d = phraseBlockSize;
+	  last_l = phraseBlockSize;
+	}
+
+	// add phrases to an existing result set
+	else if (queryMode == 3) {
+	  first_e = area.nextPhraseBlock * phraseBlockSize;
+	  area.nextPhraseBlock++;
+	  last_e = area.nextPhraseBlock * phraseBlockSize;
+	} 
+	
+	// add documents to existing result set
+	else if (queryMode == 4) {
+	  first_d = area.nextDocumentBlock * phraseBlockSize;
+	  area.nextDocumentBlock++;
+	  last_d = area.nextDocumentBlock * phraseBlockSize;
+	}
+
+	// add thesaurus links to existing result set
+	else if (queryMode == 5) {
+	  first_l = area.nextThesaurusLinkBlock * phraseBlockSize;
+	  area.nextThesaurusLinkBlock++;
+	  last_l = area.nextThesaurusLinkBlock * phraseBlockSize;
+	}
+
+	query = query + "&pfe=" + first_e + "&ple=" + last_e
+                      + "&pfd=" + first_d + "&pld=" + last_d
+                      + "&pfl=" + first_l + "&pll=" + last_l;
+
+	// Send the query to the phindcgi program 
+	System.out.println("1:sending query: " + query);
+	try {
+	    URL phindcgi = new URL(query);
+	    DataInputStream in = new DataInputStream(phindcgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    area.parseXML(data_elem);
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query to phindcgi: " + e);
+	    e.printStackTrace();
+	}
+	area.repaint();
+    }
+    
+
+    // Tidy up URLs
+    //
+    // Ensure a URL address (as string) has a protocol, host, and file.
+    //
+    // If the URL is a CGI script URL, it should be tidied up so that it is
+    // appropriate to tage attrib=value pairs on the end.  This means it
+    // must either end with a "?" or (if it contains a question-mark
+    // internally) end with a "&".
+    String tidy_URL(String address, boolean isCGI) {
+
+	// System.err.println("tidy URL: " + address);
+	
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+
+	}
+
+	// if the URL is a cgi script, make sure it has a "?" in ti,
+	// and that it ends with a "?" or "&"
+	if (isCGI) {
+	    if (address.indexOf((int) '?') == -1) { 
+		address = address + "?";
+	    } else if (!address.endsWith("?")) {
+		address = address + "&";
+	    }
+	}
+
+	return address;
+    }
+
+
+
+    // Open an arbitrary web page
+    void displayWebPage(String address, String window) {
+	try { 
+	    URL url= new URL(address);
+	    if (window.length() > 0) {
+		getAppletContext().showDocument(url, window);
+	    } else {
+		getAppletContext().showDocument(url);
+	    }
+	} catch (Exception e) { 
+	    System.out.println("Cannot open web page: " + e.toString());
+	}
+    }
+
+
+    // Get the applet parameters
+    void getParameters() {
+
+	// What is this collection called?
+	collection = parameterValue("collection");
+	System.out.println("Phind collection: " + collection);
+
+	// Which of the collection's classifiers are we using?
+	classifier = parameterValue("classifier", "1");
+	System.out.println("Phind classifier: " + classifier);
+
+	// Where is the Greenstone library
+	library_address = parameterValue("library");
+	library_address = tidy_URL(library_address, true);
+	System.out.println("Phind library: " + library_address);
+
+	// Where is the phind CGI script
+	// we assume this is relative to the greenstone library
+	phindcgi_address = parameterValue("library")+parameterValue("phindcgi");
+	phindcgi_address = tidy_URL(phindcgi_address, true);
+	System.out.println("Phind phindcgi: " + phindcgi_address);
+
+	
+	// Is there a default search term?
+	initialSearch = parameterValue("initial_search", "");
+
+	// Should we display the control panel
+	showControlPanel = true;
+	if (parameterValue("control_panel", "show").toLowerCase().equals("hide")) {
+	    showControlPanel = false;
+	}
+	
+	// Should we show a background image?
+	backdrop_address = parameterValue("backdrop", "");
+	if (backdrop_address.length() > 0) {
+	    backdrop_address = tidy_URL(backdrop_address, false);
+	    System.out.println("Phind backdrop URL: " + backdrop_address);
+
+	    try {
+		URL backdrop_url = new URL(backdrop_address);
+		backgroundImage = getImage(backdrop_url);
+		showImage = true;
+	    } catch (Exception e) { 
+		System.out.println("Phind could not load " + backdrop_address);
+		showImage = false;
+	    }
+	}
+
+	// Should we draw a border?
+	showBorder = parameterValue("border", "on").equals("off");
+
+	// Are the windows arranged vertically or horizontally
+	if (parameterValue("orientation", "vertical").toLowerCase().startsWith("hori")) {
+	    vertical = false;
+	} else {
+	    vertical = true;
+	}
+	
+	// How many phind windows are there?
+	depth = parameterValue("depth", 3);
+
+	// Result sort order
+	// Standard is "LlEeDd", expansion-first is "EeLlDd"
+	String order = parameterValue("resultorder", "standard");
+	if (!order.equals("standard")) {
+	    int next = 20;
+	    ResultItem.sortMessage = next;
+	    for (int x = 0; x < order.length(); x++) {
+		if (order.charAt(x) == ',') {
+		    next--;
+		} else if (order.charAt(x) == 'L') {
+		    ResultItem.sortLinkItem = next;
+		} else if (order.charAt(x) == 'l') {
+		    ResultItem.sortMoreLinks = next;
+		} else if (order.charAt(x) == 'E') {
+		    ResultItem.sortPhraseItem = next;
+		} else if (order.charAt(x) == 'e') {
+		    ResultItem.sortMorePhrases = next;
+		} else if (order.charAt(x) == 'D') {
+		    ResultItem.sortDocumentItem = next;
+		} else if (order.charAt(x) == 'd') {
+		    ResultItem.sortMoreDocuments = next;
+		}
+	    }
+	    System.out.println("link: " + ResultItem.sortLinkItem);
+	    System.out.println("exps: " + ResultItem.sortPhraseItem);
+	    System.out.println("docs: " + ResultItem.sortDocumentItem);
+
+	}
+
+	// How many phrases should we fetch at any given time?
+	phraseBlockSize = parameterValue("blocksize", 10);
+
+	// What font should we use?
+	fontSize = parameterValue("fontsize", 10);
+	fontName = parameterValue("fontname", "Helvetica");
+
+	// Column dimensions
+	column_1_width = parameterValue("first_column_width", 6);
+	column_2_width = parameterValue("second_column_width", column_1_width);
+	
+	// Where do we open new windows
+	searchWindowName = parameterValue("search_window", "phindsearch");
+	documentWindowName = parameterValue("document_window", "phinddoc");
+
+	// Colours
+	panel_fg = parameterValue("panel_fg", Color.black);
+	panel_bg = parameterValue("panel_bg", Color.white);
+
+	highlight_bg = parameterValue("highlight_bg", Color.yellow);
+
+	expansion_fg = parameterValue("expansion_fg", Color.black);
+	thesaurus_fg = parameterValue("thesaurus_fg", new Color(0, 100, 0));
+	document_fg = parameterValue("document_fg", Color.blue);
+
+	thesaurus_bar_fg = parameterValue("thesaurus_bar_fg", Color.black);
+	expansion_bar_fg = parameterValue("expansion_bar_fg", Color.black);
+	document_bar_fg = parameterValue("document_bar_fg", Color.black);
+
+	thesaurus_bar_bg = parameterValue("thesaurus_bar_bg", new Color(160, 160, 190));
+	expansion_bar_bg = parameterValue("expansion_bar_bg", new Color(255, 200, 200));
+	document_bar_bg = parameterValue("document_bar_bg", new Color(150, 193, 156));
+
+	column_1_fg = parameterValue("first_column_fg", Color.black);
+	column_1_bg = parameterValue("first_column_bg", new Color(235, 245, 235));
+	column_2_fg = parameterValue("second_column_fg", Color.black);
+	column_2_bg = parameterValue("second_column_bg", new Color(200, 220, 200));
+
+	message_fg = parameterValue("message_fg", Color.black);
+	message_bg = parameterValue("message_bg", Color.white);
+
+	// Colours I don't use, yet
+	// thesaurus_bg = parameterValue("thesaurus_bg", Color.white);
+	// expansion_bg = parameterValue("expansion_bg", Color.white);
+	// document_bg = parameterValue("document_bg", Color.white);
+    }
+
+    // Get the value of a parameter given its name.
+    // There are many types of parameters, hence the variety of functions.
+
+    // Get a REQUIRED string.  Stop the applet if we cannot.
+    String parameterValue(String name) {
+	try { 
+	    return getParameter(name);
+	} catch (Exception e) {
+	    System.err.println("Phind: you must give a parameter for \"" 
+			       + name + "\".  Stopping.");
+	    stop();
+	}
+	return "";
+    }
+
+    // Get an optional parameter.  Return a default if we cannot.
+    String parameterValue(String name, String defaultValue) {
+	String text = getParameter(name);
+	if (text == null) {
+	    return defaultValue;
+	}
+	System.out.println("Phind " + name + ": " + text);
+	return text;
+    }
+
+    int parameterValue(String name, int defaultValue) {
+	int value;
+	try { 
+	    value = Integer.parseInt(getParameter(name));
+	} catch (Exception e) {
+	    return defaultValue;
+	}
+	System.out.println("Phind " + name + ": " + value);
+	return value;
+    }
+
+    Color parameterValue(String name, Color defaultValue) {
+
+	String text = getParameter(name);
+	if (text == null) {
+	    return defaultValue;
+	}
+	text = text.toLowerCase();
+
+	// a number of the form "#ddffee" 
+	if (text.startsWith("#") && (text.length() == 7)) {
+	    text = text.substring(1);
+	    int r, g, b;
+	    try {
+		r = Integer.parseInt(text.substring(0,2), 16);
+		g = Integer.parseInt(text.substring(2,4), 16);
+		b = Integer.parseInt(text.substring(4), 16);
+		return new Color(r, g, b);
+	    }  catch (Exception e) {
+		return defaultValue;
+	    }
+	}
+
+	// a known Java colour string
+	else if (text.equals("black")) { return Color.black; }
+	else if (text.equals("blue")) { return Color.blue; }
+	else if (text.equals("cyan")) { return Color.cyan; }
+	else if (text.equals("darkgray")) { return Color.darkGray; }
+	else if (text.equals("gray")) { return Color.gray; }
+	else if (text.equals("green")) { return Color.green; }
+	else if (text.equals("lightgray")) { return Color.lightGray; }
+	else if (text.equals("magenta")) { return Color.magenta; }
+	else if (text.equals("orange")) { return Color.orange; }
+	else if (text.equals("pink")) { return Color.pink; }
+	else if (text.equals("red")) { return Color.red; }
+	else if (text.equals("white")) { return Color.white; }
+	else if (text.equals("yellow")) { return Color.yellow; }
+
+	return defaultValue;
+    }
+
+
+    // Control panel operations
+
+    // Initialise the control panel
+    void initialiseControlPanel() {
+	
+	if (showControlPanel) {
+	    Panel p1 = new Panel();
+	    add("North", p1);
+	    p1.setLayout(new FlowLayout(FlowLayout.CENTER, 0, 0));
+	    
+	    searchButton = new Button("Search");
+	    searchButton.setFont(boldFont);
+	    //searchButton.setEnabled(true);
+	    searchButton.addActionListener(this);
+	    p1.add(searchButton);
+	    
+	    Label tempLabel = new Label("  for");
+	    tempLabel.setFont(boldFont);
+	    p1.add(tempLabel);
+	    
+	    wordField = new TextField(12);
+	    wordField.setFont(boldFont);
+	    wordField.addActionListener(this);
+	    p1.add(wordField);
+	    
+	    Label temp2 = new Label("          ");
+	    p1.add(temp2);
+	    
+	    prevButton = new Button("Previous");
+	    prevButton.setFont(boldFont);
+	    prevButton.addActionListener(this);
+	    prevButton.setEnabled(false);
+	    
+	    p1.add(prevButton);
+	    
+	    nextButton = new Button("  Next  ");
+	    nextButton.setFont(boldFont);
+	    nextButton.addActionListener(this);
+	    nextButton.setEnabled(false);
+	    p1.add(nextButton);
+	    
+	}
+    }
+    
+    // Button and field functionality
+    
+    // Enable and disable the word field
+    void enableSearchButton() {
+	if (showControlPanel) {
+	    searchButton.setEnabled(true);
+	}
+    }
+    void disableSearchButton() {
+	if (showControlPanel) {
+	    searchButton.setEnabled(false);
+	}
+    }
+
+    // Get and set the search text in the wordField
+    String getSearchTerm() {
+	if (showControlPanel) {
+	    return wordField.getText();
+	} else {
+	    return initialSearch;
+	}
+    }
+    void setSearchTerm(String word) {
+	if (showControlPanel) {
+	    wordField.setText(word);
+	} 
+    }
+
+    // Enable or disable the "Previous" and "Next" buttons
+    void enablePreviousAndNext() {
+	if (showControlPanel) {
+	    Component c = firstDisplay.current;
+	    if (c.getClass().getName().endsWith("ResultBox")) {
+		if (((ResultBox) c).prevBoxExists()) {
+		    prevButton.setEnabled(true);
+		} else {
+		    prevButton.setEnabled(false);
+		}
+	    }
+	    
+	    c = lastDisplay.current;
+	    if (c.getClass().getName().endsWith("ResultBox")) {
+		if (((ResultBox) c).nextBoxExists()) {
+		    nextButton.setEnabled(true);
+		} else {
+		    nextButton.setEnabled(false);
+		}
+	    }
+	}
+    }
+
+    // Shift to previous box
+    //
+    // If the user clicks "Previous" then scroll up.
+    void shiftPrevious() {
+
+	Component c = firstDisplay.current;
+	if (c.getClass().getName().endsWith("ResultBox")) {
+
+	    ResultBox b = (ResultBox) c;
+	    if (b.prevBoxExists()) {
+		b = b.prev;
+
+		// empty all the displays
+		firstDisplay.emptyContents();
+
+		// add as many result boxes as there are displays
+		for (int i = 1 ; ((i <= depth) && (b != null)); i++) {
+		    lastDisplay.display(b);
+		    b.setSize(b.display.getSize());
+		    b.paintAll(b.getGraphics());
+		    b = b.next;
+		}
+	    }
+	}
+	enablePreviousAndNext();
+    }
+
+    // Shift to next box
+    //
+    // If the user clicks "Next" then scroll down if possible
+    void shiftNext() {
+
+	Component c = lastDisplay.current;
+	if (c.getClass().getName().endsWith("ResultBox")) {
+
+	    ResultBox b = (ResultBox) c;
+	    if (b.nextBoxExists()) {
+
+		// find the new "first" displayed box
+		c = firstDisplay.current;
+		b = (ResultBox) c;
+		b = b.next;
+
+		// empty all the displays
+		firstDisplay.emptyContents();
+
+		// add as many result boxes as there are displays
+		for (int i = 1 ; ((i <= depth) && (b != null)); i++) {
+		    lastDisplay.display(b);
+		    b.setSize(b.display.getSize());
+		    b.paintAll(b.getGraphics());
+		    b = b.next;
+		}
+	    }
+	}
+	enablePreviousAndNext();
+    }
+
+
+
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/PhindTitle.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/PhindTitle.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/PhindTitle.java	(revision 31634)
@@ -0,0 +1,84 @@
+/**********************************************************************
+ *
+ * PhindTitle.java -- backdrops to empty ResultDisplay panels.
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+/*********************************************************************
+
+PhindTitle is for drawing backdrops in empty ResultDisplay panels.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.applet.Applet;
+import java.awt.Canvas;
+import java.awt.Color;
+import java.awt.Dimension;
+import java.awt.Graphics;
+import java.awt.Image;
+
+public class PhindTitle extends Canvas {
+
+    Phind phind;
+    static Image backgroundImage;
+    boolean displayInfo;
+    
+    PhindTitle(Phind p) {
+	phind = p;
+    }
+
+    public void update(Graphics g) {
+	paint(g);
+    }
+
+    public void paint(Graphics g) {
+	Dimension canvasSize = getSize();
+
+	Color fore = phind.panel_fg;
+	Color back = phind.panel_bg;
+
+	// set the screen background
+	if (phind.showImage)
+	    try {
+		g.drawImage(phind.backgroundImage, 
+			    0, 0, canvasSize.width, canvasSize.height, back, null);
+	    } catch (Exception e) {
+		System.err.println("PhindTitle paint: "  + e);
+	    }
+	else {
+	    g.setColor(back);
+	    g.fillRect(0,0, canvasSize.width, canvasSize.height);
+	}
+    }
+}
+
+ 
+
+
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultBox.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultBox.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultBox.java	(revision 31634)
@@ -0,0 +1,423 @@
+/**********************************************************************
+ *
+ * ResultBox.java -- a list of phrases in the Phind java interface
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A ResultBox holds the results of a query to phindcgi.  They deal mostly
+with the information content of the query, and have methods for parsing the
+input into phrase and document items.  They have little do with display:
+ResultBoxes are shown to the user through "ResultDisplay" panels, and are
+drawn using "ResultCanvas" objects.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Panel;
+import java.awt.BorderLayout;
+import java.awt.Scrollbar;
+
+import java.awt.Label;
+import java.awt.AWTEvent;
+import java.awt.event.AdjustmentListener;
+import java.awt.event.AdjustmentEvent;
+import java.net.*;
+import java.applet.*;
+
+import java.util.Vector;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ResultBox extends Panel 
+    implements AdjustmentListener {
+    
+    // Objects at a "higher" level than this one
+    Phind phind;
+    ResultDisplay display;
+
+    // Surroundig objects
+    ResultBox prev, next;
+
+    // The components contained by this one.
+    ResultCanvas c;
+    ResultTitle t;
+    Scrollbar s;
+    Panel label;
+
+    // The key identifying the phrase displayed, with its text and the
+    // collection from which it is drawn
+    String searchKey, searchPhrase, searchCollection;
+    
+    // The total frequency, expansion frequency, and document frequency
+    // of the phrase
+    int numberOfOccurances;
+    int numberOfExpansions;
+    int numberOfDocuments;
+    int numberOfThesaurusLinks;
+
+    // The number of phrases and documents retrieved, and the number of
+    // times the user has requested more phrases or documents.
+    int expansionsRetrieved;
+    int documentsRetrieved;
+    int thesaurusLinksRetrieved;
+    int nextPhraseBlock;
+    int nextDocumentBlock;
+    int nextThesaurusLinkBlock;
+    
+    int mode;
+    final int initMode = 0;
+    public final int emptyMode = 1;
+    final int loadingMode = 2;
+    final int finishedMode = 3;
+
+    String buffer;
+    boolean finished;
+
+
+    // Create a ResultBox
+    // given details of the search that generated it.
+    ResultBox(Phind p, String collect, String key, String phrase, ResultBox rb) {
+
+	super();
+	mode = initMode;
+
+	phind = p;
+	display = null;
+	next = null;
+	prev = rb;
+	if (prev != null) prev.next = this;
+
+	searchKey = key;
+	searchPhrase = phrase;
+	searchCollection = collect;
+
+	numberOfOccurances = -1;
+	numberOfExpansions = -1;
+	numberOfDocuments = -1;
+	numberOfThesaurusLinks = -1;
+
+	expansionsRetrieved = 0;
+	documentsRetrieved = 0;
+	thesaurusLinksRetrieved = 0;
+
+	nextPhraseBlock = 1;
+	nextDocumentBlock = 1;
+	nextThesaurusLinkBlock = 1;
+
+
+	setLayout(new BorderLayout());
+
+	s = new Scrollbar(Scrollbar.VERTICAL);
+	disableScrollbar();
+	s.addAdjustmentListener(this);
+	add("East", s);
+
+	c = new ResultCanvas(this);
+	add("Center", c);
+
+	t = new ResultTitle(this);
+	add("North", t);
+
+	buffer = "";
+	finished = false;
+	mode = emptyMode;
+    }
+
+    static String describeContents(String phrase, String c) {
+	return( "\"" + phrase + "\" in " + c + ".");
+    }
+
+    String describeContents() {
+	return( describeContents(searchPhrase, searchCollection) );
+    }
+
+
+    // Reset the contents of the box
+    void resetBox( ) {
+	buffer = "";
+	finished = false;
+	c.resetCanvas();
+	disableScrollbar();
+	mode = emptyMode;
+    
+	numberOfExpansions = -1;
+	numberOfDocuments = -1;
+	numberOfThesaurusLinks = -1;
+
+	expansionsRetrieved = 0;
+	documentsRetrieved = 0;
+	thesaurusLinksRetrieved = 0;
+    }
+
+    void setStatus( String status ) {
+	phind.setStatus(status);
+    }
+
+    void disableScrollbar() {
+	if (s.isEnabled()) {
+	    s.setValues(0, 1, 0, 1);
+	    s.setUnitIncrement(1);
+	    s.setBlockIncrement(1);
+	    s.setEnabled(false);
+	}
+    }
+
+    // Are there displays previous to and after this?
+    public boolean prevBoxExists () {
+	return (prev != null);
+    }
+    public boolean nextBoxExists () {
+	return (next != null);
+    }
+
+    
+    // Look up a phrase
+    // Phrase lookups are passed on to the Phind applet itself.
+    void lookupPhrase(String key, String phrase, int queryMode) {
+	buffer = "";
+	finished = false;
+	phind.searchForPhrase(this, key, phrase, queryMode);
+	t.repaint();
+    }
+
+    /*   public void processEvent(AWTEvent event) {
+	//    System.out.println("event: " + event.toString());
+	if ( (event.target == s) && 
+	     ( (event.id == Event.SCROLL_ABSOLUTE) ||
+	       (event.id == Event.SCROLL_LINE_DOWN) ||
+	       (event.id == Event.SCROLL_LINE_UP) ||
+	       (event.id == Event.SCROLL_PAGE_DOWN) ||
+	       (event.id == Event.SCROLL_PAGE_UP) ) ) {
+	    c.repaint();
+	    
+	} else {
+	    super.processEvent(event);
+	}
+	} */  
+
+    public void adjustmentValueChanged(AdjustmentEvent evt) {
+	c.repaint();
+    }
+
+    void parseXML(Element data) {
+
+	//System.out.println("phinddata:"+data.toString());
+
+	//String id="", phrase="";
+	//int df=0, ef=0, lf=0, tf=0;
+	
+	searchKey = data.getAttribute("id");
+	
+	numberOfDocuments = Integer.valueOf(data.getAttribute("df")).intValue();
+	numberOfExpansions = Integer.valueOf(data.getAttribute("ef")).intValue();
+	numberOfThesaurusLinks = Integer.valueOf(data.getAttribute("lf")).intValue();
+	numberOfOccurances = Integer.valueOf(data.getAttribute("tf")).intValue();
+
+	//searchPhrase = "cat"; // for now
+
+	// go through children
+	Node e = data.getFirstChild();
+	while (e!=null) {
+	    String node_name = e.getNodeName();
+	    if (node_name.equals("phrase")) {
+		// the value of the phrase
+		searchPhrase = getNodeText(e);
+	    } else if (node_name.equals("expansionList")) {
+		
+		NodeList expansions = ((Element)e).getElementsByTagName("expansion");
+		for (int i=0; i<expansions.getLength(); i++) {
+		    processExpansionElement((Element)expansions.item(i));
+		}
+	    } else if (node_name.equals("documentList")) {
+			
+		NodeList documents = ((Element)e).getElementsByTagName("document");
+		for (int i=0; i<documents.getLength(); i++) {
+		    processDocumentElement((Element)documents.item(i));
+		}
+	
+	
+	    } else if (node_name.equals("thesaurusList")) {
+				
+		NodeList thesaurai = ((Element)e).getElementsByTagName("thesaurus");
+		for (int i=0; i<thesaurai.getLength(); i++) {
+		    processThesaurusElement((Element)thesaurai.item(i));
+		}
+		
+	    } else {
+		System.out.println("error in phinddata format, have unwanted node, "+node_name);
+	    }
+	    e = e.getNextSibling();
+	}
+	// finished parsing, update the getMoreDocs markers
+	c.updateMarkers();
+    }
+
+
+    /** Add an expansion tag
+     *
+     * Given a string containing an XML expansion element of the form:
+     *   <expansion num="3" id="8421" prefix="PEOPLE and" suffix="" tf="3" df="3"><suffix></suffix><prefix>PEOPLE</prefix></expansion>
+     *
+     * Create a new ResultItemPhrase for display
+     *
+     *  Return true if successful, otherwise false. */
+    boolean processExpansionElement(Element e) {
+
+	String num = "", id = "", tf = "", df = "", 
+	    prefix = "", body = "", suffix = "";
+
+	body = searchPhrase;
+
+	id = e.getAttribute("id");
+	tf = e.getAttribute("tf");
+	df = e.getAttribute("df");
+	num = e.getAttribute("num");
+
+	// get prefix child
+	Node prefix_node = getChildByTagName(e, "prefix");
+	if (prefix_node!=null) {
+	    prefix = getNodeText(prefix_node);
+	}
+	Node suffix_node = getChildByTagName(e, "suffix");
+	if (suffix_node !=null) {
+	    suffix = getNodeText(suffix_node);
+	}
+
+	ResultItemPhrase ri = new ResultItemPhrase(id, tf, df, prefix, body, suffix);
+	
+	if (c.addResultItem(ri)) {
+	    expansionsRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** Add a document tag
+     *
+     * Given an XML Element of the form:
+     *   <document num="2" hash="HASH424e64b811fdad933be69c" freq="1"><title>CONTENTS</title></document>
+     *
+     * Create a new ResultItemDocument for display
+     *
+     * Return true if successful, otherwise false. */
+
+    boolean processDocumentElement(Element d ) {
+	// why do we have num??
+	String num = "", hash = "", freq = "", title = "";
+
+	num = d.getAttribute("num");
+	freq = d.getAttribute("freq");
+	hash = d.getAttribute("hash");
+
+	Node title_node = getChildByTagName(d, "title");
+	if (title_node != null) {
+	    title = getNodeText(title_node);
+	}
+	// Create a new ResultItem and add it to the display
+	ResultItemDocument ri = new ResultItemDocument(hash, title, freq);
+	
+	if (c.addResultItem(ri)) {
+	    documentsRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** Add a thesaurus tag
+     *
+     * Given an  XML element  of the form:
+     *
+     * <thesaurus num="3" id="36506" tf="0" df="0" type="RT"><phrase>ANGLOPHONE AFRICA</phrase></thesaurus>
+     *
+     * Create a new ResultItemLink for display
+     *
+     * Return true if successful, otherwise false. */
+
+    boolean processThesaurusElement(Element t ) {
+	
+	// why do we have num??? - not used anywhere
+	String num = "", id = "", tf = "", df = "", type = "", phrase = "";
+
+	id = t.getAttribute("id");
+	tf = t.getAttribute("tf");
+	df = t.getAttribute("df");
+	type = t.getAttribute("type");
+	num = t.getAttribute("num");
+
+	Node phrase_node = getChildByTagName(t, "phrase");
+	if (phrase_node !=null) {
+	    phrase = getNodeText(phrase_node);
+	}
+	// Create a new ResultItem and add it to the display
+	ResultItemLink ri = new ResultItemLink(id, phrase, type, tf, df);
+
+	if (c.addResultItem(ri)) {
+	    thesaurusLinksRetrieved++;
+	    return true;
+	}
+	
+	return false;
+    }
+
+    /** extracts the text out of a node */
+    protected String getNodeText(Node elem) {
+	elem.normalize();
+	Node n = elem.getFirstChild();
+	while (n!=null && n.getNodeType() !=Node.TEXT_NODE) {
+	    n=n.getNextSibling();
+	}
+	if (n==null) { // no text node
+	    return "";
+	}
+	return n.getNodeValue();
+    }
+
+    /** returns the (first) child element with the given name */
+    protected Node getChildByTagName(Node n, String name) {
+
+	Node child = n.getFirstChild();
+	while (child!=null) {
+	    if (child.getNodeName().equals(name)) {
+		return child;
+	    }
+	    child = child.getNextSibling();
+	}
+	return null; //not found
+    }
+
+
+}
+
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultCanvas.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultCanvas.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultCanvas.java	(revision 31634)
@@ -0,0 +1,647 @@
+/**********************************************************************
+ *
+ * ResultCanvas.java -- a Canvas onto which a phrase list is drawn
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+The results of a query are displayed on a ResultCanvas object.  Each 
+entry in the result list is stored in a ResultItem, and the ResultCanvas 
+contains a Vector of ResultItems.
+
+Each ResultCanvas is embedded in a ResultBox alongside a Scrollbar.  
+When the ResultCanvas is drawn, it looks at the scrollbar, calculates 
+which ResultItems are visible, then draws them on the screen.
+
+**********************************************************************/
+
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Canvas;
+import java.awt.Scrollbar;
+import java.awt.Graphics;
+import java.awt.Font;
+import java.awt.FontMetrics;
+import java.awt.Dimension;
+import java.awt.Color;
+import java.util.Vector;
+import java.awt.Image;
+//import java.awt.Event;
+import java.awt.event.MouseListener;
+import java.awt.event.MouseEvent;
+import java.awt.event.MouseAdapter;
+import java.net.URL;
+import java.applet.Applet;
+import java.util.Date;
+
+public class ResultCanvas extends Canvas {
+    
+
+    // Other objects associated with this 
+    Phind phind;
+    ResultBox parent;
+    Scrollbar scrollbar;
+    
+    // fonts and font spacings to use
+    Font areaPlain, areaBold;
+    int lineSpacing;
+    
+    // The items to display on this canvas
+    int numberOfItems;
+    int firstItemDisplayed;
+    int itemSelected;
+    Vector<ResultItem> items;
+    
+    // the background image
+    public static Image backgroundImage;
+    
+    
+    // Create a ResultCanvas from the ResultBox which is its parent.
+    ResultCanvas(ResultBox p) {
+	
+	parent = p;
+	phind = p.phind;
+	scrollbar = p.s;
+	parent.disableScrollbar();
+
+	areaPlain = phind.plainFont;
+	areaBold  = phind.boldFont;
+
+	lineSpacing = phind.fontSize + 2;
+
+	items = new Vector<ResultItem>();
+	numberOfItems = 0;
+	firstItemDisplayed = 0;
+	itemSelected = -1;
+    
+	if (backgroundImage == null) {
+	    backgroundImage = p.phind.backgroundImage;
+	}
+
+	// add a mouse listener for the mouse clicks. we only want to override
+	// one method, so we use the adaptor class
+	addMouseListener(new MouseAdapter() {
+		public void mouseClicked(MouseEvent evt) {
+		    ResultCanvas.this.mouseClicked(evt);
+		}
+	    });
+	
+
+    }
+
+    void resetCanvas( ) {
+	items.removeAllElements();
+	numberOfItems = 0;
+	repaint();
+    }
+
+
+    // Add a new search result.
+    // Return true if successful, otherwise false.
+    boolean addResultItem( ResultItem item ) {
+ 
+	// Add a new result, in sorted order.
+	// First sort key is item kind (lowest first),
+	// second is frequency (highest first).
+	// This is not efficient, but I don't care right now.
+	int index = 0;
+	while ((index < numberOfItems) &&
+	       ((item.sort < items.elementAt(index).sort) ||
+		((item.sort == items.elementAt(index).sort) &&
+		 (item.frequency <= items.elementAt(index).frequency)))) {
+	    index++;
+	}
+	items.insertElementAt(item,index);
+	numberOfItems++;
+	return true;
+    }
+
+
+    // Update more phrases/documents/links markers
+    void updateMarkers() {
+	updateMorePhrasesMarker();
+	updateMoreDocumentsMarker();
+	updateMoreLinksMarker();
+    }
+
+    // Make sure the more Phrases item only appears in the list
+    // if it is meant to.
+    void updateMorePhrasesMarker() {
+    
+	 System.out.println("updateMorePhrasesMarker() ");
+	 System.out.println("expansions retrieved = "+parent.expansionsRetrieved +" num expansions = "+parent.numberOfExpansions);
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (items.elementAt(index).isMorePhrases()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.expansionsRetrieved == parent.numberOfExpansions) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.expansionsRetrieved < parent.numberOfExpansions) {
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.morePhrases);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+
+    // Make sure the more documents marker appears when required
+    void updateMoreDocumentsMarker() {
+    
+	// System.out.println("updateMoreDocumentsMarker() ");
+
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (items.elementAt(index).isMoreDocuments()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.documentsRetrieved == parent.numberOfDocuments) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.documentsRetrieved < parent.numberOfDocuments) {
+	    // there should be a marker
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.moreDocuments);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+      
+    // Make sure the more links marker appears when required
+    void updateMoreLinksMarker() {
+    
+	// System.out.println("updateMoreLinksMarker() ");
+
+	// look for a marker
+	boolean found = false;
+	int index = 0;
+	while (!found && (index < numberOfItems)) {
+	    if (items.elementAt(index).isMoreLinks()) {
+		found = true;
+	    } else {
+		index++;
+	    }
+	}
+
+	if (parent.thesaurusLinksRetrieved == parent.numberOfThesaurusLinks) {
+	    // there should be no marker
+	    if (found) {
+		items.removeElementAt(index);
+		numberOfItems--;
+	    }
+
+	} else if (parent.thesaurusLinksRetrieved < parent.numberOfThesaurusLinks) {
+	    // there should be a marker
+	    if (!found) {
+		ResultItem ri = new ResultItem(ResultItem.moreLinks);
+		addResultItem(ri);
+	    }
+	}
+    }
+
+      
+
+ 
+
+
+    public void update(Graphics g) {
+	paint(g);
+    }
+
+    public void paint(Graphics g) {
+
+	// calculate the canvas size, margins, and spacing
+	
+	// Note: the size() method is deprecated, but getSize fails in NetScape 4
+	Dimension canvasSize = getSize();
+	
+	g.setFont(areaPlain);
+	int space = g.getFontMetrics().stringWidth(" ");
+	int nought = g.getFontMetrics().stringWidth("0");
+
+	int firstColumnWidth = phind.column_1_width * nought;
+	int secondColumnWidth = phind.column_2_width * nought;
+	int columnWidth = firstColumnWidth + secondColumnWidth;
+
+	int firstColumnLeft = canvasSize.width - columnWidth;
+	int secondColumnLeft = firstColumnLeft + firstColumnWidth;
+
+	int mainLeftMargin = 0;
+	int mainRightMargin = firstColumnLeft;
+
+	// the number of items that will fit on the canvas
+	int visible = canvasSize.height / lineSpacing;
+
+	// calculate the first item to output
+	int scrollValue = scrollbar.getValue();
+	if (numberOfItems <= visible) {
+	    scrollValue = 0;
+	} else if (scrollValue > (numberOfItems - visible)) {
+	    scrollValue = numberOfItems - visible;
+	} 
+	firstItemDisplayed = scrollValue;
+
+	// Draw the phrase area
+	Color fore = phind.panel_fg;
+	Color back = phind.panel_bg;
+
+
+	// draw the background
+	if (phind.showImage) {
+	    try {
+		g.drawImage(backgroundImage, 
+			    mainLeftMargin, 0, mainRightMargin, canvasSize.height, 
+			    back, null);
+	    } catch (Exception e) {
+		phind.showImage = false;
+		System.err.println("ResultCanvas paint error: " + e); 
+		g.setColor(back);
+		g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	    }
+	} else {
+	    g.setColor(back);
+	    g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	}
+    
+	// If there are no phrases, output a brief explanation.
+	if (numberOfItems == 0) {
+	    g.drawString("No phrases match this query.", mainLeftMargin + 10, lineSpacing);
+	}
+
+	// Output each of the visible ResultItems
+	ResultItem result;
+	int tab, i, y = 0;
+	int center = mainLeftMargin 
+	    + ((mainRightMargin - mainLeftMargin 
+		- g.getFontMetrics().stringWidth(parent.searchPhrase)) / 2);
+	int thesColumn = mainLeftMargin + firstColumnWidth 
+	    + g.getFontMetrics().stringWidth("Narrower term");
+
+	String body, prefix, suffix;
+	
+	for (i = scrollValue; 
+	     (i < numberOfItems) && (y + lineSpacing < canvasSize.height); i++) {
+
+	    // Get the resultItem to output
+	    result = items.elementAt(i);
+	    
+	    // Graphics settings for drawing this line
+	    y += lineSpacing;
+	    g.setFont(areaPlain);
+
+	    // Highlight the selected phrase.
+	    if (i == itemSelected) {
+		g.setColor(phind.highlight_bg);
+		g.fillRect(mainLeftMargin, y-lineSpacing+2, mainRightMargin, lineSpacing);
+	    }
+
+	    // Draw the item
+
+	    // Draw a phrase item
+	    if (result.isPhrase()) {
+		prefix = result.prefixText();
+		body = result.mainText();
+		suffix = result.suffixText();
+		
+		g.setColor(phind.expansion_fg);
+		g.setFont(areaPlain);
+		tab = center - g.getFontMetrics().stringWidth(prefix) - space;
+		g.drawString(prefix, tab, y);
+		
+		g.setFont(areaBold);
+		g.drawString(body, center, y);
+		
+		tab = center + space + g.getFontMetrics().stringWidth(body);
+		g.setFont(areaPlain);
+		g.drawString(suffix, tab, y);
+	
+	    } 
+
+	    // Draw a document item
+	    else if (result.isDocument()){
+		body = result.mainText();
+
+		g.setColor(phind.document_fg);
+		g.setFont(areaPlain);
+		tab = (mainRightMargin - g.getFontMetrics().stringWidth(body)) / 2;
+		g.drawString(body, tab, y);
+      
+	    } 
+	
+	    // Draw a thesaurus link item
+	    else if (result.isLink()){
+		prefix = result.prefixText() + ":";
+		body = result.mainText();
+		tab = thesColumn - g.getFontMetrics().stringWidth(prefix) - space;
+
+		g.setColor(phind.thesaurus_fg);
+		g.setFont(areaPlain);
+		g.drawString(prefix, tab, y);
+		g.setFont(areaBold);
+		g.drawString(body, thesColumn, y);
+	    } 
+	    
+	    // Draw a "more phrases/documents/links" marker
+	    else if (result.isMorePhrases() 
+		     || result.isMoreDocuments() 
+		     || result.isMoreLinks()){
+		body = result.mainText();
+		
+		if (result.isMorePhrases()) {
+		    g.setColor(phind.expansion_bar_bg);
+		} else if (result.isMoreDocuments()) {
+		    g.setColor(phind.document_bar_bg);
+		} else {
+		    g.setColor(phind.thesaurus_bar_bg);		    
+		}
+		g.fillRect(mainLeftMargin, y-lineSpacing+2, mainRightMargin, lineSpacing);
+
+		if (result.isMorePhrases()) {
+		    g.setColor(phind.expansion_bar_fg);
+		} else if (result.isMoreDocuments()) {
+		    g.setColor(phind.document_bar_fg);
+		} else {
+		    g.setColor(phind.thesaurus_bar_fg);		    
+		}
+		g.setFont(areaPlain);
+		tab = (mainRightMargin - g.getFontMetrics().stringWidth(body)) / 2;
+		g.drawString(body, tab, y);
+	    } 
+	}
+
+	// Draw the frequecy columns
+
+	// column backgrounds
+	g.setColor(phind.column_1_bg);
+	g.fillRect(firstColumnLeft, 0, firstColumnWidth, canvasSize.height);
+	g.setColor(phind.column_2_bg);
+	g.fillRect(secondColumnLeft, 0, secondColumnWidth, canvasSize.height);
+
+	// fill in the numbers
+	g.setColor(phind.column_1_fg);
+	g.setFont(areaPlain);
+	y = 0;
+	String docsText, freqText;
+
+	for (i = scrollValue; 
+	     (i < numberOfItems) && (y + lineSpacing < canvasSize.height); i++) {
+	    
+	    // Get the resultItem to output
+	    result = items.elementAt(i);
+	    docsText = result.docsText();
+	    freqText = result.freqText();
+	    
+	    // Graphics settings for drawing this line
+	    y += lineSpacing;
+
+	    // Write the document frequency
+	    if (docsText.length() > 0) {
+		tab = secondColumnLeft - space - g.getFontMetrics().stringWidth(docsText);
+		g.drawString(docsText, tab, y);
+	    }
+
+	    // Write the term frequency
+	    if (freqText.length() > 0) {
+		tab = secondColumnLeft + secondColumnWidth 
+		    - space - g.getFontMetrics().stringWidth(freqText);
+		g.drawString(freqText, tab, y);
+	    }
+	}
+
+	// Adjust the scrollbar
+	if (visible >= numberOfItems) {
+	    parent.disableScrollbar();
+	} else {
+	    scrollbar.setValues(scrollValue, visible, 0, numberOfItems);
+	    // Deprecated - setPageIncrement() has been replaced by
+	    // setBlockIncrement(), but the latter doesn't work on some
+	    // browsers.  Damn.
+	    scrollbar.setBlockIncrement(visible - 1);
+	    //scrollbar.setPageIncrement(visible - 1);
+	    // Deprecated - more nonsense: enable() has been replaced by
+	    // setEnabled(true), but this doesn't work on some browsers.
+	    scrollbar.setEnabled(true);
+	    //scrollbar.enable();
+	}
+
+	
+	// draw the border
+	if (phind.showBorder) {
+	    g.setColor(phind.panel_fg);
+	    g.drawRect(0,0, canvasSize.width - 1, canvasSize.height - 1);
+	}
+    }
+
+    // handle mouse clicks in the new way. just copy gordons thing for now
+    public void mouseClicked(MouseEvent event) {
+
+	// ignore actions that occur within 1 second of the last
+	Date now = new Date();
+	// System.out.println("Click time: " + now.toString());
+	if (now.getTime() < (phind.lastQueryEndTime.getTime() + 1000)) {
+	    System.out.println("Ignoring click - too close to last query.");
+	    return;
+	}
+
+	// which Item is selected?
+	int rowSelected = event.getY() / lineSpacing;
+	itemSelected = rowSelected + firstItemDisplayed;
+	ResultItem item = items.elementAt(itemSelected);
+
+	if (itemSelected <= numberOfItems) {
+	    
+	    //User clicks on a phrase
+	    if (item.isPhrase()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+		
+		// If meta key is held down, send query to search engine
+		/*if (event.metaDown() && !phind.library_address.equals("")) {
+		    String address = phind.library_address  
+			+ "a=q&sa=text&c=" + phind.collection
+			+ "&q=%22" + item.toString().replace(' ', '+') + "%22";
+		    phind.displayWebPage(address, phind.searchWindowName);
+		    }*/
+	    } 
+	    
+	    // Click on a thesaurus link
+	    else if (item.isLink()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+		
+		// If meta key is held down, send query to search engine
+		/*if (event.metaDown() && !phind.library_address.equals("")) {
+		    String address = phind.library_address  
+			+ "a=q&sa=text&c=" + phind.collection
+			+ "&q=%22" + item.toString().replace(' ', '+') + "%22";
+		    phind.displayWebPage(address, phind.searchWindowName);
+		    }*/
+	    } 
+	    
+	    // The user clicks on a URL; display it.
+	    else if (item.isDocument()) {
+		itemSelected = itemSelected;
+		update(getGraphics());
+		
+		String address = phind.library_address 
+		    + "a=d&c=" + phind.collection 
+		    + "&d=" + item.hiddenText()
+		    + "&q=" + parent.searchPhrase.replace(' ', '+');
+		
+		phind.displayWebPage(address, phind.documentWindowName);
+	    }
+	    
+	    // When the user clicks on "get more phrases" or other marker,
+	    // we have to send a new query to the host
+	    else if (item.isMorePhrases()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 3);
+	    } else if (item.isMoreDocuments()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 4);
+	    } else if (item.isMoreLinks()){
+		parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 5);
+		
+		
+	    }
+	    repaint();
+	}
+    }
+    
+
+    // User interaction
+    //
+    // All interaction with the ResultCanvas is therough mouse clicks, and
+    // is handles in this method.  Note we ignore clicks that follow
+    // another too closely to avoid problems with slow connections.
+    /*  public boolean processEvent(Event event) {
+    
+	if (event.id == Event.MOUSE_UP) {
+
+	    // ignore actions that occur within 1 second of the last
+	    Date now = new Date();
+	    // System.out.println("Click time: " + now.toString());
+	    if (now.getTime() < (phind.lastQueryEndTime.getTime() + 1000)) {
+		System.out.println("Ignoring click - too close to last query.");
+		return true;
+	    }
+
+	    // which Item is selected?
+	    int rowSelected = event.y / lineSpacing;
+	    itemSelected = rowSelected + firstItemDisplayed;
+	    ResultItem item = (ResultItem) items.elementAt(itemSelected);
+
+	    if (itemSelected <= numberOfItems) {
+	
+		//User clicks on a phrase
+		if (item.isPhrase()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+		    parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+
+		    // If meta key is held down, send query to search engine
+		    if (event.metaDown() && !phind.library_address.equals("")) {
+			String address = phind.library_address  
+			    + "a=q&sa=text&c=" + phind.collection
+			    + "&q=%22" + item.toString().replace(' ', '+') + "%22";
+			phind.displayWebPage(address, phind.searchWindowName);
+		    }
+		} 
+
+		// Click on a thesaurus link
+		else if (item.isLink()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+		    parent.lookupPhrase(item.hiddenText(), item.toString(), 2);
+
+		    // If meta key is held down, send query to search engine
+		    if (event.metaDown() && !phind.library_address.equals("")) {
+			String address = phind.library_address  
+			    + "a=q&sa=text&c=" + phind.collection
+			    + "&q=%22" + item.toString().replace(' ', '+') + "%22";
+			phind.displayWebPage(address, phind.searchWindowName);
+		    }
+		} 
+
+		// The user clicks on a URL; display it.
+		else if (item.isDocument()) {
+		    itemSelected = itemSelected;
+		    update(getGraphics());
+
+		    String address = phind.library_address 
+			+ "a=r&c=" + phind.collection 
+			+ "&r=" + item.hiddenText()
+			+ "&q=" + parent.searchPhrase.replace(' ', '+');
+
+		    phind.displayWebPage(address, phind.documentWindowName);
+		}
+		 
+		// When the user clicks on "get more phrases" or other marker,
+		// we have to send a new query to the host
+		else if (item.isMorePhrases()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 3);
+		} else if (item.isMoreDocuments()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 4);
+		} else if (item.isMoreLinks()){
+		    parent.lookupPhrase(parent.searchKey, parent.searchPhrase, 5);
+
+
+		}
+		repaint();
+	    }
+	}
+	return true;
+	} */   
+  
+}
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultDisplay.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultDisplay.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultDisplay.java	(revision 31634)
@@ -0,0 +1,126 @@
+/**********************************************************************
+ *
+ * ResultDisplay.java -- Phrase list holder for Phind java applet
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A ResultDisplay is a Panel that can display a single Component, which will
+be a ResultBox or PhindTitle object.
+
+ResultBoxes can be "chained" together with their "prev" and "next" fields,
+allowing us to have an arbitrary number of "displays" in which to show
+phrase lists.
+
+Main methods:
+* emptyContents will remove the component and emptyAll the "next" display
+* display will add an item to the first (ie: most "prev") available box
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+import java.awt.Panel;
+import java.awt.BorderLayout;
+import java.awt.Component;
+
+class ResultDisplay extends Panel {
+
+  Phind phind;
+  ResultDisplay prev, next;
+  Component current;
+  boolean empty;
+
+  // Create a new Display
+  ResultDisplay(Phind ph, ResultDisplay pr) {
+    phind = ph;
+    prev = pr;
+    next = null;
+    if (prev != null) prev.next = this;
+    setLayout(new BorderLayout());
+
+    empty = true;
+    current = new PhindTitle(ph);
+    add(current);
+  }
+
+  // Empty the contents of the display,
+  // and all the next displays
+  void emptyContents() {
+    if (empty == false) {
+      empty = true;
+      current = new PhindTitle(phind);
+      removeAll();
+      add(current);
+    }
+    if (next != null) next.emptyContents();
+  }
+
+  // Set the contents of this display
+  Component setContents( Component r ) {
+    removeAll();
+    add("Center", r);
+    current = r;
+    empty = false;
+    if (r.getClass().getName().equals("ResultBox")) {
+      ((ResultBox) r).display = this;
+    }
+    return r;
+  }
+
+
+  
+  // Display a component in the "most previous" display
+  ResultDisplay display( Component r ) {
+
+    if (prev == null) {
+      // if there is no previous box, add the component here
+      setContents(r);
+      return this;
+
+    } else if (prev.empty == true) {
+      // if the previous box is empty, add the component to it instead
+      return prev.display(r);
+
+    } else if (empty == true) {
+      // if this box is empty, add the compnent to it
+      setContents(r);
+      return this;
+
+    } else {
+      // if this box is occupied,
+      // shift it's contents prev-wards and add here.
+      prev.display(current);
+      setContents(r);
+      return this;
+    }
+  }
+
+}
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItem.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItem.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItem.java	(revision 31634)
@@ -0,0 +1,128 @@
+/**********************************************************************
+ *
+ * ResultItemDocument.java -- a result in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+A result item holds the data describing a single line returned from a phind
+query.  The complex functionality of phrase and URL items is implemented in
+subclasses; the base class handles simpler cases like the "get more phrases" 
+marker.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItem {
+
+    // A text string describing the item
+    String text = "Generic ResultItem";
+
+    // There are several kinds of ResultItem, identified by their
+    // "kind" field, which has an integer value from the list below.
+    final static int message = 8;
+    final static int linkItem = 7;
+    final static int moreLinks = 6;
+    final static int phraseItem = 5;
+    final static int morePhrases = 4;
+    final static int documentItem = 3;
+    final static int moreDocuments = 2;
+    final static int unknownResultItem = 1;
+
+    // The kind of the item
+    int kind = unknownResultItem;
+
+    // When results are displayed, they are loosely sorted by their
+    // kind.  The user can change the sort order numbers with the 
+    // ResultOrder parameter in Phind.java.
+    static int sortMessage = 8;
+    static int sortLinkItem = 7;
+    static int sortMoreLinks = 6;
+    static int sortPhraseItem = 5;
+    static int sortMorePhrases = 4;
+    static int sortDocumentItem = 3;
+    static int sortMoreDocuments = 2;
+    static int sortUnknownResultItem = 1;
+
+    // the primary sort key
+    int sort = 0;
+    
+    // The frequency of the item (secondary sort key)
+    int frequency = 0;
+
+
+    // Create a blank ResultItem
+    ResultItem() {
+	text = "Unknown result type";
+	kind = unknownResultItem;
+	frequency = 0;
+    }
+
+    // Create a new ResultItem of some given kind
+    ResultItem(int newKind) {
+	kind = newKind;
+	frequency = 0;
+
+	if (kind == moreLinks) {
+	    text = "get more thesaurus links";
+	    sort = sortMoreLinks;
+	} else if (kind == morePhrases) {
+	    text = "get more phrases";
+	    sort = sortMorePhrases;
+	} else if (kind == moreDocuments) {
+	    text = "get more documents";
+	    sort = sortMoreDocuments;
+	} else {
+	    text = "Unknown result type";
+	    sort = sortUnknownResultItem;
+	}
+    }
+
+    // Test the type of a ResultItem
+    public boolean isLink() { return false; }
+    public boolean isMoreLinks() { return (kind == moreLinks); }
+    public boolean isPhrase() { return false; }
+    public boolean isMorePhrases() { return (kind == morePhrases); }
+    public boolean isDocument() { return false; }
+    public boolean isMoreDocuments() { return (kind == moreDocuments); }
+
+    // Return the bare text of the item
+    public String toString() { return(text.trim()); }
+
+    // Return the text of the item components
+    public String mainText() { return text; }
+    public String freqText() { 
+	if (frequency > 0) { return Integer.toString(frequency); } 
+	else { return ""; }
+    }
+    public String docsText() { return ""; }
+    public String prefixText() { return ""; }
+    public String suffixText() { return ""; }
+    public String hiddenText() { return ""; }
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemDocument.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemDocument.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemDocument.java	(revision 31634)
@@ -0,0 +1,70 @@
+/**********************************************************************
+ *
+ * ResultItemDocument.java -- a document result in the Phind applet
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a document in which a search term 
+occurs.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItemDocument extends ResultItem {
+
+    // The document's title is stored in the superclasses 
+    // text variable, and the number of times the phrase occurs
+    // in the frequency variable. 
+
+    // The document's unique object identifier (hash value).
+    String hash;
+
+    // Create a ResultItem for a document
+    ResultItemDocument(String newHash, String newTitle, String newFrequency) {
+	
+	kind = documentItem;
+	text = newTitle;
+
+	sort = sortDocumentItem;
+	frequency = Integer.valueOf(newFrequency).intValue();
+	hash = newHash;
+    }
+
+    // Is this item a document?
+    public boolean isDocument() { return true; }
+
+    // Return the bare text of the item
+    public String toString() { return(text); }
+
+    // Return the text of the item components
+    public String hiddenText() { return hash; }
+ 
+
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemLink.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemLink.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemLink.java	(revision 31634)
@@ -0,0 +1,88 @@
+/**********************************************************************
+ *
+ * ResultItemLink.java -- a thesaurus link in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a thesaurus link.
+
+**********************************************************************/
+package org.greenstone.applet.phind;
+//package org.nzdl.gsdl.Phind;
+
+public class ResultItemLink extends ResultItem {
+
+    // The phrase number, stored as a string and as a number
+    String id;
+    int symbol;
+
+    // The text of the link is stored in the inherited text field.
+
+    // The type of thesaurus link
+    String linkType;
+
+    // The total frequency is stored in the inherited frequency field.
+
+    // The document frequency of the thesaurus item
+    int documentFrequency;
+
+    // Create a ResultItem for a thesaurus link
+    ResultItemLink(String newId, String newText, String newLinkType, 
+		   String tf, String df) { 
+
+	kind = linkItem;
+	id = newId;
+	symbol = Integer.valueOf(newId).intValue();
+	text = newText.trim();
+	sort = sortLinkItem;
+	linkType = newLinkType;
+	frequency = Integer.valueOf(tf).intValue();
+	documentFrequency = Integer.valueOf(df).intValue();
+    }
+
+    // Is this item a phrase?
+    public boolean isLink() { return true; }
+
+    // Return the text of the item components
+    public String hiddenText() { return id; }
+    public String docsText() { 
+	if (documentFrequency <= 0) { return ""; }
+	else { return Integer.toString(documentFrequency); }
+    }
+    public String prefixText() { 
+	if (linkType.equals("RT")) { return "Related term"; }
+	else if (linkType.equals("BT")) { return "Broader term"; }
+	else if (linkType.equals("NT")) { return "Narrower term"; }
+	else if (linkType.equals("USE")) { return "Use"; }
+	else if (linkType.startsWith("UF")) { return "Used for"; }
+	else { return linkType; }
+    }
+}
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemPhrase.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemPhrase.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultItemPhrase.java	(revision 31634)
@@ -0,0 +1,85 @@
+/**********************************************************************
+ *
+ * ResultItemPhrase.java -- a phrase result in the Phind applet
+ *
+ * Copyright 2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+It contains information describing a phrase.
+
+**********************************************************************/
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+public class ResultItemPhrase extends ResultItem {
+
+    // The phrase number, stored as a string and as a number
+    String id;
+    int symbol;
+
+    // The text of the phrase is split into a prefix, body, and suffix.
+    // The body is the same as the parent phrase (or search term) and 
+    // the prefix and suffix are the surrounding text.
+    String prefix;
+    String body;
+    String suffix;
+
+    // The document frequency of the phrase
+    int documentFrequency;
+
+    // Create a ResultItem for a phrase
+    ResultItemPhrase(String newId, String tf, String df, 
+		     String p, String b, String s) {
+
+	kind = phraseItem;
+
+	id = newId;
+	symbol = Integer.valueOf(newId).intValue();
+
+	sort = sortPhraseItem;
+	frequency = Integer.valueOf(tf).intValue();
+	documentFrequency = Integer.valueOf(df).intValue();
+
+	prefix = p;
+	body = b;
+	suffix = s;
+	text = (prefix + " " + body + " " + suffix);
+
+    }
+
+    // Is this item a phrase?
+    public boolean isPhrase() { return true; }
+
+    // Return the text of the item components
+    public String mainText() { return body; }
+    public String docsText() { return Integer.toString(documentFrequency); }
+    public String prefixText() { return prefix; }
+    public String suffixText() { return suffix; }
+    public String hiddenText() { return id; }
+
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultTitle.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultTitle.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/ResultTitle.java	(revision 31634)
@@ -0,0 +1,162 @@
+/**********************************************************************
+ *
+ * ResultTitle.java -- describe the contents of a ResultBox
+ *
+ * Copyright 1997-2000 Gordon W. Paynter
+ * Copyright 2000 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+/*********************************************************************
+
+This class is used in the Phind java applet (Phind.java).
+
+ResultTitle is part of the result list displayed to the user.  It lists the
+phrase in the Result box, the number of expansions it occurs in, and the
+number of documents it occurs in.
+
+**********************************************************************/
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+import java.awt.Canvas;
+import java.awt.Graphics;
+import java.awt.Font;
+import java.awt.FontMetrics;
+import java.awt.Dimension;
+import java.awt.Color;
+
+
+public class ResultTitle extends Canvas {
+    
+    // Other objects associated with this 
+    Phind phind;
+    ResultBox parent;
+    ResultCanvas canvas;
+
+    // fonts and font spacings to use
+    Font plain, bold;
+    Graphics g;
+    
+    // Create a ResultTitle from the ResultBox which is its parent.
+    ResultTitle(ResultBox p) {
+	
+	parent = p;
+	phind = p.phind;
+	canvas = p.c;
+	g = getGraphics();
+	
+	plain = phind.plainFont;
+	bold  = phind.boldFont;
+	
+	// Note: the size() and resize() methods are deprecated, but the newer
+	// getSize() and setSize() methods fail in NetScape 4
+	Dimension d = getSize();
+	d.height = phind.fontSize + 10;
+	setSize(d);
+	
+	
+    }
+    
+    public void update(Graphics g) {
+	paint(g);
+    }
+    
+    public void paint(Graphics g) {
+	
+	// calculate the canvas size, margins, and spacing
+	Dimension canvasSize = getSize();
+	
+	g.setFont(plain);
+	int margin = g.getFontMetrics().stringWidth(" 8888 ");
+	int y = phind.fontSize + 5;
+	
+	int rightMargin = canvas.getSize().width;
+	int secondColumn = rightMargin - margin;
+	int firstColumn = secondColumn - margin;
+	
+	g.setColor(Color.white);
+	g.fillRect(0, 0, canvasSize.width, canvasSize.height);
+	g.setColor(Color.black);
+    
+    
+	// What is the phrase we are searching for?
+	String phrase = parent.searchPhrase.replace('+', ' ');
+	
+	// Construct the description phrase
+	String links = "";
+	if (parent.numberOfThesaurusLinks <= 0) {
+	    links = "";
+	} else if (parent.numberOfThesaurusLinks == 1) { 
+	    links = "1 link";
+	} else if (parent.thesaurusLinksRetrieved == parent.numberOfThesaurusLinks) {
+	    links = parent.numberOfThesaurusLinks + " links";
+	} else {
+	    links = parent.thesaurusLinksRetrieved + " of " 
+		+ parent.numberOfThesaurusLinks + " links";
+	}
+	
+	String expansions = "";
+	if (parent.numberOfExpansions <= 0) {
+	    expansions = "no phrases";
+	} else if (parent.numberOfExpansions == 1) { 
+	    expansions = "1 phrase";
+	} else if (parent.expansionsRetrieved == parent.numberOfExpansions) {
+	    expansions = parent.numberOfExpansions + " phrases";
+	} else {
+	    expansions = parent.expansionsRetrieved + " of " 
+		+ parent.numberOfExpansions + " phrases";
+	}
+	
+	String documents = "";
+	if (parent.numberOfDocuments <= 0) { 
+	    documents = "no documents";
+	} else if (parent.documentsRetrieved == 1) {
+	    documents = "1 document";
+	} else if (parent.documentsRetrieved == parent.numberOfDocuments) {
+	    documents = parent.numberOfDocuments + " documents";
+	} else {
+	    documents = parent.documentsRetrieved + " of " 
+		+ parent.numberOfDocuments + " documents";
+	}
+	
+	String status = "(";
+	if (parent.numberOfThesaurusLinks > 0) {
+	    status = status + links + ", ";
+	} 
+	status = status + expansions + ", " + documents + ")";
+	
+	
+	// Draw the text 
+	g.setFont(bold);
+	g.drawString(phrase, 0, y);
+	int tab = g.getFontMetrics().stringWidth(phrase + "  ");
+	
+	g.setFont(plain);
+	g.drawString(status, tab, y);
+	tab = tab + g.getFontMetrics().stringWidth(status);
+	
+	if (tab < firstColumn) {
+	    g.drawString("docs", firstColumn, y);
+	    g.drawString("freq", secondColumn, y);
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/URLUTF8Encoder.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/URLUTF8Encoder.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/applet/phind/URLUTF8Encoder.java	(revision 31634)
@@ -0,0 +1,159 @@
+/**********************************************************************
+ *
+ *
+ * URLUTF8Encoder.java -- encode UTF8 String into a URL form
+ *
+ * W3C SOFTWARE NOTICE AND LICENSE
+ *
+ * Copyright (C) 1994-2000 World Wide Web Consortium, 
+ * (Massachusetts Institute of Technology, Institut National de 
+ * Recherche en Informatique et en Automatique, Keio University). 
+ * All Rights Reserved.  http://www.w3.org/Consortium/Legal/
+ * 
+ * This W3C work (including software, documents, or other related items) is
+ * being provided by the copyright holders under the following license. By
+ * obtaining, using and/or copying this work, you (the licensee) agree that
+ * you have read, understood, and will comply with the following terms and
+ * conditions:
+ * 
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation, with or without modification, for any purpose and without
+ * fee or royalty is hereby granted, provided that you include the following
+ * on ALL copies of the software and documentation or portions thereof,
+ * including modifications, that you make:
+ * 
+ *   1. The full text of this NOTICE in a location viewable to users of the
+ *      redistributed or derivative work.
+ * 
+ *   2. Any pre-existing intellectual property disclaimers, notices, or terms
+ *      and conditions. If none exist, a short notice of the following form
+ *      (hypertext is preferred, text is permitted) should be used within the
+ *      body of any redistributed or derivative code: "Copyright (C)
+ *      [$date-of-software] World Wide Web Consortium, (Massachusetts
+ *      Institute of Technology, Institut National de Recherche en
+ *      Informatique et en Automatique, Keio University). All Rights
+ *      Reserved. http://www.w3.org/Consortium/Legal/"
+ * 
+ *   3. Notice of any changes or modifications to the W3C files, including the
+ *      date changes were made. (We recommend you provide URIs to the location
+ *      from which the code is derived.)
+ * 
+ * THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS
+ * MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT
+ * NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR
+ * PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE
+ * ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS.
+ * 
+ * COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR
+ * CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR
+ * DOCUMENTATION.
+ * 
+ * The name and trademarks of copyright holders may NOT be used in advertising
+ * or publicity pertaining to the software without specific, written prior
+ * permission. Title to copyright in this software and any associated
+ * documentation will at all times remain with copyright holders.
+ *
+ *********************************************************************/
+
+/**
+ * Provides a method to encode any string into a URL-safe
+ * form, the so-called "x-www-form-urlencoded" form.
+ * Non-ASCII characters are first encoded as sequences of
+ * two or three bytes, using the UTF-8 algorithm, before being
+ * encoded in "x-www-form-urlencoded".
+ */
+
+//package org.nzdl.gsdl.Phind;
+package org.greenstone.applet.phind;
+public class URLUTF8Encoder
+{
+
+  final static String[] hex = {
+    "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07",
+    "%08", "%09", "%0a", "%0b", "%0c", "%0d", "%0e", "%0f",
+    "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17",
+    "%18", "%19", "%1a", "%1b", "%1c", "%1d", "%1e", "%1f",
+    "%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27",
+    "%28", "%29", "%2a", "%2b", "%2c", "%2d", "%2e", "%2f",
+    "%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37",
+    "%38", "%39", "%3a", "%3b", "%3c", "%3d", "%3e", "%3f",
+    "%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47",
+    "%48", "%49", "%4a", "%4b", "%4c", "%4d", "%4e", "%4f",
+    "%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57",
+    "%58", "%59", "%5a", "%5b", "%5c", "%5d", "%5e", "%5f",
+    "%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67",
+    "%68", "%69", "%6a", "%6b", "%6c", "%6d", "%6e", "%6f",
+    "%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77",
+    "%78", "%79", "%7a", "%7b", "%7c", "%7d", "%7e", "%7f",
+    "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87",
+    "%88", "%89", "%8a", "%8b", "%8c", "%8d", "%8e", "%8f",
+    "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97",
+    "%98", "%99", "%9a", "%9b", "%9c", "%9d", "%9e", "%9f",
+    "%a0", "%a1", "%a2", "%a3", "%a4", "%a5", "%a6", "%a7",
+    "%a8", "%a9", "%aa", "%ab", "%ac", "%ad", "%ae", "%af",
+    "%b0", "%b1", "%b2", "%b3", "%b4", "%b5", "%b6", "%b7",
+    "%b8", "%b9", "%ba", "%bb", "%bc", "%bd", "%be", "%bf",
+    "%c0", "%c1", "%c2", "%c3", "%c4", "%c5", "%c6", "%c7",
+    "%c8", "%c9", "%ca", "%cb", "%cc", "%cd", "%ce", "%cf",
+    "%d0", "%d1", "%d2", "%d3", "%d4", "%d5", "%d6", "%d7",
+    "%d8", "%d9", "%da", "%db", "%dc", "%dd", "%de", "%df",
+    "%e0", "%e1", "%e2", "%e3", "%e4", "%e5", "%e6", "%e7",
+    "%e8", "%e9", "%ea", "%eb", "%ec", "%ed", "%ee", "%ef",
+    "%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
+    "%f8", "%f9", "%fa", "%fb", "%fc", "%fd", "%fe", "%ff"
+  };
+
+  /**
+   * Encode a string to the "x-www-form-urlencoded" form, enhanced
+   * with the UTF-8-in-URL proposal. This is what happens:
+   *
+   * <ul>
+   * <li><p>The ASCII characters 'a' through 'z', 'A' through 'Z',
+   *        and '0' through '9' remain the same.
+   *
+   * <li><p>The space character ' ' is converted into a plus sign '+'.
+   *
+   * <li><p>All other ASCII characters are converted into the
+   *        3-character string "%xy", where xy is
+   *        the two-digit hexadecimal representation of the character
+   *        code
+   *
+   * <li><p>All non-ASCII characters are encoded in two steps: first
+   *        to a sequence of 2 or 3 bytes, using the UTF-8 algorithm;
+   *        secondly each of these bytes is encoded as "%xx".
+   * </ul>
+   *
+   * @param s The string to be encoded
+   * @return The encoded string
+   */
+  public static String encode(String s)
+  {
+    StringBuffer sbuf = new StringBuffer();
+    int len = s.length();
+    for (int i = 0; i < len; i++) {
+      int ch = s.charAt(i);
+      if ('A' <= ch && ch <= 'Z') {		// 'A'..'Z'
+	sbuf.append((char)ch);
+      } else if ('a' <= ch && ch <= 'z') {	// 'a'..'z'
+	sbuf.append((char)ch);
+      } else if ('0' <= ch && ch <= '9') {	// '0'..'9'
+	sbuf.append((char)ch);
+      } else if (ch == ' ') {			// space
+	sbuf.append('+');
+      } else if (ch <= 0x007f) {		// other ASCII
+	sbuf.append(hex[ch]);
+      } else if (ch <= 0x07FF) {		// non-ASCII <= 0x7FF
+	sbuf.append(hex[0xc0 | (ch >> 6)]);
+	sbuf.append(hex[0x80 | (ch & 0x3F)]);
+      } else {					// 0x7FF < ch <= 0xFFFF
+	sbuf.append(hex[0xe0 | (ch >> 12)]);
+	sbuf.append(hex[0x80 | ((ch >> 6) & 0x3F)]);
+	sbuf.append(hex[0x80 | (ch & 0x3F)]);
+      }
+    }
+    return sbuf.toString();
+  }
+
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/BaseGreenstoneServlet.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/BaseGreenstoneServlet.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/BaseGreenstoneServlet.java	(revision 31634)
@@ -0,0 +1,16 @@
+package org.greenstone.gsdl3;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+
+import org.greenstone.util.GlobalProperties;
+
+public class BaseGreenstoneServlet extends HttpServlet
+{
+	public void init(ServletConfig config) throws ServletException
+	{
+		super.init(config);
+		GlobalProperties.loadGlobalProperties(config.getServletContext().getRealPath(""));
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/ClientSideServlet.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/ClientSideServlet.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/ClientSideServlet.java	(revision 31634)
@@ -0,0 +1,183 @@
+package org.greenstone.gsdl3;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.lang3.StringUtils;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.XSLTUtil;
+
+public class ClientSideServlet extends BaseGreenstoneServlet
+{
+	public void init(ServletConfig config) throws ServletException
+	{
+		super.init(config);
+	}
+
+	public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		String query_string = request.getQueryString();
+		PrintWriter w = response.getWriter();
+
+		if (query_string == null)
+			displayParamError(response);
+
+		String[] parts = StringUtils.split(query_string, "&");
+
+		String[] keys = null; // Array of keys to look up
+		String lang = "";
+		String interface_name = "";
+		String command = "";
+		String params = "";
+
+		for (String part : parts)
+		{
+
+			String[] nameval = StringUtils.split(part, '=');
+
+			if (nameval.length != 2)
+			{
+				displayParamError(response);
+				return;
+			}
+
+			String name = nameval[0];
+			String value = nameval[1];
+
+			if (name.equals("k"))
+				keys = StringUtils.split(value, ",");
+			else if (name.equals("l"))
+				lang = value;
+			else if (name.equals("i"))
+				interface_name = value;
+			else if (name.equals("c"))
+				// Alternative commands
+				command = value;
+			else if (name.equals("p"))
+				params = value;
+		}
+
+		if (keys == null && command.equals("") && params.equals(""))
+		{
+			// Not satisfiable
+			displayParamError(response);
+			return;
+		}
+
+		String bundle = "<?xml version=\"1.0\" encoding=\"iso-8859-1\" ?>\r\n<textBundle>\r\n";
+
+		if (!command.equals("") && !params.equals(""))
+		{
+
+			// Alternative commands other than dictionary lookup
+			String[] parameters = StringUtils.split(params, "|");
+			String result = null;
+
+			if (command.equals("getNumberedItem") && parameters.length == 2)
+			{
+				result = XSLTUtil.getNumberedItem(parameters[0], Integer.parseInt(parameters[1]));
+			}
+			else if (command.equals("exists") && parameters.length == 2)
+			{
+				result = XSLTUtil.exists(parameters[0], parameters[1]) ? "true" : "false";
+			}
+			else if (command.equals("isImage") && parameters.length == 1)
+			{
+				result = (XSLTUtil.isImage(parameters[0])) ? "true" : "false";
+			}
+
+			if (result != null)
+			{
+				bundle += "	<method name=\"" + command + "\" parameters=\"" + params + "\" return=\"" + result + "\" />\r\n";
+			}
+			else
+			{
+				displayParamError(response);
+				return;
+			}
+		}
+		else
+		{
+
+			for (String key : keys)
+			{
+
+				String original_key = key;
+				String[] theParts = StringUtils.split(key, "|");
+				String[] args = null;
+
+				if (theParts.length > 1)
+					args = StringUtils.split(theParts[1], ";");
+
+				key = theParts[0];
+
+				// Straight from XSLTUtils.java (src), with some modifications
+				if (key.equals("null"))
+				{
+					bundle += "	<item key=\"" + key + "\" value=\"\" />\r\n";
+					continue;
+				}
+
+				Dictionary dict = new Dictionary("interface_" + interface_name, lang);
+				String result = dict.get(key, args);
+
+				if (result == null)
+				{
+					String sep_interface_dir = interface_name + File.separatorChar + lang + File.separatorChar + "interface";
+					dict = new Dictionary(sep_interface_dir, lang);
+					result = dict.get(key, args);
+				}
+
+				if (result == null && !interface_name.equals("default"))
+				{ // not found, try the default interface
+					dict = new Dictionary("interface_default", lang);
+					result = dict.get(key, args);
+				}
+
+				if (result == null)
+				{ // not found
+					result = "_" + original_key + "_";
+				}
+
+				bundle += "	<item key=\"" + encode(original_key) + "\" value=\"" + encode(result) + "\" />\r\n";
+			}
+		}
+
+		bundle += "</textBundle>";
+
+		response.setContentType("text/xml");
+
+		w.print(bundle);
+		w.close();
+	}
+
+	private String encode(String s)
+	{
+		s = StringUtils.replace(s, "&", "&amp;");
+		s = StringUtils.replace(s, "<", "&lt;");
+		s = StringUtils.replace(s, ">", "&gt;");
+		return s;
+	}
+
+	private void displayParamError(HttpServletResponse response)
+	{
+
+		try
+		{
+			PrintWriter w = response.getWriter();
+			w.write("Invalid parameters supplied! Need key (k=), interface name (i=) and language (l=). If you specified a method call, it must be on the list of supported method calls, and you must specify a command (c=) and parameters (p=).");
+			w.close();
+		}
+		catch (Exception ex)
+		{ /* Should log here */
+		}
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/FileLoaderServlet.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/FileLoaderServlet.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/FileLoaderServlet.java	(revision 31634)
@@ -0,0 +1,188 @@
+package org.greenstone.gsdl3;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.channels.FileChannel;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.fileupload.FileItem;
+import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.util.GlobalProperties;
+
+public class FileLoaderServlet extends LibraryServlet
+{
+	public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		if (request.getMethod().equals("POST"))
+		{
+			DiskFileItemFactory fileItemFactory = new DiskFileItemFactory();
+
+			int sizeLimit = System.getProperties().containsKey("servlet.upload.filesize.limit") ? Integer.parseInt(System.getProperty("servlet.upload.filesize.limit")) : 20 * 1024 * 1024;
+
+			fileItemFactory.setSizeThreshold(sizeLimit);
+			fileItemFactory.setRepository(new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp"));
+
+			ServletFileUpload uploadHandler = new ServletFileUpload(fileItemFactory);
+
+			String storageLocation = "";
+			File uploadedFile = null;
+			Boolean ajaxUpload = false;
+			StringBuilder json = new StringBuilder("[");
+			try
+			{
+				List items = uploadHandler.parseRequest(request);
+				Iterator iter = items.iterator();
+				while (iter.hasNext())
+				{
+					FileItem current = (FileItem) iter.next();
+					if (current.isFormField())
+					{
+						if (current.getFieldName().equals(GSParams.FILE_LOCATION))
+						{
+							storageLocation = current.getString();
+						}
+
+						if (current.getFieldName().equals("ajaxUpload") && current.getString().equals("true"))
+						{
+							ajaxUpload = true;
+						}
+					}
+					else
+					{
+						File file = new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp" + File.separator + current.getName());
+						File tmpFolder = new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp");
+						if (!tmpFolder.exists())
+						{
+							tmpFolder.mkdirs();
+						}
+						current.write(file);
+
+						uploadedFile = file;
+
+						if (!json.toString().equals("["))
+						{
+							json.append(",");
+						}
+
+						json.append("{");
+						json.append("\"name\":\"" + file.getName() + "\",");
+						json.append("\"size\":\"" + file.length() + "\",");
+						json.append("\"url\":\"" + "tmp/" + current.getName() + "\",");
+						json.append("\"thumbnail_url\":\"" + "tmp/" + current.getName() + "\",");
+						json.append("\"delete_url\":\"\",");
+						json.append("\"delete_type\":\"\"");
+						json.append("}");
+					}
+				}
+				json.append("]");
+
+				if (ajaxUpload)
+				{
+					response.setContentType("application/json");
+					PrintWriter writer = response.getWriter();
+					writer.write(json.toString());
+					writer.flush();
+
+					return;
+				}
+
+				if (!storageLocation.equals("") && uploadedFile != null)
+				{
+					String[] locations = storageLocation.split(":");
+
+					for (String location : locations)
+					{
+						File toFile = new File(GlobalProperties.getGSDL3Home() + location);
+						if (toFile.exists())
+						{
+							File backupFile = new File(toFile.getAbsolutePath() + System.currentTimeMillis());
+
+							logger.info("Backing up file (" + toFile.getAbsolutePath() + ") to " + backupFile.getAbsolutePath());
+							toFile.renameTo(backupFile);
+						}
+
+						FileChannel source = null;
+						FileChannel destination = null;
+						try
+						{
+							logger.info("Moving uploaded file (" + uploadedFile.getAbsolutePath() + ") to " + toFile.getAbsolutePath());
+							source = new FileInputStream(uploadedFile).getChannel();
+							destination = new FileOutputStream(toFile).getChannel();
+							destination.transferFrom(source, 0, source.size());
+						}
+						finally
+						{
+							if (source != null)
+							{
+								source.close();
+							}
+							if (destination != null)
+							{
+								destination.close();
+							}
+						}
+
+					}
+				}
+			}
+			catch (Exception e)
+			{
+				logger.error("Exception in LibraryServlet -> " + e.getMessage());
+			}
+		}
+		else
+		{
+			Map<String, String[]> queryMap = request.getParameterMap();
+			Iterator<String> queryIter = queryMap.keySet().iterator();
+
+			while (queryIter.hasNext())
+			{
+				String q = queryIter.next();
+				if (q.equals("downloadFile"))
+				{
+					String fileLocation = queryMap.get(q)[0];
+					File fileToGet = new File(GlobalProperties.getGSDL3Home() + File.separator + fileLocation);
+
+					if (fileToGet.exists())
+					{
+						response.setContentType("application/octet-stream");
+						response.addHeader("Content-Disposition", "attachment;filename=" + fileToGet.getName());
+						FileInputStream fis = new FileInputStream(fileToGet);
+						ServletOutputStream sos = response.getOutputStream();
+
+						byte[] buffer = new byte[4096];
+						int len;
+						while ((len = fis.read(buffer)) != -1)
+						{
+							sos.write(buffer, 0, len);
+						}
+						sos.flush();
+						fis.close();
+						sos.close();
+
+						return;
+					}
+				}
+			}
+		}
+
+		super.doGet(request, response);
+	}
+
+	public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		doGet(request, response);
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/Library1.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/Library1.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/Library1.java	(revision 31634)
@@ -0,0 +1,69 @@
+/*
+ *    Library1.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.*;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+/** sample command line interface to gsdl3 site
+ *
+ * enter XML queries at the prompt
+ * @author Katherine Don
+ * @version $Revision$
+ */
+final public class Library1 {
+
+    public static void main(String args[]) {
+
+	if (args.length != 1) {
+	    System.out.println("Usage: Library1 <sitename>");
+	    System.exit(1);
+	}
+	String site_name = args[0];
+
+	MessageRouter mr = new MessageRouter();
+	mr.setSiteName(site_name);
+	mr.configure();
+	
+	BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+	String query=null;
+	String result=null;
+	while (true) {
+	    System.out.println("Please enter an XML query all on one line, or 'exit' to quit");
+	    try {
+		query = br.readLine();
+	    } catch (Exception e) {
+		System.err.println("Library1 exception:"+e.getMessage());
+	    }
+	    if (query.startsWith("exit")) {
+		System.exit(1);
+	    }
+	    System.out.println("\nProcessing query");
+	    
+	    result = mr.process(query);
+
+	    System.out.println("Result:\n"+result+"\n");
+
+
+	}
+       
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryCommandline.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryCommandline.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryCommandline.java	(revision 31634)
@@ -0,0 +1,224 @@
+/*
+ *    LibraryCommandline.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.util.GlobalProperties;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Iterator;
+
+/**
+ * A program to take cgi args from the command line and return html
+ * 
+ */
+
+final public class LibraryCommandline
+{
+	protected HashMap<String, String> saved_args = null;
+	protected GSParams params = null;
+	protected DefaultReceptionist recept = null;
+
+	public LibraryCommandline()
+	{
+		this.saved_args = new HashMap<String, String>();
+		this.params = new GSParams();
+		this.recept = new DefaultReceptionist();
+	}
+
+	public void configure(String site_name, String interface_name)
+	{
+
+		HashMap<String, Object> config_params = new HashMap<String, Object>();
+		//config_params.put(GSConstants.GSDL3_HOME, gsdl_home);
+		config_params.put(GSConstants.SITE_NAME, site_name);
+		config_params.put(GSConstants.INTERFACE_NAME, interface_name);
+		config_params.put(GSConstants.USE_CLIENT_SIDE_XSLT, false);
+
+		// new message router - create it and pass a handle to recept.
+		// the servlet wont use this directly
+		MessageRouter message_router = new MessageRouter();
+
+		message_router.setSiteName(site_name);
+		message_router.configure();
+		// new receptionist
+		recept.setConfigParams(config_params);
+		recept.setMessageRouter(message_router);
+		recept.setParams(params);
+		recept.configure();
+	}
+
+    /*
+     *  On Linux, run as:
+     *  GS3> source ./gs3-setup.sh
+     *  GS3> java -classpath "web/WEB-INF/lib/*":"lib/jni/*":"web/WEB-INF/classes/." org.greenstone.gsdl3.LibraryCommandline localsite default
+     *  Press enter to accept the default cgi-args to pass in.
+     * 
+     *  For how to include all jars in a folder into the classpath to run a java program, see
+     *  http://stackoverflow.com/questions/219585/setting-multiple-jars-in-java-classpath
+     *  http://stackoverflow.com/questions/6780678/run-class-in-jar-file
+     */
+	public static void main(String args[])
+	{
+
+	    if (System.getenv("GSDL3SRCHOME") == null) {
+		System.out.println("Before calling this script, run: source gs3-setup.sh");	
+		System.exit(1);
+	    }
+
+		if (args.length != 2)
+		{
+			System.out.println("Usage: LibraryCommandline <site name> <interface name>");
+			System.exit(1);
+		}
+
+		// force GlobalProperties to default GSDL3HOME to GSDL3SRCHOME/web if not already set
+		GlobalProperties.loadGlobalProperties("");
+
+		LibraryCommandline library = new LibraryCommandline();
+		library.configure(args[0], args[1]);
+
+		BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
+		String query = null;
+		String result = null;
+		while (true)
+		{
+			System.out.println("Please enter the  cgi args (all on one line), or 'exit' to quit (default a=p&sa=home)");
+			try
+			{
+				query = br.readLine();
+			}
+			catch (Exception e)
+			{
+				System.err.println("LibraryCommandline exception:" + e.getMessage());
+			}
+			if (query.startsWith("exit"))
+			{
+				System.exit(1);
+			}
+
+			result = library.process(query);
+
+			System.out.println(result);
+
+		}
+
+	}
+
+	protected String process(String query)
+	{
+		Element xml_message = generateRequest(query);
+		System.out.println("*********************");
+		System.out.println(XMLConverter.getPrettyString(xml_message));
+		Node xml_result = recept.process(xml_message);
+		return XMLConverter.getPrettyString(xml_result);
+	}
+
+	protected Element generateRequest(String cgiargs)
+	{
+
+	  Document msg_doc = XMLConverter.newDOM();
+		Element xml_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element xml_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PAGE, "", new UserContext());
+		xml_message.appendChild(xml_request);
+		Element xml_param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		xml_request.appendChild(xml_param_list);
+
+		// the defaults
+		String action = "p";
+		String subaction = "home";
+		String lang = saved_args.get(GSParams.LANGUAGE);
+		if (lang == null)
+		{
+			lang = "en";
+		}
+		String output = "html";
+
+		String args[] = cgiargs.split("&");
+		for (int i = 0; i < args.length; i++)
+		{
+			String arg = args[i];
+			int pos = arg.indexOf('=');
+			if (pos == -1)
+				continue;
+			String name = arg.substring(0, pos);
+			String value = arg.substring(pos + 1);
+			if (name.equals(GSParams.ACTION))
+			{
+				action = value;
+			}
+			else if (name.equals(GSParams.SUBACTION))
+			{
+				subaction = (value);
+			}
+			else if (name.equals(GSParams.LANGUAGE))
+			{
+				lang = value;
+				saved_args.put(name, value);
+			}
+			else if (name.equals(GSParams.OUTPUT))
+			{
+				output = value;
+			}
+			else if (params.shouldSave(name))
+			{
+				saved_args.put(name, value);
+			}
+			else
+			{
+				// make a param now
+				Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+				param.setAttribute(GSXML.NAME_ATT, name);
+				param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(value));
+				xml_param_list.appendChild(param);
+			}
+		}
+
+		xml_request.setAttribute(GSXML.OUTPUT_ATT, output);
+		xml_request.setAttribute(GSXML.ACTION_ATT, action);
+		xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
+		xml_request.setAttribute(GSXML.LANG_ATT, lang);
+
+		// put in all the params from the session cache
+		Set<String> params = saved_args.keySet();
+		Iterator<String> i = params.iterator();
+		while (i.hasNext())
+		{
+			String name = i.next();
+			if (name.equals(GSParams.LANGUAGE))
+				continue;
+			Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, name);
+			param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(saved_args.get(name)));
+			xml_param_list.appendChild(param);
+		}
+
+		return xml_message;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryServlet.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryServlet.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/LibraryServlet.java	(revision 31634)
@@ -0,0 +1,1193 @@
+package org.greenstone.gsdl3;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+import javax.servlet.http.HttpSessionBindingEvent;
+import javax.servlet.http.HttpSessionBindingListener;
+
+import org.apache.commons.fileupload.FileItem;
+import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.action.PageAction;
+import org.greenstone.gsdl3.comms.Communicator;
+import org.greenstone.gsdl3.comms.SOAPCommunicator;
+import org.greenstone.gsdl3.core.DefaultReceptionist;
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.core.Receptionist;
+import org.greenstone.gsdl3.service.Authentication;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.json.JSONObject;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+/**
+ * a servlet to serve the greenstone library - we are using servlets instead of
+ * cgi the init method is called only once - the first time the servlet classes
+ * are loaded. Each time a request comes in to the servlet, the session() method
+ * is called in a new thread (calls doGet/doPut etc) takes the a=p&p=home type
+ * args and builds a simple request to send to its receptionist, which returns a
+ * result in html, cos output=html is set in the request
+ * 
+ * 18/Jul/07 xiao modify to make the cached parameters collection-specific. Most
+ * of the work is done in doGet(), except adding an inner class
+ * UserSessionCache.
+ * 
+ * @see Receptionist
+ */
+public class LibraryServlet extends BaseGreenstoneServlet
+{
+	/** the receptionist to send messages to */
+	protected Receptionist recept = null;
+
+	/**
+	 * the default language - is specified by setting a servlet param, otherwise
+	 * DEFAULT_LANG is used
+	 */
+	protected String default_lang = null;
+
+  /** We record the library name for later */
+  protected String library_name = null;
+	/** Whether or not client-side XSLT support should be exposed */
+	protected boolean supports_client_xslt = false;
+
+	/**
+	 * The default default - used if a default lang is not specified in the
+	 * servlet params
+	 */
+	protected final String DEFAULT_LANG = "en";
+
+	/**
+	 * the cgi stuff - the Receptionist can add new args to this
+	 * 
+	 * its used by the servlet to determine what args to save
+	 */
+	protected GSParams params = null;
+
+	/**
+	 * user id - new one per session. This doesn't work if session state is
+	 * saved between restarts - this requires this value to be saved too.
+	 */
+	protected int next_user_id = 0;
+
+	/**
+	 * a hash that contains all the active session IDs mapped to the cached
+	 * items It is updated whenever the whole site or a particular collection is
+	 * reconfigured using the command a=s&sa=c or a=s&sa=c&c=xxx It is in the
+	 * form: sid -> (UserSessionCache object)
+	 */
+	protected Hashtable<String, UserSessionCache> session_ids_table = new Hashtable<String, UserSessionCache>();
+
+	/**
+	 * the maximum interval that the cached info remains in session_ids_table
+	 * (in seconds) This is set in web.xml
+	 */
+	protected int session_expiration = 1800;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.LibraryServlet.class.getName());
+
+	/** initialise the servlet */
+	public void init(ServletConfig config) throws ServletException
+	{
+		// always call super.init;
+		super.init(config);
+		// disable preferences - does this work anyway??
+		//System.setProperty("java.util.prefs.PreferencesFactory", "org.greenstone.gsdl3.util.DisabledPreferencesFactory");
+
+		library_name = config.getInitParameter(GSConstants.LIBRARY_NAME);
+		String interface_name = config.getInitParameter(GSConstants.INTERFACE_NAME);
+
+		String useXslt = (String) config.getInitParameter(GSConstants.USE_CLIENT_SIDE_XSLT);
+		supports_client_xslt = useXslt != null && useXslt.equals("true");
+
+		this.default_lang = config.getInitParameter(GSConstants.DEFAULT_LANG);
+		String sess_expire = config.getInitParameter(GSXML.SESSION_EXPIRATION);
+
+		if (sess_expire != null && !sess_expire.equals(""))
+		{
+			this.session_expiration = Integer.parseInt(sess_expire);
+		}
+
+		if (library_name == null || interface_name == null)
+		{
+			// must have this
+			System.err.println("initialisation parameters not all set!");
+			System.err.println(" you must have libraryname and interfacename");
+			System.exit(1);
+		}
+
+		String site_name = config.getInitParameter(GSConstants.SITE_NAME);
+		String remote_site_name = null;
+		String remote_site_type = null;
+		String remote_site_address = null;
+
+		if (site_name == null)
+		{
+			// no site, try for communicator
+			remote_site_name = config.getInitParameter("remote_site_name");
+			remote_site_type = config.getInitParameter("remote_site_type");
+			remote_site_address = config.getInitParameter("remote_site_address");
+			if (remote_site_name == null || remote_site_type == null || remote_site_address == null)
+			{
+				System.err.println("initialisation paramters not all set!");
+				System.err.println("if site_name is not set, then you must have remote_site_name, remote_site_type and remote_site_address set");
+				System.exit(1);
+			}
+		}
+
+		if (this.default_lang == null)
+		{
+			// choose english
+			this.default_lang = DEFAULT_LANG;
+		}
+
+		HashMap<String, Object> config_params = new HashMap<String, Object>();
+
+		config_params.put(GSConstants.LIBRARY_NAME, library_name);
+		config_params.put(GSConstants.INTERFACE_NAME, interface_name);
+		config_params.put(GSConstants.USE_CLIENT_SIDE_XSLT, supports_client_xslt);
+
+		if (site_name != null)
+		{
+			config_params.put(GSConstants.SITE_NAME, site_name);
+		}
+		//this.converter = new XMLConverter();
+		//this.doc = XMLConverter.newDOM();
+
+		// the receptionist -the servlet will talk to this
+		String recept_name = (String) config.getInitParameter("receptionist_class");
+		if (recept_name == null)
+		{
+			this.recept = new DefaultReceptionist();
+		}
+		else
+		{
+			try
+			{
+				this.recept = (Receptionist) Class.forName("org.greenstone.gsdl3.core." + recept_name).newInstance();
+			}
+			catch (Exception e)
+			{ // cant use this new one, so use normal one
+				System.err.println("LibraryServlet configure exception when trying to use a new Receptionist " + recept_name + ": " + e.getMessage());
+				e.printStackTrace();
+				this.recept = new DefaultReceptionist();
+			}
+		}
+		this.recept.setConfigParams(config_params);
+
+		// the receptionist uses a MessageRouter or Communicator to send its requests to. We either create a MessageRouter here for the designated site (if site_name set), or we create a Communicator for a remote site. The is given to teh Receptionist, and the servlet never talks to it again.directly.
+		if (site_name != null)
+		{
+			String mr_name = (String) config.getInitParameter("messagerouter_class");
+			MessageRouter message_router = null;
+			if (mr_name == null)
+			{ // just use the normal MR
+				message_router = new MessageRouter();
+			}
+			else
+			{ // try the specified one
+				try
+				{
+					message_router = (MessageRouter) Class.forName("org.greenstone.gsdl3.core." + mr_name).newInstance();
+				}
+				catch (Exception e)
+				{ // cant use this new one, so use normal one
+					System.err.println("LibraryServlet configure exception when trying to use a new MessageRouter " + mr_name + ": " + e.getMessage());
+					e.printStackTrace();
+					message_router = new MessageRouter();
+				}
+			}
+
+			message_router.setSiteName(site_name);
+			message_router.setLibraryName(library_name);
+			message_router.configure();
+			this.recept.setMessageRouter(message_router);
+		}
+		else
+		{
+			// talking to a remote site, create a communicator
+			Communicator communicator = null;
+			// we need to create the XML to configure the communicator
+			Document doc = XMLConverter.newDOM();
+			Element site_elem = doc.createElement(GSXML.SITE_ELEM);
+			site_elem.setAttribute(GSXML.TYPE_ATT, remote_site_type);
+			site_elem.setAttribute(GSXML.NAME_ATT, remote_site_name);
+			site_elem.setAttribute(GSXML.ADDRESS_ATT, remote_site_address);
+
+			if (remote_site_type.equals(GSXML.COMM_TYPE_SOAP_JAVA))
+			{
+				communicator = new SOAPCommunicator();
+			}
+			else
+			{
+				System.err.println("LibraryServlet.init Error: invalid Communicator type: " + remote_site_type);
+				System.exit(1);
+			}
+
+			if (!communicator.configure(site_elem))
+			{
+				System.err.println("LibraryServlet.init Error: Couldn't configure communicator");
+				System.exit(1);
+			}
+			this.recept.setMessageRouter(communicator);
+		}
+
+		// the params arg thingy
+
+		String params_name = (String) config.getInitParameter("params_class");
+		if (params_name == null)
+		{
+			this.params = new GSParams();
+		}
+		else
+		{
+			try
+			{
+				this.params = (GSParams) Class.forName("org.greenstone.gsdl3.util." + params_name).newInstance();
+			}
+			catch (Exception e)
+			{
+				System.err.println("LibraryServlet configure exception when trying to use a new params thing " + params_name + ": " + e.getMessage());
+				e.printStackTrace();
+				this.params = new GSParams();
+			}
+		}
+		// pass it to the receptionist
+		this.recept.setParams(this.params);
+		this.recept.configure();
+
+		//Allow the message router and the document to be accessed from anywhere in this servlet context
+		this.getServletContext().setAttribute("GSRouter", this.recept.getMessageRouter());
+		//this.getServletContext().setAttribute("GSDocument", this.doc);
+	}
+
+	private void logUsageInfo(HttpServletRequest request)
+	{
+		String usageInfo = "";
+
+		//session-info: get params stored in the session
+		HttpSession session = request.getSession(true);
+		Enumeration attributeNames = session.getAttributeNames();
+		while (attributeNames.hasMoreElements())
+		{
+			String name = (String) attributeNames.nextElement();
+			usageInfo += name + "=" + session.getAttribute(name) + " ";
+		}
+
+		//logged info = general-info + session-info
+		usageInfo = request.getServletPath() + " " + //serlvet
+		"[" + request.getQueryString() + "]" + " " + //the query string
+		"[" + usageInfo.trim() + "]" + " " + // params stored in a session
+		request.getRemoteAddr() + " " + //remote address
+		request.getRequestedSessionId() + " " + //session id
+		request.getHeader("user-agent") + " "; //the remote brower info
+
+		logger.info(usageInfo);
+
+	}
+
+	public class UserSessionCache implements HttpSessionBindingListener
+	{
+		String session_id = "";
+
+		/**
+		 * a hash that maps the session ID to a hashtable that maps the
+		 * coll_name to its parameters coll_name -> Hashtable (param_name ->
+		 * param_value)
+		 */
+		protected Hashtable<String, Hashtable<String, String>> coll_name_params_table = null;
+
+		public UserSessionCache(String id, Hashtable<String, Hashtable<String, String>> table)
+		{
+			session_id = id;
+			coll_name_params_table = (table == null) ? new Hashtable() : table;
+		}
+
+		protected void cleanupCache(String coll_name)
+		{
+			if (coll_name_params_table.containsKey(coll_name))
+			{
+				coll_name_params_table.remove(coll_name);
+			}
+		}
+
+		protected Hashtable<String, Hashtable<String, String>> getParamsTable()
+		{
+			return coll_name_params_table;
+		}
+
+		public void valueBound(HttpSessionBindingEvent event)
+		{
+			// Do nothing
+		}
+
+		public void valueUnbound(HttpSessionBindingEvent event)
+		{
+			if (session_ids_table.containsKey(session_id))
+			{
+				session_ids_table.remove(session_id);
+			}
+		}
+
+		public int tableSize()
+		{
+			return (coll_name_params_table == null) ? 0 : coll_name_params_table.size();
+		}
+	}
+
+	public void destroy()
+	{
+		recept.cleanUp();
+	}
+
+	public void doGetOrPost(HttpServletRequest request, HttpServletResponse response, Map<String, String[]> queryMap) throws ServletException, IOException
+	{
+		logUsageInfo(request);
+
+		if (queryMap != null)
+		{
+			Iterator<String> queryIter = queryMap.keySet().iterator();
+			boolean redirect = false;
+			String href = null;
+			String rl = null;
+			String el = null;
+
+			while (queryIter.hasNext())
+			{
+				String q = queryIter.next();
+				if (q.equals(GSParams.EXTERNAL_LINK_TYPE))
+				{
+					el = queryMap.get(q)[0];
+				}
+				else if (q.equals(GSParams.HREF))
+				{
+					href = queryMap.get(q)[0];
+					href = StringUtils.replace(href, "%2f", "/");
+					href = StringUtils.replace(href, "%7e", "~");
+					href = StringUtils.replace(href, "%3f", "?");
+					href = StringUtils.replace(href, "%3A", "\\:");
+				}
+				else if (q.equals(GSParams.RELATIVE_LINK))
+				{
+					rl = queryMap.get(q)[0];
+				}
+			}
+
+			//if query_string contains "el=direct", an href is specified, and its not a relative link, then the web page will be redirected to the external URl, otherwise a greenstone page with an external URL will be displayed
+			//"rl=0" this is an external link
+			//"rl=1" this is an internal link
+			if ((href != null) && (rl.equals("0")))
+			{// This is an external link, 
+
+				if (el.equals("framed"))
+				{
+					//TODO **** how best to change to a=p&sa=html&c=collection&url=href
+					// response.setContentType("text/xml");
+					//response.sendRedirect("http://localhost:8383/greenstone3/gs3library?a=p&sa=html&c=external&url="+href);
+				}
+				else
+				{
+					// el = '' or direct
+					//the web page is re-directed to the external URL (&el=&rl=0&href="http://...")
+					response.setContentType("text/xml");
+					response.sendRedirect(href);
+				}
+			}
+		}
+
+		// Nested Diagnostic Configurator to identify the client for
+		HttpSession session = request.getSession(true);
+		session.setMaxInactiveInterval(session_expiration);
+		String uid = (String) session.getAttribute(GSXML.USER_ID_ATT);
+		if (uid == null)
+		{
+			uid = "" + getNextUserId();
+			session.setAttribute(GSXML.USER_ID_ATT, uid);
+		}
+
+		request.setCharacterEncoding("UTF-8");
+		response.setContentType("text/html;charset=UTF-8");
+		PrintWriter out = response.getWriter();
+
+		String lang = getFirstParam(GSParams.LANGUAGE, queryMap);
+		if (lang == null || lang.equals(""))
+		{
+			// try the session cached lang
+			lang = (String) session.getAttribute(GSParams.LANGUAGE);
+			if (lang == null || lang.equals(""))
+			{
+				// still not set, use the default
+				lang = this.default_lang;
+			}
+		}
+		UserContext userContext = new UserContext();
+		userContext.setLanguage(lang);
+		userContext.setUserID(uid);
+
+		if (request.getAuthType() != null)
+		{
+			//Get the username
+			userContext.setUsername(request.getUserPrincipal().getName());
+
+			//Get the groups for the user
+			Document msg_doc = XMLConverter.newDOM();
+			Element acquireGroupMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+			Element acquireGroupRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, "GetUserInformation", userContext);
+			acquireGroupMessage.appendChild(acquireGroupRequest);
+
+			Element paramList = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			acquireGroupRequest.appendChild(paramList);
+			paramList.appendChild(GSXML.createParameter(msg_doc, GSXML.USERNAME_ATT, request.getUserPrincipal().getName()));
+
+			Element aquireGroupsResponseMessage = (Element) this.recept.process(acquireGroupMessage);
+			Element aquireGroupsResponse = (Element) GSXML.getChildByTagName(aquireGroupsResponseMessage, GSXML.RESPONSE_ELEM);
+			Element param_list = (Element) GSXML.getChildByTagName(aquireGroupsResponse, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+			if (param_list != null)
+			{
+				HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+				String groups = (String) params.get("groups");
+				userContext.setGroups(groups.split(","));
+			}
+		}
+
+		// set the lang in the session
+		session.setAttribute(GSParams.LANGUAGE, lang);
+
+		String output = getFirstParam(GSParams.OUTPUT, queryMap);
+		if (output == null || output.equals(""))
+		{
+			output = "html"; // uses html by default
+		}
+
+		// If server output, force a switch to traditional interface
+		//output = (output.equals("server")) ? "html" : output;
+
+		// Force change the output mode if client-side XSLT is supported - server vs. client
+		// BUT only if the library allows client-side transforms	
+		if (supports_client_xslt)
+		{
+			// MUST be done before the xml_message is built
+			Cookie[] cookies = request.getCookies();
+			Cookie xsltCookie = null;
+
+			// The client has cookies enabled and a value set - use it!
+			if (cookies != null)
+			{
+				for (Cookie c : cookies)
+				{
+					if (c.getName().equals("supportsXSLT"))
+					{
+						xsltCookie = c;
+						break;
+					}
+				}
+				output = (xsltCookie != null && xsltCookie.getValue().equals("true") && output.equals("html")) ? "xsltclient" : output;
+			}
+		}
+
+		// the request to the receptionist
+		Document msg_doc = XMLConverter.newDOM();
+		Element xml_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element xml_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PAGE, "", userContext);
+		xml_request.setAttribute(GSXML.OUTPUT_ATT, output);
+
+		xml_message.appendChild(xml_request);
+
+		String action = getFirstParam(GSParams.ACTION, queryMap);
+		String subaction = getFirstParam(GSParams.SUBACTION, queryMap);
+		String collection = getFirstParam(GSParams.COLLECTION, queryMap);
+		String document = getFirstParam(GSParams.DOCUMENT, queryMap);
+		String service = getFirstParam(GSParams.SERVICE, queryMap);
+
+		// We clean up the cache session_ids_table if system
+		// commands are issued (and also don't need to do caching for this request)
+		boolean should_cache = true;
+		if (action != null && action.equals(GSParams.SYSTEM_ACTION) 
+		    && !subaction.equals(GSXML.SYSTEM_TYPE_PING) 
+		    && !subaction.equals(GSXML.SYSTEM_TYPE_AUTHENTICATED_PING)) // don't 'clean' anything on a mere ping
+		{
+			should_cache = false;
+
+			// we may want to remove all collection cache info, or just a specific collection
+			boolean clean_all = true;
+			String clean_collection = null;
+			// system commands are to activate/deactivate stuff
+			// collection param is in the sc parameter.
+			// don't like the fact that it is hard coded here
+			String coll = getFirstParam(GSParams.SYSTEM_CLUSTER, queryMap);
+			if (coll != null && !coll.equals(""))
+			{
+				clean_all = false;
+				clean_collection = coll;
+			}
+			else
+			{
+				// check other system types
+				if (subaction.equals("a") || subaction.equals("d"))
+				{
+					String module_name = getFirstParam("sn", queryMap);
+					if (module_name != null && !module_name.equals(""))
+					{
+						clean_all = false;
+						clean_collection = module_name;
+					}
+				}
+			}
+			if (clean_all)
+			{
+			    // TODO
+				session_ids_table = new Hashtable<String, UserSessionCache>();
+				session.removeAttribute(GSXML.USER_SESSION_CACHE_ATT); // triggers valueUnbound(), which removes the session id from the session_ids_table
+			}
+			else
+			{
+				// just clean up info for clean_collection
+				ArrayList<UserSessionCache> cache_list = new ArrayList<UserSessionCache>(session_ids_table.values());
+				for (int i = 0; i < cache_list.size(); i++)
+				{
+					UserSessionCache cache = cache_list.get(i);
+					cache.cleanupCache(clean_collection);
+				}
+
+			}
+		}
+
+		// cache_key is the collection name, or service name
+		String cache_key = collection;
+		if (cache_key == null || cache_key.equals(""))
+		{
+			cache_key = service;
+		}
+
+		// logger.info("should_cache= " + should_cache);
+
+		//clear the collection-specific cache in the session, since we have no way to know whether this session is 
+		//about the same collection as the last session or not.
+		Enumeration attributeNames = session.getAttributeNames();
+		while (attributeNames.hasMoreElements())
+		{
+			String name = (String) attributeNames.nextElement();
+			if (!name.equals(GSXML.USER_SESSION_CACHE_ATT) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSXML.USER_ID_ATT))
+			{
+				session.removeAttribute(name);
+			}
+		}
+
+		UserSessionCache session_cache = null;
+		Hashtable<String, Hashtable<String, String>> param_table = null;
+		Hashtable<String, String> table = null;
+		String sid = session.getId();
+		if (should_cache == true && cache_key != null && !cache_key.equals(""))
+		{
+			if (session_ids_table.containsKey(sid))
+			{
+				session_cache = session_ids_table.get(sid);
+				param_table = session_cache.getParamsTable();
+				logger.info("collections in table: " + tableToString(param_table));
+				if (param_table.containsKey(cache_key))
+				{
+					//logger.info("existing table: " + collection);
+					table = param_table.get(cache_key);
+				}
+				else
+				{
+					table = new Hashtable<String, String>();
+					param_table.put(cache_key, table);
+					//logger.info("new table: " + collection);
+				}
+			}
+			else
+			{
+				param_table = new Hashtable<String, Hashtable<String, String>>();
+				table = new Hashtable<String, String>();
+				param_table.put(cache_key, table);
+				session_cache = new UserSessionCache(sid, param_table);
+				session_ids_table.put(sid, session_cache);
+				session.setAttribute(GSXML.USER_SESSION_CACHE_ATT, session_cache);
+				//logger.info("new session id");
+			}
+		}
+
+		if (action == null || action.equals(""))
+		{
+			// should we do all the following stuff if using default page?
+			// display the home page  - the default page
+			xml_request.setAttribute(GSXML.ACTION_ATT, "p");
+			xml_request.setAttribute(GSXML.SUBACTION_ATT, PageAction.HOME_PAGE);
+		}
+		else
+		{
+			xml_request.setAttribute(GSXML.ACTION_ATT, action);
+			if (subaction != null)
+			{
+				xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
+			}
+
+			//  create the param list for the greenstone request - includes
+			// the params from the current request and any others from the saved session
+			Element xml_param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			xml_request.appendChild(xml_param_list);
+			
+			if (queryMap.containsKey("s1.collection") || queryMap.containsKey("s1.group")){
+				table.remove("s1.collection");
+				table.remove("s1.group");
+			}
+			for (String name : queryMap.keySet())
+			{
+				if (!name.equals(GSParams.ACTION) && !name.equals(GSParams.SUBACTION) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSParams.OUTPUT))
+				{// we have already dealt with these
+
+					String value = "";
+					String[] values = queryMap.get(name);
+					value = values[0];
+					if (values.length > 1)
+					{
+						for (int i = 1; i < values.length; i++)
+						{
+							value += "," + values[i];
+						}
+					}
+					// either add it to the param list straight away, or save it to the session and add it later
+					if (this.params.shouldSave(name) && table != null)
+					{
+						table.put(name, value);
+					}
+					else
+					{
+						Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+						param.setAttribute(GSXML.NAME_ATT, name);
+						param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(value));
+						xml_param_list.appendChild(param);
+					}
+				}
+			}
+			//put everything in the table into the session
+			// do we need to do this? why not just put from table into param list
+			if (table != null)
+			{
+				Enumeration<String> keys = table.keys();
+				while (keys.hasMoreElements())
+				{
+					String name = keys.nextElement();
+					session.setAttribute(name, table.get(name));
+				}
+			}
+
+			// put in all the params from the session cache
+			Enumeration params = session.getAttributeNames();
+			while (params.hasMoreElements())
+			{
+				String name = (String) params.nextElement();
+
+				if (!name.equals(GSXML.USER_SESSION_CACHE_ATT) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSXML.USER_ID_ATT))
+				{
+
+					// lang and uid are stored but we dont want it in the param list cos its already in the request
+					Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+					param.setAttribute(GSXML.NAME_ATT, name);
+					String value = GSXML.xmlSafe((String) session.getAttribute(name));
+
+					// ugly hack to undo : escaping
+					value = StringUtils.replace(value, "%3A", "\\:");
+					param.setAttribute(GSXML.VALUE_ATT, value);
+					xml_param_list.appendChild(param);
+				}
+			}
+		}
+
+		if (output.equals("json"))
+		{
+			response.setContentType("application/json");
+		}
+		else if (!output.equals("html") && !output.equals("server") && !output.equals("xsltclient"))
+		{
+			response.setContentType("text/xml"); // for now use text
+		}
+
+		//Add custom HTTP headers if requested
+		String httpHeadersParam = getFirstParam(GSParams.HTTP_HEADER_FIELDS, queryMap);
+		if (httpHeadersParam != null && httpHeadersParam.length() > 0)
+		{
+			Gson gson = new Gson();
+			Type type = new TypeToken<List<Map<String, String>>>()
+			{
+			}.getType();
+			List<Map<String, String>> httpHeaders = gson.fromJson(httpHeadersParam, type);
+			if (httpHeaders != null && httpHeaders.size() > 0)
+			{
+
+				for (int j = 0; j < httpHeaders.size(); j++)
+				{
+					Map nameValueMap = httpHeaders.get(j);
+					String name = (String) nameValueMap.get("name");
+					String value = (String) nameValueMap.get("value");
+
+					if (name != null && value != null)
+					{
+						response.setHeader(name, value);
+					}
+				}
+			}
+		}
+
+		String requestedURL = request.getRequestURL().toString();
+		String baseURL = "";
+		if (requestedURL.indexOf(library_name) != -1)
+		{
+			baseURL = requestedURL.substring(0, requestedURL.indexOf(library_name));
+			xml_request.setAttribute("baseURL", baseURL);
+		}
+		String fullURL;
+		if (request.getQueryString() != null)
+		{
+			fullURL = requestedURL + "?" + request.getQueryString();
+		}
+		else
+		{
+			fullURL = requestedURL;
+		}
+
+		xml_request.setAttribute("remoteAddress", request.getRemoteAddr());
+		xml_request.setAttribute("fullURL", fullURL.replace("&", "&amp;"));
+
+		if (!runSecurityChecks(request, xml_request, userContext, out, baseURL, collection, document, queryMap))
+		{
+			return;
+		}
+
+		Node xml_result = this.recept.process(xml_message);
+		encodeURLs(xml_result, response);
+
+		String xml_string = XMLConverter.getPrettyString(xml_result);
+		
+		if (output.equals("json"))
+		{
+			try
+			{
+				JSONObject json_obj = org.json.XML.toJSONObject(xml_string);
+
+				out.println(json_obj.toString());
+			}
+			catch (Exception e)
+			{
+				e.printStackTrace();
+				out.println("Error: failed to convert output XML to JSON format");
+			}
+		}
+		else
+		{
+			out.println(xml_string);
+		}
+
+		displaySize(session_ids_table);
+	}
+
+	public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		doGetOrPost(request, response, request.getParameterMap());
+	} //end of doGet(HttpServletRequest, HttpServletResponse)
+
+	private boolean runSecurityChecks(HttpServletRequest request, Element xml_request, UserContext userContext, PrintWriter out, String baseURL, String collection, String document, Map<String, String[]> queryMap) throws ServletException
+	{
+		//Check if we need to login or logout
+		String username = getFirstParam("username", queryMap);
+		String password = getFirstParam("password", queryMap);
+		String logout = getFirstParam("logout", queryMap);
+
+		if (logout != null)
+		{
+			request.logout();
+		}
+
+		if (username != null && password != null)
+		{
+			//We are changing to another user, so log out first
+			if (request.getAuthType() != null)
+			{
+				request.logout();
+			}
+
+			//This try/catch block catches when the login request fails (e.g. The user enters an incorrect password).
+			try
+			{
+				//Try a global login first
+				password = Authentication.hashPassword(password);
+				request.login(username, password);
+			}
+			catch (Exception ex)
+			{
+				try
+				{
+					//If the global login fails then try a site-level login
+					String siteName = (String) this.recept.getConfigParams().get(GSConstants.SITE_NAME);
+					request.login(siteName + "-" + username, password);
+				}
+				catch (Exception exc)
+				{
+					//The user entered in either the wrong username or the wrong password
+				  Document loginPageDoc = XMLConverter.newDOM();
+					Element loginPageMessage = loginPageDoc.createElement(GSXML.MESSAGE_ELEM);
+					Element loginPageRequest = GSXML.createBasicRequest(loginPageDoc, GSXML.REQUEST_TYPE_PAGE, "", userContext);
+					loginPageRequest.setAttribute(GSXML.ACTION_ATT, "p");
+					loginPageRequest.setAttribute(GSXML.SUBACTION_ATT, "login");
+					loginPageRequest.setAttribute(GSXML.OUTPUT_ATT, "html");
+					loginPageRequest.setAttribute(GSXML.BASE_URL, baseURL);
+					loginPageMessage.appendChild(loginPageRequest);
+
+					Element paramList = loginPageDoc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+					loginPageRequest.appendChild(paramList);
+
+					Element messageParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
+					messageParam.setAttribute(GSXML.NAME_ATT, "loginMessage");
+					messageParam.setAttribute(GSXML.VALUE_ATT, "Either your username or password was incorrect, please try again.");
+					paramList.appendChild(messageParam);
+
+					Element urlParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
+					urlParam.setAttribute(GSXML.NAME_ATT, "redirectURL");
+					String queryString = "";
+					if (request.getQueryString() != null)
+					{
+						queryString = "?" + request.getQueryString().replace("&", "&amp;");
+					}
+					urlParam.setAttribute(GSXML.VALUE_ATT, library_name + queryString);
+					paramList.appendChild(urlParam);
+
+					Node loginPageResponse = this.recept.process(loginPageMessage);
+					out.println(XMLConverter.getPrettyString(loginPageResponse));
+
+					return false;
+				}
+			}
+		}
+
+		//If a user is logged in
+		if (request.getAuthType() != null)
+		{
+		  Element userInformation = xml_request.getOwnerDocument().createElement(GSXML.USER_INFORMATION_ELEM);
+			userInformation.setAttribute(GSXML.USERNAME_ATT, request.getUserPrincipal().getName());
+
+			Document msg_doc = XMLConverter.newDOM();
+			Element userInfoMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+			Element userInfoRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_SECURITY, "GetUserInformation", userContext);
+			userInfoMessage.appendChild(userInfoRequest);
+
+			Element paramList = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			userInfoRequest.appendChild(paramList);
+
+			Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, GSXML.USERNAME_ATT);
+			param.setAttribute(GSXML.VALUE_ATT, request.getUserPrincipal().getName());
+			paramList.appendChild(param);
+
+			Element userInformationResponse = (Element) GSXML.getChildByTagName(this.recept.process(userInfoMessage), GSXML.RESPONSE_ELEM);
+			Element responseParamList = (Element) GSXML.getChildByTagName(userInformationResponse, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (responseParamList == null)
+			{
+				logger.error("Can't get the groups for user " + request.getUserPrincipal().getName());
+			}
+			else
+			{
+				HashMap<String, Serializable> responseParams = GSXML.extractParams(responseParamList, true);
+				String groups = (String) responseParams.get(GSXML.GROUPS_ATT);
+				String editEnabled = (String) responseParams.get("editEnabled");
+
+				userInformation.setAttribute(GSXML.GROUPS_ATT, groups);
+				userInformation.setAttribute("editEnabled", (editEnabled != null) ? editEnabled : "false");
+				xml_request.appendChild(userInformation);
+			}
+		}
+
+		//If we are in a collection-related page then make sure this user is allowed to access it
+		if (collection != null && !collection.equals(""))
+		{
+			//Get the security info for this collection
+		  Document msg_doc = XMLConverter.newDOM();
+			Element securityMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+			Element securityRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_SECURITY, collection, userContext);
+			securityMessage.appendChild(securityRequest);
+			if (document != null && !document.equals(""))
+			{
+				securityRequest.setAttribute(GSXML.NODE_OID, document);
+			}
+
+			Element securityResponse = (Element) GSXML.getChildByTagName(this.recept.process(securityMessage), GSXML.RESPONSE_ELEM);
+			if (securityResponse == null)
+			{
+				return false;
+			}
+
+			ArrayList<String> groups = GSXML.getGroupsFromSecurityResponse(securityResponse);
+
+			//If guests are not allowed to access this page then check to see if the user is in a group that is allowed to access the page
+			if (!groups.contains(""))
+			{
+				boolean found = false;
+				for (String group : groups)
+				{
+					if (request.isUserInRole(group))
+					{
+						found = true;
+						break;
+					}
+				}
+
+				//The current user is not allowed to access the page so produce a login page
+				if (!found)
+				{
+				  Document loginPageDoc = XMLConverter.newDOM();
+					Element loginPageMessage = loginPageDoc.createElement(GSXML.MESSAGE_ELEM);
+					Element loginPageRequest = GSXML.createBasicRequest(loginPageDoc, GSXML.REQUEST_TYPE_PAGE, "", userContext);
+					loginPageRequest.setAttribute(GSXML.ACTION_ATT, "p");
+					loginPageRequest.setAttribute(GSXML.SUBACTION_ATT, "login");
+					loginPageRequest.setAttribute(GSXML.OUTPUT_ATT, "html");
+					loginPageRequest.setAttribute(GSXML.BASE_URL, baseURL);
+					loginPageMessage.appendChild(loginPageRequest);
+
+					Element paramList = loginPageDoc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+					loginPageRequest.appendChild(paramList);
+
+					Element messageParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
+					messageParam.setAttribute(GSXML.NAME_ATT, "loginMessage");
+					if (request.getAuthType() == null)
+					{
+						messageParam.setAttribute(GSXML.VALUE_ATT, "Please log in to view this page");
+					}
+					else
+					{
+						messageParam.setAttribute(GSXML.VALUE_ATT, "You are not in the correct group to view this page, would you like to log in as a different user?");
+					}
+					paramList.appendChild(messageParam);
+
+					Element urlParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
+					urlParam.setAttribute(GSXML.NAME_ATT, "redirectURL");
+					if (request.getQueryString() != null && request.getQueryString().length() > 0)
+					{
+						urlParam.setAttribute(GSXML.VALUE_ATT, request.getRequestURL() + "?" + request.getQueryString().replace("&", "&amp;"));
+					}
+					else
+					{
+						urlParam.setAttribute(GSXML.VALUE_ATT, request.getRequestURL().toString());
+					}
+					paramList.appendChild(urlParam);
+
+					Node loginPageResponse = this.recept.process(loginPageMessage);
+					out.println(XMLConverter.getPrettyString(loginPageResponse));
+
+					return false;
+				}
+			}
+		}
+		return true;
+	}
+
+	//a debugging method
+	private void displaySize(Hashtable<String, UserSessionCache> table)
+	{
+		if (table == null)
+		{
+			logger.info("cached table is null");
+			return;
+		}
+		if (table.size() == 0)
+		{
+			logger.info("cached table size is zero");
+			return;
+		}
+		int num_cached_coll = 0;
+		ArrayList<UserSessionCache> cache_list = new ArrayList<UserSessionCache>(table.values());
+		for (int i = 0; i < cache_list.size(); i++)
+		{
+			num_cached_coll += cache_list.get(i).tableSize();
+		}
+		logger.info("Number of sessions : total number of cached collection info = " + table.size() + " : " + num_cached_coll);
+	}
+
+	/** merely a debugging method! */
+	private String tableToString(Hashtable<String, Hashtable<String, String>> table)
+	{
+		String str = "";
+		Enumeration<String> keys = table.keys();
+		while (keys.hasMoreElements())
+		{
+			String name = keys.nextElement();
+			str += name + ", ";
+		}
+		return str;
+	}
+
+	/**
+	 * this goes through each URL and adds in a session id if needed-- its
+	 * needed if the browser doesn't accept cookies also escapes things if
+	 * needed
+	 */
+	protected void encodeURLs(Node dataNode, HttpServletResponse response)
+	{
+		if (dataNode == null)
+		{
+			return;
+		}
+
+		Element data = null;
+
+		short nodeType = dataNode.getNodeType();
+		if (nodeType == Node.DOCUMENT_NODE)
+		{
+			Document docNode = (Document) dataNode;
+			data = docNode.getDocumentElement();
+		}
+		else
+		{
+			data = (Element) dataNode;
+		}
+
+		if (data != null)
+		{
+
+			// get all the <a> elements
+			NodeList hrefs = data.getElementsByTagName("a");
+			// Instead of calculating each iteration...
+			int hrefscount = hrefs.getLength();
+
+			for (int i = 0; hrefs != null && i < hrefscount; i++)
+			{
+				Element a = (Element) hrefs.item(i);
+				// ugly hack to get rid of : in the args - interferes with session handling
+				String href = a.getAttribute("href");
+				if (!href.equals(""))
+				{
+					if (href.indexOf("?") != -1)
+					{
+						String[] parts = StringUtils.split(href, "\\?", -1);
+						if (parts.length == 1)
+						{
+							parts[0] = StringUtils.replace(parts[0], ":", "%3A");
+							href = "?" + parts[0];
+						}
+						else
+						{
+							parts[1] = StringUtils.replace(parts[1], ":", "%3A");
+							href = parts[0] + "?" + parts[1];
+						}
+
+					}
+					a.setAttribute("href", response.encodeURL(href));
+				}
+			}
+
+			// now find any submit bits - get all the <form> elements
+			NodeList forms = data.getElementsByTagName("form");
+			int formscount = forms.getLength();
+			for (int i = 0; forms != null && i < formscount; i++)
+			{
+				Element form = (Element) forms.item(i);
+				form.setAttribute("action", response.encodeURL(form.getAttribute("action")));
+			}
+			// are these the only cases where URLs occur??
+			// we should only do this for greenstone urls?
+		}
+
+	}
+
+	protected String getFirstParam(String name, Map<String, String[]> map)
+	{
+		String[] val = map.get(name);
+		if (val == null || val.length == 0)
+		{
+			return null;
+		}
+
+		return val[0];
+	}
+
+	synchronized protected int getNextUserId()
+	{
+		next_user_id++;
+		return next_user_id;
+	}
+
+	public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		//Check if we need to process a file upload
+		if (ServletFileUpload.isMultipartContent(request))
+		{
+			DiskFileItemFactory fileItemFactory = new DiskFileItemFactory();
+
+			int sizeLimit = System.getProperties().containsKey("servlet.upload.filesize.limit") ? Integer.parseInt(System.getProperty("servlet.upload.filesize.limit")) : 100 * 1024 * 1024;
+
+			File tempDir = new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp");
+			if (!tempDir.exists())
+			{
+				tempDir.mkdirs();
+			}
+
+			//We want all files to be stored on disk (hence the 0)
+			fileItemFactory.setSizeThreshold(0);
+			fileItemFactory.setRepository(tempDir);
+
+			ServletFileUpload uploadHandler = new ServletFileUpload(fileItemFactory);
+			uploadHandler.setFileSizeMax(sizeLimit);
+
+			HashMap<String, String[]> queryMap = new HashMap<String, String[]>();
+			try
+			{
+				List items = uploadHandler.parseRequest(request);
+				Iterator iter = items.iterator();
+				while (iter.hasNext())
+				{
+					FileItem current = (FileItem) iter.next();
+					if (current.isFormField())
+					{
+						queryMap.put(current.getFieldName(), new String[] { current.getString() });
+					}
+					else if (current.getName() != null && !current.getName().equals(""))
+					{
+						File file = new File(tempDir, current.getName());
+						current.write(file);
+
+						queryMap.put("md___ex.Filesize", new String[] { "" + file.length() });
+						queryMap.put("md___ex.Filename", new String[] { "" + current.getName() });
+					}
+				}
+			}
+			catch (Exception e)
+			{
+				e.printStackTrace();
+			}
+
+			doGetOrPost(request, response, queryMap);
+		}
+		else
+		{
+			doGetOrPost(request, response, request.getParameterMap());
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/OAIServer.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/OAIServer.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/OAIServer.java	(revision 31634)
@@ -0,0 +1,507 @@
+/*
+ *    OAIServer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.UnavailableException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.comms.Communicator;
+import org.greenstone.gsdl3.comms.SOAPCommunicator;
+import org.greenstone.gsdl3.core.OAIMessageRouter;
+import org.greenstone.gsdl3.core.OAIReceptionist;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OAIResumptionToken;
+import org.greenstone.gsdl3.util.OAIXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+/** a servlet to serve the OAI metadata harvesting - we are using servlets instead
+ * of cgi
+ * the init method is called only once - the first time the servlet classes
+ * are loaded. Each time a request comes in to the servlet, the session()
+ * method is called in a new thread (calls doGet/doPut etc)
+ * takes the verb= type args and builds a simple request to send to
+ * the oai receptionist, which returns a result in xml, conforming to the OAI-PMH
+ * protocol.
+ * @see Receptionist
+ */
+/**
+ * OAI server configuration instructions *
+ * 
+ */
+public class OAIServer extends BaseGreenstoneServlet
+{
+
+	/** the receptionist to send messages to */
+	protected OAIReceptionist recept = null;
+	/**
+	 * the default language - is specified by setting a servlet param, otherwise
+	 * DEFAULT_LANG is used
+	 */
+	protected String default_lang = null;
+	/**
+	 * The default default - used if a default lang is not specified in the
+	 * servlet params
+	 */
+	protected final String DEFAULT_LANG = "en";
+
+	/** A HashSet which contains all the legal verbs. */
+	protected HashSet<String> verb_set = null;
+	/**
+	 * A HashSet which contains all the legal oai keys in the key/value argument
+	 * pair.
+	 */
+	protected HashSet<String> param_set = null;
+	/**
+	 * The name of the site with which we will finally be dealing, whether it is
+	 * a local site or a remote site through a communicator.
+	 */
+	protected String site = "";
+
+  // can be overriddden in OAIConfig.xml
+	// do we output the stylesheet processing instruction?
+	protected boolean use_oai_stylesheet = true;
+	protected String oai_stylesheet = "interfaces/oai/oai2.xsl";
+
+	// there is no getQueryString() method in the HttpServletRequest returned from doPost, 
+	// since that is actually of type apache RequestFacade, and doesn't define such a method
+	protected String queryString = null;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.OAIServer.class.getName());
+
+	/**
+	 * initialise the servlet
+	 */
+	public void init(ServletConfig config) throws ServletException
+	{
+		// always call super.init, i.e., HttpServlet.;
+		super.init(config);
+		this.default_lang = config.getInitParameter(GSConstants.DEFAULT_LANG);
+
+		initVerbs();
+		initParams();
+
+		String site_name = config.getInitParameter(GSConstants.SITE_NAME);
+		String remote_site_name = null;
+		String remote_site_type = null;
+		String remote_site_address = null;
+
+		if (site_name == null)
+		{
+			// no local site, try for communicator (remote site)
+			remote_site_name = config.getInitParameter("remote_site_name");
+			remote_site_type = config.getInitParameter("remote_site_type");
+			remote_site_address = config.getInitParameter("remote_site_address");
+			if (remote_site_name == null || remote_site_type == null || remote_site_address == null)
+			{
+				logger.error("initialisation paramters not all set!");
+				logger.error("if site_name is not set, then you must have remote_site_name, remote_site_type and remote_site_address set");
+				throw new UnavailableException("OAIServer: incorrect servlet parameters");
+			}
+		}
+
+		if (this.default_lang == null)
+		{
+			// choose english
+			this.default_lang = DEFAULT_LANG;
+		}
+
+		// the receptionist -the servlet will talk to this
+		this.recept = new OAIReceptionist();
+
+		// the receptionist uses a OAIMessageRouter or Communicator to send its requests to. We either create a OAIMessageRouter here for the designated site (if site_name set), or we create a Communicator for a remote site. The is given to teh Receptionist, and the servlet never talks to it again.directly.
+		if (site_name != null)
+		{
+			//this site_name could consist of comma separated more than one site name.
+			String mr_name = (String) config.getInitParameter("messagerouter_class");
+			OAIMessageRouter message_router = null;
+			if (mr_name == null)
+			{ // just use the normal MR *********
+				message_router = new OAIMessageRouter();
+			}
+			else
+			{ // try the specified one
+				try
+				{
+					message_router = (OAIMessageRouter) Class.forName("org.greenstone.gsdl3.core." + mr_name).newInstance();
+				}
+				catch (Exception e)
+				{ // cant use this new one, so use normal one
+				  logger.error("OAIServlet configure exception when trying to use a new OAIMessageRouter " + mr_name, e);
+				  message_router = new OAIMessageRouter();
+				}
+			}
+
+			message_router.setSiteName(site_name);
+			// lots of work is done in this step; see OAIMessageRouter.java
+			if (!message_router.configure()) {
+			  throw new UnavailableException("OAIServer: Couldn't configure OAIMessageRouter");
+			}
+			this.recept.setSiteName(site_name);
+			this.recept.setMessageRouter(message_router);
+
+		}
+		else
+		{
+			// talking to a remote site, create a communicator
+			Communicator communicator = null;
+			// we need to create the XML to configure the communicator
+			Document site_doc = XMLConverter.newDOM();
+			Element site_elem = site_doc.createElement(GSXML.SITE_ELEM);
+			site_elem.setAttribute(GSXML.TYPE_ATT, remote_site_type);
+			site_elem.setAttribute(GSXML.NAME_ATT, remote_site_name);
+			site_elem.setAttribute(GSXML.ADDRESS_ATT, remote_site_address);
+
+			if (remote_site_type.equals(GSXML.COMM_TYPE_SOAP_JAVA))
+			{
+				communicator = new SOAPCommunicator();
+			}
+			else
+			{
+				logger.error("OAIServlet.init Error: invalid Communicator type: " + remote_site_type);
+				throw new UnavailableException("OAIServer: invalid communicator type");
+			}
+
+			if (!communicator.configure(site_elem))
+			{
+				logger.error("OAIServlet.init Error: Couldn't configure communicator");
+				throw new UnavailableException("OAIServer: Couldn't configure communicator");
+			}
+			this.recept.setSiteName(remote_site_name);
+			this.recept.setMessageRouter(communicator);
+		}
+
+		// Read in OAIConfig.xml (residing web/WEB-INF/classes/) and 
+		//use it to configure the receptionist. 
+		Element oai_config = OAIXML.getOAIConfigXML();
+		if (oai_config == null)
+		{
+			logger.error("Fail to parse oai config file OAIConfig.xml.");
+			throw new UnavailableException("OAIServer: Couldn't parse OAIConfig.xml");
+		}
+		// pass it to the receptionist
+		if (!this.recept.configure(oai_config)) {
+		  logger.error("Couldn't configure receptionist");
+		  throw new UnavailableException("OAIServer: Couldn't configure receptionist"); 
+		}
+		// also, we have something we want to get from here - useOAIStylesheet
+		this.configure(oai_config);
+
+	}//end of init()
+
+	private void configure(Element oai_config)
+	{
+		Element use_stylesheet_elem = (Element) GSXML.getChildByTagName(oai_config, OAIXML.USE_STYLESHEET);
+		if (use_stylesheet_elem != null)
+		{
+			String value = GSXML.getNodeText(use_stylesheet_elem);
+			if (value.equals("no"))
+			{
+				this.use_oai_stylesheet = false;
+			}
+		}
+		if (this.use_oai_stylesheet)
+		{
+			// now see if there is a custom stylesheet specified
+			Element stylesheet_elem = (Element) GSXML.getChildByTagName(oai_config, OAIXML.STYLESHEET);
+			if (stylesheet_elem != null)
+			{
+				String value = GSXML.getNodeText(stylesheet_elem);
+				if (!value.equals(""))
+				{
+					oai_stylesheet = value;
+				}
+			}
+
+		}
+	}
+
+	private void initVerbs()
+	{
+		verb_set = new HashSet<String>();
+		verb_set.add(OAIXML.GET_RECORD);
+		verb_set.add(OAIXML.LIST_RECORDS);
+		verb_set.add(OAIXML.LIST_IDENTIFIERS);
+		verb_set.add(OAIXML.LIST_SETS);
+		verb_set.add(OAIXML.LIST_METADATA_FORMATS);
+		verb_set.add(OAIXML.IDENTIFY);
+	}
+
+	private void initParams()
+	{
+		param_set = new HashSet<String>();
+		param_set.add(OAIXML.METADATA_PREFIX);
+		param_set.add(OAIXML.FROM);
+		param_set.add(OAIXML.UNTIL);
+		param_set.add(OAIXML.SET);
+		param_set.add(OAIXML.RESUMPTION_TOKEN);
+		param_set.add(OAIXML.IDENTIFIER);
+	}
+
+	private void logUsageInfo(HttpServletRequest request)
+	{
+		String usageInfo = "";
+
+		String query = (queryString == null) ? request.getQueryString() : queryString;
+
+		//logged info = general-info + session-info
+		usageInfo = request.getContextPath() + " " + //session id
+		request.getServletPath() + " " + //serlvet
+		"[" + query + "]" + " " + //the query string
+		"[" + usageInfo.trim() + "]" + " " + // params stored in a session
+		request.getRemoteAddr() + " " + //remote address
+		request.getHeader("user-agent") + " "; //the remote brower info
+
+		logger.info(usageInfo);
+	}
+
+	/**
+	 * return true if the url is in the form of baseURL?verb=...,
+	 */
+	private boolean validate(String query, String verb)
+	{
+		//Here in OAIServer, only the verbs are validated. All the validation for individual verb
+		// is taken in their doXXX() methods.
+		if (query == null || !query.startsWith(OAIXML.VERB + "="))
+		{
+			return false;
+		}
+		if (!verb_set.contains(verb))
+		{
+			return false;
+		}
+		return true;
+	}
+
+	private String getVerb(String query)
+	{
+		if (query == null)
+			return "";
+		int verb_start_index = query.indexOf("=") + 1;// first occurence of '='
+		int verb_end_index = query.indexOf("&");
+		if (verb_end_index == -1)
+		{
+			return query.substring(verb_start_index);
+		}
+		return query.substring(verb_start_index, verb_end_index);
+	}
+
+	public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+		logUsageInfo(request);
+
+		// oai always requires the content type be text/xml
+		request.setCharacterEncoding("UTF-8");
+		response.setContentType("text/xml;charset=UTF-8");
+		PrintWriter out = response.getWriter();
+
+		//
+		String lang = request.getParameter(GSParams.LANGUAGE);
+		if (lang == null || lang.equals(""))
+		{
+			// use the default
+			lang = this.default_lang;
+		}
+		//we don't get the baseURL from the http request because what we get might be different from the one known publicly due to local network redirection.
+		//For example, puka.cs.waikato.ac.nz vs www.greenstone.org
+		//String base_url = request.getRequestURL().toString();
+		
+		// if called by doPost (if this was originally a POST request), var queryString would have been set
+		String query = (queryString == null) ? request.getQueryString() : queryString;
+		queryString = null; // reset member variable, else no doGet will work as long as the server remains running
+
+		if (query!=null && query.equals("reset")) {
+		  logger.error("reset was called*******************");
+		  out.println("<?xml version='1.0' encoding='UTF-8' ?>");
+		  out.println(this.recept.process("<message><request reset='true'/></message>"));
+		  return;
+		}
+		String[] pairs = (query == null) ? null : query.split("&");//split into key/value pairs
+		
+		String verb = getVerb(query);
+		Document response_doc = XMLConverter.newDOM();
+		Element xml_response = OAIXML.createBasicResponse(response_doc, verb, pairs);
+		Element verb_elem = null;
+
+		if (validate(query, verb) == false)
+		{
+			if (verb_set.contains(verb) == false)
+			{
+				logger.error(OAIXML.BAD_VERB + ": " + query);
+				verb_elem = OAIXML.createErrorElement(response_doc, OAIXML.BAD_VERB, OAIXML.ILLEGAL_OAI_VERB);
+			}
+			else
+			{
+				//must be something else other than bad verbs caused an error, so bad argument
+				logger.error(OAIXML.BAD_ARGUMENT + ": " + query);
+				verb_elem = OAIXML.createErrorElement(response_doc, OAIXML.BAD_ARGUMENT, "");
+			}
+			xml_response.appendChild(verb_elem);
+
+			out.println("<?xml version='1.0' encoding='UTF-8' ?>");
+			if (this.use_oai_stylesheet)
+			{
+				out.println("<?xml-stylesheet type='text/xsl' href='" + this.oai_stylesheet + "' ?>\n");
+			}
+			out.println(XMLConverter.getPrettyString(xml_response));
+			return;
+		}//end of if(validate
+
+		// The query is valid, we can now
+		// compose the request message to the receptionist
+		Document request_doc = XMLConverter.newDOM();
+		Element xml_message = request_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element xml_request = request_doc.createElement(GSXML.REQUEST_ELEM);
+		// The type attribute is set to be 'oaiService' from OAIServer to OAIReceptionist.
+		//xml_request.setAttribute(GSXML.TYPE_ATT, OAIXML.OAI_SERVICE);
+		xml_request.setAttribute(GSXML.LANG_ATT, lang);
+		xml_request.setAttribute(GSXML.TO_ATT, verb);
+		addParams(xml_request, pairs);
+
+		//xml_request.setAttribute(GSXML.OUTPUT_ATT, output);????
+		xml_message.appendChild(xml_request);
+
+		Node xml_result = this.recept.process(xml_message);
+		if (xml_result == null)
+		{
+			logger.info("xml_result is null");
+			verb_elem = OAIXML.createErrorElement(response_doc, "Internal error", "");
+			xml_response.appendChild(verb_elem);
+		}
+		else
+		{
+
+			/**
+			 * All response elements are in the form (with a corresponding verb
+			 * name): <message> <response> <verb> ... <resumptionToken> .. this
+			 * is optional! </resumptionToken> </verb> </response> </message>
+			 */
+			Node res = GSXML.getChildByTagName(xml_result, GSXML.RESPONSE_ELEM);
+			if (res == null)
+			{
+				logger.info("response element in xml_result is null");
+				verb_elem = OAIXML.createErrorElement(response_doc, "Internal error", "");
+			}
+			else
+			{
+				verb_elem = GSXML.getFirstElementChild(res);
+			}
+
+			if ( verb_elem.getTagName().equals(OAIXML.ERROR))
+			{
+			  xml_response.appendChild(response_doc.importNode(verb_elem, true));
+			}
+			else if (OAIXML.oai_version.equals(OAIXML.OAI_VERSION2)) {
+			  xml_response.appendChild(response_doc.importNode(verb_elem, true));
+			}
+			else
+			{
+				GSXML.copyAllChildren(xml_response, verb_elem);
+			}
+		}
+		out.println("<?xml version='1.0' encoding='UTF-8' ?>");
+		if (this.use_oai_stylesheet)
+		{
+			out.println("<?xml-stylesheet type='text/xsl' href='" + this.oai_stylesheet + "' ?>\n");
+		}
+		out.println(XMLConverter.getPrettyString(xml_response));
+		return;
+	}
+
+	/** append parameter elements to the request sent to the receptionist */
+	public void addParams(Element request, String[] pairs)
+	{
+	  Document doc = request.getOwnerDocument();
+		// no params apart from the verb
+		if (pairs == null || pairs.length < 2)
+			return;
+
+		/**
+		 * the request xml is composed in the form: <request> <param name=.../>
+		 * <param name=.../> </request> (No paramList element in between).
+		 */
+		for (int i = 1; i < pairs.length; i++)
+		{
+			//the first pair in pairs is the verb=xxx
+			int index = pairs[i].indexOf("=");
+			if (index != -1)
+			{ //just a double check
+			  Element param = GSXML.createParameter(doc, pairs[i].substring(0, index), OAIXML.oaiDecode(pairs[i].substring(index + 1)));
+			  request.appendChild(param);
+			}
+		}
+	}
+
+	// For OAI version 2.0, validation tests indicated that POST needs to be supported. Some
+	// modification was required in order to ensure that the request is passed intact to doGet()
+	public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+	{
+
+		// the post method returns a wrapper of type RequestFacade by apache and there
+		// is no getQueryString() method defined for it. Therefore, need to work this out
+		// manually before calling doGet(request, response) so that doGet can work as before.
+
+		queryString = "";
+		Iterator parameter_entries = request.getParameterMap().entrySet().iterator();
+		while (parameter_entries.hasNext())
+		{
+			Map.Entry param_entry = (Map.Entry) parameter_entries.next();
+			String[] paramVals = (String[]) param_entry.getValue();
+			if (paramVals != null)
+			{
+				if (paramVals.length > 0)
+				{
+					logger.error("POST request received: " + param_entry.getKey() + " - " + paramVals[0]);
+					queryString = queryString + "&" + param_entry.getKey() + "=" + paramVals[0];
+				}
+			}
+		}
+		if (queryString.length() > 0)
+		{
+			queryString = queryString.substring(1);
+			//queryString = OAIXML.oaiEncode(queryString);
+		}
+		if (queryString.equals(""))
+		{
+			queryString = null;
+		}
+		doGet(request, response);
+	}
+
+
+	public void destroy()
+	{
+		recept.cleanUp();
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/SOAPServerlocalsite.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/SOAPServerlocalsite.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/SOAPServerlocalsite.java	(revision 31634)
@@ -0,0 +1,91 @@
+/*
+ *    SOAPServer.java.in: a template for a new SOAPServer
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import java.io.File;
+
+/**
+ * The server side of a SOAP connection
+ *
+ * @author Katherine Don
+ * @see <a href="http://www.w3.org/TR/SOAP/">Simple Object Access Protocol (SOAP) 1.1 </a>
+ */
+
+public class SOAPServerlocalsite
+{
+    
+    /** The message router we're talking to */
+    protected MessageRouter mr=null;
+    /** the name of the site we are serving */
+    protected String site_name = "localsite";
+
+    /** The no-args constructor */
+    public SOAPServerlocalsite() {
+
+	String gsdl3_home = GlobalProperties.getGSDL3Home();
+	if (gsdl3_home == null || gsdl3_home.equals("")) {
+	    System.err.println("Couldn't access GSDL3Home from GlobalProerties.getGSDL3HOME, can't initialize the SOAP Server.");
+	    return;
+	}
+	String site_home = GSFile.siteHome(gsdl3_home, this.site_name);
+	
+	File site_file = new File(site_home);
+	if (!site_file.isDirectory()) {
+	    System.err.println("The site directory "+site_file.getPath()+" doesn't exist. Can't initialize the SOAP Server.");
+	    return;
+	}   	
+	mr = new MessageRouter();
+	mr.setSiteName(this.site_name);
+	mr.configure();
+    }
+    
+    
+    public Element [] process (Element [] xml_in) {
+	Element [] result = new Element[xml_in.length];
+	for (int i=0; i<xml_in.length; i++) {
+	    Element req = xml_in[i];
+	    // get rid of the obligatory namespace that axis needs
+	    String tag_name = req.getTagName();
+	    String namespace="";
+	    if (tag_name.indexOf(':')!= -1) {
+		namespace = tag_name.substring(0, tag_name.indexOf(':'));
+		tag_name = tag_name.substring(tag_name.indexOf(':')+1);	
+	    }
+	    Element new_req = GSXML.duplicateWithNewName(req.getOwnerDocument(), req, tag_name, true);
+	    Node n = mr.process(new_req);
+	    Element r = GSXML.nodeToElement(n);
+	    // add the namespace back on
+	    //Element new_res = r;
+	    //if (!namespace.equals("")) {
+	    //	new_res = GSXML.duplicateWithNewName(r.getOwnerDocument(), r, namespace+r.getTagName(), true);
+	    //}
+	    result[i] = r;
+	}
+	return result;
+    }   
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/TransformingLibrary.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/TransformingLibrary.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/TransformingLibrary.java	(revision 31634)
@@ -0,0 +1,101 @@
+/*
+ *    TransformingLibrary.java
+ *    Copyright (C) 2012 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3;
+
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLTransformer;
+import org.greenstone.util.GlobalProperties;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+
+import java.io.File;
+
+
+/**
+ * A program to take an GS3 xml file and an XSLT file from the command line and return html
+ * @author ak19
+ */
+final public class TransformingLibrary
+{
+    // GS3> java -classpath "web/WEB-INF/lib/*":"lib/jni/*":"web/WEB-INF/classes/" org.greenstone.gsdl3.TransformingLibrary localsite default C1.xml CL1.xslt 2>&1 | less
+
+    /**
+     *  On Linux, run as:
+     *  GS3&gt; java -classpath "web/WEB-INF/lib/*":"lib/jni/*":"web/WEB-INF/classes/" org.greenstone.gsdl3.TransformingLibrary localsite default inputXML inputXSLT
+     *  where inputXML is generated by o=xml and inputXSLT is generated by o=skinandlibdoc
+     *
+     *  Relative paths in xsl:imports made by the input xslt file were solved as described in 
+     *  @see http://stackoverflow.com/questions/3699860/resolving-relative-paths-when-loading-xslt-files
+     *
+     *  For how to include all jars in a folder into the classpath to run a java program, see
+     *  @see http://stackoverflow.com/questions/219585/setting-multiple-jars-in-java-classpath
+     *  @see http://stackoverflow.com/questions/6780678/run-class-in-jar-file
+     */
+    public static void main(String args[])
+    {
+	String usageMsg = "Usage: TransformingLibrary <site name> <interface name> <xml file> <xslt file>";
+	
+	if (System.getenv("GSDL3SRCHOME") == null) {
+	    System.out.println("Before calling this script, run: source gs3-setup.sh\n" + usageMsg);
+	    System.exit(1);
+	}
+	// force GlobalProperties to default GSDL3HOME to GSDL3SRCHOME/web if not already set
+	// need GSDL3HOME set for the XMLTransformer.transformer(XSLTfile, XMLfile, interfaceName, DocType)
+	GlobalProperties.loadGlobalProperties("");
+	
+	
+	if (args.length < 3 || args.length > 4) {
+	    System.out.println(usageMsg);
+	    System.exit(1);
+	}
+	
+	File xmlFile = new File(args[2]);
+	if (!xmlFile.exists()) {
+	    System.out.println("File " + args[2] + " does not exist");
+	    System.out.println(usageMsg);
+	    System.exit(1);
+	}
+	
+	File xsltFile = null;
+	if(args.length == 4) {
+	    xsltFile = new File(args[3]);
+	    if (!xsltFile.exists()) {
+		System.out.println("XSLT file " + args[3] + " does not exist");
+		System.out.println(usageMsg);
+		System.exit(1);
+	    }
+	}
+
+	
+	String siteName = args[0];
+	String interfaceName = args[1];
+	
+	XMLTransformer transformer = new XMLTransformer();
+	// Can now call the custom transform() method to do the transformation while 
+	// also dealing with relative paths in <xsl:import>s in the input XSLT file
+	Node resultNode = transformer.transform(xsltFile, xmlFile, interfaceName, null);
+	Element resultElement = resultNode.getOwnerDocument().getDocumentElement();
+	System.out.println("*********************");
+	System.out.println(GSXML.elementToString(resultElement, true));
+	
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/Action.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/Action.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/Action.java	(revision 31634)
@@ -0,0 +1,266 @@
+package org.greenstone.gsdl3.action;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/** base class for Actions */
+abstract public class Action
+{
+
+	/** the system set up variables */
+	protected HashMap<String, Object> config_params = null;
+	
+	
+	/** a converter class to parse XML and create Docs */
+	protected XMLConverter converter = null;
+	/**
+	 * a reference to the message router that it must talk to to get info. it
+	 * may be a communicator acting as a proxy, but it doesn't care about that
+	 */
+	protected ModuleInterface mr = null;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.Action.class.getName());
+
+	public Action()
+	{
+		this.converter = new XMLConverter();
+	}
+
+	/** the config variables must be set before configure is called */
+	public void setConfigParams(HashMap<String, Object> params)
+	{
+		this.config_params = params;
+	}
+
+	/** sets the message router */
+	public void setMessageRouter(ModuleInterface m)
+	{
+		this.mr = m;
+	}
+
+	public boolean configure()
+	{
+		// does nothing yet
+		return true;
+	}
+
+	/**
+	 * process takes an xml representation of cgi args and returns the page of
+	 * results - may be in html/xml/other depending on the output att of the
+	 * request
+	 */
+	public String process(String xml_in)
+	{
+
+		Document message_doc = this.converter.getDOM(xml_in);
+		if (message_doc == null)
+		{
+			logger.error("Couldn't parse request");
+			logger.error(xml_in);
+			return null;
+		}
+		Node result = process(message_doc);
+		return this.converter.getString(result);
+	}
+
+	/** the main process method - must be implemented in subclass */
+	abstract public Node process(Node xml_in);
+
+	/**
+	 * tell the param class what its arguments are if an action has its own
+	 * arguments, this should add them to the params object - particularly
+	 * important for args that should not be saved
+	 */
+	public boolean addActionParameters(GSParams params)
+	{
+		return true;
+	}
+
+  protected void getRequiredMetadataNames(Element format, HashSet<String> meta_names) {
+    GSXSLT.findExtraMetadataNames(format, meta_names);
+  }
+
+
+	protected Element createMetadataParamList(Document doc, HashSet<String> metadata_names)
+	{
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		Element param = null;
+		Iterator<String> i = metadata_names.iterator();
+		while (i.hasNext())
+		{
+			String name = i.next();
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param_list.appendChild(param);
+			param.setAttribute(GSXML.NAME_ATT, "metadata");
+			param.setAttribute(GSXML.VALUE_ATT, name);
+
+		}
+		return param_list;
+	}
+
+	protected boolean processErrorElements(Element message, Element page)
+	{
+		NodeList error_nodes = message.getElementsByTagName(GSXML.ERROR_ELEM);
+		if (error_nodes.getLength() == 0)
+		{
+			return false;
+		}
+		Document owner = page.getOwnerDocument();
+		for (int i = 0; i < error_nodes.getLength(); i++)
+		{
+			page.appendChild(owner.importNode(error_nodes.item(i), true));
+		}
+		return true;
+	}
+
+	/**
+	 * Takes an XML element and adds the metadata of the current site to it.
+	 * Useful for adding the current site's metadata to a response before
+	 * sending it
+	 * 
+	 * @param element
+	 *            the element to add site metadata to
+	 * @param lang
+	 *            the current language
+	 * @param uid
+	 *            the current user id
+	 */
+	protected void addSiteMetadata(Element element, UserContext userContext)
+	{
+		Document doc = element.getOwnerDocument();
+		
+		//ADD SITE METADATA
+		Element metadata_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		//create the element to put the params in
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		// want to get metadataList and displayItemList
+		GSXML.addParameterToList(param_list, GSXML.SUBSET_PARAM , GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParameterToList(param_list, GSXML.SUBSET_PARAM , GSXML.DISPLAY_TEXT_ELEM  + GSXML.LIST_MODIFIER);
+		
+		metadata_request.appendChild(param_list);
+		//create the message
+		Element metadata_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		metadata_message.appendChild(metadata_request);
+		//get response
+		Element metadata_response_message = (Element) this.mr.process(metadata_message);
+		//drill down to response
+		Element metadata_response = (Element) GSXML.getChildByTagName(metadata_response_message, GSXML.RESPONSE_ELEM);
+		//merge in metadata
+		// *************** need to merge the displayItem lists too
+		GSXML.mergeMetadataLists(element, metadata_response);
+		GSXML.mergeSpecifiedLists(element, metadata_response, GSXML.DISPLAY_TEXT_ELEM);
+	}
+
+	protected void addInterfaceOptions(Element elem)
+	{
+		Document doc = elem.getOwnerDocument();
+		
+		Element documentOptionList = doc.createElement("interfaceOptions");
+		for (Object key : this.config_params.keySet())
+		{
+			Element option = doc.createElement("option");
+			option.setAttribute(GSXML.NAME_ATT, (String) key);
+			option.setAttribute(GSXML.VALUE_ATT, this.config_params.get(key).toString());
+			documentOptionList.appendChild(option);
+		}
+		elem.appendChild(elem.getOwnerDocument().importNode(documentOptionList, true));
+	}
+
+	protected Element getFormatInfo(String to, UserContext userContext)
+	{
+		// Eclipse call hierarchy shows the element returned from this method is 
+		// subsequently used in a 'importNode'.  For this reason it is safe here
+		// for call up our own document DOM.  
+		//
+		// If this pattern changes for any reason, then the DOM will need to be
+		// passed in as a parameter
+		
+		Document doc = XMLConverter.newDOM();
+		
+		Element mr_format_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_format_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT, to, userContext);
+		mr_format_message.appendChild(mr_format_request);
+
+		// process the message
+		Element mr_response_message = (Element) this.mr.process(mr_format_message);
+		// the response
+
+		Element format_response = (Element) GSXML.getChildByTagName(mr_response_message, GSXML.RESPONSE_ELEM);
+
+		Element format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+		if (format_elem != null)
+		{
+			Element global_format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.GLOBAL_FORMAT_ELEM);
+			if (global_format_elem != null)
+			{
+				GSXSLT.mergeFormatElements(format_elem, global_format_elem, false);
+			}
+		}
+		return format_elem;
+	}
+
+	protected String getTextString(String key, String lang, String dictionary, String[] args)
+	{
+	  logger.error("lang = "+lang);
+		if (dictionary != null)
+		{
+			// just try the one specified dictionary
+		  Dictionary dict = new Dictionary(dictionary, lang);
+			String result = dict.get(key, args);
+			if (result == null)
+			{ // not found
+				return "_" + key + "_";
+			}
+			return result;
+		}
+
+		// otherwise we try class names for dictionary names
+		String class_name = this.getClass().getName();
+		class_name = class_name.substring(class_name.lastIndexOf('.') + 1);
+		Dictionary dict = new Dictionary(class_name, lang);
+		String result = dict.get(key, args);
+		if (result != null)
+		{
+			return result;
+		}
+
+		// we have to try super classes
+		Class c = this.getClass().getSuperclass();
+		while (result == null && c != null)
+		{
+			class_name = c.getName();
+			class_name = class_name.substring(class_name.lastIndexOf('.') + 1);
+			if (class_name.equals("ServiceRack"))
+			{
+				// this is as far as we go
+				break;
+			}
+			dict = new Dictionary(class_name, lang);
+			result = dict.get(key, args);
+			c = c.getSuperclass();
+		}
+		if (result == null)
+		{
+			return "_" + key + "_";
+		}
+		return result;
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/AppletAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/AppletAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/AppletAction.java	(revision 31634)
@@ -0,0 +1,144 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.log4j.*;
+
+/** action class for handling applets */
+public class AppletAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.AppletAction.class.getName());
+
+	public Node process(Node message_node)
+	{
+
+		Element message = GSXML.nodeToElement(message_node);
+		Document doc = message.getOwnerDocument();
+		
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+		result.appendChild(page_response);
+
+		// get the collection and service params
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		// request_type is display (d) or request (r)
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		if (!request_type.equals("d") && !request_type.equals("r"))
+		{
+			logger.error("AppletAction Error: the rt arg should be either d or r, instead it was " + request_type + "!");
+			return result;
+		}
+
+		String collection = (String) params.get(GSParams.COLLECTION);
+		boolean coll_specified = true;
+		String service_name = (String) params.get(GSParams.SERVICE);
+		UserContext userContext = new UserContext(request);
+		String to = null;
+		if (collection == null || collection.equals(""))
+		{
+			coll_specified = false;
+			to = service_name;
+		}
+		else
+		{
+			to = GSPath.appendLink(collection, service_name);
+		}
+
+		if (request_type.equals("r"))
+		{
+			// we are processing stuff for the applet send a message to the service, type="query", and take out the something element, and return that as our result - the applet must take xml
+
+			Element mr_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+			mr_message.appendChild(mr_request);
+			// just append all the params for now - should filter out unneeded ones
+			mr_request.appendChild(doc.importNode(cgi_param_list, true));
+
+			// process the request
+			Element mr_response = (Element) this.mr.process(mr_message);
+			// get the applet data out and pass it back as is. 
+			String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.APPLET_DATA_ELEM);
+			Element applet_info = GSXML.getFirstElementChild(GSXML.getNodeByPath(mr_response, path));
+			//Element applet_info = (Element)GSXML.getNodeByPath(mr_response, path).getFirstChild();
+			return applet_info;
+
+		}
+
+		// get the applet description, and the collection info if a collection is specified
+
+		Element mr_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element applet_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+		mr_message.appendChild(applet_request);
+
+		Element mr_response = (Element) this.mr.process(mr_message);
+
+		// add in the applet info
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.SERVICE_ELEM);
+		//path = GSPath.appendLink(path, GSXML.APPLET_ELEM);
+		Element service_elem = (Element) doc.importNode(GSXML.getNodeByPath(mr_response, path), true);
+		Element applet_elem = (Element) GSXML.getChildByTagName(service_elem, GSXML.APPLET_ELEM);
+		// must handle any params that have values that are not 
+		// necessarily known by the service
+		// should this be done here or by web receptionist??
+		// cant really have an applet without web?
+		editLocalParams(applet_elem, (String) config_params.get(GSConstants.LIBRARY_NAME), collection);
+		page_response.appendChild(service_elem);
+
+		//append site metadata
+		addSiteMetadata(page_response, userContext);
+		//addInterfaceOptions(page_response);
+		return result;
+
+	}
+
+	/**
+	 * this method looks through the PARAMs of the applet description for
+	 * 'library' or 'collection'. If found, the params are set to the
+	 * appropriate values should this be done here or in the receptionist?
+	 */
+	protected void editLocalParams(Element description, String library_name, String full_collection_name)
+	{
+
+		Node child = description.getFirstChild();
+		while (child != null)
+		{
+			if (child.getNodeType() == Node.ELEMENT_NODE)
+			{
+				String param = child.getNodeName();
+				if (param.equals("PARAM") || param.equals("param"))
+				{
+					String name = ((Element) child).getAttribute("NAME");
+					if (name == null || name.equals(""))
+					{
+						// somethings wrong!!
+					}
+					else if (name.equals("library"))
+					{
+						((Element) child).setAttribute("VALUE", library_name);
+					}
+					else if (name.equals("collection"))
+					{
+						((Element) child).setAttribute("VALUE", full_collection_name);
+					}
+
+				}
+			}
+			child = child.getNextSibling();
+		}
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/BrowseAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/BrowseAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/BrowseAction.java	(revision 31634)
@@ -0,0 +1,338 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.UserContext;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+//NOTE: this class not used at present!!!!!
+/** action for classifier browsing */
+public class BrowseAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.BrowseAction.class.getName());
+
+	public static final String CLASSIFIER_ARG = "cl";
+	public static final String SIBLING_ARG = "sib";
+
+	/** process the request */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+		Document doc = message.getOwnerDocument();
+		
+		// get the request - assume only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// the result
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element response = classifierBrowse(request);
+		result.appendChild(response);
+		return result;
+	}
+
+	protected Element classifierBrowse(Element request)
+	{
+		Document doc = request.getOwnerDocument();
+		
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// extract the params from the cgi-request, and check that we have a coll specified
+		Element cgi_paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_paramList, false);
+
+		String service_name = (String) params.get(GSParams.SERVICE);
+		String collection = (String) params.get(GSParams.COLLECTION);
+		if (collection == null || collection.equals(""))
+		{
+			logger.error("classifierBrowse, need to specify a collection!");
+			return page_response;
+
+		}
+
+		//whether to retrieve siblings or not
+		boolean get_siblings = false;
+		String sibs = (String) params.get(SIBLING_ARG);
+		if (sibs != null && sibs.equals("1"))
+		{
+			get_siblings = true;
+		}
+
+		UserContext userContext = new UserContext(request);
+		String to = GSPath.appendLink(collection, service_name);
+
+		// the first part of the response is the service description 
+		// for now get this again from the service. 
+		// this should be cached somehow later on. 
+
+		Element info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+		info_message.appendChild(info_request);
+
+		// also get the format stuff now if there is some
+		Element format_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT, to, userContext);
+		info_message.appendChild(format_request);
+		// process the requests
+
+		Element info_response = (Element) this.mr.process(info_message);
+
+		// the two responses
+		NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		Element service_response = (Element) responses.item(0);
+		Element format_response = (Element) responses.item(1);
+
+		Element service_description = (Element) GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM);
+		page_response.appendChild(doc.importNode(service_description, true));
+
+		//append site metadata
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+
+		// if rt=d, then we are just displaying the service
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		if (request_type.equals("d"))
+		{
+			//return the page that we have so far
+			return page_response;
+		}
+
+		// get the node that the user has clicked on
+		String classifier_node = (String) params.get(CLASSIFIER_ARG);
+
+		// if the node is not defined, return the page that we have so far
+		if (classifier_node == null || classifier_node.equals(""))
+		{
+			return page_response;
+		}
+
+		// the id of the classifier is the top id of the selected node
+		String top_id = OID.getTop(classifier_node);
+
+		HashSet<String> metadata_names = new HashSet<String>();
+
+		// add the format info into the response
+		Element format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+		if (format_elem != null)
+		{
+			// find the one for the classifier we are in
+			Element this_format = GSXML.getNamedElement(format_elem, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+			if (this_format == null)
+			{
+				this_format = (Element) GSXML.getChildByTagName(format_elem, GSXML.DEFAULT_ELEM);
+			}
+			if (this_format != null)
+			{
+				Element global_format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.GLOBAL_FORMAT_ELEM);
+				if(global_format_elem != null)
+				{
+					GSXSLT.mergeFormatElements(this_format, global_format_elem, false);
+				}
+				Element new_format = GSXML.duplicateWithNewName(doc, this_format, GSXML.FORMAT_ELEM, false);
+				// set teh format type
+				new_format.setAttribute(GSXML.TYPE_ATT, "browse");
+
+				page_response.appendChild(new_format);
+				getRequiredMetadataNames(new_format, metadata_names);
+			}
+		}
+		
+		Element extraMetaListElem = (Element) GSXML.getChildByTagName(request, GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
+		if(extraMetaListElem != null)
+		{
+			NodeList extraMetaList = extraMetaListElem.getElementsByTagName(GSXML.EXTRA_METADATA);
+			for(int i = 0; i < extraMetaList.getLength(); i++)
+			{
+				metadata_names.add(((Element)extraMetaList.item(i)).getAttribute(GSXML.NAME_ATT));
+			}
+		}
+
+		logger.info("extracted meta names, " + metadata_names.toString());
+		// get the browse structure for the selected node
+		Element classify_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element classify_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		classify_message.appendChild(classify_request);
+
+		//Create a parameter list to specify the required structure information
+		// for now, always get ancestors and children
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		classify_request.appendChild(param_list);
+		Element param = doc.createElement(GSXML.PARAM_ELEM);
+		param_list.appendChild(param);
+		param.setAttribute(GSXML.NAME_ATT, "structure");
+		param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+		param = doc.createElement(GSXML.PARAM_ELEM);
+		param_list.appendChild(param);
+		param.setAttribute(GSXML.NAME_ATT, "structure");
+		param.setAttribute(GSXML.VALUE_ATT, "children");
+		if (get_siblings)
+		{
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param_list.appendChild(param);
+			param.setAttribute(GSXML.NAME_ATT, "structure");
+			param.setAttribute(GSXML.VALUE_ATT, "siblings");
+		}
+
+		// put the classifier node into a classifier node list
+		Element classifier_list = doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element classifier = doc.createElement(GSXML.CLASS_NODE_ELEM);
+		classifier.setAttribute(GSXML.NODE_ID_ATT, classifier_node);
+		classifier_list.appendChild(classifier);
+		classify_request.appendChild(classifier_list);
+
+		// process the request
+		Element classify_response = (Element) this.mr.process(classify_message);
+		// get the structure element
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+		path = GSPath.appendLink(path, GSXML.NODE_STRUCTURE_ELEM);
+		// assume that we always get back the top level CL1 node - this becomes the page_classifier node
+		path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+		Element cl_structure = (Element) GSXML.getNodeByPath(classify_response, path);
+		if (cl_structure == null)
+		{
+			logger.error("classifier structure request returned no structure");
+			return page_response;
+		}
+
+		// add the classifier node as the page classifier 
+		Element page_classifier = GSXML.duplicateWithNewName(doc, cl_structure, GSXML.CLASSIFIER_ELEM, true);
+		page_response.appendChild(page_classifier);
+		page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+
+		// get the metadata for each classifier node, 
+		// then for each document node
+
+		Element metadata_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		boolean did_classifier = false;
+		boolean did_documents = false;
+
+		// if there are classifier nodes 
+		// create a metadata request for the classifier, and add it to 
+		// the the message
+		NodeList cl_nodes = page_classifier.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+
+		if (cl_nodes.getLength() > 0)
+		{
+			did_classifier = true;
+			Element cl_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to + "MetadataRetrieve", userContext);
+			metadata_message.appendChild(cl_meta_request);
+
+			Element new_cl_nodes_list = doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+			cl_meta_request.appendChild(new_cl_nodes_list);
+
+			for (int c = 0; c < cl_nodes.getLength(); c++)
+			{
+
+				Element cl = doc.createElement(GSXML.CLASS_NODE_ELEM);
+				cl.setAttribute(GSXML.NODE_ID_ATT, ((Element) cl_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+				new_cl_nodes_list.appendChild(cl);
+			}
+
+			// create and add in the param list - for now get all the metadata
+			// should be based on info sent in from the recept, and the 
+			// format stuff
+			Element cl_param_list = null;
+			if (metadata_names.isEmpty())
+			{
+				cl_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				Element p = doc.createElement(GSXML.PARAM_ELEM);
+				cl_param_list.appendChild(p);
+				p.setAttribute(GSXML.NAME_ATT, "metadata");
+				p.setAttribute(GSXML.VALUE_ATT, "Title");
+			}
+			else
+			{
+				cl_param_list = createMetadataParamList(doc,metadata_names);
+			}
+
+			cl_meta_request.appendChild(cl_param_list);
+
+		}
+
+		// if there are document nodes in the classification (happens 
+		// sometimes), create a second request for document metadata and 
+		// append to the message
+		NodeList doc_nodes = page_classifier.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+		if (doc_nodes.getLength() > 0)
+		{
+			did_documents = true;
+			Element doc_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, GSPath.appendLink(collection, "DocumentMetadataRetrieve"), userContext);
+			metadata_message.appendChild(doc_meta_request);
+
+			Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			doc_meta_request.appendChild(doc_list);
+
+			for (int c = 0; c < doc_nodes.getLength(); c++)
+			{
+
+				Element d = doc.createElement(GSXML.DOC_NODE_ELEM);
+				d.setAttribute(GSXML.NODE_ID_ATT, ((Element) doc_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+				doc_list.appendChild(d);
+			}
+
+			// create and add in the param list - add all for now
+			Element doc_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			Element p = doc.createElement(GSXML.PARAM_ELEM);
+			doc_param_list.appendChild(p);
+			p.setAttribute(GSXML.NAME_ATT, "metadata");
+			p.setAttribute(GSXML.VALUE_ATT, "all");
+			doc_meta_request.appendChild(doc_param_list);
+
+		}
+
+		// process the metadata requests
+		Element metadata_response = (Element) this.mr.process(metadata_message);
+		if (did_classifier)
+		{
+			// the classifier one will be the first response
+			// add the metadata lists for each node back into the 
+			// page_classifier nodes
+			path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+			NodeList meta_response_cls = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+			for (int i = 0; i < cl_nodes.getLength(); i++)
+			{
+				GSXML.mergeMetadataLists(cl_nodes.item(i), meta_response_cls.item(i));
+			}
+		}
+		if (did_documents)
+		{
+			NodeList meta_response_docs = null;
+			if (!did_classifier)
+			{
+				// its the first response
+				path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				meta_response_docs = GSXML.getNodeByPath(metadata_response, path).getChildNodes();
+			}
+			else
+			{ // its the second response
+				meta_response_docs = GSXML.getChildByTagName(metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1), GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER).getChildNodes();
+			}
+
+			for (int i = 0; i < doc_nodes.getLength(); i++)
+			{
+				GSXML.mergeMetadataLists(doc_nodes.item(i), meta_response_docs.item(i));
+			}
+		}
+
+		logger.debug("(BrowseAction) Page:\n" + this.converter.getPrettyString(page_response));
+		return page_response;
+	}
+
+	protected Element unknownBrowse(Element page, Element request, String browse_type)
+	{
+		logger.error("unknown browse subtype: " + browse_type);
+		return null;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DepositorAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DepositorAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DepositorAction.java	(revision 31634)
@@ -0,0 +1,389 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.commons.io.FileUtils;
+import org.greenstone.gsdl3.util.DerbyWrapper;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+public class DepositorAction extends Action
+{
+	//Sub actions
+	private final String DE_RETRIEVE_WIZARD = "getwizard";
+	private final String DE_DEPOSIT_FILE = "depositfile";
+	private final String DE_CLEAR_CACHE = "clearcache";
+	private final String DE_CLEAR_DATABASE = "cleardatabase";
+
+	public Node process(Node message)
+	{
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		Document doc = request.getOwnerDocument();
+		
+		UserContext uc = new UserContext((Element) request);
+		String currentUsername = uc.getUsername();
+
+		Element responseMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element response = GSXML.createBasicResponse(doc, this.getClass().getSimpleName());
+		responseMessage.appendChild(response);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collection = (String) params.get("c");
+
+		int pageNum = -1;
+		boolean pageNumParseFail = false;
+		try
+		{
+			pageNum = Integer.parseInt(((String) params.get("dePage")));
+		}
+		catch (Exception ex)
+		{
+			pageNumParseFail = true;
+		}
+
+		int prevPageNum = -1;
+		boolean prevPageNumFail = false;
+		try
+		{
+			prevPageNum = Integer.parseInt((String) params.get("currentPage"));
+		}
+		catch (Exception ex)
+		{
+			prevPageNumFail = true;
+		}
+
+		DerbyWrapper database = new DerbyWrapper(GlobalProperties.getGSDL3Home() + File.separatorChar + "etc" + File.separatorChar + "usersDB");
+		if (pageNumParseFail)
+		{
+			try
+			{
+				pageNum = Integer.parseInt(database.getUserData(currentUsername, "DE___" + collection + "___CACHED_PAGE"));
+			}
+			catch (Exception ex)
+			{
+				pageNum = 1;
+			}
+		}
+
+		int highestVisitedPage = -1;
+		String result = "";
+		int counter = 1;
+		while (result != null)
+		{
+			result = database.getUserData(currentUsername, "DE___" + collection + "___" + counter + "___VISITED_PAGE");
+			if (result != null)
+			{
+				counter++;
+			}
+		}
+		highestVisitedPage = counter - 1;
+		if (highestVisitedPage == 0)
+		{
+			highestVisitedPage = 1;
+		}
+
+		if (pageNum > highestVisitedPage + 1)
+		{
+			pageNum = highestVisitedPage + 1;
+		}
+
+		database.addUserData(currentUsername, "DE___" + collection + "___" + pageNum + "___VISITED_PAGE", "VISITED");
+
+		String subaction = ((Element) request).getAttribute(GSXML.SUBACTION_ATT);
+		if (subaction.toLowerCase().equals(DE_RETRIEVE_WIZARD))
+		{
+			//Save given metadata
+			StringBuilder saveString = new StringBuilder("[");
+			Iterator<String> paramIter = params.keySet().iterator();
+			while (paramIter.hasNext())
+			{
+				String paramName = paramIter.next();
+				if (paramName.startsWith("md___"))
+				{
+					Object paramValue = params.get(paramName);
+
+					if (paramValue instanceof String)
+					{
+						saveString.append("{name:\"" + paramName + "\", value:\"" + (String) paramValue + "\"},");
+					}
+					else if (paramValue instanceof HashMap)
+					{
+						HashMap<String, String> subMap = (HashMap<String, String>) paramValue;
+						Iterator<String> subKeyIter = subMap.keySet().iterator();
+						while (subKeyIter.hasNext())
+						{
+							String subName = subKeyIter.next();
+							saveString.append("{name:\"" + paramName + "." + subName + "\", value:\"" + subMap.get(subName) + "\"},");
+						}
+					}
+				}
+			}
+			if (saveString.length() > 2)
+			{
+				saveString.deleteCharAt(saveString.length() - 1);
+				saveString.append("]");
+
+				if (!prevPageNumFail)
+				{
+					database.addUserData(currentUsername, "DE___" + collection + "___" + prevPageNum + "___CACHED_VALUES", saveString.toString());
+				}
+			}
+
+			//Construct the xsl
+			Document compiledDepositorFile = null;
+			try
+			{
+				compiledDepositorFile = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+			Document depositorBaseFile = GSXSLT.mergedXSLTDocumentCascade("depositor/depositor.xsl", (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), (ArrayList<String>) this.config_params.get(GSConstants.BASE_INTERFACES), false);
+
+			Element numOfPagesElement = GSXML.getNamedElement(depositorBaseFile.getDocumentElement(), "xsl:variable", "name", "numOfPages");
+			int numberOfPages = Integer.parseInt(numOfPagesElement.getTextContent());
+
+			compiledDepositorFile.appendChild(compiledDepositorFile.importNode(depositorBaseFile.getDocumentElement(), true));
+
+			ArrayList<Document> pageDocs = new ArrayList<Document>();
+			ArrayList<String> pageNames = new ArrayList<String>();
+			for (int i = 1; i <= numberOfPages; i++)
+			{
+				Document page = GSXSLT.mergedXSLTDocumentCascade("depositor/de_page" + i + ".xsl", (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), (ArrayList<String>) this.config_params.get(GSConstants.BASE_INTERFACES), false);
+				pageDocs.add(page);
+
+				Element pageTitleElem = (Element) GSXML.getNamedElement(page.getDocumentElement(), "xsl:variable", "name", "title");
+				pageNames.add(pageTitleElem.getTextContent());
+
+				Element wizardPageElem = (Element) GSXML.getNamedElement(page.getDocumentElement(), "xsl:template", "name", "wizardPage");
+				wizardPageElem.setAttribute("name", "wizardPage" + i);
+				compiledDepositorFile.getDocumentElement().appendChild(compiledDepositorFile.importNode(wizardPageElem, true));
+			}
+
+			//Create the wizard bar
+			Element wizardBarTemplate = GSXML.getNamedElement(compiledDepositorFile.getDocumentElement(), "xsl:template", "name", "wizardBar");
+			Element wizardBar = compiledDepositorFile.createElement("ul");
+			wizardBar.setAttribute("id", "wizardBar");
+			wizardBarTemplate.appendChild(wizardBar);
+
+			for (int i = 0; i < pageNames.size(); i++)
+			{
+				String pageName = pageNames.get(i);
+				Element pageLi = compiledDepositorFile.createElement("li");
+				if (pageNum == i + 1)
+				{
+					pageLi.setAttribute("class", "wizardStepLink ui-state-active ui-corner-all");
+				}
+				else if (i + 1 > highestVisitedPage + 1 && i + 1 > pageNum + 1)
+				{
+					pageLi.setAttribute("class", "wizardStepLink ui-state-disabled ui-corner-all");
+				}
+				else
+				{
+					pageLi.setAttribute("class", "wizardStepLink ui-state-default ui-corner-all");
+				}
+				Element link = compiledDepositorFile.createElement("a");
+				pageLi.appendChild(link);
+
+				link.setAttribute(GSXML.HREF_ATT, "javascript:;");
+				link.setAttribute("page", "" + (i + 1));
+				link.appendChild(compiledDepositorFile.createTextNode(pageName));
+				wizardBar.appendChild(pageLi);
+			}
+
+			//Add a call-template call to the appropriate page in the xsl
+			Element mainDePageElem = GSXML.getNamedElement(compiledDepositorFile.getDocumentElement(), "xsl:template", "match", "/page");
+			Element wizardContainer = GSXML.getNamedElement(mainDePageElem, "div", "id", "wizardContainer");
+			Element formContainer = GSXML.getNamedElement(wizardContainer, "form", "name", "depositorform");
+			Element callToPage = compiledDepositorFile.createElement("xsl:call-template");
+			callToPage.setAttribute("name", "wizardPage" + pageNum);
+			formContainer.appendChild(callToPage);
+
+			Element cachedValueElement = doc.createElement("cachedValues");
+			response.appendChild(cachedValueElement);
+			try
+			{
+				for (int i = pageNum; i > 0; i--)
+				{
+					Element page = doc.createElement("pageCache");
+					page.setAttribute("pageNum", "" + i);
+					String cachedValues = database.getUserData(currentUsername, "DE___" + collection + "___" + i + "___CACHED_VALUES");
+					if (cachedValues != null)
+					{
+						page.appendChild(doc.createTextNode(cachedValues));
+						cachedValueElement.appendChild(page);
+					}
+				}
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+
+			try
+			{
+				Transformer transformer = TransformerFactory.newInstance().newTransformer();
+
+				File newFileDir = new File(GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + this.config_params.get(GSConstants.SITE_NAME) + File.separator + "collect" + File.separator + collection + File.separator + "transform" + File.separator + "depositor");
+				newFileDir.mkdirs();
+
+				File newFile = new File(newFileDir, File.separator + "compiledDepositor.xsl");
+
+				//initialize StreamResult with File object to save to file
+				StreamResult sresult = new StreamResult(new FileWriter(newFile));
+				DOMSource source = new DOMSource(compiledDepositorFile);
+				transformer.transform(source, sresult);
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+			database.closeDatabase();
+		}
+		else if (subaction.toLowerCase().equals(DE_DEPOSIT_FILE))
+		{
+			String fileToAdd = (String) params.get("fileToAdd");
+			File tempFile = new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp" + File.separator + fileToAdd);
+			if (tempFile.exists())
+			{
+				File newFileLocationDir = new File(GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + this.config_params.get(GSConstants.SITE_NAME) + File.separator + "collect" + File.separator + collection + File.separator + "import" + File.separator + fileToAdd);
+				if (!newFileLocationDir.exists())
+				{
+					newFileLocationDir.mkdir();
+				}
+				File newFileLocation = new File(newFileLocationDir, fileToAdd);
+
+				try
+				{
+					FileUtils.copyFile(tempFile, newFileLocation);
+				}
+				catch (Exception ex)
+				{
+					ex.printStackTrace();
+					GSXML.addError(responseMessage, "Failed to copy the deposited file into the collection.");
+					return responseMessage;
+				}
+
+				HashMap<String, String> metadataMap = new HashMap<String, String>();
+				for (int i = pageNum; i > 0; i--)
+				{
+					String cachedValues = database.getUserData(currentUsername, "DE___" + collection + "___" + i + "___CACHED_VALUES");
+					if (cachedValues != null)
+					{
+						Type type = new TypeToken<List<Map<String, String>>>()
+						{
+						}.getType();
+
+						Gson gson = new Gson();
+						List<Map<String, String>> metadataList = gson.fromJson(cachedValues, type);
+						for (Map<String, String> metadata : metadataList)
+						{
+							metadataMap.put(metadata.get("name"), metadata.get("value"));
+						}
+					}
+				}
+
+				String xmlString = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\"><DirectoryMetadata><FileSet>";
+				xmlString += "<FileName>.*</FileName><Description>";
+				for (String key : metadataMap.keySet())
+				{
+					xmlString += "<Metadata name=\"" + key.substring("MD___".length()) + "\" mode=\"accumulate\">" + metadataMap.get(key) + "</Metadata>";
+				}
+				xmlString += "</Description></FileSet></DirectoryMetadata>";
+
+				File metadataFile = new File(GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + this.config_params.get(GSConstants.SITE_NAME) + File.separator + "collect" + File.separator + collection + File.separator + "import" + File.separator + fileToAdd + File.separator + "metadata.xml");
+
+				try
+				{
+					BufferedWriter bw = new BufferedWriter(new FileWriter(metadataFile));
+					bw.write(xmlString);
+					bw.close();
+				}
+				catch (Exception ex)
+				{
+					ex.printStackTrace();
+				}
+
+				Element buildMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+				Element buildRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, "ImportCollection", uc);
+				buildMessage.appendChild(buildRequest);
+
+				Element paramListElem = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				buildRequest.appendChild(paramListElem);
+
+				Element collectionParam = doc.createElement(GSXML.PARAM_ELEM);
+				paramListElem.appendChild(collectionParam);
+				collectionParam.setAttribute(GSXML.NAME_ATT, GSXML.COLLECTION_ATT);
+				collectionParam.setAttribute(GSXML.VALUE_ATT, collection);
+
+				Element documentsParam = doc.createElement(GSXML.PARAM_ELEM);
+				paramListElem.appendChild(documentsParam);
+				documentsParam.setAttribute(GSXML.NAME_ATT, "documents");
+				documentsParam.setAttribute(GSXML.VALUE_ATT, fileToAdd);
+
+				Element buildResponseMessage = (Element) this.mr.process(buildMessage);
+
+				response.appendChild(doc.importNode(buildResponseMessage, true));
+			}
+		}
+		else if (subaction.toLowerCase().equals(DE_CLEAR_CACHE))
+		{
+			database.clearUserDataWithPrefix(currentUsername, "DE___");
+		}
+		else if (subaction.toLowerCase().equals(DE_CLEAR_DATABASE))
+		{
+			database.clearUserData();
+			database.clearTrackerData();
+		}
+		else
+		{
+			Element depositorPage = doc.createElement("depositorPage");
+			response.appendChild(depositorPage);
+
+			Element collList = getCollectionsInSite(doc);
+			depositorPage.appendChild(doc.importNode(collList, true));
+		}
+
+		return responseMessage;
+	}
+
+	public Element getCollectionsInSite(Document doc) 
+	{
+		Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", new UserContext());
+		message.appendChild(request);
+		Element responseMessage = (Element) this.mr.process(message);
+
+		Element response = (Element) GSXML.getChildByTagName(responseMessage, GSXML.RESPONSE_ELEM);
+		Element collectionList = (Element) GSXML.getChildByTagName(response, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+
+		return collectionList;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DocumentAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DocumentAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/DocumentAction.java	(revision 31634)
@@ -0,0 +1,1278 @@
+/*
+ *    DocumentAction.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.action;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.log4j.*;
+
+/** Action class for retrieving Documents via the message router */
+public class DocumentAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.DocumentAction.class.getName());
+
+	// this is used to specify that the sibling nodes of a selected one should be obtained
+	public static final String SIBLING_ARG = "sib";
+	public static final String GOTO_PAGE_ARG = "gp";
+	public static final String ENRICH_DOC_ARG = "end";
+	public static final String EXPAND_DOCUMENT_ARG = "ed";
+	public static final String EXPAND_CONTENTS_ARG = "ec";
+	public static final String REALISTIC_BOOK_ARG = "book";
+
+	/**
+	 * if this is set to true, when a document is displayed, any annotation type
+	 * services (enrich) will be offered to the user as well
+	 */
+	protected boolean provide_annotations = false;
+
+	protected boolean highlight_query_terms = false;
+
+	public boolean configure()
+	{
+		super.configure();
+		String highlight = (String) config_params.get("highlightQueryTerms");
+		if (highlight != null && highlight.equals("true"))
+		{
+			highlight_query_terms = true;
+		}
+		String annotate = (String) config_params.get("displayAnnotationService");
+		if (annotate != null && annotate.equals("true"))
+		{
+			provide_annotations = true;
+		}
+		return true;
+	}
+
+	public Node process(Node message_node)
+	{
+		// for now, no subaction eventually we may want to have subactions such as text assoc or something ?
+
+		Element message = GSXML.nodeToElement(message_node);
+		Document doc = message.getOwnerDocument();
+		
+		// the response
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+		result.appendChild(page_response);
+
+		// get the request - assume only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		Element cgi_paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_paramList, false);
+
+		// just in case there are some that need to get passed to the services
+		HashMap service_params = (HashMap) params.get("s0");
+
+		String collection = (String) params.get(GSParams.COLLECTION);
+		String document_id = (String) params.get(GSParams.DOCUMENT);
+		if (document_id != null && document_id.equals(""))
+		{
+			document_id = null;
+		}
+		String href = (String) params.get(GSParams.HREF);//for an external link : get the href URL if it is existing in the params list 
+		if (href != null && href.equals(""))
+		{
+			href = null;
+		}
+		String rl = (String) params.get(GSParams.RELATIVE_LINK);//for an external link : get the rl value if it is existing in the params list
+		if (document_id == null && href == null)
+		{
+			logger.error("no document specified!");
+			return result;
+		}
+		if (rl != null && rl.equals("0"))
+		{
+			// this is a true external link, we should have been directed to a different page or action
+			logger.error("rl value was 0, shouldn't get here");
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//append site metadata
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+
+		// get the additional data needed for the page
+		getBackgroundData(page_response, collection, userContext);
+		Element format_elem = (Element) GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);
+
+		if (format_elem != null) {
+		  // lets look for param defaults set in config file
+		  NodeList param_defaults = format_elem.getElementsByTagName("paramDefault");
+		  for (int i=0; i<param_defaults.getLength(); i++) {
+		    Element p = (Element)param_defaults.item(i);
+		    String name = p.getAttribute(GSXML.NAME_ATT);
+		    if (params.get(name) ==null) {
+		      // wasn't set from interface
+		      String value = p.getAttribute(GSXML.VALUE_ATT);
+		      params.put(name, value );
+		      // also add into request param xml so that xslt knows it too
+		      GSXML.addParameterToList(cgi_paramList, name, value);
+		    }
+		  }
+		}
+		String document_type = (String) params.get(GSParams.DOCUMENT_TYPE);
+		if (document_type != null && document_type.equals(""))
+		{
+			//document_type = "hierarchy";
+			document_type = null; // we'll get it later if not already specified
+		}
+		//whether to retrieve siblings or not
+		boolean get_siblings = false;
+		String sibs = (String) params.get(SIBLING_ARG);
+		if (sibs != null && sibs.equals("1"))
+		{
+			get_siblings = true;
+		}
+
+		String doc_id_modifier = "";
+		String sibling_num = (String) params.get(GOTO_PAGE_ARG);
+		if (sibling_num != null && !sibling_num.equals(""))
+		{
+			// we have to modify the doc name
+			doc_id_modifier = "." + sibling_num + ".ss";
+		}
+
+		boolean expand_document = false;
+		String ed_arg = (String) params.get(EXPAND_DOCUMENT_ARG);
+		if (ed_arg != null && ed_arg.equals("1"))
+		{
+			expand_document = true;
+		}
+
+		boolean expand_contents = false;
+		if (expand_document)
+		{ // we always expand the contents with the text
+			expand_contents = true;
+		}
+		else
+		{
+			String ec_arg = (String) params.get(EXPAND_CONTENTS_ARG);
+			if (ec_arg != null && ec_arg.equals("1"))
+			{
+				expand_contents = true;
+			}
+		}
+
+		// UserContext userContext = new UserContext(request);
+
+		// //append site metadata
+		// addSiteMetadata(page_response, userContext);
+		// addInterfaceOptions(page_response);
+
+		// // get the additional data needed for the page
+		// getBackgroundData(page_response, collection, userContext);
+		// Element format_elem = (Element) GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);
+
+		// the_document is where all the doc info - structure and metadata etc
+		// is added into, to be returned in the page
+		Element the_document = doc.createElement(GSXML.DOCUMENT_ELEM);
+		page_response.appendChild(the_document);
+
+		// create a basic doc list containing the current node
+		Element basic_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
+		basic_doc_list.appendChild(current_doc);
+		if (document_id != null)
+		{
+			current_doc.setAttribute(GSXML.NODE_ID_ATT, document_id + doc_id_modifier);
+		}
+		else
+		{
+			current_doc.setAttribute(GSXML.HREF_ID_ATT, href);
+			// do we need this??
+			current_doc.setAttribute(GSXML.ID_MOD_ATT, doc_id_modifier);
+		}
+
+		if (document_type == null)
+		{
+			document_type = getDocumentType(basic_doc_list, collection, userContext, page_response);
+		}
+		if (document_type == null)
+		{
+		    logger.debug("doctype is null, setting to simple");
+		    document_type = GSXML.DOC_TYPE_SIMPLE;
+		}
+		
+		the_document.setAttribute(GSXML.DOC_TYPE_ATT, document_type);
+		
+
+		// Create a parameter list to specify the required structure information
+		Element ds_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (service_params != null)
+		{
+			GSXML.addParametersToList(ds_param_list, service_params);
+		}
+
+		Element ds_param = null;
+		boolean get_structure = false;
+		boolean get_structure_info = false;
+		if (document_type.equals(GSXML.DOC_TYPE_PAGED))
+		{
+			get_structure_info = true;
+
+			if (expand_contents)
+			{
+				ds_param = doc.createElement(GSXML.PARAM_ELEM);
+				ds_param_list.appendChild(ds_param);
+				ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+				ds_param.setAttribute(GSXML.VALUE_ATT, "entire");
+			}
+
+			// get the info needed for paged naviagtion
+			ds_param = doc.createElement(GSXML.PARAM_ELEM);
+			ds_param_list.appendChild(ds_param);
+			ds_param.setAttribute(GSXML.NAME_ATT, "info");
+			ds_param.setAttribute(GSXML.VALUE_ATT, "numSiblings");
+			ds_param = doc.createElement(GSXML.PARAM_ELEM);
+			ds_param_list.appendChild(ds_param);
+			ds_param.setAttribute(GSXML.NAME_ATT, "info");
+			ds_param.setAttribute(GSXML.VALUE_ATT, "numChildren");
+			ds_param = doc.createElement(GSXML.PARAM_ELEM);
+			ds_param_list.appendChild(ds_param);
+			ds_param.setAttribute(GSXML.NAME_ATT, "info");
+			ds_param.setAttribute(GSXML.VALUE_ATT, "siblingPosition");
+
+			if (get_siblings)
+			{
+				ds_param = doc.createElement(GSXML.PARAM_ELEM);
+				ds_param_list.appendChild(ds_param);
+				ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+				ds_param.setAttribute(GSXML.VALUE_ATT, "siblings");
+			}
+
+		}
+		else if (document_type.equals(GSXML.DOC_TYPE_HIERARCHY) || document_type.equals(GSXML.DOC_TYPE_PAGED_HIERARCHY))
+		{
+			get_structure = true;
+			if (expand_contents)
+			{
+				ds_param = doc.createElement(GSXML.PARAM_ELEM);
+				ds_param_list.appendChild(ds_param);
+				ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+				ds_param.setAttribute(GSXML.VALUE_ATT, "entire");
+			}
+			else
+			{
+				// get the info needed for table of contents
+				ds_param = doc.createElement(GSXML.PARAM_ELEM);
+				ds_param_list.appendChild(ds_param);
+				ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+				ds_param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+				ds_param = doc.createElement(GSXML.PARAM_ELEM);
+				ds_param_list.appendChild(ds_param);
+				ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+				ds_param.setAttribute(GSXML.VALUE_ATT, "children");
+				if (get_siblings)
+				{
+					ds_param = doc.createElement(GSXML.PARAM_ELEM);
+					ds_param_list.appendChild(ds_param);
+					ds_param.setAttribute(GSXML.NAME_ATT, "structure");
+					ds_param.setAttribute(GSXML.VALUE_ATT, "siblings");
+				}
+			}
+		}
+		else
+		{
+		  // we dont need any structure
+		}
+
+		boolean has_dummy = false;
+		if (get_structure || get_structure_info)
+		{
+
+			// Build a request to obtain the document structure
+			Element ds_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			String to = GSPath.appendLink(collection, "DocumentStructureRetrieve");// Hard-wired?
+			Element ds_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+			ds_message.appendChild(ds_request);
+			ds_request.appendChild(ds_param_list);
+
+			// add the node list we created earlier
+			ds_request.appendChild(basic_doc_list);
+
+			// Process the document structure retrieve message
+			Element ds_response_message = (Element) this.mr.process(ds_message);
+			if (processErrorElements(ds_response_message, page_response))
+			{
+				return result;
+			}
+
+			// get the info and print out
+			String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+			path = GSPath.appendLink(path, "nodeStructureInfo");
+			Element ds_response_struct_info = (Element) GSXML.getNodeByPath(ds_response_message, path);
+			// get the doc_node bit 
+			if (ds_response_struct_info != null)
+			{
+				the_document.appendChild(doc.importNode(ds_response_struct_info, true));
+			}
+			path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+			path = GSPath.appendLink(path, GSXML.NODE_STRUCTURE_ELEM);
+			Element ds_response_structure = (Element) GSXML.getNodeByPath(ds_response_message, path);
+
+			if (ds_response_structure != null)
+			{
+				// add the contents of the structure bit into the_document
+				NodeList structs = ds_response_structure.getChildNodes();
+				for (int i = 0; i < structs.getLength(); i++)
+				{
+					the_document.appendChild(doc.importNode(structs.item(i), true));
+				}
+			}
+			else
+			{
+				// no structure nodes, so put in a dummy doc node
+				Element doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+				if (document_id != null)
+				{
+					doc_node.setAttribute(GSXML.NODE_ID_ATT, document_id);
+				}
+				else
+				{
+					doc_node.setAttribute(GSXML.HREF_ID_ATT, href);
+
+				}
+				the_document.appendChild(doc_node);
+				has_dummy = true;
+			}
+		}
+		else
+		{ // a simple type - we dont have a dummy node for simple
+			// should think about this more
+			// no structure request, so just put in a dummy doc node
+			Element doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+			if (document_id != null)
+			{
+				doc_node.setAttribute(GSXML.NODE_ID_ATT, document_id);
+			}
+			else
+			{
+				doc_node.setAttribute(GSXML.HREF_ID_ATT, href);
+			}
+			the_document.appendChild(doc_node);
+			has_dummy = true;
+		}
+
+		// Build a request to obtain some document metadata
+		Element dm_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		String to = GSPath.appendLink(collection, "DocumentMetadataRetrieve"); // Hard-wired?
+		Element dm_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		dm_message.appendChild(dm_request);
+		// Create a parameter list to specify the required metadata information
+
+		HashSet<String> meta_names = new HashSet<String>();
+		meta_names.add("Title"); // the default
+		if (format_elem != null)
+		{
+			getRequiredMetadataNames(format_elem, meta_names);
+		}
+
+		Element extraMetaListElem = (Element) GSXML.getChildByTagName(request, GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
+		if (extraMetaListElem != null)
+		{
+			NodeList extraMetaList = extraMetaListElem.getElementsByTagName(GSXML.EXTRA_METADATA);
+			for (int i = 0; i < extraMetaList.getLength(); i++)
+			{
+				meta_names.add(((Element) extraMetaList.item(i)).getAttribute(GSXML.NAME_ATT));
+			}
+		}
+
+		Element dm_param_list = createMetadataParamList(doc,meta_names);
+		if (service_params != null)
+		{
+			GSXML.addParametersToList(dm_param_list, service_params);
+		}
+
+		dm_request.appendChild(dm_param_list);
+
+		// create the doc node list for the metadata request
+		Element dm_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		dm_request.appendChild(dm_doc_list);
+
+		// Add each node from the structure response into the metadata request
+		NodeList doc_nodes = the_document.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+		for (int i = 0; i < doc_nodes.getLength(); i++)
+		{
+			Element doc_node = (Element) doc_nodes.item(i);
+			String doc_node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+
+			// Add the documentNode to the list
+			Element dm_doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+			dm_doc_list.appendChild(dm_doc_node);
+			dm_doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_node_id);
+			dm_doc_node.setAttribute(GSXML.NODE_TYPE_ATT, doc_node.getAttribute(GSXML.NODE_TYPE_ATT));
+			if (document_id == null){
+				dm_doc_node.setAttribute(GSXML.HREF_ID_ATT, href );
+				}
+
+		}
+
+		// we also want a metadata request to the top level document to get
+		// assocfilepath - this could be cached too
+		Element doc_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		dm_message.appendChild(doc_meta_request);
+		Element doc_meta_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (service_params != null)
+		{
+			GSXML.addParametersToList(doc_meta_param_list, service_params);
+		}
+
+		doc_meta_request.appendChild(doc_meta_param_list);
+		Element doc_param = doc.createElement(GSXML.PARAM_ELEM);
+		doc_meta_param_list.appendChild(doc_param);
+		doc_param.setAttribute(GSXML.NAME_ATT, "metadata");
+		doc_param.setAttribute(GSXML.VALUE_ATT, "assocfilepath");
+
+		// create the doc node list for the metadata request
+		Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		doc_meta_request.appendChild(doc_list);
+
+		Element doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+		// the node we want is the root document node
+		if (document_id != null)
+		{
+			doc_node.setAttribute(GSXML.NODE_ID_ATT, document_id + ".rt");
+		}
+		/*else
+		{
+			doc_node.setAttribute(GSXML.HREF_ID_ATT, href);// + ".rt");
+			// can we assume that href is always a top level doc??
+			//doc_node.setAttribute(GSXML.ID_MOD_ATT, ".rt");
+			//doc_node.setAttribute("externalURL", has_rl);
+		}*/
+		doc_list.appendChild(doc_node);
+
+		Element dm_response_message = (Element) this.mr.process(dm_message);
+		if (processErrorElements(dm_response_message, page_response))
+		{
+			return result;
+		}
+
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element dm_response_doc_list = (Element) GSXML.getNodeByPath(dm_response_message, path);
+
+		// Merge the metadata with the structure information
+		NodeList dm_response_docs = dm_response_doc_list.getChildNodes();
+		for (int i = 0; i < doc_nodes.getLength(); i++)
+		{
+			GSXML.mergeMetadataLists(doc_nodes.item(i), dm_response_docs.item(i));
+		}
+		// get the top level doc metadata out
+		Element doc_meta_response = (Element) dm_response_message.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1);
+		Element top_doc_node = (Element) GSXML.getNodeByPath(doc_meta_response, "documentNodeList/documentNode");
+		GSXML.mergeMetadataLists(the_document, top_doc_node);
+
+		// Build a request to obtain some document content
+		Element dc_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		to = GSPath.appendLink(collection, "DocumentContentRetrieve"); // Hard-wired?
+		Element dc_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		dc_message.appendChild(dc_request);
+
+		// Create a parameter list to specify the request parameters - empty for now
+		Element dc_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (service_params != null)
+		{
+			GSXML.addParametersToList(dc_param_list, service_params);
+		}
+
+		dc_request.appendChild(dc_param_list);
+
+		// get the content
+		// the doc list for the content request is the same as the one for the structure request unless we want the whole document, in which case its the same as for the metadata request.
+		if (expand_document)
+		{
+			dc_request.appendChild(dm_doc_list);
+		}
+		else
+		{
+			dc_request.appendChild(basic_doc_list);
+		}
+		Element dc_response_message = (Element) this.mr.process(dc_message);
+		if (processErrorElements(dc_response_message, page_response))
+		{
+			return result;
+		}
+		Element dc_response_doc_list = (Element) GSXML.getNodeByPath(dc_response_message, path);
+
+		if (expand_document)
+		{
+			// Merge the content with the structure information
+			NodeList dc_response_docs = dc_response_doc_list.getChildNodes();
+			for (int i = 0; i < doc_nodes.getLength(); i++)
+			{
+				Node content = GSXML.getChildByTagName((Element) dc_response_docs.item(i), GSXML.NODE_CONTENT_ELEM);
+				if (content != null)
+				{
+					if (highlight_query_terms)
+					{
+					  String node_id = ((Element)doc_nodes.item(i)).getAttribute(GSXML.NODE_ID_ATT);
+					  content = highlightQueryTerms(request, node_id, (Element) content);
+					}
+					
+					doc_nodes.item(i).appendChild(doc.importNode(content, true));
+				}
+				//GSXML.mergeMetadataLists(doc_nodes.item(i), dm_response_docs.item(i));
+			}
+			if (has_dummy && document_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+			  Element dummy_node = (Element) doc_nodes.item(0);
+			  the_document.removeChild(dummy_node);
+			  the_document.setAttribute(GSXML.NODE_ID_ATT, dummy_node.getAttribute(GSXML.NODE_ID_ATT));
+			  NodeList dummy_children = dummy_node.getChildNodes();
+			  for (int i = dummy_children.getLength() - 1; i >= 0; i--)
+			    {
+			      // special case as we don't want more than one metadata list
+			      if (dummy_children.item(i).getNodeName().equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER))
+				{
+				  GSXML.mergeMetadataFromList(the_document, dummy_children.item(i));
+				}
+			      else
+				{
+				  the_document.appendChild(dummy_children.item(i));
+				}
+			    }
+			}
+		}
+		else
+		{
+			//path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+			Element dc_response_doc = (Element) GSXML.getChildByTagName(dc_response_doc_list, GSXML.DOC_NODE_ELEM);
+			Element dc_response_doc_content = (Element) GSXML.getChildByTagName(dc_response_doc, GSXML.NODE_CONTENT_ELEM);
+			//Element dc_response_doc_external = (Element) GSXML.getChildByTagName(dc_response_doc, "external");
+
+			if (dc_response_doc_content == null)
+			{
+				// no content to add
+				if (dc_response_doc.getAttribute("external").equals("true"))
+				{
+
+					//if (dc_response_doc_external != null)
+					//{
+					String href_id = dc_response_doc.getAttribute(GSXML.HREF_ID_ATT);
+
+					the_document.setAttribute("selectedNode", href_id);
+					the_document.setAttribute("external", href_id);
+				}
+				return result;
+			}
+			if (highlight_query_terms)
+			{
+				dc_response_doc.removeChild(dc_response_doc_content);
+
+				dc_response_doc_content = highlightQueryTerms(request, null, dc_response_doc_content);
+				dc_response_doc.appendChild(dc_response_doc.getOwnerDocument().importNode(dc_response_doc_content, true));
+			}
+
+			if (provide_annotations)
+			{
+				String service_selected = (String) params.get(ENRICH_DOC_ARG);
+				if (service_selected != null && service_selected.equals("1"))
+				{
+					// now we can modifiy the response doc if needed
+					String enrich_service = (String) params.get(GSParams.SERVICE);
+					// send a message to the service
+					Element enrich_message = doc.createElement(GSXML.MESSAGE_ELEM);
+					Element enrich_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, enrich_service, userContext);
+					enrich_message.appendChild(enrich_request);
+					// check for parameters
+					HashMap e_service_params = (HashMap) params.get("s1");
+					if (e_service_params != null)
+					{
+						Element enrich_pl = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+						GSXML.addParametersToList(enrich_pl, e_service_params);
+						enrich_request.appendChild(enrich_pl);
+					}
+					Element e_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+					enrich_request.appendChild(e_doc_list);
+					e_doc_list.appendChild(doc.importNode(dc_response_doc, true));
+
+					Node enrich_response = this.mr.process(enrich_message);
+
+					String[] links = { GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM, GSXML.NODE_CONTENT_ELEM };
+					path = GSPath.createPath(links);
+					dc_response_doc_content = (Element) GSXML.getNodeByPath(enrich_response, path);
+
+				}
+			} // if provide_annotations
+
+			// use the returned id rather than the sent one cos there may have
+			// been modifiers such as .pr that are removed.
+			String modified_doc_id = dc_response_doc.getAttribute(GSXML.NODE_ID_ATT);
+			the_document.setAttribute("selectedNode", modified_doc_id);
+			if (has_dummy)
+			{
+				// change the id if necessary and add the content
+				Element dummy_node = (Element) doc_nodes.item(0);
+
+				dummy_node.setAttribute(GSXML.NODE_ID_ATT, modified_doc_id);
+				dummy_node.appendChild(doc.importNode(dc_response_doc_content, true));
+				// hack for simple type
+				if (document_type.equals(GSXML.DOC_TYPE_SIMPLE))
+				{
+					// we dont want the internal docNode, just want the content and metadata in the document
+					// rethink this!!
+					the_document.removeChild(dummy_node);
+
+					NodeList dummy_children = dummy_node.getChildNodes();
+					//for (int i=0; i<dummy_children.getLength(); i++) {
+					for (int i = dummy_children.getLength() - 1; i >= 0; i--)
+					{
+						// special case as we don't want more than one metadata list
+						if (dummy_children.item(i).getNodeName().equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER))
+						{
+							GSXML.mergeMetadataFromList(the_document, dummy_children.item(i));
+						}
+						else
+						{
+							the_document.appendChild(dummy_children.item(i));
+						}
+					}
+				}
+
+				the_document.setAttribute(GSXML.NODE_ID_ATT, modified_doc_id);
+			}
+			else
+			{
+				// Merge the document content with the metadata and structure information
+				for (int i = 0; i < doc_nodes.getLength(); i++)
+				{
+					Node dn = doc_nodes.item(i);
+					String dn_id = ((Element) dn).getAttribute(GSXML.NODE_ID_ATT);
+					if (dn_id.equals(modified_doc_id))
+					{
+						dn.appendChild(doc.importNode(dc_response_doc_content, true));
+						break;
+					}
+				}
+			}
+		}
+		//logger.debug("(DocumentAction) Page:\n" + GSXML.xmlNodeToString(result));
+		return result;
+	}
+
+	/**
+	 * tell the param class what its arguments are if an action has its own
+	 * arguments, this should add them to the params object - particularly
+	 * important for args that should not be saved
+	 */
+	public boolean addActionParameters(GSParams params)
+	{
+		params.addParameter(GOTO_PAGE_ARG, false);
+		params.addParameter(ENRICH_DOC_ARG, false);
+		params.addParameter(EXPAND_DOCUMENT_ARG, false);
+		params.addParameter(EXPAND_CONTENTS_ARG, false);
+		params.addParameter(REALISTIC_BOOK_ARG, false);
+
+		return true;
+	}
+
+	/**
+	 * this method gets the collection description, the format info, the list of
+	 * enrich services, etc - stuff that is needed for the page, but is the same
+	 * whatever the query is - should be cached
+	 */
+	protected boolean getBackgroundData(Element page_response, String collection, UserContext userContext)
+	{
+		Document doc = page_response.getOwnerDocument();
+		
+		// create a message to process - contains requests for the collection 
+		// description, the format element, the enrich services on offer
+		// these could all be cached
+		Element info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		String path = GSPath.appendLink(collection, "DocumentContentRetrieve");
+		// the format request - ignore for now, where does this request go to??
+		Element format_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT, path, userContext);
+		info_message.appendChild(format_request);
+
+		// the enrich_services request - only do this if provide_annotations is true
+
+		if (provide_annotations)
+		{
+			Element enrich_services_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+			enrich_services_request.setAttribute(GSXML.INFO_ATT, "serviceList");
+			info_message.appendChild(enrich_services_request);
+		}
+
+		Element info_response = (Element) this.mr.process(info_message);
+
+		// the collection is the first response
+		NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		Element format_resp = (Element) responses.item(0);
+
+		Element format_elem = (Element) GSXML.getChildByTagName(format_resp, GSXML.FORMAT_ELEM);
+		if (format_elem != null)
+		{
+			Element global_format_elem = (Element) GSXML.getChildByTagName(format_resp, GSXML.GLOBAL_FORMAT_ELEM);
+			if (global_format_elem != null)
+			{
+				GSXSLT.mergeFormatElements(format_elem, global_format_elem, false);
+			}
+
+			// set the format type
+			format_elem.setAttribute(GSXML.TYPE_ATT, "display");
+			page_response.appendChild(doc.importNode(format_elem, true));
+		}
+
+		if (provide_annotations)
+		{
+			Element services_resp = (Element) responses.item(1);
+
+			// a new message for the mr
+			Element enrich_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			NodeList e_services = services_resp.getElementsByTagName(GSXML.SERVICE_ELEM);
+			boolean service_found = false;
+			for (int j = 0; j < e_services.getLength(); j++)
+			{
+				if (((Element) e_services.item(j)).getAttribute(GSXML.TYPE_ATT).equals("enrich"))
+				{
+					Element s = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, ((Element) e_services.item(j)).getAttribute(GSXML.NAME_ATT), userContext);
+					enrich_message.appendChild(s);
+					service_found = true;
+				}
+			}
+			if (service_found)
+			{
+				Element enrich_response = (Element) this.mr.process(enrich_message);
+
+				NodeList e_responses = enrich_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+				Element service_list = doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+				for (int i = 0; i < e_responses.getLength(); i++)
+				{
+					Element e_resp = (Element) e_responses.item(i);
+					Element e_service = (Element) doc.importNode(GSXML.getChildByTagName(e_resp, GSXML.SERVICE_ELEM), true);
+					e_service.setAttribute(GSXML.NAME_ATT, e_resp.getAttribute(GSXML.FROM_ATT));
+					service_list.appendChild(e_service);
+				}
+				page_response.appendChild(service_list);
+			}
+		} // if provide_annotations
+		return true;
+
+	}
+
+	protected String getDocumentType(Element basic_doc_list, String collection, UserContext userContext, Element page_response)
+	{
+		Document doc = basic_doc_list.getOwnerDocument();
+		
+		Element ds_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		String to = GSPath.appendLink(collection, "DocumentStructureRetrieve");// Hard-wired?
+		Element ds_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		ds_message.appendChild(ds_request);
+
+		// Create a parameter list to specify the required structure information
+		Element ds_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element ds_param = doc.createElement(GSXML.PARAM_ELEM);
+		ds_param_list.appendChild(ds_param);
+		ds_param.setAttribute(GSXML.NAME_ATT, "info");
+		ds_param.setAttribute(GSXML.VALUE_ATT, "documentType");
+
+		ds_request.appendChild(ds_param_list);
+
+		// add the node list we created earlier
+		ds_request.appendChild(basic_doc_list);
+
+		// Process the document structure retrieve message
+		Element ds_response_message = (Element) this.mr.process(ds_message);
+		if (processErrorElements(ds_response_message, page_response))
+		{
+			return null;
+		}
+
+		String[] links = { GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM, "nodeStructureInfo" };
+		String path = GSPath.createPath(links);
+		Element info_elem = (Element) GSXML.getNodeByPath(ds_response_message, path);
+		if (info_elem == null) {
+		    return null;
+		}
+		Element doctype_elem = GSXML.getNamedElement(info_elem, "info", "name", "documentType");
+		if (doctype_elem != null)
+		{
+			String doc_type = doctype_elem.getAttribute("value");
+			return doc_type;
+		}
+		return null;
+	}
+
+	/**
+	 * this involves a bit of a hack to get the equivalent query terms - has to
+	 * requery the query service - uses the last selected service name. (if it
+	 * ends in query). should this action do the query or should it send a
+	 * message to the query action? but that will involve lots of extra stuff.
+	 * also doesn't handle phrases properly - just highlights all the terms
+	 * found in the text.
+	 */
+  protected Element highlightQueryTerms(Element request, String current_node_id, Element dc_response_doc_content)
+	{
+		Document doc = request.getOwnerDocument();
+		
+		// do the query again to get term info 
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		HashMap previous_params = (HashMap) params.get("p");
+		if (previous_params == null)
+		{
+			return dc_response_doc_content;
+		}
+		String service_name = (String) previous_params.get(GSParams.SERVICE);
+		if (service_name == null || !service_name.endsWith("Query"))
+		{ // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
+			logger.debug("invalid service, not doing highlighting");
+			return dc_response_doc_content;
+		}
+		String collection = (String) params.get(GSParams.COLLECTION);
+		UserContext userContext = new UserContext(request);
+		String to = GSPath.appendLink(collection, service_name);
+
+		Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		mr_query_message.appendChild(mr_query_request);
+
+		// paramList
+		HashMap service_params = (HashMap) params.get("s1");
+
+		Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParametersToList(query_param_list, service_params);
+		if (current_node_id != null) {
+		  GSXML.addParameterToList(query_param_list, "hldocOID", current_node_id);
+		} else {
+		  GSXML.addParameterToList(query_param_list, "hldocOID", (String) params.get(GSParams.DOCUMENT));
+		}
+		mr_query_request.appendChild(query_param_list);
+		// do the query
+		Element mr_query_response = (Element) this.mr.process(mr_query_message);
+		String pathNode = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.NODE_CONTENT_ELEM);
+		Element highlighted_Node = (Element) GSXML.getNodeByPath(mr_query_response, pathNode);
+		// For SOLR, the above query may come back with a nodeContent element, which is the hldocOID section content, with search terms marked up. We send it back to the documnetContentRetrieve service so that resolveTextMacros can be applied, and it can be properly encased in documentNode etc elements
+		if (highlighted_Node != null)
+		{
+			// Build a request to process highlighted text
+			
+			Element hl_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+			Element dc_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+			hl_message.appendChild(dc_request);
+
+			// Create a parameter list to specify the request parameters - empty for now
+			Element dc_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			dc_request.appendChild(dc_param_list);
+
+			// get the content
+			Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			dc_request.appendChild(doc_list);
+			Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
+			doc_list.appendChild(current_doc);
+			current_doc.setAttribute(GSXML.NODE_ID_ATT, (String) params.get(GSParams.DOCUMENT));
+			//Append highlighted content to request for processing
+			dc_request.appendChild(doc.importNode(highlighted_Node, true));
+			Element hl_response_message = (Element) this.mr.process(hl_message);
+		
+			//Get results
+			NodeList contentList = hl_response_message.getElementsByTagName(GSXML.NODE_CONTENT_ELEM);
+			Element content = (Element) contentList.item(0);	
+			return content;
+		}
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
+		Element query_term_list_element = (Element) GSXML.getNodeByPath(mr_query_response, path);
+		if (query_term_list_element == null)
+		{
+			// no term info
+			logger.error("No query term information.\n");
+			return dc_response_doc_content;
+		}
+
+		String content = GSXML.getNodeText(dc_response_doc_content);
+
+		String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
+
+		HashSet<String> query_term_variants = new HashSet<String>();
+		NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
+		if (equivalent_terms_nodelist == null || equivalent_terms_nodelist.getLength() == 0)
+		{
+			NodeList terms_nodelist = query_term_list_element.getElementsByTagName("term");
+			if (terms_nodelist != null && terms_nodelist.getLength() > 0)
+			{
+				for (int i = 0; i < terms_nodelist.getLength(); i++)
+				{
+					String termValue = ((Element) terms_nodelist.item(i)).getAttribute("name");
+					String termValueU = null;
+					String termValueL = null;
+
+					if (termValue.length() > 1)
+					{
+						termValueU = termValue.substring(0, 1).toUpperCase() + termValue.substring(1);
+						termValueL = termValue.substring(0, 1).toLowerCase() + termValue.substring(1);
+					}
+					else
+					{
+						termValueU = termValue.substring(0, 1).toUpperCase();
+						termValueL = termValue.substring(0, 1).toLowerCase();
+					}
+
+					query_term_variants.add(termValueU);
+					query_term_variants.add(termValueL);
+				}
+			}
+		}
+		else
+		{
+			for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++)
+			{
+				Element equivalent_terms_element = (Element) equivalent_terms_nodelist.item(i);
+				String[] equivalent_terms = GSXML.getAttributeValuesFromList(equivalent_terms_element, GSXML.NAME_ATT);
+				for (int j = 0; j < equivalent_terms.length; j++)
+				{
+					query_term_variants.add(equivalent_terms[j]);
+				}
+			}
+		}
+
+		ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy = new ArrayList<ArrayList<HashSet<String>>>();
+
+		Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
+		String performed_query = GSXML.getNodeText(query_element) + " ";
+
+		ArrayList<HashSet<String>> phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+		int term_start = 0;
+		boolean in_term = false;
+		boolean in_phrase = false;
+		for (int i = 0; i < performed_query.length(); i++)
+		{
+			char character = performed_query.charAt(i);
+			boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
+
+			// Has a query term just started?
+			if (in_term == false && is_character_letter_or_digit == true)
+			{
+				in_term = true;
+				term_start = i;
+			}
+
+			// Or has a term just finished?
+			else if (in_term == true && is_character_letter_or_digit == false)
+			{
+				in_term = false;
+				String term = performed_query.substring(term_start, i);
+
+				Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
+				if (term_element != null)
+				{
+
+					HashSet<String> phrase_query_p_term_x_variants = new HashSet<String>();
+
+					NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
+					if (term_equivalent_terms_nodelist == null || term_equivalent_terms_nodelist.getLength() == 0)
+					{
+						String termValueU = null;
+						String termValueL = null;
+
+						if (term.length() > 1)
+						{
+							termValueU = term.substring(0, 1).toUpperCase() + term.substring(1);
+							termValueL = term.substring(0, 1).toLowerCase() + term.substring(1);
+						}
+						else
+						{
+							termValueU = term.substring(0, 1).toUpperCase();
+							termValueL = term.substring(0, 1).toLowerCase();
+						}
+
+						phrase_query_p_term_x_variants.add(termValueU);
+						phrase_query_p_term_x_variants.add(termValueL);
+					}
+					else
+					{
+						for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++)
+						{
+							Element term_equivalent_terms_element = (Element) term_equivalent_terms_nodelist.item(j);
+							String[] term_equivalent_terms = GSXML.getAttributeValuesFromList(term_equivalent_terms_element, GSXML.NAME_ATT);
+							for (int k = 0; k < term_equivalent_terms.length; k++)
+							{
+								phrase_query_p_term_x_variants.add(term_equivalent_terms[k]);
+							}
+						}
+					}
+					phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
+
+					if (in_phrase == false)
+					{
+						phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+						phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+					}
+				}
+			}
+			// Watch for phrases (surrounded by quotes)
+			if (character == '\"')
+			{
+				// Has a phrase just started?
+				if (in_phrase == false)
+				{
+					in_phrase = true;
+				}
+				// Or has a phrase just finished?
+				else if (in_phrase == true)
+				{
+					in_phrase = false;
+					phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+				}
+
+				phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+			}
+		}
+
+		return highlightQueryTermsInternal(doc, content, query_term_variants, phrase_query_term_variants_hierarchy);
+	}
+
+	/**
+	 * Highlights query terms in a piece of text.
+	 */
+	private Element highlightQueryTermsInternal(Document doc, String content, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+	{
+		// Convert the content string to an array of characters for speed
+		char[] content_characters = new char[content.length()];
+		content.getChars(0, content.length(), content_characters, 0);
+
+		// Now skim through the content, identifying word matches
+		ArrayList<WordMatch> word_matches = new ArrayList<WordMatch>();
+		int word_start = 0;
+		boolean in_word = false;
+		boolean preceding_word_matched = false;
+		boolean inTag = false;
+		for (int i = 0; i < content_characters.length; i++)
+		{
+			//We don't want to find words inside HTML tags
+			if (content_characters[i] == '<')
+			{
+				inTag = true;
+				continue;
+			}
+			else if (inTag && content_characters[i] == '>')
+			{
+				inTag = false;
+			}
+			else if (inTag)
+			{
+				continue;
+			}
+
+			boolean is_character_letter_or_digit = Character.isLetterOrDigit(content_characters[i]);
+
+			// Has a word just started?
+			if (in_word == false && is_character_letter_or_digit == true)
+			{
+				in_word = true;
+				word_start = i;
+			}
+
+			// Or has a word just finished?
+			else if (in_word == true && is_character_letter_or_digit == false)
+			{
+				in_word = false;
+
+				// Check if the word matches any of the query term equivalents
+				String word = new String(content_characters, word_start, (i - word_start));
+				if (query_term_variants.contains(word))
+				{
+					// We have found a matching word, so remember its location
+					word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
+					preceding_word_matched = true;
+				}
+				else
+				{
+					preceding_word_matched = false;
+				}
+			}
+		}
+
+		// Don't forget the last word...
+		if (in_word == true)
+		{
+			// Check if the word matches any of the query term equivalents
+			String word = new String(content_characters, word_start, (content_characters.length - word_start));
+			if (query_term_variants.contains(word))
+			{
+				// We have found a matching word, so remember its location
+				word_matches.add(new WordMatch(word, word_start, content_characters.length, preceding_word_matched));
+			}
+		}
+
+		ArrayList<Integer> highlight_start_positions = new ArrayList<Integer>();
+		ArrayList<Integer> highlight_end_positions = new ArrayList<Integer>();
+
+		// Deal with phrases now
+		ArrayList<PartialPhraseMatch> partial_phrase_matches = new ArrayList<PartialPhraseMatch>();
+		for (int i = 0; i < word_matches.size(); i++)
+		{
+			WordMatch word_match = word_matches.get(i);
+
+			// See if any partial phrase matches are extended by this word
+			if (word_match.preceding_word_matched)
+			{
+				for (int j = partial_phrase_matches.size() - 1; j >= 0; j--)
+				{
+					PartialPhraseMatch partial_phrase_match = partial_phrase_matches.remove(j);
+					ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(partial_phrase_match.query_phrase_number);
+					HashSet phrase_query_p_term_x_variants = (HashSet) phrase_query_p_term_variants_list.get(partial_phrase_match.num_words_matched);
+					if (phrase_query_p_term_x_variants.contains(word_match.word))
+					{
+						partial_phrase_match.num_words_matched++;
+
+						// Has a complete phrase match occurred?
+						if (partial_phrase_match.num_words_matched == phrase_query_p_term_variants_list.size())
+						{
+							// Check for overlaps by looking at the previous highlight range
+							if (!highlight_end_positions.isEmpty())
+							{
+								int last_highlight_index = highlight_end_positions.size() - 1;
+								int last_highlight_end = highlight_end_positions.get(last_highlight_index).intValue();
+								if (last_highlight_end > partial_phrase_match.start_position)
+								{
+									// There is an overlap, so remove the previous phrase match
+									int last_highlight_start = highlight_start_positions.remove(last_highlight_index).intValue();
+									highlight_end_positions.remove(last_highlight_index);
+									partial_phrase_match.start_position = last_highlight_start;
+								}
+							}
+
+							highlight_start_positions.add(new Integer(partial_phrase_match.start_position));
+							highlight_end_positions.add(new Integer(word_match.end_position));
+						}
+						// No, but add the partial match back into the list for next time
+						else
+						{
+							partial_phrase_matches.add(partial_phrase_match);
+						}
+					}
+				}
+			}
+			else
+			{
+				partial_phrase_matches.clear();
+			}
+
+			// See if this word is at the start of any of the phrases
+			for (int p = 0; p < phrase_query_term_variants_hierarchy.size(); p++)
+			{
+				ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(p);
+				HashSet phrase_query_p_term_1_variants = (HashSet) phrase_query_p_term_variants_list.get(0);
+				if (phrase_query_p_term_1_variants.contains(word_match.word))
+				{
+					// If this phrase is just one word long, we have a complete match
+					if (phrase_query_p_term_variants_list.size() == 1)
+					{
+						highlight_start_positions.add(new Integer(word_match.start_position));
+						highlight_end_positions.add(new Integer(word_match.end_position));
+					}
+					// Otherwise we have the start of a potential phrase match
+					else
+					{
+						partial_phrase_matches.add(new PartialPhraseMatch(word_match.start_position, p));
+					}
+				}
+			}
+		}
+
+		// Now add the annotation tags into the document at the correct points
+		Element content_element = doc.createElement(GSXML.NODE_CONTENT_ELEM);
+
+		int last_wrote = 0;
+		for (int i = 0; i < highlight_start_positions.size(); i++)
+		{
+			int highlight_start = highlight_start_positions.get(i).intValue();
+			int highlight_end = highlight_end_positions.get(i).intValue();
+
+			// Print anything before the highlight range
+			if (last_wrote < highlight_start)
+			{
+				String preceding_text = new String(content_characters, last_wrote, (highlight_start - last_wrote));
+				content_element.appendChild(doc.createTextNode(preceding_text));
+			}
+
+			// Print the highlight text, annotated
+			if (highlight_end > last_wrote)
+			{
+				String highlight_text = new String(content_characters, highlight_start, (highlight_end - highlight_start));
+				Element annotation_element = GSXML.createTextElement(doc, "annotation", highlight_text);
+				annotation_element.setAttribute("type", "query_term");
+				content_element.appendChild(annotation_element);
+				last_wrote = highlight_end;
+			}
+		}
+
+		// Finish off any unwritten text
+		if (last_wrote < content_characters.length)
+		{
+			String remaining_text = new String(content_characters, last_wrote, (content_characters.length - last_wrote));
+			content_element.appendChild(doc.createTextNode(remaining_text));
+		}
+		return content_element;
+	}
+
+	static private class WordMatch
+	{
+		public String word;
+		public int start_position;
+		public int end_position;
+		public boolean preceding_word_matched;
+
+		public WordMatch(String word, int start_position, int end_position, boolean preceding_word_matched)
+		{
+			this.word = word;
+			this.start_position = start_position;
+			this.end_position = end_position;
+			this.preceding_word_matched = preceding_word_matched;
+		}
+	}
+
+	static private class PartialPhraseMatch
+	{
+		public int start_position;
+		public int query_phrase_number;
+		public int num_words_matched;
+
+		public PartialPhraseMatch(int start_position, int query_phrase_number)
+		{
+			this.start_position = start_position;
+			this.query_phrase_number = query_phrase_number;
+			this.num_words_matched = 1;
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/FormatAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/FormatAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/FormatAction.java	(revision 31634)
@@ -0,0 +1,153 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.util.GlobalProperties;
+
+// XML classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document; 
+
+// other java stuff
+import java.io.File;
+import java.util.*;
+import java.io.StringReader;
+import org.xml.sax.InputSource;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import java.io.*;
+import org.xml.sax.*;
+import javax.xml.parsers.SAXParserFactory; 
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringWriter;
+
+import org.apache.log4j.*;
+
+public class FormatAction extends Action {
+    
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.FormatAction.class.getName());
+    XMLTransformer transformer = null;
+
+    String tempVal = "";
+
+    /** process a request */
+    public Node process (Node message_node) {
+	
+	    Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+    	// assume only one request
+	    Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+	
+    	String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+    	UserContext userContext = new UserContext(request);
+	    // get the param list
+    	Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+    	Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	    String coll = (String)params.get(GSParams.COLLECTION); //SYSTEM_CLUSTER);
+        String service = (String)params.get(GSParams.SERVICE);
+        String classifier = (String)params.get("cl");
+
+        String to = "";
+        if (coll!=null && !coll.equals("")) {
+            to = coll;
+        }
+
+        Element mr_request_message = doc.createElement(GSXML.MESSAGE_ELEM);
+        Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT_STRING, to, userContext);
+
+        mr_request.setAttribute("service", service);
+        mr_request.setAttribute("subaction", subaction);
+        //if(classifier != null)
+        mr_request.setAttribute("classifier", classifier);
+
+        mr_request_message.appendChild(mr_request);
+	
+        String format_string = (String)params.get("data");
+    
+        Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+        Iterator<String> it = params.keySet().iterator();
+
+        if(subaction.equals("saveDocument"))
+        {
+            Element format = doc.createElement(GSXML.FORMAT_STRING_ELEM);
+            try{
+                DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+                DocumentBuilder builder = factory.newDocumentBuilder();
+                InputSource is = new InputSource( new StringReader( format_string ) );
+                Document d = (Document) builder.parse( is );
+                Node n1 = d.getFirstChild();
+
+                Element format_statement = (Element) doc.importNode(n1, true);
+                format.appendChild(format_statement);
+                mr_request.appendChild(format);
+            } catch( Exception ex ) {
+                logger.error("There was an exception "+ex);
+
+                StringWriter sw = new StringWriter();
+                PrintWriter pw = new PrintWriter(sw, true);
+                ex.printStackTrace(pw);
+                pw.flush();
+                sw.flush();
+                logger.error(sw.toString());
+            }
+
+        }
+
+        else
+        {
+
+            try {
+                Document d = this.converter.getDOM(format_string); 
+
+                // Call XSLT to transform document to xml format string
+                XMLTransformer transformer = new XMLTransformer();
+                String style = GSFile.interfaceStylesheetFile(GlobalProperties.getGSDL3Home(),(String)this.config_params.get(GSConstants.INTERFACE_NAME), "formatString.xsl");
+                //logger.error("Style doc is "+style+", compared to /research/sjb48/greenstone3/web/interfaces/oran/transform/formatString.xsl");
+                Document style_doc = this.converter.getDOM(new File(style), "UTF-8");
+                //Document style_doc = this.converter.getDOM(new File("/research/sjb48/greenstone3/web/interfaces/oran/transform/formatString.xsl"), "UTF-8");  /*************************/
+
+                if(style_doc == null)
+                    logger.error("style_doc is null");
+
+                logger.error("About to transform");
+                Node transformed = (Node) transformer.transform(style_doc, d);  
+                
+                if(transformed.getNodeType() == Node.DOCUMENT_NODE)
+                    transformed = ((Document)transformed).getDocumentElement(); 
+
+                Element format = doc.createElement(GSXML.FORMAT_STRING_ELEM);
+                format.appendChild(doc.importNode(transformed,true));
+                mr_request.appendChild(format); 
+                logger.error("Transformed: "+transformed);
+
+            } catch( Exception ex ) {
+                logger.error("There was an exception "+ex);
+
+                StringWriter sw = new StringWriter();
+                PrintWriter pw = new PrintWriter(sw, true);
+                ex.printStackTrace(pw);
+                pw.flush();
+                sw.flush();
+                logger.error(sw.toString());
+            }
+        }
+
+        Node response_message = this.mr.process(mr_request_message);
+	
+        result.appendChild(GSXML.duplicateWithNewName(doc, (Element)GSXML.getChildByTagName(response_message, GSXML.RESPONSE_ELEM), GSXML.RESPONSE_ELEM, true));
+        return result;
+	
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GS2BrowseAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GS2BrowseAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GS2BrowseAction.java	(revision 31634)
@@ -0,0 +1,502 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.UserContext;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/** action for GS2 style classifier browsing */
+public class GS2BrowseAction extends Action
+{
+
+	public static final String CLASSIFIER_ARG = "cl";
+
+	public static final String DESCENDANTS_ARG = "descendants";
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.GS2BrowseAction.class.getName());
+
+	/** process the request */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// get the request - assume only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// the result
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element response = classifierBrowse(request);
+		result.appendChild(response);
+		return result;
+	}
+
+	protected Element classifierBrowse(Element request)
+	{
+		Document doc = request.getOwnerDocument();
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// extract the params from the cgi-request, and check that we have a coll specified
+		Element cgi_paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_paramList, false);
+
+		String service_name = (String) params.get(GSParams.SERVICE);
+		String collection = (String) params.get(GSParams.COLLECTION);
+		if (collection == null || collection.equals(""))
+		{
+			logger.error("classifierBrowse, need to specify a collection!");
+			return page_response;
+		}
+
+		boolean get_descendants = false;
+		String descendants_str = (String) params.get(DESCENDANTS_ARG);
+		if (descendants_str != null && descendants_str.equals("1"))
+		{
+			get_descendants = true;
+		}
+
+		UserContext userContext = new UserContext(request);
+		String to = GSPath.appendLink(collection, service_name);
+
+		// the first part of the response is the service description 
+		// for now get this again from the service. 
+		// this should be cached somehow later on. 
+
+		Element info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+		info_message.appendChild(info_request);
+
+		// also get the format stuff now if there is some
+		Element format_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT, to, userContext);
+		info_message.appendChild(format_request);
+		// process the requests
+
+		Element info_response = (Element) this.mr.process(info_message);
+
+		// the two responses
+		NodeList responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		Element service_response = (Element) responses.item(0);
+		Element format_response = (Element) responses.item(1);
+
+		Element service_description = (Element) GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM);
+		page_response.appendChild(doc.importNode(service_description, true));
+
+		//append site metadata
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+
+		// if rt=d, then we are just displaying the service
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		if (request_type.equals("d"))
+		{
+			//return the page that we have so far
+			return page_response;
+		}
+
+		// get the node that the user has clicked on
+		String classifier_node = (String) params.get(CLASSIFIER_ARG);
+
+		// if the node is not defined, return the page that we have so far
+		if (classifier_node == null || classifier_node.equals(""))
+		{
+			return page_response;
+		}
+
+		// the id of the classifier is the top id of the selected node
+		String top_id = OID.getTop(classifier_node);
+		HashSet<String> doc_meta_names = new HashSet<String>();
+		HashSet<String> class_meta_names = new HashSet<String>();
+		// add in the defaults
+		doc_meta_names.add("Title");
+		doc_meta_names.add("root_assocfilepath");
+		class_meta_names.add("Title");
+
+		// add the format info into the response
+		Element format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.FORMAT_ELEM);
+		if (format_elem != null)
+		{
+			// find the one for the classifier we are in
+			Element this_format = GSXML.getNamedElement(format_elem, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+			if (this_format == null)
+			{
+				this_format = (Element) GSXML.getChildByTagName(format_elem, GSXML.DEFAULT_ELEM);
+			}
+
+			if (this_format != null)
+			{
+				Element global_format_elem = (Element) GSXML.getChildByTagName(format_response, GSXML.GLOBAL_FORMAT_ELEM);
+				if (global_format_elem != null)
+				{
+					GSXSLT.mergeFormatElements(this_format, global_format_elem, false);
+				}
+
+				Element new_format = GSXML.duplicateWithNewName(doc, this_format, GSXML.FORMAT_ELEM, false);
+				extractMetadataNames(new_format, doc_meta_names, class_meta_names);
+
+				Element extraMetaListElem = (Element) GSXML.getChildByTagName(request, GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
+				if (extraMetaListElem != null)
+				{
+					NodeList extraMetaList = extraMetaListElem.getElementsByTagName(GSXML.EXTRA_METADATA);
+					for (int i = 0; i < extraMetaList.getLength(); i++)
+					{
+						doc_meta_names.add(((Element) extraMetaList.item(i)).getAttribute(GSXML.NAME_ATT));
+					}
+				}
+				
+				// set the format type
+				new_format.setAttribute(GSXML.TYPE_ATT, "browse");
+
+				page_response.appendChild(new_format);
+			}
+		}
+
+		// find out if this classifier is horizontal at top
+		Element class_list = (Element) GSXML.getChildByTagName(service_description, GSXML.CLASSIFIER_ELEM + GSXML.LIST_MODIFIER);
+		Element this_classifier = GSXML.getNamedElement(class_list, GSXML.CLASSIFIER_ELEM, GSXML.NAME_ATT, top_id);
+
+		// get the browse structure for the selected node
+		Element classify_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element classify_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		classify_message.appendChild(classify_request);
+
+		//Create a parameter list to specify the required structure information
+		// for now, always get ancestors and children
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		classify_request.appendChild(param_list);
+		Element param = doc.createElement(GSXML.PARAM_ELEM);
+		param_list.appendChild(param);
+		param.setAttribute(GSXML.NAME_ATT, "structure");
+		param.setAttribute(GSXML.VALUE_ATT, "ancestors");
+		param = doc.createElement(GSXML.PARAM_ELEM);
+		param_list.appendChild(param);
+		param.setAttribute(GSXML.NAME_ATT, "structure");
+		param.setAttribute(GSXML.VALUE_ATT, "children");
+
+		if (get_descendants)
+		{
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param_list.appendChild(param);
+			param.setAttribute(GSXML.NAME_ATT, "structure");
+			param.setAttribute(GSXML.VALUE_ATT, "descendants");
+		}
+
+
+		// put the classifier node into a classifier node list
+		Element classifier_list = doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element classifier = doc.createElement(GSXML.CLASS_NODE_ELEM);
+		classifier.setAttribute(GSXML.NODE_ID_ATT, classifier_node);
+		classifier_list.appendChild(classifier);
+		classify_request.appendChild(classifier_list);
+
+		// process the request
+		Element classify_response = (Element) this.mr.process(classify_message);
+
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element class_node_list = (Element) GSXML.getNodeByPath(classify_response, path);
+
+		path = GSPath.appendLink(GSXML.CLASS_NODE_ELEM, GSXML.NODE_STRUCTURE_ELEM);
+		// assume that we always get back the top level CL1 node - this becomes the page_classifier node
+		path = GSPath.appendLink(path, GSXML.CLASS_NODE_ELEM);
+		Element cl_structure = (Element) GSXML.getNodeByPath(class_node_list, path);
+		if (cl_structure == null)
+		{
+			logger.error("classifier structure request returned no structure");
+			return page_response;
+		}
+
+		//If the user is viewing a horizontal classifier then we need to get extra information
+		if (cl_structure.getAttribute(GSXML.CHILD_TYPE_ATT).equals(GSXML.HLIST))
+		{
+			//If we have a horizontal classifier and we have had the top-level node requested (e.g. CL1, CL2 etc.)
+			//then we want to get the children of the first classifier node (e.g. the children of CL2.1) 
+			if (OID.isTop(classifier_node))
+			{
+				boolean firstChildIsClassifierNode = false;
+				NodeList classifierChildrenNodes = GSXML.getChildrenByTagName(cl_structure, GSXML.CLASS_NODE_ELEM);
+				for (int i = 0; i < classifierChildrenNodes.getLength(); i++)
+				{
+					Element currentChild = (Element) classifierChildrenNodes.item(i);
+					if (currentChild.getAttribute(GSXML.NODE_ID_ATT).endsWith(".1"))
+					{
+						firstChildIsClassifierNode = true;
+					}
+				}
+
+				if (firstChildIsClassifierNode)
+				{
+					Element childStructure = getClassifierStructureFromID(doc, classifier_node + ".1", request, collection, service_name);
+
+					Element replacementElem = null;
+					NodeList childClassifierNodes = childStructure.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+					for (int i = 0; i < childClassifierNodes.getLength(); i++)
+					{
+						Element currentElem = (Element) childClassifierNodes.item(i);
+						if (currentElem.getAttribute(GSXML.NODE_ID_ATT).equals(classifier_node + ".1"))
+						{
+							replacementElem = currentElem;
+							break;
+						}
+					}
+
+					NodeList nodesToSearch = cl_structure.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+					for (int i = 0; i < nodesToSearch.getLength(); i++)
+					{
+						Element currentElem = (Element) nodesToSearch.item(i);
+						if (currentElem.getAttribute(GSXML.NODE_ID_ATT).equals(classifier_node + ".1"))
+						{
+							Element parent = (Element) currentElem.getParentNode();
+							parent.insertBefore(parent.getOwnerDocument().importNode(replacementElem, true), currentElem); // wrong doc node!!!
+							parent.removeChild(currentElem);
+							break;
+						}
+					}
+				}
+			}
+			//If we have a horizontal classifier and we have NOT had the top-level node requested then we need to 
+			//make sure we get the full list of top-level children to display (e.g. if the user has requested
+			//CL2.1.1 we also need to make sure we have CL2.2, CL2.3, CL2.4 etc.)
+			else
+			{
+				Element childStructure = getClassifierStructureFromID(doc, OID.getTop(classifier_node), request, collection, service_name);
+
+				String[] idParts = classifier_node.split("\\.");
+				String idToSearchFor = idParts[0] + "." + idParts[1];
+
+				Element replacementElem = null;
+				NodeList childClassifierNodes = cl_structure.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+				for (int i = 0; i < childClassifierNodes.getLength(); i++)
+				{
+					Element currentElem = (Element) childClassifierNodes.item(i);
+					if (currentElem.getAttribute(GSXML.NODE_ID_ATT).equals(idToSearchFor))
+					{
+						replacementElem = currentElem;
+						break;
+					}
+				}
+
+				if (replacementElem != null)
+				{
+					NodeList nodesToSearch = childStructure.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+					for (int i = 0; i < nodesToSearch.getLength(); i++)
+					{
+						Element currentElem = (Element) nodesToSearch.item(i);
+						if (currentElem.getAttribute(GSXML.NODE_ID_ATT).equals(idToSearchFor))
+						{
+							Element parent = (Element) currentElem.getParentNode();
+							parent.insertBefore(parent.getOwnerDocument().importNode(replacementElem,true), currentElem);
+							parent.removeChild(currentElem);
+							break;
+						}
+					}
+
+					cl_structure = childStructure;
+				}
+			}
+		}
+
+		Element page_classifier = null;
+		// add the single classifier node as the page classifier 
+		page_classifier = GSXML.duplicateWithNewName(doc, cl_structure, GSXML.CLASSIFIER_ELEM, true);
+		page_response.appendChild(page_classifier);
+		page_classifier.setAttribute(GSXML.NAME_ATT, top_id);
+
+		// get the metadata for each classifier node, 
+		// then for each document node
+
+		Element metadata_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		boolean did_classifier = false;
+		boolean did_documents = false;
+
+		// if there are classifier nodes 
+		// create a metadata request for the classifier, and add it to 
+		// the the message
+		NodeList cl_nodes = page_classifier.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+
+		if (cl_nodes.getLength() > 0)
+		{
+			did_classifier = true;
+			Element cl_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to + "MetadataRetrieve", userContext);
+			metadata_message.appendChild(cl_meta_request);
+
+			Element new_cl_nodes_list = doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+			cl_meta_request.appendChild(new_cl_nodes_list);
+
+			for (int c = 0; c < cl_nodes.getLength(); c++)
+			{
+
+				Element cl = doc.createElement(GSXML.CLASS_NODE_ELEM);
+				cl.setAttribute(GSXML.NODE_ID_ATT, ((Element) cl_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+				new_cl_nodes_list.appendChild(cl);
+			}
+
+			// create and add in the param list - for now get all the metadata
+			// should be based on info sent in from the recept, and the 
+			// format stuff
+			Element cl_param_list = createMetadataParamList(doc,class_meta_names);
+			cl_meta_request.appendChild(cl_param_list);
+
+		}
+
+		// if there are document nodes in the classification (happens 
+		// sometimes), create a second request for document metadata and 
+		// append to the message
+		NodeList doc_nodes = page_classifier.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+		if (doc_nodes.getLength() > 0)
+		{
+			did_documents = true;
+			Element doc_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, GSPath.appendLink(collection, "DocumentMetadataRetrieve"), userContext);
+			metadata_message.appendChild(doc_meta_request);
+
+			Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			doc_meta_request.appendChild(doc_list);
+
+			for (int c = 0; c < doc_nodes.getLength(); c++)
+			{
+
+				Element d = doc.createElement(GSXML.DOC_NODE_ELEM);
+				d.setAttribute(GSXML.NODE_ID_ATT, ((Element) doc_nodes.item(c)).getAttribute(GSXML.NODE_ID_ATT));
+				doc_list.appendChild(d);
+			}
+
+			// create and add in the param list - add all for now
+			Element doc_param_list = createMetadataParamList(doc,doc_meta_names);
+			doc_meta_request.appendChild(doc_param_list);
+
+		}
+
+		// process the metadata requests
+		Element metadata_response = (Element) this.mr.process(metadata_message);
+		if (did_classifier)
+		{
+			// the classifier one will be the first response
+			// add the metadata lists for each node back into the 
+			// page_classifier nodes
+			path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+			Node pathNode = GSXML.getNodeByPath(metadata_response, path);
+			if (pathNode == null)
+			{
+				return page_response;
+			}
+			//NodeList meta_response_cls = (Element)pathNode.getChildNodes(); // can't handle empty elements from converting formatted strings (with empty newlines) into XML
+			NodeList meta_response_cls = ((Element) pathNode).getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+			for (int i = 0; i < cl_nodes.getLength(); i++)
+			{
+				GSXML.mergeMetadataLists(cl_nodes.item(i), meta_response_cls.item(i));
+			}
+		}
+
+		if (did_documents)
+		{
+			NodeList meta_response_docs = null;
+			if (!did_classifier)
+			{
+				// its the first response
+				path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				Node pathNode = GSXML.getNodeByPath(metadata_response, path);
+				if (pathNode == null)
+				{
+					return page_response;
+				}
+
+				meta_response_docs = pathNode.getChildNodes();
+
+			}
+			else
+			{ // its the second response
+				Node nodes = GSXML.getChildByTagName(metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1), GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				if (nodes == null)
+				{
+					return page_response;
+				}
+				meta_response_docs = nodes.getChildNodes();
+			}
+
+			for (int i = 0; i < doc_nodes.getLength(); i++)
+			{
+				GSXML.mergeMetadataLists(doc_nodes.item(i), meta_response_docs.item(i));
+			}
+		}
+
+		logger.debug("(GS2BrowseAction) Page:\n" + this.converter.getPrettyString(page_response));
+		return page_response;
+	}
+
+	private Element getClassifierStructureFromID(Document doc, String id, Element request, String collection, String service_name)
+	{
+		UserContext userContext = new UserContext(request);
+		String to = GSPath.appendLink(collection, service_name);
+
+		Element firstClassifierNodeChildrenMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element firstClassifierNodeChildrenRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		firstClassifierNodeChildrenMessage.appendChild(firstClassifierNodeChildrenRequest);
+
+		Element paramList = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		firstClassifierNodeChildrenRequest.appendChild(paramList);
+
+		Element ancestorParam = doc.createElement(GSXML.PARAM_ELEM);
+		paramList.appendChild(ancestorParam);
+		ancestorParam.setAttribute(GSXML.NAME_ATT, "structure");
+		ancestorParam.setAttribute(GSXML.VALUE_ATT, "ancestors");
+
+		Element childrenParam = doc.createElement(GSXML.PARAM_ELEM);
+		paramList.appendChild(childrenParam);
+		childrenParam.setAttribute(GSXML.NAME_ATT, "structure");
+		childrenParam.setAttribute(GSXML.VALUE_ATT, "children");
+
+		Element classifierToGetList = doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element classifierToGet = doc.createElement(GSXML.CLASS_NODE_ELEM);
+		classifierToGet.setAttribute(GSXML.NODE_ID_ATT, id);
+		classifierToGetList.appendChild(classifierToGet);
+		firstClassifierNodeChildrenRequest.appendChild(classifierToGetList);
+
+		Element firstClassifierNodeChildrenResponse = (Element) this.mr.process(firstClassifierNodeChildrenMessage);
+
+		String nsPath = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element topClassifierNode = (Element) GSXML.getNodeByPath(firstClassifierNodeChildrenResponse, nsPath);
+		nsPath = GSPath.appendLink(GSXML.CLASS_NODE_ELEM, GSXML.NODE_STRUCTURE_ELEM);
+		nsPath = GSPath.appendLink(nsPath, GSXML.CLASS_NODE_ELEM);
+		Element childStructure = (Element) GSXML.getNodeByPath(topClassifierNode, nsPath);
+
+		return childStructure;
+	}
+
+	protected void extractMetadataNames(Element new_format, HashSet<String> doc_meta_names, HashSet<String> class_meta_names)
+	{
+		NodeList templates = new_format.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "template");
+		for (int i = 0; i < templates.getLength(); i++)
+		{
+			Element template = (Element) templates.item(i);
+			String match = template.getAttribute(GSXML.MATCH_ATT);
+			String name = template.getAttribute(GSXML.NAME_ATT);
+			if (match != null && match.length() > 0)
+			{
+				if (match.startsWith("documentNode"))
+				{
+					getRequiredMetadataNames(template, doc_meta_names);
+				}
+				else if (match.startsWith("classifierNode")) // not match.equals, as we want to match nodes like: classifierNode[@classifierStyle = 'VList']
+				{
+					getRequiredMetadataNames(template, class_meta_names);
+				}
+			}
+			else
+			{
+				getRequiredMetadataNames(template, doc_meta_names);
+				getRequiredMetadataNames(template, class_meta_names);
+			}
+		}
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GeneralAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GeneralAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/GeneralAction.java	(revision 31634)
@@ -0,0 +1,201 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashMap;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Result;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class GeneralAction extends Action
+{
+
+	/** process a request */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// the result
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+		result.appendChild(page_response);
+
+		// assume only one request
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		logger.debug(" request=" + this.converter.getString(request));
+
+		UserContext userContext = new UserContext(request);
+
+		// get the param list
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		if (params.get("configChangeName") != null && params.get("configChangeValue") != null)
+		{
+			String optionName = (String) params.get("configChangeName");
+			String optionValue = (String) params.get("configChangeValue");
+
+			changeConfig(optionName, optionValue);
+		}
+
+		String service_name = (String) params.get(GSParams.SERVICE);
+		String cluster_name = (String) params.get(GSParams.CLUSTER);
+		String response_only_p = (String) params.get(GSParams.RESPONSE_ONLY);
+		boolean response_only = false;
+		if (response_only_p != null)
+		{
+			response_only = (response_only_p.equals("1") ? true : false);
+		}
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		// what is carried out depends on the request_type
+		// if rt=d, then a describe request is done,
+		// is rt=r, a request and then a describe request is done
+		// if rt=s, a status request is done.
+
+		// if ro=1, then this calls for a process only page - we do the request
+		// (rt should be r or s) and just give the response straight back
+		// without any page processing
+
+		// where to send requests
+		String to;
+		if (cluster_name != null)
+		{
+			to = GSPath.appendLink(cluster_name, service_name);
+		}
+		else
+		{
+			to = service_name;
+		}
+
+		if (request_type.equals("r") || request_type.equals("s") || request_type.equals("ro"))
+		{
+			//do the request
+			Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+
+			if (request_type.equals("s"))
+			{
+				mr_query_request.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_STATUS);
+			}
+
+			mr_query_message.appendChild(mr_query_request);
+
+			Element param_list = null;
+			// add in the service params - except the ones only used by the action
+			HashMap service_params = (HashMap) params.get("s1");
+			if (service_params != null)
+			{
+				param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				GSXML.addParametersToList(param_list, service_params);
+				mr_query_request.appendChild(param_list);
+			}
+
+			Element userInformation = (Element) GSXML.getChildByTagName(request, GSXML.USER_INFORMATION_ELEM);
+			if (userInformation != null)
+			{
+				mr_query_request.appendChild(doc.importNode(userInformation, true));
+			}
+			mr_query_request.setAttribute("remoteAddress", request.getAttribute("remoteAddress"));
+
+			Element mr_query_response = (Element) this.mr.process(mr_query_message);
+			Element result_response = (Element) GSXML.getChildByTagName(mr_query_response, GSXML.RESPONSE_ELEM);
+
+			if (response_only)
+			{
+				// just send the reponse as is
+				addSiteMetadata(result_response, userContext);
+				addInterfaceOptions(result_response);
+				return result_response;
+			}
+			if (result_response != null)
+			{
+				// else append the contents of the response to the page 
+				GSXML.copyAllChildren(page_response, result_response);
+			}
+		}
+
+		// another part of the page is the service description
+
+		// request the service info for the selected service - should be cached
+		Element mr_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+		mr_info_message.appendChild(mr_info_request);
+		Element mr_info_response = (Element) this.mr.process(mr_info_message);
+
+		String path = GSXML.RESPONSE_ELEM;
+		path = GSPath.appendLink(path, GSXML.SERVICE_ELEM);
+
+		Node desNode = GSXML.getNodeByPath(mr_info_response, path);
+		if (desNode != null)
+		{
+			page_response.appendChild((Element) doc.importNode(desNode, true));
+		}
+
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+
+		return result;
+	}
+
+	protected void changeConfig(String optionName, String optionValue)
+	{
+		if (this.config_params.get(optionName) != null)
+		{
+			this.config_params.put(optionName, optionValue);
+
+			File interfaceConfigFile = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get("interface_name"))));
+
+			Document interfaceXML = null;
+			try
+			{
+				DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+				DocumentBuilder db = dbf.newDocumentBuilder();
+				interfaceXML = db.parse(interfaceConfigFile);
+				Element topElement = interfaceXML.getDocumentElement();
+				Element optionListElem = (Element) GSXML.getChildByTagName(topElement, "optionList");
+
+				NodeList optionList = optionListElem.getElementsByTagName("option");
+
+				for (int i = 0; i < optionList.getLength(); i++)
+				{
+					Element currentOption = (Element) optionList.item(i);
+					if (currentOption.getAttribute(GSXML.NAME_ATT) != null && currentOption.getAttribute(GSXML.NAME_ATT).equals(optionName))
+					{
+						currentOption.setAttribute(GSXML.VALUE_ATT, optionValue);
+					}
+				}
+
+				DOMSource source = new DOMSource(interfaceXML);
+				Result xmlresult = new StreamResult(interfaceConfigFile);
+
+				Transformer transformer = TransformerFactory.newInstance().newTransformer();
+				transformer.transform(source, xmlresult);
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+		}
+		else
+		{
+			logger.error("Could not set param \"" + optionName + "\" to \"" + optionValue + "\" because that option does not exist.");
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/NoCollQueryAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/NoCollQueryAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/NoCollQueryAction.java	(revision 31634)
@@ -0,0 +1,251 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.log4j.*;
+
+/**
+ * action class for queries this is used when querying isn't collection
+ * specific, but it occurs across all collections in the site. The service
+ * description is assumed to be known by the xslt so we dont ask for it. we just
+ * pass all the service params to the TextQuery service of all the collections
+ */
+public class NoCollQueryAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.NoCollQueryAction.class.getName());
+
+	/**
+	 * process - processes a request.
+	 */
+	public Node process(Node message_node)
+	{
+
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// get the request - assume there is only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// create the return message
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		// for now we only have one type of query - subaction not used
+		Element response = basicQuery(request);
+		result.appendChild(doc.importNode(response, true));
+		return result;
+	}
+
+	/**
+	 * a generic query handler this gets the service description, does the query
+	 * (just passes all the params to the service, then gets the titles for any
+	 * results
+	 */
+	protected Element basicQuery(Element request)
+	{
+		// the result
+		Document doc = request.getOwnerDocument();
+		
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// extract the params from the cgi-request, and check that we have a coll specified
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		UserContext userContext = new UserContext(request);
+		if (request_type.equals("d"))
+		{
+			// display the query page
+			// the only info we need to return is the collection list cos the xslt does teh rest
+
+			Element coll_list = getCollectionList(doc, userContext);
+			page_response.appendChild(doc.importNode(coll_list, true));
+			return page_response;
+
+		}
+
+		// else we have a query
+		String service_name = (String) params.get(GSParams.SERVICE);
+		if (service_name == null || service_name.equals(""))
+		{
+			service_name = "TextQuery";
+		}
+		String query_coll_list = (String) params.get(GSParams.COLLECTION);
+
+		if (query_coll_list == null || query_coll_list.equals(""))
+		{
+			logger.error("no collections were specified!");
+			Element coll_list = getCollectionList(doc,userContext);
+			page_response.appendChild(doc.importNode(coll_list, true));
+			return page_response;
+		}
+
+		// service paramList
+		HashMap service_params = (HashMap) params.get("s1");
+		if (service_params == null)
+		{ // no query
+			return page_response;
+		}
+		Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParametersToList(query_param_list, service_params);
+
+		// we will do a query for each coll
+		String[] colls = query_coll_list.split(",");
+
+		Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		for (int i = 0; i < colls.length; i++)
+		{
+			String to = GSPath.appendLink(colls[i], service_name);
+			Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+			mr_query_message.appendChild(mr_query_request);
+			mr_query_request.appendChild(query_param_list.cloneNode(true));
+
+		}
+
+		// do the query
+		Element mr_query_response = (Element) this.mr.process(mr_query_message);
+
+		// get the Title metadata for each node - need Node title and root title
+		Element mr_meta_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		NodeList responses = mr_query_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		for (int j = 0; j < responses.getLength(); j++)
+		{
+			Element document_list = (Element) GSXML.getChildByTagName((Element) responses.item(j), GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			if (document_list != null)
+			{
+				String coll_name = extractCollName(((Element) responses.item(j)).getAttribute(GSXML.FROM_ATT));
+				String path = GSPath.appendLink(coll_name, "DocumentMetadataRetrieve");
+				Element mr_meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, path, userContext);
+				mr_meta_message.appendChild(mr_meta_request);
+				mr_meta_request.appendChild(doc.importNode(document_list, true));
+				// metadata params 
+				Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				Element param = GSXML.createParameter(doc, "metadata", "Title");
+				param_list.appendChild(param);
+				param = GSXML.createParameter(doc, "metadata", "root_Title");
+				param_list.appendChild(param);
+				mr_meta_request.appendChild(param_list);
+
+			}
+		}
+
+		// do the request
+		Element mr_meta_response = (Element) this.mr.process(mr_meta_message);
+
+		// the result 
+		Element result_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		page_response.appendChild(result_doc_list);
+
+		responses = mr_meta_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		for (int j = 0; j < responses.getLength(); j++)
+		{
+			Element document_list = (Element) GSXML.getChildByTagName((Element) responses.item(j), GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			String coll_name = extractCollName(((Element) responses.item(j)).getAttribute(GSXML.FROM_ATT));
+
+			mergeDocLists(result_doc_list, document_list, coll_name);
+		}
+
+		return page_response;
+	}
+
+	protected String extractCollName(String path)
+	{
+		return GSPath.removeLastLink(path);
+	}
+
+	protected void mergeDocLists(Element result_list, Element from_list, String collection)
+	{
+
+		Document owner = result_list.getOwnerDocument();
+		Node child = from_list.getFirstChild();
+		while (child != null && child.getNodeType() == Node.ELEMENT_NODE)
+		{
+			((Element) child).setAttribute("collection", collection);
+			result_list.appendChild(owner.importNode(child, true));
+			child = child.getNextSibling();
+		}
+
+	}
+
+	protected Element getCollectionList(Document doc, UserContext userContext)
+	{
+
+		// first, get the message router info
+		Element coll_list_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element coll_list_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		coll_list_message.appendChild(coll_list_request);
+		Element coll_list_response = (Element) this.mr.process(coll_list_message);
+		if (coll_list_response == null)
+		{
+			logger.error("couldn't query the message router!");
+			return null;
+		}
+
+		// second, get the display info for each collection 
+		NodeList colls = coll_list_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+
+		Element coll_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element param = GSXML.createParameter(doc, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		coll_param_list.appendChild(param);
+		// we will send all the requests in a single message
+		Element metadata_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		for (int i = 0; i < colls.getLength(); i++)
+		{
+			Element c = (Element) colls.item(i);
+			String name = c.getAttribute(GSXML.NAME_ATT);
+
+			Element metadata_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, name, userContext);
+			metadata_request.appendChild(coll_param_list.cloneNode(true));
+			metadata_message.appendChild(metadata_request);
+		}
+
+		Element metadata_response = (Element) this.mr.process(metadata_message);
+
+		NodeList coll_responses = metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		// check that have same number of responses as collections
+		if (colls.getLength() != coll_responses.getLength())
+		{
+			logger.error("didn't get a response for each collection - somethings gone wrong!");
+			// for now, dont use the metadata
+		}
+		else
+		{
+			for (int i = 0; i < colls.getLength(); i++)
+			{
+				Element c1 = (Element) colls.item(i);
+				Element c2 = (Element) coll_responses.item(i);
+				if (c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.FROM_ATT)))
+				{
+					//add the collection data into the original response
+					GSXML.mergeElements(c1, (Element) GSXML.getChildByTagName(c2, GSXML.COLLECTION_ELEM));
+				}
+				else
+				{
+					logger.error("response does not correspond to request!");
+				}
+
+			}
+		}
+
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		Element response = (Element) GSXML.getNodeByPath(coll_list_response, path);
+		return response;
+
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/PageAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/PageAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/PageAction.java	(revision 31634)
@@ -0,0 +1,430 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class PageAction extends Action
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.PageAction.class.getName());
+
+	public static final String HOME_PAGE = "home";
+	public static final String ABOUT_PAGE = "about";
+	public static final String PREFS_PAGE = "pref";
+	public static final String GLI4GS3_PAGE = "gli4gs3";
+
+	public Node process(Node message_node)
+	
+	{
+		Element message = GSXML.nodeToElement(message_node);
+		Document doc = XMLConverter.newDOM();
+	    
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		String collection = "";
+		if (paramList != null)
+		{
+			HashMap<String, Serializable> params = GSXML.extractParams(paramList, false);
+			if (params != null && params.get(GSParams.COLLECTION) != null)
+			{
+				collection = (String) params.get(GSParams.COLLECTION);
+			}
+		}
+
+		// the page name is the subaction
+		String page_name = request.getAttribute(GSXML.SUBACTION_ATT);
+		if (page_name.equals(""))
+		{ // if no page specified, assume home page
+			page_name = HOME_PAGE;
+		}
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element response;
+		if (page_name.equals(HOME_PAGE))
+		{
+		  response = homePage(request);
+			//} else if (page_name.equals(ABOUT_PAGE)) {
+		}
+		else if (page_name.equals(ABOUT_PAGE) || page_name.equals(PREFS_PAGE))
+		{
+		  response = aboutPage(request);
+			//}else if (page_name.equals(PREFS_PAGE)) {
+			//response = prefsPage(request);
+		}
+		else if (page_name.equals(GLI4GS3_PAGE))
+		{
+		  response = gli4gs3Page(request);
+		}
+		else
+		{ // unknown page
+		  response = unknownPage(request);
+		}
+
+		Element formatMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element formatRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_FORMAT, collection, new UserContext(request));
+		formatMessage.appendChild(formatRequest);
+		Element formatResponseMessage = (Element) this.mr.process(formatMessage);
+		Element formatResponse = (Element) GSXML.getChildByTagName(formatResponseMessage, GSXML.RESPONSE_ELEM);
+
+		Element globalFormat = (Element) GSXML.getChildByTagName(formatResponse, GSXML.FORMAT_ELEM);
+		if (globalFormat != null)
+		{
+		  response.appendChild(response.getOwnerDocument().importNode(globalFormat, true));
+		}
+
+		result.appendChild(doc.importNode(response, true));
+		logger.debug("page action result: " + this.converter.getPrettyString(result));
+
+		return result;
+	}
+
+  protected Element homePage(Element request)
+	{
+	  Document doc = XMLConverter.newDOM();
+		
+
+		UserContext userContext = new UserContext(request);
+		// first, get the message router info
+		Element info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		//Create param list
+		Element param_list_element = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		info_request.appendChild(param_list_element);
+		//Describe params without collectionlist. Collectionlist provided by CollectionGroup service
+		GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM, GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM, GSXML.SITE_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		info_message.appendChild(info_request);
+		//Send request to message router
+		Element info_response_message = (Element) this.mr.process(info_message);
+		//Check if it is not null
+		if (info_response_message == null)
+		{
+			logger.error(" couldn't query the message router!");
+			return null;
+		}
+		//Check if it is not null
+		Element info_response = (Element) GSXML.getChildByTagName(info_response_message, GSXML.RESPONSE_ELEM);
+		if (info_response == null)
+		{
+			logger.error("couldn't query the message router!");
+			return null;
+		}
+		
+		Element resp_service_list = (Element) GSXML.getChildByTagName(info_response, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		
+		if (resp_service_list == null) {
+			logger.error("No services available. Couldn't query the message router!");
+			return null;
+		}
+		Element groupInfoService = GSXML.getNamedElement(resp_service_list, GSXML.SERVICE_ELEM, GSXML.TYPE_ATT,
+				GSXML.SERVICE_TYPE_GROUPINFO);
+		if (groupInfoService != null) {
+			// Prepare request for CollectionGroup service to get current
+			// collections and groups list
+			Element group_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			Element group_info_request = GSXML.createBasicRequest(doc, GSXML.TO_ATT,
+					groupInfoService.getAttribute(GSXML.NAME_ATT), userContext);
+			group_info_message.appendChild(group_info_request);
+			//Append group request if exists
+			Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (paramList != null) {
+				group_info_request.appendChild(doc.importNode(paramList, true));
+			}
+			Element group_info_response_message = (Element) this.mr.process(group_info_message);
+			Element group_info_response = (Element) GSXML.getChildByTagName(group_info_response_message,
+					GSXML.RESPONSE_ELEM);
+			Element collection_list = (Element) GSXML.getChildByTagName(group_info_response,
+					GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+			// Add Collection List from CollectionGroup Service to response
+			// from message router
+			info_response = (Element) doc.importNode(info_response, true);
+			if (collection_list != null) {
+				info_response.appendChild(doc.importNode(collection_list, true));
+			}
+			Element group_list = (Element) GSXML.getChildByTagName(group_info_response,
+					GSXML.GROUP_ELEM + GSXML.LIST_MODIFIER);
+			if (group_list != null) {
+				info_response.appendChild(doc.importNode(group_list, true));
+			}
+			Element path_list = (Element) GSXML.getChildByTagName(group_info_response,
+					GSXML.PATH_ELEM + GSXML.LIST_MODIFIER);
+			if (path_list != null) {
+				info_response.appendChild(doc.importNode(path_list, true));
+			}
+			// Send message to groupInfoType Services
+		} else {
+			// If no service with type SERVICE_TYPE_GROUPINFO could be provided
+			// request message router for all available collections
+			GSXML.addParameterToList(param_list_element, GSXML.SUBSET_PARAM,
+					GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+			info_response_message = (Element) this.mr.process(info_message);
+
+			if (info_response_message == null) {
+				logger.error(" couldn't query the message router!");
+				return null;
+			}
+			info_response = (Element) GSXML.getChildByTagName(info_response_message, GSXML.RESPONSE_ELEM);
+			if (info_response == null) {
+				logger.error("couldn't query the message router!");
+				return null;
+			}
+		}
+
+		// second, get the metadata for each collection - we only want specific
+		// elements but for now, we'll just get it all
+		Element collection_list = (Element) GSXML.getChildByTagName(info_response, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		//logger.debug(GSXML.xmlNodeToString(collection_list));
+		if (collection_list != null)
+		{
+			NodeList colls = GSXML.getChildrenByTagName(collection_list, GSXML.COLLECTION_ELEM);
+			if (colls.getLength() > 0)
+			{
+				sendMultipleRequests(doc, colls, null, GSXML.REQUEST_TYPE_DESCRIBE, userContext);
+			}
+		}
+
+		// get metadata for any services
+		Element service_list = (Element) GSXML.getChildByTagName(info_response, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		if (service_list != null)
+		{
+			NodeList services = GSXML.getChildrenByTagName(service_list, GSXML.SERVICE_ELEM);
+			if (services.getLength() > 0)
+			{
+				sendMultipleRequests(doc, services, null, GSXML.REQUEST_TYPE_DESCRIBE, userContext);
+			}
+		}
+
+		// get metadata for service clusters
+		Element cluster_list = (Element) GSXML.getChildByTagName(info_response, GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+		if (cluster_list != null)
+		{
+			NodeList clusters = GSXML.getChildrenByTagName(cluster_list, GSXML.CLUSTER_ELEM);
+			if (clusters.getLength() > 0)
+			{
+				sendMultipleRequests(doc, clusters, null, GSXML.REQUEST_TYPE_DESCRIBE, userContext);
+
+			}
+		}
+
+		addSiteMetadata(info_response, userContext);
+		addInterfaceOptions(info_response);
+		// all the components have been merged into info_response
+		return info_response;
+
+	} // homePage
+
+	protected Element aboutPage(Element request)
+	{
+	  Document doc = XMLConverter.newDOM();
+		
+		UserContext userContext = new UserContext(request);
+		// extract the params from the cgi-request, 
+		Element cgi_paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_paramList, false);
+
+		String coll_name = (String) params.get(GSParams.COLLECTION);
+		if (coll_name == null || coll_name.equals(""))
+		{
+			logger.error("about page requested with no collection or cluster specified!");
+			// return an empty response
+			Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+			addSiteMetadata(response, userContext);
+			addInterfaceOptions(response);
+			return response;
+		}
+
+		// get the collection or cluster description
+		Element coll_about_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		Element coll_about_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, coll_name, userContext);
+		coll_about_message.appendChild(coll_about_request);
+		Element coll_about_response = (Element) this.mr.process(coll_about_message);
+
+		// add collection type attribute to paramList
+		String col_type = "";
+		NodeList collect_elem = coll_about_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		if (collect_elem.getLength() != 0)
+		{
+			for (int i = 0; i < collect_elem.getLength(); i++)
+			{
+				Element e = (Element) collect_elem.item(i);
+				col_type = e.getAttribute(GSXML.TYPE_ATT);
+			}
+		}
+		else
+		{
+			logger.error(GSXML.COLLECTION_ELEM + " element is null");
+		}
+
+		NodeList paramList = request.getElementsByTagName(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (paramList.getLength() != 0)
+		{
+			for (int i = 0; i < paramList.getLength(); i++)
+			{
+				Element e = (Element) paramList.item(i);
+				Element ct = GSXML.createParameter(request.getOwnerDocument(), GSParams.COLLECTION_TYPE, col_type.equalsIgnoreCase("mg") ? "0" : "1");
+				e.appendChild(ct);
+			}
+		}
+		else
+		{
+			logger.info("paramList is null!!");
+		}
+
+		if (coll_about_response == null)
+		{
+			return null;
+		}
+
+		// second, get the info for each service - we only want display items 
+		// but for now, we'll just get it all
+		NodeList services = coll_about_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+		if (services.getLength() > 0)
+		{
+			sendMultipleRequests(doc, services, coll_name, GSXML.REQUEST_TYPE_DESCRIBE, userContext);
+		}
+
+		Element response = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.RESPONSE_ELEM);
+		//add the site metadata
+		addSiteMetadata(response, userContext);
+		addInterfaceOptions(response);
+		return response;
+	}
+
+	//protected Element prefsPage(Element request) {
+
+	//	return null;
+	//}
+
+	/** if we dont know the page type, use this method */
+	protected Element unknownPage(Element request)
+	{
+	  Document doc = XMLConverter.newDOM();
+		
+		UserContext userContext = new UserContext(request);
+		String page_name = request.getAttribute(GSXML.SUBACTION_ATT);
+
+		// extract the params from the cgi-request, 
+		Element cgi_paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_paramList, false);
+
+		String coll_name = (String) params.get(GSParams.COLLECTION);
+		if (coll_name == null || coll_name.equals(""))
+		{
+			// just return an empty response
+			Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+			addSiteMetadata(response, userContext);
+			addInterfaceOptions(response);
+			return response;
+		}
+
+		// else get the coll description - actually this is the same as for the about page - should we merge these two methods??
+
+		// if there is a service specified should we get the service description instead??
+		// get the collection or cluster description
+		Element coll_about_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		Element coll_about_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, coll_name, userContext);
+		coll_about_message.appendChild(coll_about_request);
+
+		Element coll_about_response = (Element) this.mr.process(coll_about_message);
+
+		Element response = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.RESPONSE_ELEM);
+
+		//add the site metadata
+		addSiteMetadata(response, userContext);
+		addInterfaceOptions(response);
+
+		return response;
+
+	}
+
+	protected boolean sendMultipleRequests(Document doc, NodeList items, String path_prefix, String request_type, UserContext userContext)
+	{
+		// we will send all the requests in a single message
+		Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+		for (int i = 0; i < items.getLength(); i++)
+		{
+			Element c = (Element) items.item(i);
+			String path = c.getAttribute(GSXML.NAME_ATT);
+			if (path_prefix != null)
+			{
+				path = GSPath.appendLink(path_prefix, path);
+			}
+			Element request = GSXML.createBasicRequest(doc, request_type, path, userContext);
+			message.appendChild(request);
+		}
+
+		Element response_message = (Element) this.mr.process(message);
+
+		NodeList responses = response_message.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		// check that have same number of responses as requests
+		if (items.getLength() != responses.getLength())
+		{
+			logger.error("didn't get a response for each request - somethings gone wrong!");
+			return false;
+		}
+
+		for (int i = 0; i < items.getLength(); i++)
+		{
+			Element c1 = (Element) items.item(i);
+			Element c2 = (Element) GSXML.getChildByTagName((Element) responses.item(i), c1.getTagName());
+			if (c1 != null && c2 != null && c1.getAttribute(GSXML.NAME_ATT).endsWith(c2.getAttribute(GSXML.NAME_ATT)))
+			{
+				//add the new data into the original element
+				GSXML.mergeElements(c1, c2);
+			}
+			else
+			{
+				logger.debug(" response does not correspond to request!");
+			}
+
+		}
+
+		return true;
+
+	}
+
+	protected Element gli4gs3Page(Element request)
+	{
+	  Document doc = XMLConverter.newDOM();
+		
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String uid = request.getAttribute(GSXML.USER_ID_ATT);
+
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+		Element applet_elem = doc.createElement("Applet");
+		page_response.appendChild(applet_elem);
+		applet_elem.setAttribute("ARCHIVE", "SignedGatherer.jar");
+		applet_elem.setAttribute("CODE", "org.greenstone.gatherer.GathererApplet");
+		applet_elem.setAttribute("CODEBASE", "applet");
+		applet_elem.setAttribute("HEIGHT", "50");
+		applet_elem.setAttribute("WIDTH", "380");
+		Element gwcgi_param_elem = doc.createElement("PARAM");
+		gwcgi_param_elem.setAttribute("name", "gwcgi");
+		String library_name = GlobalProperties.getGSDL3WebAddress();
+		gwcgi_param_elem.setAttribute("value", library_name);
+		applet_elem.appendChild(gwcgi_param_elem);
+
+		Element gsdl3_param_elem = doc.createElement("PARAM");
+		gsdl3_param_elem.setAttribute("name", "gsdl3");
+		gsdl3_param_elem.setAttribute("value", "true");
+		applet_elem.appendChild(gsdl3_param_elem);
+
+		return page_response;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/ProcessAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/ProcessAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/ProcessAction.java	(revision 31634)
@@ -0,0 +1,154 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+import java.io.Serializable;
+
+public class ProcessAction extends Action
+{
+
+	/** process a request */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// the result
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+		result.appendChild(page_response);
+
+		// assume only one request
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// get the param list
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+		String service_name = (String) params.get(GSParams.SERVICE);
+		String cluster_name = (String) params.get(GSParams.CLUSTER);
+		String response_only_p = (String) params.get(GSParams.RESPONSE_ONLY);
+		boolean response_only = false;
+		if (response_only_p != null)
+		{
+			response_only = (response_only_p.equals("1") ? true : false);
+		}
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		UserContext userContext = new UserContext(request);
+		// what is carried out depends on the request_type
+		// if rt=d, then a describe request is done,
+		// is rt=r, a request and then a describe request is done
+		// if rt=s, a status request is done.
+
+		// if ro=1, then this calls for a process only page - we do the request
+		// (rt should be r or s) and just give the response straight back
+		// without any page processing
+
+		// where to send requests
+		String to;
+		if (cluster_name != null)
+		{
+
+			to = GSPath.appendLink(cluster_name, service_name);
+		}
+		else
+		{
+			to = service_name;
+		}
+
+		if (!request_type.equals("d"))
+		{
+			// if rt=s or rt=r, do the request
+
+			Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			String request_type_att;
+			Element param_list = null;
+			if (request_type.equals("s"))
+			{ // status
+				request_type_att = GSXML.REQUEST_TYPE_STATUS;
+				param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				Element param = doc.createElement(GSXML.PARAM_ELEM);
+				param.setAttribute(GSXML.NAME_ATT, GSParams.PROCESS_ID);
+				param.setAttribute(GSXML.VALUE_ATT, (String) params.get(GSParams.PROCESS_ID));
+				param_list.appendChild(param);
+			}
+			else
+			{
+				request_type_att = GSXML.REQUEST_TYPE_PROCESS;
+				// add in the service params - except the ones only used by the action
+				HashMap service_params = (HashMap) params.get("s1");
+				if (service_params != null)
+				{
+					param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+					GSXML.addParametersToList(param_list, service_params);
+				}
+
+			}
+			Element mr_query_request = GSXML.createBasicRequest(doc, request_type_att, to, userContext);
+			if (param_list != null)
+			{
+				mr_query_request.appendChild(param_list);
+			}
+			mr_query_message.appendChild(mr_query_request);
+
+			Element mr_query_response = (Element) this.mr.process(mr_query_message);
+			Element result_response = (Element) GSXML.getChildByTagName(mr_query_response, GSXML.RESPONSE_ELEM);
+
+			if (response_only)
+			{
+				// just send the reponse as is
+				return result_response;
+			}
+
+			// else append the contents of the response to the page - just the status elem for now
+			Element status = (Element) GSXML.getChildByTagName(result_response, GSXML.STATUS_ELEM);
+			page_response.appendChild(doc.importNode(status, true));
+		}
+
+		// another part of the page is the service description
+
+		// request the service info for the selected service - should be cached
+		Element mr_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+		mr_info_message.appendChild(mr_info_request);
+		Element mr_info_response = (Element) this.mr.process(mr_info_message);
+
+		String path = GSXML.RESPONSE_ELEM;
+		path = GSPath.appendLink(path, GSXML.SERVICE_ELEM);
+		Element description = (Element) doc.importNode(GSXML.getNodeByPath(mr_info_response, path), true);
+
+		page_response.appendChild(description);
+
+		return result;
+	}
+
+	protected Element getServiceParamList(Element cgi_param_list)
+	{
+		Document doc = cgi_param_list.getOwnerDocument();
+		
+		Element new_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element param;
+		NodeList cgi_params = cgi_param_list.getChildNodes();
+		for (int i = 0; i < cgi_params.getLength(); i++)
+		{
+			Element p = (Element) cgi_params.item(i);
+			String name = p.getAttribute(GSXML.NAME_ATT);
+			if (name.equals(GSParams.SERVICE) || name.equals(GSParams.REQUEST_TYPE) || name.equals(GSParams.CLUSTER))
+			{
+				continue;
+			}
+			// else add it in to the list
+			new_param_list.appendChild(doc.importNode(p, true));
+		}
+		return new_param_list;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/QueryAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/QueryAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/QueryAction.java	(revision 31634)
@@ -0,0 +1,431 @@
+package org.greenstone.gsdl3.action;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/** action class for queries */
+public class QueryAction extends Action
+{
+
+  public static final String HITS_PER_PAGE = "hitsPerPage";
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.QueryAction.class.getName());
+  
+	/**
+	 * process - processes a request.
+	 */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// get the request - assume there is only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// create the return message
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element response = basicQuery(request);
+		result.appendChild(doc.importNode(response, true));
+		return result;
+	}
+
+	/**
+	 * a generic query handler this gets the service description, does the query
+	 * (just passes all the params to the service, then gets the titles for any
+	 * results
+	 */
+	protected Element basicQuery(Element request)
+	{
+		// the result
+		Document doc = request.getOwnerDocument();
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// extract the params from the cgi-request, and check that we have a coll specified
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		String request_type = (String) params.get(GSParams.REQUEST_TYPE);
+		String service_name = (String) params.get(GSParams.SERVICE);
+		String collection = (String) params.get(GSParams.COLLECTION);
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		// collection may be null or empty when we are doing cross coll services
+		if (collection == null || collection.equals(""))
+		{
+			collection = null;
+		}
+
+		UserContext userContext = new UserContext(request);
+		String to = service_name;
+		if (collection != null)
+		{
+			to = GSPath.prependLink(to, collection);
+		}
+
+		// get the format info - there may be global format info in the collection that searching needs
+		Element format_elem = getFormatInfo(to, userContext);
+		// set the format type
+		format_elem.setAttribute(GSXML.TYPE_ATT, "search");
+		// for now just add to the response
+		page_response.appendChild(doc.importNode(format_elem, true));
+
+		//if (request_type.indexOf("d") != -1)
+		//{
+		// get the service description
+			// we have been asked for the service description
+			Element mr_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+			Element mr_info_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+			mr_info_message.appendChild(mr_info_request);
+
+			// process the message
+			Element mr_info_response = (Element) this.mr.process(mr_info_message);
+			// the response
+
+			Element service_response = (Element) GSXML.getChildByTagName(mr_info_response, GSXML.RESPONSE_ELEM);
+
+			Element service_description = (Element) doc.importNode(GSXML.getChildByTagName(service_response, GSXML.SERVICE_ELEM), true);
+
+
+			// have we been asked to return it as part of the response?
+			if (request_type.indexOf("d") != -1) {
+			  page_response.appendChild(service_description);
+			  addCollectionsHierarchy(page_response,userContext);
+			}
+			//}
+		boolean does_paging = false;
+		Element meta_list =(Element) GSXML.getChildByTagName(service_description, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		if (meta_list != null) {
+		  String value = GSXML.getMetadataValue(meta_list, "does_paging");
+		  if (value.equals("true")) {
+		    does_paging = true;
+		  }
+		}
+
+		if (does_paging == false) {
+		  // we will do the paging, so lets add in a hitsPerPage param to the service
+		  addHitsParamToService(doc, service_description, lang);
+		}
+		if (request_type.indexOf("r") == -1)
+		{
+			// just a display request, no actual processing to do
+			//append site metadata
+			addSiteMetadata(page_response, userContext);
+			addInterfaceOptions(page_response);
+			return page_response;
+		}
+
+		// check that we have some service params
+		HashMap service_params = (HashMap) params.get("s1");
+		if (service_params == null)
+		{ // no query
+			//append site metadata
+			addSiteMetadata(page_response, userContext);
+			addInterfaceOptions(page_response);
+			return page_response;
+		}
+
+		// create the query request
+		Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		mr_query_message.appendChild(mr_query_request);
+
+		Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		GSXML.addParametersToList(query_param_list, service_params);
+		mr_query_request.appendChild(query_param_list);
+
+		// do the query
+		Element mr_query_response = (Element) this.mr.process(mr_query_message);
+
+		// check for errors
+		if (processErrorElements(mr_query_response, page_response))
+		{
+			//append site metadata
+			addSiteMetadata(page_response, userContext);
+			addInterfaceOptions(page_response);
+			return page_response;
+		}
+
+		Element query_response = (Element) GSXML.getChildByTagName(mr_query_response, GSXML.RESPONSE_ELEM);
+		Element query_result_metadata_list = (Element) GSXML.getChildByTagName(query_response, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		
+		if (query_result_metadata_list == null)
+		{
+			logger.error("No query result metadata.\n");
+		}
+		else
+		{ // add it into the page response
+			page_response.appendChild(doc.importNode(query_result_metadata_list, true));
+		}
+
+		Element query_term_info_list = (Element) GSXML.getChildByTagName(query_response, GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
+		if (query_term_info_list == null)
+		{
+			logger.error("No query term information.\n");
+		}
+		else
+		{ // add it into the page response
+			page_response.appendChild(doc.importNode(query_term_info_list, true));
+		}
+
+		Element facet_list = (Element) GSXML.getChildByTagName(query_response, GSXML.FACET_ELEM + GSXML.LIST_MODIFIER);
+		if (facet_list == null)
+		{
+			logger.error("No query term information.\n");
+		}
+		else
+		{ // add it into the page response
+			page_response.appendChild(doc.importNode(facet_list, true));
+		}
+
+		// check that there are some documents - for now check the list, but later should use a numdocs metadata elem	
+		Element document_list = (Element) GSXML.getChildByTagName(query_response, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		// documentList not present if no docs found
+		if (document_list == null)
+		{
+			// add in a dummy doc node list - used by the display. need to think about this
+			page_response.appendChild(doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER));
+			//append site metadata
+			addSiteMetadata(page_response, userContext);
+			addInterfaceOptions(page_response);
+			return page_response;
+		}
+
+		// now we check to see if there is metadata already - some search services return predefined metadata. if there is some, don't do a metadata request
+		NodeList doc_metadata = document_list.getElementsByTagName(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		if (doc_metadata.getLength() > 0)
+		{
+		  // why are we not paging these results?????
+		  // append the doc list to the result
+			page_response.appendChild(doc.importNode(document_list, true));
+			//append site metadata
+			addSiteMetadata(page_response, userContext);
+			addInterfaceOptions(page_response);
+			return page_response;
+		}
+
+		// get the metadata elements needed from the format statement if any
+		HashSet<String> metadata_names = new HashSet<String>();
+		metadata_names.add("Title");
+		// we already got the format element earlier
+		if (format_elem != null)
+		{
+		  getRequiredMetadataNames(format_elem, metadata_names);
+		}
+
+		// paging of the results is done here - we filter the list to remove unwanted entries before retrieving metadata
+		Element filtered_doc_list;
+		if (does_paging) {
+		  filtered_doc_list = (Element)doc.importNode(document_list, true);
+		} else {
+		  filtered_doc_list = filterDocList(doc, params, service_params, document_list);
+		}
+		// do the metadata request on the filtered list
+		Element mr_metadata_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		to = "DocumentMetadataRetrieve";
+		if (collection != null)
+		{
+			to = GSPath.prependLink(to, collection);
+		}
+		Element mr_metadata_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		mr_metadata_message.appendChild(mr_metadata_request);
+
+		// just get all for now - the receptionist should perhaps pass in some
+		// metadata that it wants, and QueryAction should look through the format stuff to see if there is any other?
+
+		Element extraMetaListElem = (Element) GSXML.getChildByTagName(request, GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
+		if(extraMetaListElem != null)
+		{
+			NodeList extraMetaList = extraMetaListElem.getElementsByTagName(GSXML.EXTRA_METADATA);
+			for(int i = 0; i < extraMetaList.getLength(); i++)
+			{
+				metadata_names.add(((Element)extraMetaList.item(i)).getAttribute(GSXML.NAME_ATT));
+			}
+		}
+
+		Element dm_param_list = createMetadataParamList(doc,metadata_names);
+
+		mr_metadata_request.appendChild(dm_param_list);
+
+		// add in the doc node list too
+		mr_metadata_request.appendChild(filtered_doc_list);
+
+		Element mr_metadata_response = (Element) this.mr.process(mr_metadata_message);
+
+		Element query_result_snippet_list = (Element) GSXML.getChildByTagName(query_response, GSXML.HL_SNIPPET_ELEM + GSXML.LIST_MODIFIER);
+		
+		// check for errors
+		processErrorElements(mr_metadata_response, page_response);
+
+		Element metadata_response = (Element) GSXML.getChildByTagName(mr_metadata_response, GSXML.RESPONSE_ELEM);
+
+		Element query_result_document_list = (Element) GSXML.getChildByTagName(metadata_response, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				
+		if (query_result_document_list != null)
+		{
+			page_response.appendChild(doc.importNode(query_result_document_list, true));
+			if (query_result_snippet_list != null) 
+			{
+				page_response.appendChild(doc.importNode(query_result_snippet_list,true));
+			}
+		}
+		
+		//logger.debug("Query page:\n" + this.converter.getPrettyString(page_response));
+		//append site metadata
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+		return page_response;
+	}
+
+	private void addCollectionsHierarchy(Element page_response, UserContext userContext) {
+		Document doc = page_response.getOwnerDocument();
+		String collectionsHierarchy = "CollectionsHierarchy";
+		if (checkServiceAvailable(userContext, collectionsHierarchy)){
+			Element groupQueryMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+			Element groupQueryRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, collectionsHierarchy, userContext);
+			groupQueryMessage.appendChild(groupQueryRequest);
+			Element groupQueryResult = (Element) this.mr.process(groupQueryMessage);
+			if (groupQueryResult == null){
+				return;
+			}
+			Element groupQueryResponse = (Element) GSXML.getChildByTagName(groupQueryResult, GSXML.RESPONSE_ELEM);
+			if (groupQueryResponse == null){
+				return;
+			}
+			Element hierarchy = (Element) GSXML.getChildByTagName(groupQueryResponse, GSXML.HIERARCHY_ELEM);
+			page_response.appendChild(doc.importNode(hierarchy, true));
+		}
+		
+		
+	}
+
+	private boolean checkServiceAvailable(UserContext userContext, String collectionsHierarchy) {
+		
+		Document doc = XMLConverter.newDOM();
+		Element infoMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element infoRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		Element paramList = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		infoRequest.appendChild(paramList);
+		GSXML.addParameterToList(paramList, GSXML.SUBSET_PARAM, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		infoMessage.appendChild(infoRequest);
+		Element responseMessage = (Element) this.mr.process(infoMessage);
+		if (responseMessage == null)
+		{
+			logger.error("couldn't query the message router!");
+			return false;
+		}
+		NodeList serviceLists = responseMessage.getElementsByTagName(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		if (serviceLists == null || serviceLists.getLength() == 0){
+			logger.error("No service List in response message from message router!");
+			return false;
+		}
+		Element serviceList = (Element) serviceLists.item(0);
+		Element groupInfoService = GSXML.getNamedElement(serviceList, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, collectionsHierarchy);
+		if (groupInfoService == null){
+			logger.error("service " + collectionsHierarchy + " unavailable");
+			return false;
+		}
+		return true;
+	}
+
+	/** this filters out some of the doc results for result paging */
+	protected Element filterDocList(Document doc, HashMap<String, Serializable> params, HashMap service_params, Element orig_doc_list)
+	{
+
+	  String hits_pp = (String) service_params.get(HITS_PER_PAGE);
+	 
+		int hits = 20;
+		if (hits_pp != null && !hits_pp.equals(""))
+		{
+		  if (hits_pp.equals("all")) {
+		    hits = -1;
+		  } else {
+			try
+			{
+				hits = Integer.parseInt(hits_pp);
+			}
+			catch (Exception e)
+			{
+				hits = 20;
+			}
+		  }
+		}
+		if (hits == -1)
+		{ // all
+			return (Element) doc.importNode(orig_doc_list, true);
+		}
+		NodeList result_docs = orig_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+
+		int num_docs = result_docs.getLength();
+		if (num_docs <= hits)
+		{
+			// too few docs to do paging
+			return (Element) doc.importNode(orig_doc_list, true);
+		}
+
+		// now we need our own doc list
+		Element result_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+
+		String start_p = (String) service_params.get("startPage");
+		if (start_p == null)
+		{
+			start_p = (String) params.get("startPage");
+		}
+
+		int start = 1;
+		if (start_p != null && !start_p.equals(""))
+		{
+			try
+			{
+				start = Integer.parseInt(start_p);
+			}
+			catch (Exception e)
+			{
+				start = 1;
+			}
+		}
+		int start_from = (start - 1) * hits;
+		int end_at = (start * hits) - 1;
+
+		if (start_from > num_docs)
+		{
+			// something has gone wrong
+			return result_list;
+		}
+
+		if (end_at > num_docs)
+		{
+			end_at = num_docs - 1;
+		}
+		// now we finally have the docs numbers to use
+		for (int i = start_from; i <= end_at; i++)
+		{
+			result_list.appendChild(doc.importNode(result_docs.item(i), true));
+		}
+
+		return result_list;
+	}
+
+  protected boolean addHitsParamToService(Document doc, Element service_description, String lang) {
+    Element param_list = (Element)GSXML.getChildByTagName(service_description, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+    Element param = GSXML.createParameterDescription(doc, HITS_PER_PAGE, getTextString("param." + HITS_PER_PAGE, lang, "ServiceRack", null), GSXML.PARAM_TYPE_INTEGER, "20", null, null);
+    Element query_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, "query");
+    if (query_param != null) {
+      param_list.insertBefore(param, query_param);
+    } else {
+      param_list.appendChild(param);
+    }
+    return true;
+  }
+  
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/RSSAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/RSSAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/RSSAction.java	(revision 31634)
@@ -0,0 +1,71 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+
+// other java stuff
+import java.util.*;
+import java.io.Serializable;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringWriter;
+
+import org.apache.log4j.*;
+
+public class RSSAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.RSSAction.class.getName());
+
+	/** process a request */
+	public Node process(Node message_node)
+	{
+		
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// assume only one request
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		UserContext userContext = new UserContext(request);
+		// get the param list
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		String service_name = "RSSFeed"; // RSSFeed service of RSSRetrieve.java
+		String collection = (String) params.get(GSParams.COLLECTION);
+		String to = GSPath.prependLink(service_name, collection); // collection/RSSFeed
+		// Get baseUrl for links in RSS Feed
+		String baseUrl = request.getAttribute("baseURL");
+		
+		
+		// the first part of the response is the service description 
+		// for now get this again from the service. 
+		// this should be cached somehow later on. 
+    
+		Element mr_request_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element rss_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		//Pass baseURL to processRSSFeed
+		rss_request.setAttribute("baseURL", baseUrl);
+		mr_request_message.appendChild(rss_request);
+		
+		Element rss_response = (Element) this.mr.process(mr_request_message);
+		rss_response = (Element) GSXML.getChildByTagName(rss_response, GSXML.RESPONSE_ELEM); // just the response tag
+		// NEED ERROR PROCESSING ?
+
+		// siteMeta and interfaceOptions are unnecessary, as rss.xsl is going to remove it anyway
+		// but may be handy when doing o=xml to view the original xml as it came through from the GS server
+		addSiteMetadata(rss_response, userContext);
+		addInterfaceOptions(rss_response);
+
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		result.appendChild(doc.importNode(rss_response, true));
+		return result;
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/SystemAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/SystemAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/SystemAction.java	(revision 31634)
@@ -0,0 +1,150 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+
+// other java stuff
+import java.util.*;
+
+import java.io.Serializable;
+
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringWriter;
+
+import org.apache.log4j.*;
+
+public class SystemAction extends Action
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.action.SystemAction.class.getName());
+
+	String tempVal = "";
+
+	/** process a request */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// assume only one request
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+		UserContext userContext = new UserContext(request);
+		// get the param list
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		String coll = (String) params.get(GSParams.SYSTEM_CLUSTER);
+
+		String to = "";
+		if (coll != null && !coll.equals(""))
+		{
+			to = coll;
+		}
+		else if(subaction.equals("authenticated-ping")) {
+			to = "RemoteAuthentication"; // not "Authentication/RemoteAuthentication": MessageRouter knows to map the RemoteAuthentication service to the Authentication module
+		}
+
+		Element mr_request_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_SYSTEM, to, userContext);
+		mr_request_message.appendChild(mr_request);
+
+		Element system = doc.createElement(GSXML.SYSTEM_ELEM);
+		mr_request.appendChild(system);
+
+		// will need to change the following if can do more than one system request at once
+		if (subaction.equals("c"))
+		{ // configure
+			system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_CONFIGURE);
+			String info = (String) params.get(GSParams.SYSTEM_SUBSET);
+			system.setAttribute(GSXML.SYSTEM_SUBSET_ATT, info);
+		}
+		else if (subaction.equals("ping")) { // can ping the server or a collection
+			String name = (String) params.get(GSParams.SYSTEM_MODULE_NAME);
+			
+			if(name != null && !name.equals("")) { 
+				// Pinging a collection (or module) with ?a=s&sa=ping&st=collection&sn=<colName>
+				// is a collection-level (servicecluster/module level) ping
+				
+				String type = (String) params.get(GSParams.SYSTEM_MODULE_TYPE);
+				if(type == null || type.equals("")) { 
+					type = "collection"; // if the st=collection was omitted, assume collection
+				}				
+				// ping action set to moduleType=Collection and moduleName=colName
+				system.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, name);
+				system.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, type);
+				system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_PING);
+			} // else SYSTEM_MODULE_NAME given by the "sn" GSParam is null or empty 
+			  // meaning server-level ping: ?a=s&sa=ping
+			  
+			system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_PING);
+		}
+		else if (subaction.equals("authenticated-ping")) { // can check whether a given username and password authenticates
+		
+			String username = (String) params.get(GSParams.UN);
+			String password = (String) params.get(GSParams.PW);
+			
+			
+			system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_AUTHENTICATED_PING);
+			system.setAttribute(GSXML.USERNAME_ATT, username);
+			system.setAttribute(GSXML.PASSWORD_ATT, password);
+			
+			if(params.containsKey("col")) {//params.containsKey(GSParams.COLLECTION)) {
+				String collection = (String) params.get("col");//(String) params.get(GSParams.COLLECTION);
+				system.setAttribute(GSXML.COLLECTION_ATT, collection);
+			}
+			
+		}
+		
+		//else if (subaction.equals("is-persistent")){
+		//	system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ISPERSISTENT);
+		//}
+		else
+		{
+			String name = (String) params.get(GSParams.SYSTEM_MODULE_NAME);
+			String type = (String) params.get(GSParams.SYSTEM_MODULE_TYPE);
+
+			system.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, name);
+			system.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, type);
+
+			if (subaction.equals("d"))
+			{ // delete
+				system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_DEACTIVATE);
+
+			}
+			else if (subaction.equals("a"))
+			{ // add
+				system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ACTIVATE);
+			}
+			else
+			{
+				// create the default response
+				// for now just have an error
+				logger.error("bad subaction type");
+				Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+				result.appendChild(page_response);
+
+				return result;
+			}
+		}
+
+		Node response_message = this.mr.process(mr_request_message);
+
+		Element response = GSXML.duplicateWithNewName(doc, (Element) GSXML.getChildByTagName(response_message, GSXML.RESPONSE_ELEM), GSXML.RESPONSE_ELEM, true);
+		addSiteMetadata(response, userContext);
+		addInterfaceOptions(response);
+
+		result.appendChild(response);
+		return result;
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/XMLDocumentAction.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/XMLDocumentAction.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/action/XMLDocumentAction.java	(revision 31634)
@@ -0,0 +1,121 @@
+package org.greenstone.gsdl3.action;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.service.TEIRetrieve;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+import java.util.Map;
+import java.util.Iterator;
+import java.io.File;
+import java.io.Serializable;
+
+/** action class for retrieving parts of XML documents */
+public class XMLDocumentAction extends Action
+{
+
+	/**
+	 * process - processes a request.
+	 */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+	    Document doc = message.getOwnerDocument();
+	    
+		// get the request - assume there is only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+
+		// create the return message
+		Element result = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element page_response = doc.createElement(GSXML.RESPONSE_ELEM);
+		result.appendChild(page_response);
+
+		UserContext userContext = new UserContext(request);
+		addSiteMetadata(page_response, userContext);
+		addInterfaceOptions(page_response);
+
+		// extract the params from the cgi-request, and check that we have a coll specified
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+
+		String collection = (String) params.get(GSParams.COLLECTION);
+		if (collection == null || collection.equals(""))
+		{
+			return result;
+		}
+		String doc_name = (String) params.get(GSParams.DOCUMENT);
+		if (doc_name == null || doc_name.equals(""))
+		{
+			return result;
+		}
+
+		// subaction used to decide if we are returning content or structure
+		String document_mode = request.getAttribute(GSXML.SUBACTION_ATT);
+		String to = null;
+		if (document_mode.equals("text"))
+		{
+			to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+		}
+		else if (document_mode.equals("toc"))
+		{
+			to = GSPath.appendLink(collection, "DocumentStructureRetrieve");
+		}
+		else
+		{ // the default is text
+			to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+		}
+
+		// make the request to the collection
+		Element mr_message = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		Element ret_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		mr_message.appendChild(ret_request);
+
+		Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		ret_request.appendChild(doc_list);
+		Element docelem = doc.createElement(GSXML.DOC_NODE_ELEM);
+		docelem.setAttribute(GSXML.NODE_ID_ATT, doc_name);
+		doc_list.appendChild(docelem);
+
+		// also add in a request for the Title metadata
+		to = GSPath.appendLink(collection, "DocumentMetadataRetrieve");
+		Element meta_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		// copy the doc list
+		meta_request.appendChild(doc_list.cloneNode(true));
+		// add in a metadata param
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		meta_request.appendChild(param_list);
+		Element param = GSXML.createParameter(doc, "metadata", "root_Title");
+		param_list.appendChild(param);
+
+		// add the request to the message
+		mr_message.appendChild(meta_request);
+
+		Element ret_response = (Element) this.mr.process(mr_message);
+		String[] links = { GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM };
+		String path = GSPath.createPath(links);
+		Element doc_node = (Element) doc.importNode(GSXML.getNodeByPath(ret_response, path), true);
+		page_response.appendChild(doc_node);
+
+		// get the metadata list
+		Element meta_response = (Element) ret_response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(1);
+		String[] mlinks = { GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.DOC_NODE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER };
+		path = GSPath.createPath(mlinks);
+
+		Element meta_list = (Element) GSXML.getNodeByPath(meta_response, path);
+		if (meta_list != null)
+		{
+			doc_node.appendChild(doc.importNode(meta_list, true));
+		}
+		return result;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/CollectionConstructor.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/CollectionConstructor.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/CollectionConstructor.java	(revision 31634)
@@ -0,0 +1,150 @@
+package org.greenstone.gsdl3.build;
+
+import java.io.File;
+
+import org.greenstone.gsdl3.util.*;
+//import java.util.Thread
+import javax.swing.event.EventListenerList;
+import org.w3c.dom.Element;
+
+/** base class for collection construction */
+public abstract class CollectionConstructor extends Thread
+{
+
+	/** the site in which building is to take place */
+	protected String site_home = null;
+	/** the name of the site */
+	protected String site_name = null;
+	/** the name of the collection */
+	protected String collection_name = null;
+	/** the stage of construction */
+	protected int process_type = -1;
+	/** other arguments/parameters for the construction process - in a paramList */
+	protected Element process_params = null;
+	/** the list of listeners for the process */
+	protected EventListenerList listeners = null;
+	/** A flag used to determine if this process has been asked to cancel. */
+	protected boolean cancel = false; // Not really used (in any way that works)
+	/** Stores the name of the manifest file (if one is needed) */
+	protected String manifest_file = null;
+	/** The URL params constructed as a query string, representing the CGI QUERY_STRING to the process */
+	protected String query_string = null;
+
+	public CollectionConstructor(String name)
+	{
+		super(name);
+		this.listeners = new EventListenerList();
+	}
+
+	/**
+	 * carry out any set up stuff - returns false if couldn't set up properly
+	 */
+	public boolean configure()
+	{
+		return true;
+	}
+
+    // this method never gets called. And, the way subclass GS2PerlConstructor.runPerlCommand() was originally
+    // coded, setting cancel to true never had any effect anyway in stopping any perl command that was run.
+	public void stopAction()
+	{
+		this.cancel = true;
+	}
+
+	public void setActionType(int type)
+	{
+		this.process_type = type;
+	}
+
+	public void setSiteHome(String site_home)
+	{
+		this.site_home = site_home;
+
+		File siteHomeFile = new File(site_home);
+		this.site_name = siteHomeFile.getName();
+	}
+
+	public void setCollectionName(String coll_name)
+	{
+		this.collection_name = coll_name;
+	}
+
+    	public void setQueryString(String querystring)
+	{
+		this.query_string = querystring;
+	}
+
+	public void setProcessParams(Element params)
+	{
+		this.process_params = params;
+	}
+	
+	public void setManifestFile(String manifestFile)
+	{
+		this.manifest_file = manifestFile;
+	}
+
+	public boolean addListener(ConstructionListener listener)
+	{
+		this.listeners.add(ConstructionListener.class, listener);
+		return true;
+	}
+
+	public boolean removeListener(ConstructionListener listener)
+	{
+		this.listeners.remove(ConstructionListener.class, listener);
+		return true;
+	}
+
+	protected void sendProcessBegun(ConstructionEvent evt)
+	{
+		Object[] concerned = this.listeners.getListenerList();
+		for (int i = 0; i < concerned.length; i += 2)
+		{
+			if (concerned[i] == ConstructionListener.class)
+			{
+				((ConstructionListener) concerned[i + 1]).processBegun(evt);
+			}
+		}
+	}
+
+	protected void sendProcessComplete(ConstructionEvent evt)
+	{
+		Object[] concerned = this.listeners.getListenerList();
+		for (int i = 0; i < concerned.length; i += 2)
+		{
+			if (concerned[i] == ConstructionListener.class)
+			{
+				((ConstructionListener) concerned[i + 1]).processComplete(evt);
+			}
+		}
+	}
+
+	protected void sendProcessStatus(ConstructionEvent evt)
+	{
+
+		Object[] concerned = this.listeners.getListenerList();
+		for (int i = 0; i < concerned.length; i += 2)
+		{
+			if (concerned[i] == ConstructionListener.class)
+			{
+				((ConstructionListener) concerned[i + 1]).processStatus(evt);
+			}
+		}
+	}
+
+	protected void sendMessage(ConstructionEvent evt)
+	{
+
+		Object[] concerned = this.listeners.getListenerList();
+		for (int i = 0; i < concerned.length; i += 2)
+		{
+			if (concerned[i] == ConstructionListener.class)
+			{
+				((ConstructionListener) concerned[i + 1]).message(evt);
+			}
+		}
+	}
+
+	abstract public void run();
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructCollection.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructCollection.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructCollection.java	(revision 31634)
@@ -0,0 +1,140 @@
+package org.greenstone.gsdl3.build;
+import org.greenstone.gsdl3.util.*;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import java.io.File;
+
+/** the class containing the main program for collection building - is essentially a wrapper around a CollectionConstructor
+ * @see CollectionConstructor
+ */
+public class ConstructCollection 
+    implements ConstructionListener {
+    
+    static public void main(String [] args) {
+
+	String site_home = null;
+	String coll_name = null;
+	int process_mode = -1;
+	if (args.length < 4) {
+	    printUsage();
+	    return;
+	}
+
+	// parse the args
+
+	XMLConverter converter = new XMLConverter();
+	Document doc = converter.newDOM();
+	Element option_list = doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	Element option;
+
+	for (int i=0; i<args.length-1; i++) {
+	    String key = args[i];
+	    if (!key.startsWith("-")) {
+		continue; // ignore options that dont start with '-'
+	    }
+	    key = key.substring(1);
+	    String val = args[i+1];
+	    if (val.startsWith("-")) {
+		val=null;
+	    } else {
+		i++;
+	    }
+	    
+	    if (key.equals("site")) {
+		site_home = val;
+	    } else if (key.equals("mode")) {
+		if (val==null) {
+		    continue;
+		}
+		if (val.equals("new")) {
+		    process_mode = GS2PerlConstructor.NEW;
+		} else if (val.equals("import")) {
+		    process_mode = GS2PerlConstructor.IMPORT;
+		} else if (val.equals("build")) {
+		    process_mode = GS2PerlConstructor.BUILD;
+		} else if (val.equals("activate")) {
+		    process_mode = GS2PerlConstructor.ACTIVATE;
+		}
+
+	    } else { // an option to pass to the builder
+ 		option = doc.createElement(GSXML.PARAM_ELEM);
+		option.setAttribute(GSXML.NAME_ATT, key);
+		if (val!=null) {
+		    option.setAttribute(GSXML.VALUE_ATT, val);
+		}
+		option_list.appendChild(option);
+	    }
+	}
+	String last_arg = args[args.length-1];
+	if (last_arg.startsWith("-")) { // should be coll name
+	    System.out.println("ERROR: the last arg should be the collection name!");
+	    printUsage();
+	    return;
+	}
+	
+	coll_name = last_arg;
+	
+	// check that we have the required fields
+	if (site_home==null || process_mode == -1 || coll_name==null) {
+	    System.out.println("ERROR: you have not specified all the necessary args!");
+	    printUsage();
+	    return;
+	}
+
+	// check that the collection name is valid - ie the directory exists if this is not a new one, or doesn't exist if we are creating a new one
+	File coll_dir = new File(GSFile.collectionBaseDir(site_home, coll_name));
+	if (process_mode != GS2PerlConstructor.NEW && !coll_dir.exists()) {
+	    System.out.println("ERROR: Invalid collection ("+coll_name+").");
+	    printUsage();
+	    return;
+	} else if (process_mode == GS2PerlConstructor.NEW && 
+		   coll_dir.exists()) {
+	    System.out.println("ERROR: there is already a collection named "+coll_name);
+	    printUsage();
+	    return;
+	}
+	
+   
+	ConstructCollection processor = new ConstructCollection();
+	GS2PerlConstructor constructor = new GS2PerlConstructor("perl_build");
+	if (!constructor.configure()) {
+	    System.out.println("couldn't configure the constructor!!");
+	    return;
+	}
+	constructor.setCollectionName(coll_name);
+	constructor.setSiteHome(site_home);
+	constructor.setProcessParams(option_list);
+	constructor.setActionType(process_mode);
+	constructor.addListener(processor);
+	constructor.start();
+    }
+    
+    static protected void printUsage() {
+	System.out.println("Usage: the following arguments need to be supplied:\n -site <site-home> -mode new|import|build|activate [options] <coll-name>");
+	System.out.println("Options available are:");
+	System.out.println("-maxdocs N\t process at most N documents\n"+
+			   "-gs2\t use greenstone 2 style building");
+    }
+
+    /** This event handler used to signify that a task has been started */
+    public void processBegun(ConstructionEvent evt) {
+	System.out.println("begun: "+evt.getMessage());
+    }
+    /** This event handler used to signify that a task has been completed */
+    public void processComplete(ConstructionEvent evt){
+	System.out.println("complete: "+evt.getMessage());
+    }
+    /** This event handler used to send status updates as the task is progressing */
+    public void processStatus(ConstructionEvent evt){
+	System.out.println(evt.getMessage());
+    }
+    /**  This event handler used to send any other messages to the listeners */
+    public void message(ConstructionEvent evt){
+	System.out.println(evt.getMessage());
+    }
+
+
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionEvent.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionEvent.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionEvent.java	(revision 31634)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3.build;
+
+import java.awt.AWTEvent;     
+
+public class ConstructionEvent 
+    extends AWTEvent {
+    
+    /** the event id for these events - use a random number thats
+	not reserved*/
+    private static final int EVENT_ID = RESERVED_ID_MAX+77;
+    /** The status associated with the event. */
+    private final int status;
+    /** Any message associated with this event. */
+    private final String message;
+    /* Constructor.
+     * @param source The <strong>CollectionConstructor</strong> that fired this message.
+     * @param status The status code for this event          
+     * @param message A <strong>String</strong> representing any message attatched with this event.
+     */
+    public ConstructionEvent(Object source, int status, String message) {
+	super(source, EVENT_ID);
+	this.message = message;
+	this.status = status;
+    }
+    /** Gets the message associated with this event.
+     * @return The message as a <strong>String</strong> or <i>null</i>. 
+     */
+    public String getMessage() {
+	return this.message;
+    }
+    /** Gets the status associated with this event. This status can then be matched back to the constants in <strong>GShell</strong>.
+     * @return An <strong>int</strong> signifying the process status.
+     */
+    public int getStatus() {
+	return this.status;
+    }
+    /** returns a String representation of the event */
+    public String toString() {
+	return "org.greenstone.gsdl3.build.ConstructionEvent[" + this.message
++ "," + this.status + "]";
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionListener.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionListener.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/ConstructionListener.java	(revision 31634)
@@ -0,0 +1,16 @@
+package org.greenstone.gsdl3.build;
+
+import java.util.EventListener;
+
+public interface ConstructionListener 
+    extends EventListener {
+
+    /** This event handler used to signify that a task has been started */
+    public void processBegun(ConstructionEvent evt);
+    /** This event handler used to signify that a task has been completed */
+    public void processComplete(ConstructionEvent evt);
+    /** This event handler used to send status updates as the task is progressing */
+    public void processStatus(ConstructionEvent evt);
+    /**  This event handler used to send any other messages to the listeners */
+    public void message(ConstructionEvent evt);
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlConstructor.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlConstructor.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlConstructor.java	(revision 31634)
@@ -0,0 +1,827 @@
+package org.greenstone.gsdl3.build;
+
+// greenstome classes
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.util.Misc;
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.util.SafeProcess;
+
+// xml classes
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+//general java classes
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Vector;
+
+import org.apache.log4j.*;
+
+/**
+ * CollectionConstructor class for greenstone 2 compatible building it uses the
+ * perl scripts to do the building stuff
+ */
+public class GS2PerlConstructor extends CollectionConstructor implements SafeProcess.ExceptionHandler
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.build.GS2PerlConstructor.class.getName());
+
+	public static final int NEW = 0;
+	public static final int IMPORT = 1;
+	public static final int BUILD = 2;
+	public static final int ACTIVATE = 3;
+	public static final int MODIFY_METADATA_SERVER = 4;
+
+	/**
+	 * gsdlhome for greenstone 2 - we use the perl modules and building scripts
+	 * from there
+	 */
+	protected String gsdl2home = null;
+	/** gsdlhome for gsdl3 - shouldn't need this eventually ?? */
+	protected String gsdl3home = null;
+	/** gsdlos for greenstone 2 */
+	protected String gsdlos = null;
+	/** the path environment variable */
+	protected String path = null;
+
+
+	public GS2PerlConstructor(String name)
+	{
+		super(name);
+	}
+
+	/** retrieves the necessary environment variables */
+	public boolean configure()
+	{
+		// try to get the environment variables
+		this.gsdl3home = GlobalProperties.getGSDL3Home();
+		this.gsdl2home = this.gsdl3home + File.separator + ".." + File.separator + "gs2build";
+		this.gsdlos = Misc.getGsdlOS();
+            
+		this.path = System.getenv("PATH");
+
+		if (this.gsdl2home == null)
+		{
+			System.err.println("You must have gs2build installed, and GSDLHOME set for GS2Perl building to work!!");
+			return false;
+		}
+		if (this.gsdl3home == null || this.gsdlos == null)
+		{
+			System.err.println("You must have GSDL3HOME and GSDLOS set for GS2Perl building to work!!");
+			return false;
+		}
+		if (this.path == null)
+		{
+			System.err.println("You must have the PATH set for GS2Perl building to work!!");
+			return false;
+		}
+		return true;
+	}
+
+	public void run()
+	{
+		String msg;
+		ConstructionEvent evt;
+		if (this.process_type == -1)
+		{
+			msg = "Error: you must set the action type";
+			evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+			sendMessage(evt);
+			return;
+		}
+		if (this.site_home == null)
+		{
+			msg = "Error: you must set site_home";
+			evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+			sendMessage(evt);
+			return;
+		}
+		if (this.process_type != NEW && this.collection_name == null)
+		{
+			msg = "Error: you must set collection_name";
+			evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+			sendMessage(evt);
+			return;
+		}
+
+		switch (this.process_type)
+		{
+		case NEW:
+			newCollection();
+			break;
+		case IMPORT:
+			importCollection();
+			break;
+		case BUILD:
+			buildCollection();
+			break;
+		case ACTIVATE:
+			activateCollection();
+			break;
+		case MODIFY_METADATA_SERVER:
+			modifyMetadataForCollection();
+			break;
+		default:
+			msg = "wrong type of action specified!";
+			evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
+			sendMessage(evt);
+			break;
+		}
+	}
+
+	protected void newCollection()
+	{
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: new collection."));
+		Vector<String> command = new Vector<String>();
+		command.add("gs2_mkcol.pl");
+		command.add("-site");
+		command.add(this.site_home);
+		command.add("-collectdir");
+		command.add(GSFile.collectDir(this.site_home));
+		command.addAll(extractParameters(this.process_params));
+		command.add(this.collection_name);
+		String[] command_str = {};
+		command_str = command.toArray(command_str);
+		if (runPerlCommand(command_str))
+		{
+			// success!! - need to send the final completed message
+			sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		} // else an error message has already been sent, do nothing
+
+	}
+
+	protected void importCollection()
+	{
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: import collection."));
+		Vector<String> command = new Vector<String>();
+
+		String perlPath = GlobalProperties.getProperty("perl.path", "perl");
+		if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
+		{
+			perlPath = perlPath + File.separator;
+		}
+
+		command.add(perlPath + "perl");
+		command.add("-S");
+		command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "import.pl");
+		if (this.manifest_file != null)
+		{
+			command.add("-keepold");
+			command.add("-manifest");
+			command.add(this.manifest_file);
+		}
+		command.add("-site");
+		command.add(this.site_name);
+		command.add("-collectdir");
+		command.add(GSFile.collectDir(this.site_home));
+		command.addAll(extractParameters(this.process_params));
+		command.add(this.collection_name);
+		String[] command_str = {};
+		command_str = command.toArray(command_str);
+
+		if (runPerlCommand(command_str))
+		{
+			// success!! - need to send the final completed message
+			sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		} // else an error message has already been sent, do nothing
+	}
+
+	protected void buildCollection()
+	{
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: build collection."));
+		Vector<String> command = new Vector<String>();
+
+		String perlPath = GlobalProperties.getProperty("perl.path", "perl");
+		if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
+		{
+			perlPath = perlPath + File.separator;
+		}
+
+		command.add(perlPath + "perl");
+		command.add("-S");
+		command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "incremental-buildcol.pl");
+		command.add("-incremental");
+		command.add("-builddir");
+		command.add(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator +"index");
+		command.add("-site");
+		command.add(this.site_name);
+		command.add("-collectdir");
+		command.add(GSFile.collectDir(this.site_home));
+//		command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly
+		command.addAll(extractParameters(this.process_params));
+		command.add(this.collection_name);
+
+		String[] command_str = {};
+		command_str = command.toArray(command_str);
+
+		if (runPerlCommand(command_str))
+		{
+			// success!! - need to send the final completed message
+			sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		}// else an error message has already been sent, do nothing
+	}
+
+	protected void activateCollection()
+	{
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: activate collection."));
+
+		// first check that we have a building directory
+		// (don't want to bother running activate.pl otherwise)
+		File build_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
+		if (!build_dir.exists())
+		{
+			sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "build dir doesn't exist!"));
+			return;
+		}
+
+		/*
+
+		// move building to index
+		File index_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
+		if (index_dir.exists())
+		{
+			sendMessage(new ConstructionEvent(this, GSStatus.INFO, "deleting index directory"));
+			GSFile.deleteFile(index_dir);
+			if (index_dir.exists())
+			{
+				sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index directory still exists!"));
+				return;
+			}
+		}
+
+		GSFile.moveDirectory(build_dir, index_dir);
+		if (!index_dir.exists())
+		{
+			sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index dir wasn't created!"));
+		}
+
+		// success!!  - need to send the final completed message
+		sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		*/
+
+		// Running activate.pl instead of making java move building to index as above
+		// circumvents the issue of the jdbm .lg log file (managed by TransactionManager) 
+		// in index dir not getting deleted at times. The perl code is able to delete this
+		// sucessfully consistently during testing, whereas java at times is unable to delete it.
+		Vector<String> command = new Vector<String>();
+
+		String perlPath = GlobalProperties.getProperty("perl.path", "perl");
+		if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
+		{
+			perlPath = perlPath + File.separator;
+		}
+
+		command.add(perlPath + "perl");
+		command.add("-S");
+		command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "activate.pl");
+		command.add("-incremental");
+		command.add("-builddir");
+		command.add(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator +"index");
+		command.add("-site");
+		command.add(this.site_name);
+		command.add("-collectdir");
+		command.add(GSFile.collectDir(this.site_home));
+//		command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly. Shouldn't be added for incremental building
+		command.add("-skipactivation"); // gsdl3/util/GS2Construct does the activation and reactivation
+		command.addAll(extractParameters(this.process_params));
+		command.add(this.collection_name);
+
+		String[] command_str = {};
+		command_str = command.toArray(command_str);
+
+		if (runPerlCommand(command_str))
+		{
+			// success!! - need to send the final completed message
+			sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		}// else an error message has already been sent, do nothing	    
+
+	}
+
+
+    protected void modifyMetadataForCollection()
+	{
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection metadata: modifyMetadata (set or remove meta) for collection."));
+
+		Vector<String> command = new Vector<String>();
+
+		String perlPath = GlobalProperties.getProperty("perl.path", "perl");
+		if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
+		{
+			perlPath = perlPath + File.separator;
+		}
+
+		String cgi_directory = GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi";
+		command.add(perlPath + "perl");
+		command.add("-S");
+		//command.add(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi" + File.separator + "metadata-server.pl");
+		command.add(cgi_directory + File.separator + "metadata-server.pl");
+		
+		// Need to set QUERY_STRING and REQUEST_METHOD=GET in environment
+		// Also set GS3_AUTHENTICATED, to allow running metadata-server.pl with mod (set and remove) commands
+		// http://www.cgi101.com/class/ch3/text.html
+		String[] envvars = {
+		    "QUERY_STRING=" + this.query_string,
+		    "REQUEST_METHOD=GET",
+		    "GS3_AUTHENTICATED=true" // how do we set the env var without having to assign it a value?
+		};
+
+		String[] command_str = {};
+		command_str = command.toArray(command_str);
+
+		// http://www.cgi101.com/class/ch3/text.html
+		// setenv QUERY_STRING and REQUEST_METHOD = GET.
+		if (runPerlCommand(command_str, envvars, new File(cgi_directory)))
+				   //new File(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi")))
+		{
+			// success!! - need to send the final completed message
+			sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
+		}// else an error message has already been sent, do nothing	    
+
+	}
+
+	/** extracts all the args from the xml and returns them in a Vector */
+	protected Vector<String> extractParameters(Element param_list)
+	{
+
+		Vector<String> args = new Vector<String>();
+		if (param_list == null)
+		{
+			return args; // return an empty vector
+		}
+		NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+
+		for (int i = 0; i < params.getLength(); i++)
+		{
+			Element p = (Element) params.item(i);
+			String name = p.getAttribute(GSXML.NAME_ATT);
+			String value = p.getAttribute(GSXML.VALUE_ATT);
+			if (!name.equals(""))
+			{
+				args.add("-" + name);
+				if (!value.equals(""))
+				{
+					args.add(value);
+				}
+			}
+		}
+
+		return args;
+	}
+
+    protected SafeProcess createPerlProcess(String[] command, String[] envvars, File dir) {
+	int sepIndex = this.gsdl3home.lastIndexOf(File.separator);
+	String srcHome = this.gsdl3home.substring(0, sepIndex);
+	
+	ArrayList<String> args = new ArrayList<String>();
+	args.add("GSDLHOME=" + this.gsdl2home);
+	args.add("GSDL3HOME=" + this.gsdl3home);
+	args.add("GSDL3SRCHOME=" + srcHome);
+	args.add("GSDLOS=" + this.gsdlos);
+	args.add("GSDL-RUN-SETUP=true");
+	args.add("PERL_PERTURB_KEYS=0");
+	
+	if(envvars != null) {
+	    for(int i = 0; i < envvars.length; i++) {
+		args.add(envvars[i]);
+	    }
+	}
+	
+	for (String a : System.getenv().keySet()) {
+	    args.add(a + "=" + System.getenv(a));
+	}
+
+	SafeProcess perlProcess 
+	    = new SafeProcess(command, args.toArray(new String[args.size()]), dir); //  dir can be null
+	
+	return perlProcess;
+    }
+
+    // If you want to run a Perl command without doing GS2PerlConstructor's custom logging in the build log
+    // The use the runSimplePerlCommand() versions, which use the default behaviour of running a SafeProcess
+    protected boolean runSimplePerlCommand(String[] command) {	
+	return runSimplePerlCommand(command, null, null);
+    }
+    
+    protected boolean runSimplePerlCommand(String[] command, String[] envvars, File dir) {
+	boolean success = false;	
+	
+	String command_str = "";
+	for (int i = 0; i < command.length; i++) {
+	    command_str = command_str + command[i] + " ";
+	}
+	
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = " + command_str));
+	
+	///logger.info("### Running simple command = " + command_str);
+
+	// This is where we create and run our perl process safely
+	SafeProcess perlProcess = createPerlProcess(command, envvars, dir); //  dir can be null
+	
+	perlProcess.setExceptionHandler(this);
+
+	sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
+
+	int exitVal = perlProcess.runProcess(); // uses default processing of the perl process' iostreams provided by SafeProcess
+
+	if (exitVal == 0) {
+	    success = true;
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
+	} else {
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));	    
+	    success = false; // explicit
+	}
+	
+	return success;
+    }
+
+
+    /** returns true if completed correctly, false otherwise 
+     * Building operations call runPerlCommand which sends build output to collect/log/build_log.#*.txt
+     */
+    protected boolean runPerlCommand(String[] command) {
+	return runPerlCommand(command, null, null);
+    }
+
+    	
+    protected boolean runPerlCommand(String[] command, String[] envvars, File dir)
+    {
+	boolean success = true;
+	
+	String command_str = "";
+	for (int i = 0; i < command.length; i++) {
+	    command_str = command_str + command[i] + " ";
+	}
+	
+	sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = " + command_str));
+	
+	///logger.info("### Running logged command = " + command_str);
+
+	// This is where we create and run our perl process safely
+	SafeProcess perlProcess = createPerlProcess(command, envvars, dir); //  dir can be null
+	
+	sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
+	
+	File logDir = new File(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log");
+	if (!logDir.exists()) {
+	    logDir.mkdir();
+	}
+	
+	// Only from Java 7+: Try-with-Resources block will safely close the BufferedWriter
+	// https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
+	BufferedWriter bw = null;
+	try {
+
+	    bw = new BufferedWriter(new FileWriter(new File(logDir, "build_log." + (System.currentTimeMillis()) + ".txt")));
+	
+	    bw.write("Document Editor Build \n");		
+	    bw.write("Command = " + command_str + "\n");
+	    
+	    // handle each incoming line from stdout and stderr streams, and any exceptions that occur then
+	    SafeProcess.CustomProcessHandler processOutHandler
+		= new SynchronizedProcessHandler(bw, SafeProcess.STDOUT);
+	    SafeProcess.CustomProcessHandler processErrHandler
+		= new SynchronizedProcessHandler(bw, SafeProcess.STDERR);
+	    
+	    // GS2PerlConstructor will do further handling of exceptions that may occur during the perl
+	    // process (including if writing something to the process' inputstream, not that we're doing that for this perlProcess)
+	    perlProcess.setExceptionHandler(this); 
+	    
+	    // finally, execute the process
+	    
+	    // Captures the std err of a program and pipes it into
+	    // std in of java, as before.
+
+	    perlProcess.runProcess(null, processOutHandler, processErrHandler); // use default procIn handling
+	    
+	// The original runPerlCommand() code had an ineffective check for whether the cmd had been cancelled
+	// midway through executing the perl, as condition of a while loop reading from stderr and stdout.
+	// We don't include the cancel check here, as superclass CollectionConstructor.stopAction(), which set
+	// this.cancel to true, never got called anywhere.
+	// But I think a proper cancel of our perl process launched by this GS2PerlConstructor Thread object
+	// and of the worker threads it launches, could be implemented with interrupts. See:
+	// http://stackoverflow.com/questions/6859681/better-way-to-signal-other-thread-to-stop
+	// https://docs.oracle.com/javase/tutorial/essential/concurrency/interrupt.html
+	// https://docs.oracle.com/javase/7/docs/api/java/lang/Thread.html#interrupted()
+	// https://praveer09.github.io/technology/2015/12/06/understanding-thread-interruption-in-java/
+	// The code that calls GS2PerlConstructor.stopAction() should also call GSPerlConstructor.interrupt()
+	// Then in SafeProcess.runProcess(), I think the waitFor() will throw an InterruptedException()
+	// This can be caught and interrupt() called on SafeProcess' workerthreads, 
+	// Any workerthreads' run() methods that block (IO, loops) can test this.isInterrupted() 
+	// and can break out of any loops and release resources in finally.
+	// Back in SafeProcess.runProcess, the InterruptedException catch block will be followed by finally
+	// that will clear up any further resources and destroy the process forcibly if it hadn't been ended.
+
+	} catch(IOException e) {
+	    e.printStackTrace();
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred " + e.toString()));
+	} finally {
+	    SafeProcess.closeResource(bw);
+	}
+
+	if (!this.cancel) {
+	    // Now display final message based on exit value		    
+	    
+	    if (perlProcess.getExitValue() == 0) { //status = OK;
+		
+		sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
+		success = true;
+
+	    } else { //status = ERROR;
+		sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));
+		success = false;
+		
+	    }
+	} else { // cancelled. The code would never come here, including in the old version of runPerlCommand
+	    // but leaving this here for an exact port of the old runPerlCommand to the new one which
+	    // uses SafeProcess. Also to allow the cancel functionality in future
+	    
+	    // I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine.
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.HALTED, "killing the process"));
+	    success = false;
+	}
+	// we're done, but we don't send a process complete message here cos there might be stuff to do after this has finished.
+	//return true;		
+	return success;
+    }
+	
+
+    // this method is blocking again, on Windows when adding user comments
+    protected boolean old_runPerlCommand(String[] command, String[] envvars, File dir)
+	{
+		boolean success = true;
+	
+		int sepIndex = this.gsdl3home.lastIndexOf(File.separator);
+		String srcHome = this.gsdl3home.substring(0, sepIndex);
+
+		ArrayList<String> args = new ArrayList<String>();
+		args.add("GSDLHOME=" + this.gsdl2home);
+		args.add("GSDL3HOME=" + this.gsdl3home);
+		args.add("GSDL3SRCHOME=" + srcHome);
+		args.add("GSDLOS=" + this.gsdlos);
+		args.add("GSDL-RUN-SETUP=true");
+		args.add("PERL_PERTURB_KEYS=0");
+
+		if(envvars != null) {
+		    for(int i = 0; i < envvars.length; i++) {
+			args.add(envvars[i]);
+		    }
+		}
+
+		for (String a : System.getenv().keySet())
+		{
+			args.add(a + "=" + System.getenv(a));
+		}
+
+		String command_str = "";
+		for (int i = 0; i < command.length; i++)
+		{
+			command_str = command_str + command[i] + " ";
+		}
+
+		// logger.info("### old runPerlCmd, command = " + command_str);
+
+		sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = " + command_str));
+		Process prcs = null;
+		BufferedReader ebr = null;
+		BufferedReader stdinbr = null;
+		try
+		{
+			Runtime rt = Runtime.getRuntime();
+			sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
+			prcs = (dir == null) 
+			    ? rt.exec(command, args.toArray(new String[args.size()]))
+			    : rt.exec(command, args.toArray(new String[args.size()]), dir);
+
+			InputStreamReader eisr = new InputStreamReader(prcs.getErrorStream());
+			InputStreamReader stdinisr = new InputStreamReader(prcs.getInputStream());
+			ebr = new BufferedReader(eisr);
+			stdinbr = new BufferedReader(stdinisr);
+			// Captures the std err of a program and pipes it into
+			// std in of java
+
+			File logDir = new File(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log");
+			if (!logDir.exists())
+			{
+				logDir.mkdir();
+			}
+
+			BufferedWriter bw = new BufferedWriter(new FileWriter(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log" + File.separator + "build_log." + (System.currentTimeMillis()) + ".txt"));
+			bw.write("Document Editor Build \n");
+
+			bw.write("Command = " + command_str + "\n");
+
+			String eline = null;
+			String stdinline = null;
+			while (((eline = ebr.readLine()) != null || (stdinline = stdinbr.readLine()) != null) && !this.cancel)
+			{
+				if (eline != null)
+				{
+					//System.err.println("ERROR: " + eline);
+					bw.write(eline + "\n");
+					sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, eline));
+				}
+				if (stdinline != null)
+				{
+					//System.err.println("OUT: " + stdinline);
+					bw.write(stdinline + "\n");
+					sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, stdinline));
+				}
+			}
+			SafeProcess.closeResource(bw); 
+			
+			if (!this.cancel)
+			{
+				// Now display final message based on exit value
+				prcs.waitFor();
+
+				if (prcs.exitValue() == 0)
+				{	
+					//status = OK;
+					sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
+					
+					success = true;
+				}
+				else
+				{
+					//status = ERROR;
+					sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));
+
+					//return false;
+					success = false;
+
+				}
+			}
+			else
+			{
+				// I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine.
+				sendProcessStatus(new ConstructionEvent(this, GSStatus.HALTED, "killing the process"));
+				//prcs.getOutputStream().close();
+				//prcs.destroy();
+				////status = ERROR;
+				
+				//return false;
+				success = false;
+			}
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred " + e.toString()));
+		} finally { 
+			// http://steveliles.github.io/invoking_processes_from_java.html
+			// http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+			// http://mark.koli.ch/leaky-pipes-remember-to-close-your-streams-when-using-javas-runtimegetruntimeexec
+		
+			 if( prcs != null ) {
+				SafeProcess.closeResource(prcs.getErrorStream());
+				SafeProcess.closeResource(prcs.getOutputStream());
+				SafeProcess.closeResource(prcs.getInputStream());
+				prcs.destroy();
+			}
+		
+			SafeProcess.closeResource(ebr);
+			SafeProcess.closeResource(stdinbr);
+		}
+
+		// we're done, but we don't send a process complete message here cos there might be stuff to do after this has finished.
+		//return true;		
+		return success;
+	}
+
+
+    // From interface SafeProcess.ExceptionHandler
+    // Called when an exception happens during the running of our perl process. However,
+    // exceptions when reading from our perl process' stderr and stdout streams are handled by
+    // SynchronizedProcessHandler.gotException() below, since they happen in separate threads
+    // from this one (the one from which the perl process is run).
+    public synchronized void gotException(Exception e) {
+
+	// do what original runPerlCommand() code always did when an exception occurred
+	// when running the perl process:
+	e.printStackTrace();
+	sendProcessStatus(new ConstructionEvent(this,GSStatus.ERROR, 
+						"Exception occurred " + e.toString()));
+    }
+    
+    // Each instance of this class is run in its own thread by class SafeProcess.InputGobbler.
+    // This class deals with each incoming line from the perl process' stderr or stdout streams. One
+    // instance of this class for each stream. However, since multiple instances of this CustomProcessHandler
+    // could be (and in fact, are) writing to the same file in their own threads, several objects, not just
+    // the bufferedwriter object, needed to be made threadsafe.
+    // This class also handles exceptions during the running of the perl process.
+    // The runPerlCommand code originally would do a sendProcessStatus on each exception, so we ensure
+    // we do that here too, to continue original behaviour. These calls are also synchronized to make their
+    // use of the EventListeners threadsafe.
+    protected class SynchronizedProcessHandler extends SafeProcess.CustomProcessHandler
+    {
+	private final BufferedWriter bwHandle; // needs to be final to synchronize on the object
+		
+	public SynchronizedProcessHandler(BufferedWriter bw, int src) {
+	    super(src); // will set this.source to STDERR or STDOUT
+	    this.bwHandle = bw; // caller will close bw, since many more than one
+	                        // SynchronizedProcessHandlers are using it
+	}
+
+	public void run(Closeable inputStream) {
+	    InputStream is = (InputStream) inputStream;
+
+	    BufferedReader br = null;
+	    try {
+		br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+		String line=null;
+		while ( (line = br.readLine()) != null ) {
+		    
+		    if(Thread.currentThread().isInterrupted()) { // should we not instead check if SafeProcess thread was interrupted?
+			System.err.println("Got interrupted when reading lines from process err/out stream.");
+			break; // will go to finally block
+		    }
+		    
+		    ///System.out.println("@@@ GOT LINE: " + line);
+
+
+		    //if(this.source == SafeProcess.STDERR) {
+		    ///System.err.println("ERROR: " + line);
+		    //} else {
+		    ///System.err.println("OUT: " + line);
+		    //}	
+		    
+
+		    this.gotLine(line); // synchronized
+		    
+		    /*
+		    try {
+			synchronized(bwHandle) { // get a lock on the writer handle, then write
+			    
+			    bwHandle.write(line + "\n");
+			} 
+		    } catch(IOException ioe) {
+			String msg = (this.source == SafeProcess.STDERR) ? "stderr" : "stdout";
+			msg = "Exception when writing out a line read from perl process' " + msg + " stream.";
+			GS2PerlConstructor.logger.error(msg, ioe);
+		    }
+		    
+		    // this next method is thread safe since only synchronized methods are invoked.
+		    // and only immutable (final) vars are used. 
+		    // NO, What about the listeners???
+		    sendProcessStatus(new ConstructionEvent(GS2PerlConstructor.this, GSStatus.CONTINUING, line));		    
+		    */
+		}
+	    } catch (IOException ioe) { // problem with reading in from process with BufferedReader br
+
+		String msg = (this.source == SafeProcess.STDERR) ? "stderr" : "stdout";
+		msg = "Got exception when processing the perl process' " + msg + " stream.";
+		GS2PerlConstructor.logger.error(msg, ioe);
+		// now do what the original runPerlCommand() code always did:
+		ioe.printStackTrace();
+		logException(ioe); // synchronized
+
+	    } catch (Exception e) { // problem with BufferedWriter bwHandle on processing each line
+		e.printStackTrace();
+		logException(e); // synchronized
+	    } finally {
+		SafeProcess.closeResource(br);
+	    }
+	}
+
+	// trying to keep synchronized methods as short as possible
+	private synchronized void logException(Exception e) {
+	    sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred " + e.toString()));
+	}
+
+	// trying to keep synchronized methods as short as possible
+	private synchronized void gotLine(String line) throws Exception {
+
+	    // BufferedWriter writes may not be atomic
+	    // http://stackoverflow.com/questions/9512433/is-writer-an-atomic-method
+	    // Choosing to put try-catch outside of sync block, since it's okay to give up lock on exception
+	    // http://stackoverflow.com/questions/14944551/it-is-better-to-have-a-synchronized-block-inside-a-try-block-or-a-try-block-insi
+	    try {					    
+		bwHandle.write(line + "\n");	
+	    
+///		GS2PerlConstructor.logger.info("@@@ WROTE LINE: " + line);
+
+		// this next method is thread safe since only synchronized methods are invoked.
+		// and only immutable (final) vars are used.
+		sendProcessStatus(new ConstructionEvent(GS2PerlConstructor.this, GSStatus.CONTINUING, line));
+	    
+	    } catch(IOException ioe) { // can't throw Exceptions, but are forced to handle Exceptions here
+		// since our method definition doesn't specify a throws list.
+		// "All methods on Logger are multi-thread safe", see
+		// http://stackoverflow.com/questions/14211629/java-util-logger-write-synchronization
+		
+		String msg = (this.source == SafeProcess.STDERR) ? "stderr" : "stdout";
+		msg = "IOException when writing out a line read from perl process' " + msg + " stream.";
+		msg += "\nGot line: " + line + "\n";
+		throw new Exception(msg, ioe);
+	    }		    
+	}
+
+    } // end inner class SynchronizedProcessHandler
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlListener.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlListener.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/GS2PerlListener.java	(revision 31634)
@@ -0,0 +1,62 @@
+package org.greenstone.gsdl3.build;
+
+public class GS2PerlListener 
+    implements ConstructionListener {
+
+    /** a buffer holding all the messages */
+    protected StringBuffer log=null;
+    /** a buffer holding only the latest update */
+    protected StringBuffer update = null;
+
+    /** a status code */
+    protected int status = -1;
+    /** whether the process this is listeneing to is finshed */
+    protected boolean finished = false;
+
+    public GS2PerlListener() {
+	this.log = new StringBuffer();
+	this.update = new StringBuffer();
+    }
+
+    public boolean isFinished() {
+	return this.finished;
+    }
+    public String getLog() {
+	return this.log.toString();
+    }
+    public int getStatus() {
+	return this.status;
+    }
+    synchronized public String getUpdate() {
+	this.log.append(this.update);
+	String tmp = this.update.toString();
+	this.update.delete(0, this.update.length());
+	return tmp;
+    }
+
+    // do we need to synchronize the methods below?
+
+    /** This event handler used to signify that a task has been started */
+    synchronized public void processBegun(ConstructionEvent evt) {
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+	
+    }
+    /** This event handler used to signify that a task has been completed */
+    synchronized public void processComplete(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+    /** This event handler used to send status updates as the task is progressing */
+    synchronized public void processStatus(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+    /**  This event handler used to send any other messages to the listeners */
+    synchronized public void message(ConstructionEvent evt){
+	this.status = evt.getStatus();
+	this.update.append(evt.getMessage()+"\n");
+    }
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/StatusDisplay.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/StatusDisplay.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/build/StatusDisplay.java	(revision 31634)
@@ -0,0 +1,188 @@
+package org.greenstone.gsdl3.build;
+
+import java.applet.Applet;
+import java.awt.*;
+import java.awt.event.*;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.apache.xerces.parsers.DOMParser;
+import java.net.URL;
+import java.io.DataInputStream;
+
+
+/** StatusDisplay - a java applet thats sends a message at regular intervals 
+ * to a pre specified address (the library param) and then displays the 
+ * status messages received.
+ * to use, embed in a web page using something like: (note the cgi args may differ - see process.xsl for details )
+ * <applet code="org.greenstone.gsdl3.build.StatusDisplay.class" codebase='lib/java' archive='gsdl3.jar, xercesImpl.jar, jaxp.jar, xml-apis.jar' width='537' height='100'>The status display applet.<param name='library' value='mylibrary?a=pr&rt=s&c=build&o=xml&ro=1&s=ImportCollection&l=en&coll=demo'/></applet>
+ * the cgi params for the library param are dependent on what action/service 
+ *  is using the applet
+ */
+public class StatusDisplay extends Applet 
+    implements Runnable {
+
+    /** the display area for the status messages */
+    protected TextArea display;
+    /** how long to delay between requests */
+    protected int delay=3000; // 3 secs
+    /** the thread used */
+    protected Thread updator_thread;
+    /** the url for the requests */ 
+    protected URL status_cgi = null;
+    /** if true, stops sending requests, and cant be unset */
+    protected boolean completed = false;
+    // init is called if you leave the page and go back - need to somehow retrieve all the previous messages that have been obtained cos we may have left the page and gone back to it during an import or something. dont want to lose all the info.
+    public void init() {
+	//Create the text field and make it uneditable.
+	this.display = new TextArea();
+	this.display.setEditable(false);
+	this.display.setBackground(Color.white);
+	//Set the layout manager so that the text field will be
+	//as wide as possible.
+	setLayout(new java.awt.GridLayout(1,0));
+
+	//Add the text field to the applet.
+	add(this.display);
+	validate();
+
+	// add the initial status text
+	String initial_text = parameterValue("initial_text");
+	addItem(initial_text);
+
+	// check the initial error state - there may be no more messages to come
+	this.completed = GSStatus.isCompleted(Integer.parseInt(parameterValue("initial_code")));
+		
+	// we always do the same query
+	String library = parameterValue("library");
+	library = tidy_URL(library);
+	try {
+	    this.status_cgi = new URL(library);
+	} catch (Exception e) {
+	    System.out.println("Error creating URL for "+library+": "+e.getMessage());
+	}
+    }
+    
+    public void start() {
+	if (this.completed) {
+	    // do nothing
+	} else {
+	    if (this.updator_thread == null) {
+		this.updator_thread = new Thread(this);
+	    }
+	    this.updator_thread.start();
+	}
+    }
+    public void stop() {
+	this.updator_thread = null;
+    }
+    public void run() {
+	
+	if (this.completed) { // do nothing. dont know if we need this here
+	    return;
+	}
+        //Just to be nice, lower this thread's priority
+        //so it can't interfere with other processing going on.
+        Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
+
+        //Remember which thread we are.
+        Thread current_thread = Thread.currentThread();
+
+        //This is the  loop.
+        while (current_thread == this.updator_thread && !this.completed) {
+	    queryServer();
+            this.display.repaint();
+	    
+            //Delay for the specified time
+            try {
+		Thread.sleep(this.delay);
+            } catch (InterruptedException e) {
+                break;
+            }
+        }
+    }
+
+    /** add some more text to the display    */
+    void addItem(String new_text) {
+	String t = this.display.getText()+"\n"+new_text;
+	this.display.setText(t);
+	this.display.setCaretPosition(t.length()); // so that you always see the 
+	// most recent entries - ie so that it scrolls down automatically
+	repaint();
+    }
+    
+    void queryServer() {
+
+	// Send the query 
+	System.out.println("sending query: " + this.status_cgi); // this appears in the java console
+	try {
+	    DataInputStream in = new DataInputStream(this.status_cgi.openStream());
+	    DOMParser parser = new DOMParser();
+	    parser.parse(new InputSource(in));
+	    Document data_doc = parser.getDocument();
+	    Element data_elem = data_doc.getDocumentElement();
+	    Element status_elem = (Element)GSXML.getChildByTagName(data_elem, GSXML.STATUS_ELEM);
+	    if (status_elem != null) {
+		String status = status_elem.getAttribute(GSXML.STATUS_ERROR_CODE_ATT);
+		System.out.println("received status "+status);
+		this.completed = GSStatus.isCompleted(Integer.parseInt(status));
+		addItem(GSXML.getNodeText(status_elem));
+	    } else {
+		System.out.println("Error - no status receieved");
+		this.completed=true;
+	    }
+	    in.close();
+	} catch (Exception e) {
+	    System.err.println( "Error sending query ("+this.status_cgi+"): " + e);
+	}
+
+    }
+  
+    // Get a REQUIRED string.  Stop the applet if we cannot.
+    String parameterValue(String name) {
+	try { 
+	    return getParameter(name);
+	} catch (Exception e) {
+	    System.err.println("StatusDisplay: you must give a parameter for \""+ name + "\".  Stopping.");
+	    stop();
+	}
+	return "";
+    }
+
+    /**Tidy up URLs
+     *
+     * Ensure a URL address (as string) has a protocol, host, and file.
+     */
+    String tidy_URL(String address) {
+
+	// make sure the URL has protocol, host, and file
+	if (address.startsWith("http")) {
+	    // the address has all the necessary components
+	} else if (address.startsWith("/")) {
+	    // there is not protocol and host
+	    URL document = getDocumentBase();
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    address = "http://" + document.getHost() + port  + address;
+	} else {
+	    // this URL is relative to the directory the document is in
+	    URL document = getDocumentBase();
+	    String directory = document.getFile();
+	    int end = directory.lastIndexOf('/');
+	    String port = "";
+	    if (document.getPort()!=-1) {
+		port = ":" + document.getPort();
+	    }
+	    directory = directory.substring(0,end + 1);
+	    address = "http://" + document.getHost() + port + directory + address;
+
+	}
+
+	return address;
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/Collection.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/Collection.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/Collection.java	(revision 31634)
@@ -0,0 +1,922 @@
+/*
+ *    Collection.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.collection;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.CustomClassLoader;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.OAIXML;
+import org.greenstone.gsdl3.util.SimpleMacroResolver;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.util.XMLTransformer;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * Represents a collection in Greenstone. A collection is an extension of a
+ * ServiceCluster - it has local data that the services use.
+ * 
+ * @author Katherine Don
+ * @see ModuleInterface
+ */
+public class Collection extends ServiceCluster
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.collection.Collection.class.getName());
+
+	/** is this collection being tidied and therefore can support realistic book view? */
+	protected boolean useBook = false;
+	/**
+	 * is this collection public or private - public collections will
+	 * appear on the home page, whereas private collections won't
+	 */
+	protected boolean is_public = true;
+	/** collection type : mg, mgpp or lucene */
+	protected String col_type = "";
+	/** database type : gdbm, jdbm or sqlite */
+	protected String db_type = "";
+	/** time when this collection was built Used by RSS */
+	protected long lastmodified = 0;
+	/** earliestDatestamp of this collection. Used by OAI/RSS */
+        protected long earliestDatestamp = 0;
+
+	/** Stores the default accessibility of guest users */
+	protected boolean _publicAccess = true;
+	/** Stores the scope of any security rules (either collection or document) */
+	protected boolean _securityScopeCollection = true;
+
+	protected HashMap<String, ArrayList<Element>> _documentSets = new HashMap<String, ArrayList<Element>>();
+	protected ArrayList<HashMap<String, ArrayList<String>>> _securityExceptions = new ArrayList<HashMap<String, ArrayList<String>>>();
+
+	protected XMLTransformer transformer = null;
+	/**
+	 * A class loader that knows about the collection resources directory can
+	 * put properties files, dtds etc in here
+	 */
+	//CustomClassLoader class_loader = null;
+
+	/** same as setClusterName */
+	public void setCollectionName(String name)
+	{
+		setClusterName(name);
+	}
+
+	public Collection()
+	{
+		super();
+		this.description = this.desc_doc.createElement(GSXML.COLLECTION_ELEM);
+	}
+
+	/**
+	 * Configures the collection.
+	 * 
+	 * gsdlHome and collectionName must be set before configure is called.
+	 * 
+	 * the file buildcfg.xml is located in gsdlHome/collect/collectionName
+	 * collection metadata is obtained, and services loaded.
+	 * 
+	 * @return true/false on success/fail
+	 */
+	public boolean configure()
+	{
+		if (this.site_home == null || this.cluster_name == null)
+		{
+			logger.error("Collection: site_home and collection_name must be set before configure called!");
+			return false;
+		}
+		// set up the class loader
+		this.class_loader = new CustomClassLoader(this.getClass().getClassLoader(), GSFile.collectionResourceDir(this.site_home, this.cluster_name));
+
+		macro_resolver.addMacro("_httpcollection_", this.site_http_address + "/collect/" + this.cluster_name);
+
+		Element coll_config_xml = loadCollConfigFile();
+		if (coll_config_xml == null) {
+		  logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", " + "Couldn't load collection config file");
+
+		  return false;
+		}
+		Element build_config_xml = loadBuildConfigFile();
+
+		if (build_config_xml == null)
+		{
+		  logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", " + "Couldn't load build config file");
+
+		  return false;
+		}
+
+		GSXSLT.modifyCollectionConfigForDebug(coll_config_xml);
+		// get the collection type attribute
+		Element search = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.SEARCH_ELEM);
+		if (search != null)
+		{
+			col_type = search.getAttribute(GSXML.TYPE_ATT);
+		}
+
+		Element browse = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.INFODB_ELEM);
+		if (browse != null)
+		{
+			db_type = browse.getAttribute(GSXML.TYPE_ATT);
+		}
+		else
+		{
+			db_type = "gdbm"; //Default database type
+		}
+
+		this.description.setAttribute(GSXML.TYPE_ATT, col_type);
+		this.description.setAttribute(GSXML.DB_TYPE_ATT, db_type);
+
+		_globalFormat = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.FORMAT_ELEM);
+		// process the metadata and display items and default library params
+		super.configureLocalData(coll_config_xml);
+		super.configureLocalData(build_config_xml);
+		// get extra collection specific stuff
+		findAndLoadInfo(coll_config_xml, build_config_xml);
+
+		loadSecurityInformation(coll_config_xml);
+
+		// now do the services
+		configureServiceRacks(coll_config_xml, build_config_xml);
+
+		return true;
+
+	}
+
+	public boolean useBook()
+	{
+		return useBook;
+	}
+
+	public boolean isPublic()
+	{
+		return is_public;
+	}
+
+	// Used by OAI Receptionist and RSSRetrieve 
+	public long getLastmodified()
+	{
+		return lastmodified;
+	}
+
+	// used by the OAIReceptionist and RSSRetrieve
+	public long getEarliestDatestamp()
+	{
+		return earliestDatestamp;
+	}
+
+	/**
+	 * load in the collection config file into a DOM Element
+	 */
+	protected Element loadCollConfigFile()
+	{
+
+		File coll_config_file = new File(GSFile.collectionConfigFile(this.site_home, this.cluster_name));
+
+		if (!coll_config_file.exists())
+		{
+			return null;
+		}
+		// get the xml
+		Document coll_config_doc = this.converter.getDOM(coll_config_file, CONFIG_ENCODING);
+		Element coll_config_elem = null;
+		if (coll_config_doc != null)
+		{
+			coll_config_elem = coll_config_doc.getDocumentElement();
+		}
+		return coll_config_elem;
+
+	}
+
+	/**
+	 * load in the collection build config file into a DOM Element
+	 */
+	protected Element loadBuildConfigFile()
+	{
+		File build_config_file = new File(GSFile.collectionBuildConfigFile(this.site_home, this.cluster_name));
+		if (!build_config_file.exists())
+		{
+			logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", " + build_config_file + " does not exist");
+			return null;
+		}
+		Document build_config_doc = this.converter.getDOM(build_config_file, CONFIG_ENCODING);
+		Element build_config_elem = null;
+		if (build_config_doc != null)
+		{
+			build_config_elem = build_config_doc.getDocumentElement();
+		}
+
+		lastmodified = build_config_file.lastModified();
+
+		return build_config_elem;
+	}
+
+	/**
+	 * find the metadata and display elems from the two config files and add it
+	 * to the appropriate lists
+	 */
+	protected boolean findAndLoadInfo(Element coll_config_xml, Element build_config_xml)
+	{
+	  addMetadata("httpPath", this.site_http_address + "/collect/" + this.cluster_name);
+
+
+		//check whether the html are tidy or not
+		Element import_list = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.IMPORT_ELEM);
+		if (import_list != null)
+		{
+			Element plugin_list = (Element) GSXML.getChildByTagName(import_list, GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER);
+			//addPlugins(plugin_list);
+			if (plugin_list != null)
+			{
+				Element plugin_elem = (Element) GSXML.getNamedElement(plugin_list, GSXML.PLUGIN_ELEM, GSXML.NAME_ATT, "HTMLPlugin");
+				if (plugin_elem != null)
+				{
+					//get the option
+					Element option_elem = (Element) GSXML.getNamedElement(plugin_elem, GSXML.PARAM_OPTION_ELEM, GSXML.NAME_ATT, "-use_realistic_book");
+					if (option_elem != null)
+					{
+						useBook = true;
+					}
+				}
+			}
+		}
+		String tidy = (useBook == true ? "tidy" : "untidy");
+		addMetadata("tidyoption", tidy);
+
+		
+		if (this.metadata_list != null)
+		{
+		  // check whether we are public or not
+			Element meta_elem = (Element) GSXML.getNamedElement(this.metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "public");
+			if (meta_elem != null)
+			{
+				String value = GSXML.getValue(meta_elem).toLowerCase().trim();
+				if (value.equals("false"))
+				{
+					is_public = false;
+				}
+			}
+			// earliest datestamp is the time the collection was created.
+			meta_elem = (Element) GSXML.getNamedElement(this.metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, OAIXML.EARLIEST_DATESTAMP);
+			if (meta_elem != null) {
+			  String earliestDatestampStr = GSXML.getValue(meta_elem);
+			  if (!earliestDatestampStr.equals("")) {
+			      earliestDatestamp = Long.parseLong(earliestDatestampStr) * 1000; // stored in seconds, convert to milliseconds
+			  }
+			}
+		
+		}
+		return true;
+	}
+
+	protected void loadSecurityInformation(Element coll_config_xml)
+	{
+		Element securityBlock = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.SECURITY_ELEM);
+
+		if (securityBlock == null)
+		{
+			return;
+		}
+
+		String disabled = securityBlock.getAttribute(GSXML.DISABLED_ATT);
+		if (!disabled.equals("")) {
+		  // security block has been disabled.
+		  logger.warn("Security block has been disabled. Not implementing any security for collection "+this.cluster_name);
+		  return;
+		}
+		String scope = securityBlock.getAttribute(GSXML.SCOPE_ATT);
+		String defaultAccess = securityBlock.getAttribute(GSXML.DEFAULT_ACCESS_ATT);
+
+		if (defaultAccess.toLowerCase().equals("public"))
+		{
+			_publicAccess = true;
+		}
+		else if (defaultAccess.toLowerCase().equals("private"))
+		{
+			_publicAccess = false;
+		}
+		else
+		{
+			logger.warn("Default access for collection " + this.cluster_name + " is neither public or private, assuming public");
+		}
+
+		if (scope.toLowerCase().equals("collection"))
+		{
+			_securityScopeCollection = true;
+		}
+		else if (scope.toLowerCase().equals("documents") || scope.toLowerCase().equals("document"))
+		{
+			_securityScopeCollection = false;
+		}
+		else
+		{
+			logger.warn("Security scope is neither collection or document, assuming collection");
+		}
+
+		NodeList exceptions = GSXML.getChildrenByTagName(securityBlock, GSXML.EXCEPTION_ELEM);
+
+		if (exceptions.getLength() > 0)
+		{
+			if (!_securityScopeCollection)
+			{
+				NodeList documentSetElems = GSXML.getChildrenByTagName(securityBlock, GSXML.DOCUMENT_SET_ELEM);
+				for (int i = 0; i < documentSetElems.getLength(); i++)
+				{
+					Element documentSet = (Element) documentSetElems.item(i);
+					String setName = documentSet.getAttribute(GSXML.NAME_ATT);
+					NodeList matchStatements = GSXML.getChildrenByTagName(documentSet, GSXML.MATCH_ELEM);
+					ArrayList<Element> matchStatementList = new ArrayList<Element>();
+					for (int j = 0; j < matchStatements.getLength(); j++)
+					{
+						matchStatementList.add((Element) matchStatements.item(j));
+					}
+					_documentSets.put(setName, matchStatementList);
+				}
+			}
+
+			for (int i = 0; i < exceptions.getLength(); i++)
+			{
+				HashMap<String, ArrayList<String>> securityException = new HashMap<String, ArrayList<String>>();
+				ArrayList<String> exceptionGroups = new ArrayList<String>();
+				ArrayList<String> exceptionSets = new ArrayList<String>();
+
+				Element exception = (Element) exceptions.item(i);
+				NodeList groups = GSXML.getChildrenByTagName(exception, GSXML.GROUP_ELEM);
+				for (int j = 0; j < groups.getLength(); j++)
+				{
+					Element group = (Element) groups.item(j);
+					String groupName = group.getAttribute(GSXML.NAME_ATT);
+					exceptionGroups.add(groupName);
+				}
+				NodeList docSets = GSXML.getChildrenByTagName(exception, GSXML.DOCUMENT_SET_ELEM);
+				for (int j = 0; j < docSets.getLength(); j++)
+				{
+					Element docSet = (Element) docSets.item(j);
+					String docSetName = docSet.getAttribute(GSXML.NAME_ATT);
+					exceptionSets.add(docSetName);
+				}
+				if (_securityScopeCollection) {
+				  // we don't add in any exceptions that have document sets
+				  if (!exceptionSets.isEmpty()) {
+				    continue;
+				  }
+				}
+				securityException.put("groups", exceptionGroups);
+				securityException.put("sets", exceptionSets);
+				_securityExceptions.add(securityException);
+			}
+		}
+	}
+
+	protected boolean configureServiceRacks(Element coll_config_xml, Element build_config_xml)
+	{
+		clearServices();
+		Element service_list = (Element) GSXML.getChildByTagName(build_config_xml, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+		if (service_list != null)
+		{
+			configureServiceRackList(service_list, coll_config_xml);
+		}
+		// collection Config may also contain manually added service racks
+		service_list = (Element) GSXML.getChildByTagName(coll_config_xml, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+		if (service_list != null)
+		{
+			configureServiceRackList(service_list, build_config_xml);
+		}
+		return true;
+	}
+
+	/**
+	 * do a configure on only part of the collection
+	 */
+	protected boolean configureSubset(String subset)
+	{
+
+		// need the coll config files
+		Element coll_config_elem = loadCollConfigFile();
+		Element build_config_elem = loadBuildConfigFile();
+		if (coll_config_elem == null || build_config_elem == null)
+		{
+			// wont be able to do any of the requests
+			return false;
+		}
+
+		if (subset.equals(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER))
+		{
+			return configureServiceRacks(coll_config_elem, build_config_elem);
+		}
+
+		if (subset.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER) || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER) || subset.equals(GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER))
+		{
+			configureLocalData(coll_config_elem);
+			configureLocalData(build_config_elem);
+			return findAndLoadInfo(coll_config_elem, build_config_elem);
+
+		}
+
+		logger.error("Collection: cant process system request, configure " + subset);
+		return false;
+	}
+
+	/**
+	 * handles requests made to the ServiceCluster itself
+	 * 
+	 * @param req
+	 *            - the request Element- <request>
+	 * @return the result Element - should be <response>
+	 */
+  protected Element processMessage(Document result_doc, Element request)
+	{
+		String type = request.getAttribute(GSXML.TYPE_ATT);
+		if (type.equals(GSXML.REQUEST_TYPE_FORMAT_STRING))
+		{
+		  return processFormatStringRequest(result_doc, request);
+		}
+		else if (type.equals(GSXML.REQUEST_TYPE_SECURITY))
+		{
+		  return processSecurityRequest(result_doc, request);
+		}
+		else if (type.equals(GSXML.REQUEST_TYPE_FORMAT))
+		{
+
+			Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+			response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+			response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_FORMAT);
+			if (_globalFormat != null)
+			{
+				response.appendChild(result_doc.importNode(_globalFormat, true));
+			}
+			return response;
+		}
+		// unknown type
+		return super.processMessage(result_doc, request);
+
+	}
+
+  protected Element processSecurityRequest(Document result_doc, Element request)
+	{
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_SECURITY);
+
+		String oid = request.getAttribute("oid");
+		if (oid.contains("."))
+		{
+			oid = oid.substring(0, oid.indexOf("."));
+		}
+
+		ArrayList<String> groups = getPermittedGroups(oid);
+
+		Element groupList = result_doc.createElement(GSXML.GROUP_ELEM + GSXML.LIST_MODIFIER);
+		response.appendChild(groupList);
+
+		for (String groupName : groups)
+		{
+			Element group = result_doc.createElement(GSXML.GROUP_ELEM);
+			groupList.appendChild(group);
+			group.setAttribute(GSXML.NAME_ATT, groupName);
+		}
+		return response;
+	}
+
+	protected ArrayList<String> getPermittedGroups(String oid)
+	{
+		ArrayList<String> groups = new ArrayList<String>();
+
+		if (_securityScopeCollection)
+		{
+			if (_publicAccess)
+			{
+				groups.add("");
+			}
+			else
+			{
+				for (HashMap<String, ArrayList<String>> exception : _securityExceptions)
+				{
+					for (String group : exception.get("groups"))
+					{
+						groups.add(group);
+					}
+				}
+			}
+		}
+		else
+		{
+			if (oid != null && !oid.equals(""))
+			{
+				boolean inSet = false;
+				for (HashMap<String, ArrayList<String>> exception : _securityExceptions) {
+				  
+				  ArrayList<String> exceptionSets = exception.get("sets");
+				   if (exceptionSets.size() == 0) {
+				    inSet = true;
+				    for (String group : exception.get("groups"))
+				      {
+					groups.add(group);
+				      }
+				   }
+				   else {
+				      for (String setName : exception.get("sets"))
+					{
+					  if (documentIsInSet(oid, setName))
+					    {
+					      inSet = true;
+					      for (String group : exception.get("groups"))
+						{
+						  groups.add(group);
+						}
+					      break;
+					    }
+					}
+				   }
+				}
+			
+			
+				  
+				if (!inSet && _publicAccess)
+				  {// our doc was not part of any exception, so it must be public
+					groups.add("");
+				}
+			}
+			else // if we are not doing a request with an oid, then free to access
+			{
+				groups.add("");
+			}
+		}
+
+		return groups;
+	}
+
+	protected boolean documentIsInSet(String oid, String setName)
+	{
+		ArrayList<Element> matchStatements = _documentSets.get(setName);
+		if (matchStatements == null || matchStatements.size() == 0)
+		{
+			return false;
+		}
+
+		for (Element currentMatchStatement : matchStatements)
+		{
+			String fieldName = currentMatchStatement.getAttribute(GSXML.FIELD_ATT);
+			if (fieldName == null || fieldName.equals(""))
+			{
+				fieldName = "oid";
+			}
+
+			String type = currentMatchStatement.getAttribute(GSXML.TYPE_ATT);
+			if (type == null || type.equals(""))
+			{
+				type = "match";
+			}
+
+			String fieldValue = "";
+			if (!fieldName.equals("oid"))
+			{
+				fieldValue = getFieldValue(oid, fieldName);
+				if (fieldValue == null)
+				{
+					return false;
+				}
+			}
+			else
+			{
+				fieldValue = oid;
+			}
+
+			String matchValue = GSXML.getNodeText(currentMatchStatement);
+			if (type.equals("match"))
+			{
+				if (matchValue.equals(fieldValue))
+				{
+					return true;
+				}
+			}
+			else if (type.equals("regex"))
+			{
+				if (fieldValue.matches(matchValue))
+				{
+					return true;
+				}
+			}
+			else
+			{
+				logger.warn("Unknown type of match specified in security block of collection " + this.cluster_name + ".");
+			}
+		}
+
+		return false;
+	}
+
+	protected String getFieldValue(String oid, String fieldName)
+	{
+	  Document msg_doc = XMLConverter.newDOM();
+		Element metadataMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element metadataRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, this.cluster_name + "/DocumentMetadataRetrieve", new UserContext());
+		metadataMessage.appendChild(metadataRequest);
+
+		Element paramList = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		metadataRequest.appendChild(paramList);
+
+		Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
+		paramList.appendChild(param);
+
+		param.setAttribute(GSXML.NAME_ATT, "metadata");
+		param.setAttribute(GSXML.VALUE_ATT, fieldName);
+
+		Element docList = msg_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		metadataRequest.appendChild(docList);
+
+		Element doc = msg_doc.createElement(GSXML.DOC_NODE_ELEM);
+		docList.appendChild(doc);
+
+		doc.setAttribute(GSXML.NODE_ID_ATT, oid);
+
+		Element response = (Element) this.router.process(metadataMessage);
+		NodeList metadataElems = response.getElementsByTagName(GSXML.METADATA_ELEM);
+
+		if (metadataElems.getLength() > 0)
+		{
+			Element metadata = (Element) metadataElems.item(0);
+			return GSXML.getNodeText(metadata);
+		}
+
+		return null;
+	}
+
+  protected Element processFormatStringRequest(Document result_doc, Element request)
+	{
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_FORMAT_STRING);
+		response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+
+		String subaction = request.getAttribute("subaction");
+		String service = request.getAttribute("service");
+
+		String classifier = null;
+		if (service.equals("ClassifierBrowse"))
+		{
+			classifier = request.getAttribute("classifier");
+		}
+
+		// check for version file
+		String directory = new File(GSFile.collectionConfigFile(this.site_home, this.cluster_name)).getParent() + File.separator;
+
+		String version_filename = "";
+		if (service.equals("ClassifierBrowse"))
+			version_filename = directory + "browse_" + classifier + "_format_statement_version.txt";
+		else
+			version_filename = directory + "query_format_statement_version.txt";
+
+		File version_file = new File(version_filename);
+
+		if (subaction.equals("update"))
+		{
+			Element format_element = (Element) GSXML.getChildByTagName(request, GSXML.FORMAT_STRING_ELEM);
+			//String format_string = GSXML.getNodeText(format_element);
+			Element format_statement = (Element) format_element.getFirstChild();
+
+			String version_number = "1";
+			BufferedWriter writer;
+
+			try
+			{
+
+				if (version_file.exists())
+				{
+					// Read version
+					BufferedReader reader = new BufferedReader(new FileReader(version_filename));
+					version_number = reader.readLine();
+					int aInt = Integer.parseInt(version_number) + 1;
+					version_number = Integer.toString(aInt);
+					reader.close();
+				}
+				else
+				{
+					// Create
+					version_file.createNewFile();
+					writer = new BufferedWriter(new FileWriter(version_filename));
+					writer.write(version_number);
+					writer.close();
+				}
+
+				// Write version file
+				String format_statement_filename = "";
+
+				if (service.equals("ClassifierBrowse"))
+					format_statement_filename = directory + "browse_" + classifier + "_format_statement_v" + version_number + ".txt";
+				else
+					format_statement_filename = directory + "query_format_statement_v" + version_number + ".txt";
+
+				// Write format statement
+				String format_string = this.converter.getString(format_statement); //GSXML.xmlNodeToString(format_statement);
+				writer = new BufferedWriter(new FileWriter(format_statement_filename));
+				writer.write(format_string);
+				writer.close();
+
+				// Update version number
+				writer = new BufferedWriter(new FileWriter(version_filename));
+				writer.write(version_number);
+				writer.close();
+
+			}
+			catch (IOException e)
+			{
+				logger.error("IO Exception " + e);
+			}
+		}
+
+		if (subaction.equals("saveDocument"))
+		{
+			Element format_element = (Element) GSXML.getChildByTagName(request, GSXML.FORMAT_STRING_ELEM);
+			//String format_string = GSXML.getNodeText(format_element);
+			// Get display tag
+			Element display_format = (Element) format_element.getFirstChild();
+
+			String collection_config = directory + "collectionConfig.xml";
+			Document config = this.converter.getDOM(new File(collection_config), "UTF-8");
+
+			Node current_node = GSXML.getChildByTagName(config, "CollectionConfig");
+
+			// Get display child
+			if (GSXML.getChildByTagName(current_node, "display") == null)
+			{
+				// well then create a format tag
+				Element display_tag = config.createElement("display");
+				current_node = (Node) current_node.appendChild(display_tag);
+			}
+			else
+			{
+				current_node = GSXML.getChildByTagName(current_node, "display");
+			}
+
+			if (GSXML.getChildByTagName(current_node, "format") == null)
+			{
+				// well then create a format tag
+				Element format_tag = config.createElement("format");
+				current_node.appendChild(format_tag);
+			}
+
+			current_node.replaceChild(config.importNode(display_format, true), GSXML.getChildByTagName(current_node, "format"));
+
+			String new_config = this.converter.getString(config);
+
+			new_config = StringUtils.replace(new_config, "&lt;", "<");
+			new_config = StringUtils.replace(new_config, "&gt;", ">");
+			new_config = StringUtils.replace(new_config, "&quot;", "\"");
+
+			try
+			{
+				// Write to file (not original! for now)
+				BufferedWriter writer = new BufferedWriter(new FileWriter(collection_config + ".new"));
+				writer.write(new_config);
+				writer.close();
+			}
+			catch (IOException e)
+			{
+				logger.error("IO Exception " + e);
+			}
+		}
+
+		if (subaction.equals("save"))
+		{
+			Element format_element = (Element) GSXML.getChildByTagName(request, GSXML.FORMAT_STRING_ELEM);
+			Element format_statement = (Element) format_element.getFirstChild();
+
+			try
+			{
+				// open collectionConfig.xml and read in to w3 Document
+				String collection_config = directory + "collectionConfig.xml";
+				Document config = this.converter.getDOM(new File(collection_config), "UTF-8");
+
+				//String tag_name = "";
+				int k;
+				int index;
+				Element elem;
+				Node current_node = GSXML.getChildByTagName(config, "CollectionConfig");
+				NodeList current_node_list;
+
+				if (service.equals("ClassifierBrowse"))
+				{
+					//tag_name = "browse";
+					// if CLX then need to look in <classifier> X then <format>
+					// default is <browse><format>
+
+					current_node = GSXML.getChildByTagName(current_node, "browse");
+
+					// find CLX
+					if (classifier != null)
+					{
+						current_node_list = GSXML.getChildrenByTagName(current_node, "classifier");
+						index = Integer.parseInt(classifier.substring(2)) - 1;
+
+						// index should be given by X-1
+						current_node = current_node_list.item(index);
+						// what if classifier does not have a format tag?
+						if (GSXML.getChildByTagName(current_node, "format") == null)
+						{
+							// well then create a format tag
+							Element format_tag = config.createElement("format");
+							current_node.appendChild(format_tag);
+						}
+					}
+					else
+					{
+						// To support all classifiers, set classifier to null?  There is the chance here that the format tag does not exist
+						if (GSXML.getChildByTagName(current_node, "format") == null)
+						{
+							// well then create a format tag
+							Element format_tag = config.createElement("format");
+							current_node.appendChild(format_tag);
+						}
+					}
+				}
+				else if (service.equals("AllClassifierBrowse"))
+				{
+					current_node = GSXML.getChildByTagName(current_node, "browse");
+					if (GSXML.getChildByTagName(current_node, "format") == null)
+					{
+						// well then create a format tag
+						Element format_tag = config.createElement("format");
+						current_node.appendChild(format_tag);
+					}
+				}
+				else
+				{
+					// look in <format> with no attributes
+					current_node_list = GSXML.getChildrenByTagName(current_node, "search");
+					for (k = 0; k < current_node_list.getLength(); k++)
+					{
+						current_node = current_node_list.item(k);
+						// if current_node has no attributes then break
+						elem = (Element) current_node;
+						if (elem.hasAttribute("name") == false)
+							break;
+					}
+				}
+
+				current_node.replaceChild(config.importNode(format_statement, true), GSXML.getChildByTagName(current_node, "format"));
+
+				// Now convert config document to string for writing to file
+				String new_config = this.converter.getString(config);
+
+				new_config = StringUtils.replace(new_config, "&lt;", "<");
+				new_config = StringUtils.replace(new_config, "&gt;", ">");
+				new_config = StringUtils.replace(new_config, "&quot;", "\"");
+
+				// Write to file (not original! for now)
+				BufferedWriter writer = new BufferedWriter(new FileWriter(collection_config + ".new"));
+				writer.write(new_config);
+				writer.close();
+
+			}
+			catch (Exception ex)
+			{
+				logger.error("There was an exception " + ex);
+
+				StringWriter sw = new StringWriter();
+				PrintWriter pw = new PrintWriter(sw, true);
+				ex.printStackTrace(pw);
+				pw.flush();
+				sw.flush();
+				logger.error(sw.toString());
+			}
+
+		}
+
+		return response;
+	}
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/OAICollection.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/OAICollection.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/OAICollection.java	(revision 31634)
@@ -0,0 +1,187 @@
+/*
+ *    OAICollection.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.collection;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.service.ServiceRack;
+import org.greenstone.gsdl3.service.OAIPMH;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.OAIXML;
+import org.greenstone.gsdl3.util.SimpleMacroResolver;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.util.XMLTransformer;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * Represents a collection for the OAI server. This is a cut down version of Collection, as we
+ * only want to load the OAIPMH service rack, not any of the others
+ * 
+ */
+public class OAICollection extends Collection
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.collection.OAICollection.class.getName());
+
+	/** does this collection provide the OAI service */
+	protected boolean has_oai = false;
+
+  /** a reference to the OAIPMH service rack */
+  protected OAIPMH oai_service_rack = null;
+
+	/**
+	 * Configures the collection.
+	 * 
+	 * site_home and cluster_name  must be set before configure is called.
+	 * 
+	 * collection metadata is obtained, and services loaded.
+	 * 
+	 * @return true/false on success/fail
+	 */
+	public boolean configure()
+	{
+		if (this.site_home == null || this.cluster_name == null)
+		{
+			logger.error("Collection: site_home and collection_name must be set before configure called!");
+			return false;
+		}
+
+		macro_resolver.addMacro("_httpcollection_", this.site_http_address + "/collect/" + this.cluster_name);
+
+		Element coll_config_xml = loadCollConfigFile();
+		if (coll_config_xml == null) {
+		  logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", " + "Couldn't load collection config file");
+
+		  return false;
+		}
+		Element build_config_xml = loadBuildConfigFile();
+		if (build_config_xml == null)
+		{
+		  logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", " + "Couldn't load build config file");
+
+		  return false;
+		}
+		GSXSLT.modifyCollectionConfigForDebug(coll_config_xml);
+
+		// process the metadata and display items and default library params
+		super.configureLocalData(coll_config_xml);
+		super.configureLocalData(build_config_xml);
+		// get extra collection specific stuff
+		findAndLoadInfo(coll_config_xml, build_config_xml);
+
+		// load up the OAIPMH serviceRack
+		configureServiceRacks(coll_config_xml, build_config_xml);
+
+		return true;
+
+	}
+
+
+	/**
+	 * whether this collection has OAIPMH services
+	 */
+	public boolean hasOAI()
+	{
+		return has_oai;
+	}
+
+  /** add any extra info for collection from OAIConfig.xml */
+  public boolean configureOAI(Element oai_config) {
+    // just pass the element to each service - should only be one
+    return this.oai_service_rack.configureOAI(oai_config);
+  }
+
+  /** override this to only load up OAIPMH serviceRack  - don't need all the rest of them for oai*/
+	protected boolean configureServiceRackList(Element service_rack_list, Element extra_info)
+	{
+
+	  // find the OAIPMH service
+	  Element oai_service_xml = GSXML.getNamedElement(service_rack_list, GSXML.SERVICE_CLASS_ELEM, GSXML.NAME_ATT, "OAIPMH");
+	  if (oai_service_xml == null) {
+	    return false;
+	  }
+	  
+	  // the xml request to send to the serviceRack to query what services it provides
+	  Document doc = XMLConverter.newDOM();
+	  Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+	  Element request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", new UserContext());
+	  message.appendChild(request);
+
+	  this.oai_service_rack = new OAIPMH();
+	  this.oai_service_rack.setSiteHome(this.site_home);
+	  this.oai_service_rack.setSiteAddress(this.site_http_address);
+	  this.oai_service_rack.setClusterName(this.cluster_name);
+	  this.oai_service_rack.setServiceCluster(this);
+	  this.oai_service_rack.setMessageRouter(this.router);
+	  // pass the xml node to the service for configuration
+	  if (this.oai_service_rack.configure(oai_service_xml, extra_info)) {
+	    
+	    // find out the supported service types for this service module
+	    Node types = this.oai_service_rack.process(message);
+	    NodeList typenodes = ((Element) types).getElementsByTagName(GSXML.SERVICE_ELEM);
+	    
+	    for (int j = 0; j < typenodes.getLength(); j++)
+	      {
+		String service = ((Element) typenodes.item(j)).getAttribute(GSXML.NAME_ATT);
+		
+		if (service_map.get(service) != null)
+		  {
+		    char extra = '0';
+		    String new_service = service + extra;
+		    
+		    while (service_map.get(new_service) != null)
+		      {
+			extra++;
+			new_service = service + extra;
+		      }
+		    this.service_name_map.put(new_service, service);
+		    service = new_service;
+		    ((Element) typenodes.item(j)).setAttribute(GSXML.NAME_ATT, service);
+		  }
+		this.service_map.put(service, this.oai_service_rack);
+		// also add info to the ServiceInfo XML element
+		this.service_list.appendChild(this.desc_doc.importNode(typenodes.item(j), true));
+	      }
+	    has_oai = true;
+	    return true;
+	  }
+	  
+
+	  return false;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/ServiceCluster.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/ServiceCluster.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/ServiceCluster.java	(revision 31634)
@@ -0,0 +1,949 @@
+/*
+ *    ServiceCluster.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+// leave the package name as is for now - should be changed to something better
+// cluster? groups?
+package org.greenstone.gsdl3.collection;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.service.ServiceRack;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.DisplayItemUtil;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.SimpleMacroResolver;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/* ServiceCluster - a groups of services that are related in some way
+ * Implements ModuleInterface. Contains a list of services provided by the cluster, along with metadata about the cluster itself.
+ * a collection is a special type of cluster
+ *  @see ModuleInterface
+ */
+public class ServiceCluster implements ModuleInterface
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.collection.ServiceCluster.class.getName());
+
+	protected static final String CONFIG_ENCODING = "utf-8";
+
+	protected static final String DEFAULT_LANG = "en"; // hack for now, should be read from the coll cfg file? or site cfg file for cluster
+
+	/** base directory for the site that this cluster belongs to */
+	protected String site_home = null;
+	/** http address of the site that this cluster belongs to */
+	protected String site_http_address = null;
+	/** The name of the cluster - for a collection, this is the collection name */
+	protected String cluster_name = null;
+ 
+	/** a reference to the message router */
+	protected MessageRouter router = null;
+	/**
+	 * The map of services.
+	 * 
+	 * Maps Services to ServiceRack objects
+	 * 
+	 * @see ServiceRack
+	 * 
+	 */
+	protected HashMap<String, ServiceRack> service_map = null;
+	/**
+	 * maps pseudo service names to real service names - needed if we have two
+	 * services with the same name for one collection
+	 */
+	protected HashMap<String, String> service_name_map = null;
+
+	/** XML converter for String to DOM and vice versa */
+	protected XMLConverter converter = null;
+  /** a MacroResolver for resolving macros in displayItems */
+  protected SimpleMacroResolver macro_resolver = null;
+
+	/** container doc for description elements 
+	 only use this document for creating the below stored lists. */
+        protected Document desc_doc = null;
+	/** list of services */
+	protected Element service_list = null;
+	/** list of metadata - all metadata, regardless of language goes in here . metadata should be language neutral. Language specific strings are displayItems*/
+	protected Element metadata_list = null;
+  /** language specific display items */
+
+	protected Element display_item_list = null;
+  /** extra stuff */
+  protected Element extra_info = null;
+  /** default values for servlet params */
+  protected Element library_param_list = null;
+	/** the element that will have any descriptions passed back in */
+	protected Element description = null;
+
+	/** list of plugin */
+	//protected Element plugin_item_list = null;
+
+	protected Element _globalFormat = null;
+
+  	/**
+	 * A class loader that knows where to find resources 
+	 * put properties files, dtds etc in here
+	 */
+	protected ClassLoader class_loader = null;
+
+	public void setSiteHome(String home)
+	{
+		this.site_home = home;
+	}
+
+	public void setSiteAddress(String address)
+	{
+		this.site_http_address = address;
+	}
+
+	public void cleanUp()
+	{
+		Iterator<ServiceRack> i = this.service_map.values().iterator();
+		while (i.hasNext())
+		{
+			ServiceRack s = i.next();
+			s.cleanUp();
+		}
+	}
+
+	public void setClusterName(String name)
+	{
+		this.cluster_name = name;
+		this.description.setAttribute(GSXML.NAME_ATT, name);
+	}
+
+	public void setMessageRouter(MessageRouter m)
+	{
+		this.router = m;
+	}
+
+	public ServiceCluster()
+	{
+		this.service_map = new HashMap<String, ServiceRack>();
+		this.service_name_map = new HashMap<String, String>();
+		this.converter = new XMLConverter();
+		this.macro_resolver = new SimpleMacroResolver();
+		this.desc_doc = XMLConverter.newDOM();
+		this.description = this.desc_doc.createElement(GSXML.CLUSTER_ELEM);
+		this.display_item_list = this.desc_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		this.metadata_list = this.desc_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		this.library_param_list = this.desc_doc.createElement(GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER);
+		this.service_list = this.desc_doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		this.extra_info = this.desc_doc.createElement(GSXML.EXTRA_INFO_ELEM);
+		//this.plugin_item_list = this.desc_doc.createElement(GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER);
+	}
+
+	/**
+	 * Configures the cluster.
+	 * 
+	 * gsdlHome and clusterName must be set before configure is called.
+	 * 
+	 * reads the site configuration file, and configures itself this calls
+	 * configure(Element) with the XML element node from the config file.
+	 * configure(Element) should be used if the config file has already been
+	 * parsed. This method will work with any subclass.
+	 * 
+	 * This is called by ServiceCluster itself when asked to do a reconfigure
+	 * @return true if configure successful, false otherwise.
+	 */
+	public boolean configure()
+	{
+
+		if (this.site_home == null || this.cluster_name == null)
+		{
+			logger.error("site_home and cluster_name must be set before configure called!");
+			return false;
+		}
+		logger.info("configuring service cluster");
+		macro_resolver.addMacro("_httpsite_", this.site_http_address);
+		// read the site configuration file
+		File config_file = new File(GSFile.siteConfigFile(this.site_home));
+
+		if (!config_file.exists())
+		{
+			logger.error("couldn't configure cluster: " + this.cluster_name + ", " + config_file + " does not exist");
+			return false;
+		}
+
+		Document doc = this.converter.getDOM(config_file, CONFIG_ENCODING);
+		if (doc == null)
+		{
+			logger.error("couldn't parse config file " + config_file.getPath());
+			return false;
+		}
+
+		// get the appropriate service cluster element
+		Element cluster_list = (Element) GSXML.getChildByTagName(doc.getDocumentElement(), GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+		Element sc = GSXML.getNamedElement(cluster_list, GSXML.CLUSTER_ELEM, GSXML.NAME_ATT, this.cluster_name);
+
+		// this is probably a reconfigure, so clear all previous info
+		clearServices();
+		clearLocalData();
+		return this.configure(sc);
+	}
+
+  /** this is called by configure(), but also by MR when it is loading up all the service clusters */
+	public boolean configure(Element service_cluster_info)
+	{
+	  configureLocalData(service_cluster_info);
+		// //get the plugin info
+		// Element import_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.IMPORT_ELEM);
+		// if (import_list != null)
+		// {
+		// 	Element plugin_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER);
+		// 	if (plugin_list != null)
+		// 	{
+		// 		if (!addPlugins(plugin_list))
+		// 		{
+
+		// 			logger.error("couldn't configure the plugins");
+		// 		}
+		// 	}
+		// }
+
+		// do the service racks
+		// empty the service map in case this is a reconfigure
+		//clearServices();
+		Element service_rack_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+		if (service_rack_list == null)
+		{
+			// is this an error? could you ever have a service cluster
+			// without service racks???
+			logger.error(cluster_name+" has no service racks!!");
+		}
+		else
+		{
+
+			if (!configureServiceRackList(service_rack_list, null))
+			{
+				logger.error("couldn't configure "+cluster_name+" service racks!!");
+				return false;
+			}
+		}
+
+		return true;
+	}
+
+    protected void configureLocalData(Element service_cluster_info) {
+		// get the metadata - for now just add it to the list
+		Element meta_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		if (meta_list != null)
+		{
+			if (!addMetadata(meta_list))
+			{
+
+				logger.error(" couldn't configure the metadata");
+			}
+		}
+
+		// get the display info
+		Element display_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		if (display_list != null)
+		{
+		  resolveMacros(display_list);
+		  if (!DisplayItemUtil.storeDisplayItems(this.display_item_list, display_list))
+			{
+
+				logger.error("couldn't configure the display items");
+			}
+		}
+
+		// get the servlet params
+		Element param_list = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER);
+		if (param_list != null) {
+		  if (!addLibraryParams(param_list)) {
+		    logger.error("couldn't configure the library param list");
+		  }
+		}
+		
+		// get any extra info
+		Element info = (Element) GSXML.getChildByTagName(service_cluster_info, GSXML.EXTRA_INFO_ELEM);
+		if (info != null) {
+		  if (!addExtraInfo(info)) {
+		    logger.error("couldn't add extra info");
+		  }
+		}
+
+    }
+	/**
+	 * adds metadata from a metadataList into the metadata_list xml
+	 */
+	protected boolean addMetadata(Element metadata_list)
+	{
+		if (metadata_list == null)
+			return false;
+		NodeList metanodes = metadata_list.getElementsByTagName(GSXML.METADATA_ELEM);
+		if (metanodes.getLength() > 0)
+		{
+			for (int k = 0; k < metanodes.getLength(); k++)
+			{
+				this.metadata_list.appendChild(this.desc_doc.importNode(metanodes.item(k), true));
+			}
+		}
+
+		return true;
+	}
+  /** adds an individual metadata element into the list */
+  protected boolean addMetadata(String name, String value) {
+    return GSXML.addMetadata(this.metadata_list, name, value);
+  }
+  
+  /** in displayItemList, end up with the following for each named displayItem
+      <displayItem name="">
+        <displayItem name="" lang="">value</displayItem>
+        <displayItem name="" lang="">value</displayItem>
+      </displayItem>
+  */
+	protected boolean addDisplayItemsOld(Element display_list)
+	{
+
+		if (display_list == null)
+			return false;
+		NodeList displaynodes = display_list.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (displaynodes.getLength() > 0)
+		{
+			for (int k = 0; k < displaynodes.getLength(); k++)
+			{
+				Element d = (Element) displaynodes.item(k);
+				String name = d.getAttribute(GSXML.NAME_ATT);
+				Element this_item = GSXML.getNamedElement(this.display_item_list, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, name);
+				if (this_item == null)
+				{
+					this_item = this.desc_doc.createElement(GSXML.DISPLAY_TEXT_ELEM);
+					this_item.setAttribute(GSXML.NAME_ATT, name);
+					this.display_item_list.appendChild(this_item);
+				}
+
+				this_item.appendChild(this.desc_doc.importNode(d, true));
+			}
+		}
+
+		return true;
+	}
+
+  protected boolean addExtraInfo(Element info) {
+    if (info == null) {
+      return false;
+    }
+    NodeList children = info.getChildNodes();
+    for(int i=0; i<children.getLength(); i++) {
+      this.extra_info.appendChild(this.desc_doc.importNode(children.item(i), true));
+    }
+    return true;
+  }
+
+	protected boolean resolveMacros(Element display_list)
+	{
+		if (display_list == null)
+			return false;
+		NodeList displaynodes = display_list.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (displaynodes.getLength() > 0)
+		{
+		  //String http_site = this.site_http_address;
+		  //String http_collection = this.site_http_address + "/collect/" + this.cluster_name;
+			for (int k = 0; k < displaynodes.getLength(); k++)
+			{
+				Element d = (Element) displaynodes.item(k);
+				String text = GSXML.getNodeText(d);
+				text= macro_resolver.resolve(text);
+				//text = StringUtils.replace(text, "_httpsite_", http_site);
+				//text = StringUtils.replace(text, "_httpcollection_", http_collection);
+				GSXML.setNodeText(d, text);
+			}
+		}
+		return true;
+	}
+
+  	/**
+	 * adds library params from libraryParamList into library_param_list xml
+	 */
+	protected boolean addLibraryParams(Element param_list)
+	{
+		if (param_list == null)
+			return false;
+		NodeList paramnodes = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+		if (paramnodes.getLength() > 0)
+		{
+			for (int k = 0; k < paramnodes.getLength(); k++)
+			{
+				this.library_param_list.appendChild(this.desc_doc.importNode(paramnodes.item(k), true));
+			}
+		}
+
+		return true;
+	}
+	protected void clearServices()
+	{
+	  cleanUp();
+		service_map.clear();
+		service_name_map.clear();
+		this.service_list = this.desc_doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+	}
+
+  protected void clearLocalData() {
+    this.description = this.desc_doc.createElement(GSXML.CLUSTER_ELEM);
+    this.display_item_list = this.desc_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+    this.metadata_list = this.desc_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    this.library_param_list = this.desc_doc.createElement(GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER);
+ 
+  }
+	/**
+	 * creates and configures all the services - extra_info is some more xml
+	 * that is passed to teh service - eg used for coll config files for
+	 * Collection
+	 */
+	protected boolean configureServiceRackList(Element service_rack_list, Element extra_info)
+	{
+
+		// create all the services
+		NodeList nodes = service_rack_list.getElementsByTagName(GSXML.SERVICE_CLASS_ELEM);
+		if (nodes.getLength() == 0)
+		{
+			logger.error("ServiceCluster configuration error: cluster " + this.cluster_name + " has no service modules!");
+			return false;
+		}
+
+		// the xml request to send to the serviceRacks to query what
+		// services they provide
+		// can send same message to each service rack
+		Document doc = XMLConverter.newDOM();
+		Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", new UserContext());
+		message.appendChild(request);
+		for (int i = 0; i < nodes.getLength(); i++)
+		{
+
+
+			Element n = (Element) nodes.item(i);
+			String servicetype = n.getAttribute(GSXML.NAME_ATT);
+
+			ServiceRack s = null;
+
+			try
+			{
+				// try for a default service in standard package
+				s = (ServiceRack) Class.forName("org.greenstone.gsdl3.service." + servicetype).newInstance();
+			}
+			catch (Exception e)
+			{
+			}
+			if (s == null)
+			{
+				try
+				{
+					// name as is, in case package is already specified
+					s = (ServiceRack) Class.forName(servicetype).newInstance();
+				}
+				catch (Exception e)
+				{
+				}
+			}
+
+			if (s == null)
+			{
+				logger.error("Couldn't get an instance of class " + servicetype + ", or org.greenstone.gsdl3.service." + servicetype);
+				continue;
+			}
+
+			if (_globalFormat != null)
+			{
+				s.setGlobalFormat(_globalFormat);
+			}
+
+			s.setSiteHome(this.site_home);
+			s.setSiteAddress(this.site_http_address);
+			s.setClusterName(this.cluster_name);
+			s.setServiceCluster(this);
+			s.setMessageRouter(this.router);
+			// pass the xml node to the service for configuration
+			if (s.configure(n, extra_info))
+			{
+				// find out the supported service types for this service module
+				Node types = s.process(message);
+				NodeList typenodes = ((Element) types).getElementsByTagName(GSXML.SERVICE_ELEM);
+
+				for (int j = 0; j < typenodes.getLength(); j++)
+				{
+					String service = ((Element) typenodes.item(j)).getAttribute(GSXML.NAME_ATT);
+
+					if (service_map.get(service) != null)
+					{
+						char extra = '0';
+						String new_service = service + extra;
+
+						while (service_map.get(new_service) != null)
+						{
+							extra++;
+							new_service = service + extra;
+						}
+						this.service_name_map.put(new_service, service);
+						service = new_service;
+						((Element) typenodes.item(j)).setAttribute(GSXML.NAME_ATT, service);
+					}
+					this.service_map.put(service, s);
+					// also add info to the ServiceInfo XML element
+					this.service_list.appendChild(this.desc_doc.importNode(typenodes.item(j), true));
+				}
+			}
+		}
+
+		return true;
+
+	}
+
+	/**
+	 * Process an XML document - uses Strings just calls process(Node).
+	 * 
+	 * @param in
+	 *            the Document to process - a string
+	 * @return the resultant document as a string - contains any error messages
+	 * @see String
+	 */
+	public String process(String in)
+	{
+
+		Document doc = this.converter.getDOM(in);
+
+		Node res = process(doc);
+		return this.converter.getString(res);
+
+	}
+
+	/**
+	 * process XML as Node
+	 * 
+	 */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+
+		NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+		Document mess_doc = message.getOwnerDocument();
+		Document result_doc = XMLConverter.newDOM();
+		Element mainResult = result_doc.createElement(GSXML.MESSAGE_ELEM);
+		if (requests.getLength() == 0)
+		{
+			logger.error("no requests for cluster:" + this.cluster_name);
+			// no requests
+			return mainResult; // for now
+		}
+		for (int i = 0; i < requests.getLength(); i++)
+		{
+			Element request = (Element) requests.item(i);
+			String to = request.getAttribute(GSXML.TO_ATT);
+
+			// the cluster name should be first, check, then remove
+			String clustername = GSPath.getFirstLink(to);
+			if (!clustername.equals(this.cluster_name))
+			{
+				logger.error("cluster name wrong! was " + clustername + " should have been " + this.cluster_name);
+				continue; // ignore this request
+			}
+			to = GSPath.removeFirstLink(to);
+			request.setAttribute(GSXML.TO_ATT, to);
+
+			if (to.equals(""))
+			{ // this command is for me
+			  Element response = processMessage(result_doc, request);
+				mainResult.appendChild(response);
+
+			}
+			else
+			{ // the request is for one of my services
+				String service = GSPath.getFirstLink(to);
+
+				if (!this.service_map.containsKey(service))
+				{
+					logger.error("non-existant service, " + service + ", specified!");
+					continue;
+				}
+				String real_service = service;
+				if (this.service_name_map.containsKey(service))
+				{
+					real_service = this.service_name_map.get(service);
+					// need to change the to att in the request - give the real service name
+					to = request.getAttribute(GSXML.TO_ATT);
+					String old_to = to;
+					to = GSPath.replaceFirstLink(to, real_service);
+					request.setAttribute(GSXML.TO_ATT, to);
+				}
+				// have to pass the request to the service
+				Element single_message = mess_doc.createElement(GSXML.MESSAGE_ELEM);
+				single_message.appendChild(request);
+
+				Node response_message = this.service_map.get(service).process(single_message);
+				if (response_message != null)
+				{
+					Element response = (Element) GSXML.getChildByTagName(response_message, GSXML.RESPONSE_ELEM);
+					String from = response.getAttribute(GSXML.FROM_ATT);
+					if (!real_service.equals(service))
+					{
+						// replace the real service name with the pseudo service name
+						from = GSPath.replaceFirstLink(from, service);
+						// also need to do it in the service itself
+						// shoudl this be done here??
+						Element service_elem = (Element) GSXML.getChildByTagName(response, GSXML.SERVICE_ELEM);
+						if (service_elem != null)
+						{
+							service_elem.setAttribute(GSXML.NAME_ATT, service);
+						}
+					}
+					from = GSPath.prependLink(from, this.cluster_name);
+					response.setAttribute(GSXML.FROM_ATT, from);
+					mainResult.appendChild(result_doc.importNode(response, true));
+				}
+
+			} // else
+
+		} // for each request
+		return mainResult;
+	}
+
+	/**
+	 * handles requests made to the ServiceCluster itself
+	 * 
+	 * @param req
+	 *            - the request Element- <request>
+	 * @return the result Element - should be <response>
+	 */
+  protected Element processMessage(Document result_doc, Element request)
+	{
+
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+		String type = request.getAttribute(GSXML.TYPE_ATT);
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		response.setAttribute(GSXML.TYPE_ATT, type);
+
+		if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE))
+		{
+			// create the collection element
+		  Element description = (Element) result_doc.importNode(this.description, false);
+			// set collection type : mg, mgpp, lucene or solr
+			//description.setAttribute(GSXML.TYPE_ATT, col_type);
+			//description.setAttribute(GSXML.DB_TYPE_ATT, db_type);
+
+			response.appendChild(description);
+			// check the param list
+			Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (param_list == null)
+			{
+			  Element di_list = result_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+			  description.appendChild(di_list);
+			  DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+				description.appendChild(result_doc.importNode(this.service_list, true));
+				description.appendChild(result_doc.importNode(this.metadata_list, true));
+				description.appendChild(result_doc.importNode(this.library_param_list, true));
+				description.appendChild(result_doc.importNode(this.extra_info, true));
+				//description.appendChild(this.plugin_item_list);
+				return response;
+			}
+
+			// go through the param list and see what components are wanted
+			NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+			for (int i = 0; i < params.getLength(); i++)
+			{
+
+				Element param = (Element) params.item(i);
+				// Identify the structure information desired
+				if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM))
+				{
+					String info = param.getAttribute(GSXML.VALUE_ATT);
+					if (info.equals(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER))
+					{
+					  description.appendChild(result_doc.importNode(this.service_list, true));
+					}
+					else if (info.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER))
+					{
+					  description.appendChild(result_doc.importNode(this.metadata_list, true));
+					}
+					else if (info.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+					{
+					  Element di_list = result_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+					  description.appendChild(di_list);
+					  DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+
+					}
+					else if (info.equals(GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER))
+					{
+					  description.appendChild(result_doc.importNode(this.library_param_list, true));
+					}
+					else if (info.equals(GSXML.EXTRA_INFO_ELEM)) {
+					  description.appendChild(result_doc.importNode(this.extra_info, true));
+					}
+				}
+			}
+			return response;
+		}
+		/*
+		 * if (type.equals(GSXML.REQUEST_TYPE_FORMAT_STRING)) {
+		 * logger.error("Received format string request"); String service =
+		 * request.getAttribute("service"); logger.error("Service is " +
+		 * service); String classifier = null;
+		 * if(service.equals("ClassifierBrowse")) { classifier =
+		 * request.getAttribute("classifier"); logger.error("Classifier is " +
+		 * classifier); } Element format_element = (Element)
+		 * GSXML.getChildByTagName(request, GSXML.FORMAT_STRING_ELEM); String
+		 * format_string = GSXML.getNodeText(format_element);
+		 * logger.error("Format string: " + format_string);
+		 * logger.error("Config file location = " +
+		 * GSFile.collectionConfigFile(this.site_home, this.cluster_name));
+		 * 
+		 * // check for version file
+		 * 
+		 * String directory = new
+		 * File(GSFile.collectionConfigFile(this.site_home,
+		 * this.cluster_name)).getParent() + File.pathSeparator;
+		 * logger.error("Directory is " + directory);
+		 * 
+		 * String version_filename = "";
+		 * 
+		 * if(service.equals("ClassifierBrowse")) version_filename = directory +
+		 * "browse_"+classifier+"_format_statement_version.txt"; else
+		 * version_filename = directory + "query_format_statement_version.txt";
+		 * 
+		 * File version_file = new File(version_filename);
+		 * logger.error("Version filename is " + version_filename);
+		 * 
+		 * String version_number = "1"; BufferedWriter writer; // = new
+		 * BufferedWriter(new FileWriter(version_filename)); //RandomAccessFile
+		 * version_file_random_access;
+		 * 
+		 * try{
+		 * 
+		 * if(version_file.exists()) { // Read version BufferedReader reader =
+		 * new BufferedReader(new FileReader(version_filename));
+		 * //version_file_random_access = new RandomAccessFile(version_file,
+		 * "r"); //logger.error(" //version_number =
+		 * version_file_random_access.readInt(); version_number =
+		 * reader.readLine(); int aInt = Integer.parseInt(version_number) + 1;
+		 * version_number = Integer.toString(aInt); reader.close();
+		 * //version_file_random_access.close(); } else{ // Create
+		 * version_file.createNewFile(); // write 1 to file writer = new
+		 * BufferedWriter(new FileWriter(version_filename));
+		 * //version_file_random_access = new RandomAccessFile(version_file,
+		 * "w"); //version_file_random_access.writeInt(version_number);
+		 * writer.write(version_number); writer.close();
+		 * //version_file_random_access.close(); }
+		 * 
+		 * // Write version file String format_statement_filename = "";
+		 * 
+		 * if(service.equals("ClassifierBrowse")) format_statement_filename =
+		 * directory + "browse_"+classifier+"_format_statement_v" +
+		 * version_number + ".txt"; else format_statement_filename = directory +
+		 * "query_format_statement_v" + version_number + ".txt";
+		 * 
+		 * logger.error("Format statement filename is " +
+		 * format_statement_filename);
+		 * 
+		 * writer = new BufferedWriter(new
+		 * FileWriter(format_statement_filename)); writer.write(format_string);
+		 * writer.close();
+		 * 
+		 * // Update version number //version_file_random_access = new
+		 * RandomAccessFile(version_file, "w");
+		 * //version_file_random_access.writeInt(version_number);
+		 * //version_file_random_access.close();
+		 * 
+		 * writer = new BufferedWriter(new FileWriter(version_filename));
+		 * //version_file_random_access = new RandomAccessFile(version_file,
+		 * "w"); //version_file_random_access.writeInt(version_number);
+		 * writer.write(version_number); writer.close();
+		 * 
+		 * 
+		 * 
+		 * } catch (IOException e) { logger.error("IO Exception "+e);
+		 * //System.exit(1); }
+		 * 
+		 * 
+		 * }
+		 */
+
+		if (type.equals(GSXML.REQUEST_TYPE_SYSTEM))
+		{
+			response = processSystemRequest(request);
+		}
+		else
+		{ // unknown type
+			logger.error("Can't handle request of type " + type);
+
+		}
+		return response;
+	}
+
+	protected Element processSystemRequest(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_SYSTEM);
+
+		// a list of system requests - should put any error messages
+		// or success messages into response
+		NodeList commands = request.getElementsByTagName(GSXML.SYSTEM_ELEM);
+		String message = null;
+		for (int i = 0; i < commands.getLength(); i++)
+		{
+			// all the commands should be Elements
+			Element elem = (Element) commands.item(i);
+			String action = elem.getAttribute(GSXML.TYPE_ATT);
+			if (action.equals(GSXML.SYSTEM_TYPE_CONFIGURE))
+			{
+				String subset = elem.getAttribute(GSXML.SYSTEM_SUBSET_ATT);
+				if (subset.equals(""))
+				{
+					// need to reconfigure the service cluster
+
+					if (this.configure())
+					{
+						Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, this.cluster_name + " reconfigured");
+						response.appendChild(s);
+
+					}
+					else
+					{
+						Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, this.cluster_name + " could not be reconfigured");
+						response.appendChild(s);
+					}
+				}
+				else if (this.configureSubset(subset))
+				{
+					Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, this.cluster_name + " " + subset + " reconfigured");
+					response.appendChild(s);
+				}
+				else
+				{
+					Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, this.cluster_name + " " + subset + " could not be reconfigured");
+					response.appendChild(s);
+				}
+				continue;
+			} // configure action
+
+			String module_name = elem.getAttribute(GSXML.SYSTEM_MODULE_NAME_ATT);
+			String module_type = elem.getAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT);
+			if (action.equals(GSXML.SYSTEM_TYPE_ACTIVATE))
+			{
+				Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, "activate action not yet implemented - does it even make sense in this context??");
+				response.appendChild(s);
+			}
+			else if (action.equals(GSXML.SYSTEM_TYPE_DEACTIVATE))
+			{
+				if (module_type.equals(GSXML.SERVICE_ELEM))
+				{
+					// deactivate the service
+					// remove from service_map
+					this.service_map.remove(module_name);
+					Element service_elem = GSXML.getNamedElement(this.service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, module_name);
+					service_list.removeChild(service_elem);
+					message = module_type + ": " + module_name + " deactivated";
+				}
+				else
+				{
+					message = "Can't deactivate " + module_type + " type modules!";
+				}
+				Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, message);
+				response.appendChild(s);
+			}
+			else
+			{
+				logger.error("Can't process system request, action " + action);
+				continue;
+			}
+		} // for each command
+		return response;
+	}
+
+	/**
+	 * do a configure on only part of the collection
+	 */
+	protected boolean configureSubset(String subset)
+	{
+
+		File configFile = new File(GSFile.siteConfigFile(this.site_home));
+		if (!configFile.exists())
+		{
+			logger.error("site config file: " + configFile.getPath() + " not found!");
+			// wont be able to do any of the requests
+			return false;
+
+		}
+
+		Document site_config_doc = this.converter.getDOM(configFile);
+		if (site_config_doc == null)
+		{
+			logger.error("could not read in site config file: " + configFile.getPath());
+			return false;
+		}
+
+		Element site_config_elem = site_config_doc.getDocumentElement();
+		Element cluster_config_elem = GSXML.getNamedElement((Element) GSXML.getChildByTagName(site_config_elem, GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER), GSXML.CLUSTER_ELEM, GSXML.NAME_ATT, this.cluster_name);
+		if (cluster_config_elem == null)
+		{
+			logger.error("site config file: " + configFile.getPath() + " has no element for cluster " + this.cluster_name);
+			// wont be able to do any of teh requests
+			return false;
+
+		}
+		if (subset.equals(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER))
+		{
+			Element service_rack_list = (Element) GSXML.getChildByTagName(cluster_config_elem, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+			clearServices();
+			return configureServiceRackList(service_rack_list, null);
+		}
+		else if (subset.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER))
+		{
+			this.metadata_list = this.desc_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+			Element metadata_list = (Element) GSXML.getChildByTagName(cluster_config_elem, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+			return addMetadata(metadata_list);
+		}
+		// else if (subset.equals(GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER))
+		// {
+		// 	this.plugin_item_list = this.doc.createElement(GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER);
+		// 	Element import_list = (Element) GSXML.getChildByTagName(cluster_config_elem, GSXML.IMPORT_ELEM);
+		// 	if (import_list != null)
+		// 	{
+		// 		Element plugin_item_list = (Element) GSXML.getChildByTagName(cluster_config_elem, GSXML.PLUGIN_ELEM + GSXML.LIST_MODIFIER);
+		// 		return addPlugins(plugin_item_list);
+		// 	}
+		// 	else
+		// 		return false;
+		// }
+		else
+		{
+			logger.error("cannot process system request, configure " + subset);
+			return false;
+		}
+
+	}
+
+	public HashMap<String, ServiceRack> getServiceMap()
+	{
+		return service_map;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/XMLCollection.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/XMLCollection.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/collection/XMLCollection.java	(revision 31634)
@@ -0,0 +1,133 @@
+package org.greenstone.gsdl3.collection;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+import org.greenstone.gsdl3.service.*;
+
+
+// java XML classes we're using
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.io.File;
+import java.util.HashMap;
+
+import org.apache.log4j.*;
+
+/* for a collection that hasn't been built with greenstone build stuff. expects a documentList in the collectionConfig file, and it stores this. this is where doc level metadata comes from */
+public class XMLCollection 
+    extends Collection {
+
+     static Logger logger = Logger.getLogger(org.greenstone.gsdl3.collection.XMLCollection.class.getName());
+
+    protected Element document_list = null;
+    /** overwrite this to keep the document list
+     * find the metadata and display elems from the two config files and add it to the appropriate lists
+     */
+    protected boolean findAndLoadInfo(Element coll_config_xml, 
+				      Element build_config_xml){
+
+	// add metadata to stored metadata list from collConfig and buildConfig
+	Element meta_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+	meta_list = (Element)GSXML.getChildByTagName(build_config_xml, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	addMetadata(meta_list);
+
+	addMetadata("httpPath", this.site_http_address+"/collect/"+this.cluster_name);
+
+	// display stuff
+	Element display_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER);
+	if (display_list != null) {
+	  resolveMacros(display_list);
+	  DisplayItemUtil.storeDisplayItems(this.display_item_list, display_list);
+	}
+
+	// are we a private collection??
+	if (this.metadata_list != null) {
+	  
+	  Element meta_elem = (Element) GSXML.getNamedElement(this.metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "public");
+	  if (meta_elem != null) {
+	    
+	    String value = GSXML.getValue(meta_elem).toLowerCase().trim();
+	    if (value.equals("false")) {
+	      is_public = false;
+	    }
+	  }
+	}
+	Element config_doc_list = (Element)GSXML.getChildByTagName(coll_config_xml, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+	if (config_doc_list != null) {
+	    document_list = (Element)desc_doc.importNode(config_doc_list, true);
+	} else {
+	    document_list = desc_doc.createElement(GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+	}
+	return true;
+
+    }
+
+    /** handles requests made to the ServiceCluster itself 
+     *
+     * @param req - the request Element- <request>
+     * @return the result Element - should be <response>
+     */
+  protected Element processMessage(Document response_doc, Element request) {
+
+	Element response = response_doc.createElement(GSXML.RESPONSE_ELEM);
+	response.setAttribute(GSXML.FROM_ATT, this.cluster_name);
+	String type = request.getAttribute(GSXML.TYPE_ATT);
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	response.setAttribute(GSXML.TYPE_ATT, type);
+	
+	if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE)) {
+	    // create the collection element
+	  Element description = (Element)response_doc.importNode(this.description, false);
+	    response.appendChild(description);
+	    // check the param list
+	    Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    if (param_list == null) {
+	      Element di_list = response_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+	      description.appendChild(di_list);
+	      DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+	      
+		description.appendChild(response_doc.importNode(this.service_list, true));
+		description.appendChild(response_doc.importNode(this.metadata_list, true));
+		description.appendChild(response_doc.importNode(this.library_param_list, true));
+		description.appendChild(response_doc.importNode(this.document_list, true));
+		return response;
+	    }
+	    
+	    // go through the param list and see what components are wanted
+	    NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+	    for (int i=0; i<params.getLength(); i++) {
+		
+		Element param = (Element)params.item(i);
+		// Identify the structure information desired
+		if (param.getAttribute(GSXML.NAME_ATT) == GSXML.SUBSET_PARAM ) {
+		    String info = param.getAttribute(GSXML.VALUE_ATT);
+		    if (info.equals(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(response_doc.importNode(this.service_list, true));
+		    } else if (info.equals(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(response_doc.importNode(metadata_list, true));
+		    } else if (info.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) {
+		      Element di_list = response_doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		      description.appendChild(di_list);
+		      DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+		
+			
+		    } else if (info.equals(GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER)) {
+			description.appendChild(response_doc.importNode(this.document_list, true));
+		    } else if (info.equals(GSXML.LIBRARY_PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+		      
+			description.appendChild(response_doc.importNode(this.library_param_list, true));
+		    }
+		
+		}
+	    }
+	    return response;
+	}
+	return super.processMessage(response_doc, request);
+	    
+    }
+  
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/Communicator.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/Communicator.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/Communicator.java	(revision 31634)
@@ -0,0 +1,69 @@
+/*
+ *    Communicator.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.comms;
+
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.*;
+
+//XML packages
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+/** Communicator - base class for Modules that talk via some protocol to other modules
+ *
+ * can be used by a MessageRouter - in this case set localSiteName
+ * can be used by any other module - in this case it has no local site - dont set localsiteName. the setting of localSiteName affects the handling of teh to and from fields in the message. 
+ * @author Katherine Don
+ * @version $Revision$
+ */
+abstract public class Communicator
+    implements ModuleInterface {
+
+    /** name of local site */
+    protected String local_site_name_ = null;
+
+    /** name of site to connect to */
+    protected String remote_site_name_=null;
+
+    /** converter for String to DOM and vice versa */
+    protected XMLConverter converter_= null;
+
+    public Communicator() {
+	converter_ = new XMLConverter();
+    }
+    public void cleanUp() {}
+
+    public void setLocalSiteName(String name) {
+	local_site_name_ = name;
+    }
+    /** this should be done as part of configure */
+    public void setRemoteSiteName(String name) {
+	remote_site_name_ = name;
+    }
+    /** configures the Communicator using the <site> element */
+    abstract public boolean configure(Element site_elem); 
+
+    public String process(String xml_in) {
+	Node n = converter_.getDOM(xml_in);
+	Node result = process(n);
+	return converter_.getString(result);	
+    }
+
+    abstract public Node process(Node xml_in_node);
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/SOAPCommunicator.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/SOAPCommunicator.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/comms/SOAPCommunicator.java	(revision 31634)
@@ -0,0 +1,175 @@
+/*
+ *    SOAPCommunicator.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.comms;
+
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+
+// classes needed for SOAP stuff
+import java.net.URL;
+import java.net.MalformedURLException;
+import java.util.Vector;
+import org.apache.axis.client.Call;
+import org.apache.axis.client.Service;
+import org.apache.axis.message.SOAPBodyElement;
+import javax.xml.namespace.QName;
+
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element; 
+
+import org.apache.log4j.*;
+
+/*
+ * The Client side of a SOAP Connection
+ *
+ * @author Katherine Don
+ * @version $Revision$
+ * @see <a href="http://www.w3.org/TR/SOAP/">Simple Object Access Protocol (SOAP) 1.1 </a>
+ */
+public class SOAPCommunicator
+    extends Communicator {
+
+      static Logger logger = Logger.getLogger(org.greenstone.gsdl3.comms.SOAPCommunicator.class.getName());
+
+    /** address of site to connect to */
+    protected String remote_site_address_=null;
+
+    /** the call object that does the SOAP talking */
+    private Call call_ = null;
+
+    /** The no-args constructor */
+    public SOAPCommunicator() {	
+    }
+        
+    public boolean configure(Element site_elem) {
+	
+	String type = site_elem.getAttribute(GSXML.TYPE_ATT);
+	if (!type.equals(GSXML.COMM_TYPE_SOAP_JAVA)) {
+	    logger.error("wrong type of site");
+	    return false;
+	}
+	remote_site_name_ = site_elem.getAttribute(GSXML.NAME_ATT);
+	if (remote_site_name_.equals("")) {
+	    logger.error("must have name attribute in site element");
+	    return false;
+	}
+	remote_site_address_ = site_elem.getAttribute(GSXML.ADDRESS_ATT);
+	String local_site_name = site_elem.getAttribute(GSXML.LOCAL_SITE_ATT);
+	if (remote_site_address_.equals("") && local_site_name.equals("")) {
+	    logger.error("must have address or localSite attributes in site element");
+	    return false;
+	}
+	if (remote_site_address_.equals("")) {
+	    remote_site_address_ = GlobalProperties.getGSDL3WebAddress()+"/services/"+local_site_name;
+	}
+
+	try {
+	    Service service = new Service();
+	    call_ = (Call) service.createCall();
+	    call_.setTargetEndpointAddress( new java.net.URL(remote_site_address_) );
+	} catch (Exception e) {
+	    logger.error("SOAPCommunicator.configure() Error: Exception occurred "+e);
+	    return false;
+	}
+	return true;
+    }
+
+    public Node process(Node message_node) {
+	
+	Element message = GSXML.nodeToElement(message_node);
+
+	NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+	if (requests.getLength()==0) {
+	    // no requests
+	    return null;
+	}
+
+	// if the communicator is part of a site, it needs to edit the to and from fields, otherwise it just passes the message on, as is
+	// for now, use local_site_name_ as the flag.
+	if (local_site_name_!=null) { // no local site
+	    
+	    for (int i=0;i<requests.getLength(); i++) {
+		Element e = (Element)requests.item(i);
+		String to = e.getAttribute(GSXML.TO_ATT);
+		to = GSPath.removeFirstLink(to); // remove the server name, should check that it is present
+		e.setAttribute(GSXML.TO_ATT, to);
+		String from = e.getAttribute(GSXML.FROM_ATT);
+		from = GSPath.appendLink(from, local_site_name_);
+		e.setAttribute(GSXML.FROM_ATT, from);
+	    }
+	}
+	// else do nothing to the requests, just pass it on
+
+	// the soap impl needs a namespace for the top level element
+	Element message_to_send = message;
+	if (message.getNamespaceURI() == null) {
+	    message_to_send = GSXML.duplicateWithNewNameNS(message.getOwnerDocument(), message, "gs3:"+message.getTagName(), "urn:foo", true);
+	}
+	
+	// do the soap query
+	Element result = null;
+	try {
+	    SOAPBodyElement[] input = new SOAPBodyElement[1];
+	    input[0] = new SOAPBodyElement(message_to_send);
+	    Vector output = (Vector) call_.invoke( input );
+	    SOAPBodyElement elem = (SOAPBodyElement) output.get(0);
+	    result = elem.getAsDOM();
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    return null;
+	}
+	
+	if (local_site_name_ != null) {// have to modify the from field
+	    
+	    NodeList responses = result.getElementsByTagName(GSXML.RESPONSE_ELEM);
+	    for (int i=0;i<responses.getLength(); i++) {
+		Element e = (Element)responses.item(i);
+		String from = e.getAttribute(GSXML.FROM_ATT);
+		from = GSPath.prependLink(from, remote_site_name_);
+		e.setAttribute(GSXML.FROM_ATT, from);
+	    }
+	} // else return as is
+	
+	return result;
+    }
+
+    public static void main (String [] args) {
+	SOAPCommunicator comm = new SOAPCommunicator();
+	XMLConverter converter = new XMLConverter();
+	String message = "<site name=\"localsite\" address=\"http://kanuka.cs.waikato.ac.nz:7070/axis/services/localsite\" type=\"soap\"/>";
+	Element site_elem = converter.getDOM(message).getDocumentElement();
+	comm.configure(site_elem);
+	message = "<message><request type=\"describe\" to=\"\" lang=\"en\"/></message>";
+	Element request_elem = converter.getDOM(message).getDocumentElement();
+	Node response = comm.process(request_elem);
+	
+	logger.error("response was "+converter.getPrettyString(response));
+    }
+}
+
+
+
+
+
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/DefaultReceptionist.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/DefaultReceptionist.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/DefaultReceptionist.java	(revision 31634)
@@ -0,0 +1,200 @@
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+// other java classes
+import java.io.File;
+import java.util.HashMap;
+import java.util.Enumeration;
+
+import org.apache.log4j.*;
+
+/** The default greenstone receptionist - needs some extra info for each page */
+public class DefaultReceptionist extends TransformingReceptionist
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.DefaultReceptionist.class.getName());
+
+	/**
+	 * add in the collection description to the page, then for each service, add
+	 * in the service description
+	 */
+	protected void addExtraInfo(Element page)
+	{
+		super.addExtraInfo(page);
+		// the document for the page
+		Document doc = page.getOwnerDocument();
+		// a new document to create messages to send to MR
+		Document msg_doc = XMLConverter.newDOM();
+		Element page_request = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+		// if it is a system request, then we don't bother with this.
+		String action = page_request.getAttribute(GSXML.ACTION_ATT);
+		if (action.equals("s"))
+		{
+			logger.error("HACK: don't ask for coll info if system action");
+			return;
+		}
+		logger.debug("add extra info, page request=" + this.converter.getString(page_request));
+		// is a collection defined?
+		Element param_list = (Element) GSXML.getChildByTagName(page_request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{ // must be the original home page
+			logger.debug(" no param list, assuming home page");
+			return;
+		}
+		Element coll_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSParams.COLLECTION);
+		if (coll_param == null)
+		{
+			logger.debug(" coll param is null, returning");
+			return;
+		}
+
+		// see if the collection/cluster element is already there
+		String coll_name = coll_param.getAttribute(GSXML.VALUE_ATT);
+		UserContext userContext = new UserContext(page_request);
+
+		if (coll_name.equals(""))
+		{
+			Element pc_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, "p.c");
+			if (pc_param != null)
+			{
+				coll_name = pc_param.getAttribute(GSXML.VALUE_ATT);
+			}
+		}
+
+		boolean get_service_description = false;
+		Element page_response = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+		if (this.language_list != null)
+		{
+		  page_response.appendChild(doc.importNode(this.language_list, true));
+		}
+		Element coll_description = (Element) GSXML.getChildByTagName(page_response, GSXML.COLLECTION_ELEM);
+		if (coll_description == null)
+		{
+			// try cluster
+			coll_description = (Element) GSXML.getChildByTagName(page_response, GSXML.CLUSTER_ELEM);
+		}
+		if (coll_description == null)
+		{
+		  logger.debug("getting the coll description");
+			// we dont have one yet - get it
+
+			Element coll_about_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+			Element coll_about_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, coll_name, userContext);
+			coll_about_message.appendChild(coll_about_request);
+
+			Node coll_about_response_message = this.mr.process(coll_about_message);
+			Element coll_about_response = (Element) GSXML.getChildByTagName(coll_about_response_message, GSXML.RESPONSE_ELEM);
+			if (coll_about_response == null)
+			{
+				return;
+			}
+			coll_description = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.COLLECTION_ELEM);
+			if (coll_description == null)
+			{ // may be a cluster
+				coll_description = (Element) GSXML.getChildByTagName(coll_about_response, GSXML.CLUSTER_ELEM);
+			}
+
+			if (coll_description == null)
+			{
+				logger.error(" no collection description, returning");
+				return;
+			}
+			// have found one, append it to the page response
+			coll_description = (Element) doc.importNode(coll_description, true);
+			page_response.appendChild(coll_description);
+			get_service_description = true;
+		}
+		
+
+		// have got a coll description
+		// now get the dispay info for the services
+		Element service_list = (Element) GSXML.getChildByTagName(coll_description, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		if (service_list == null)
+		{
+			logger.error(" no service list, returning");
+			// something weird has gone wrong
+			return;
+		}
+
+		NodeList services = service_list.getElementsByTagName(GSXML.SERVICE_ELEM);
+		if (services.getLength() == 0)
+		{
+			logger.error("DefaultReceoptionist: no services found for colllection/cluster " + coll_name);
+			return;
+		}
+		// check one service for display items
+		if (!get_service_description)
+		{
+			// we dont know yet if we need to get these
+			int i = 1;
+			Element test_s = (Element) services.item(0);
+			while (i < services.getLength() && (test_s.getAttribute(GSXML.TYPE_ATT).equals(GSXML.SERVICE_TYPE_RETRIEVE) || test_s.getAttribute(GSXML.TYPE_ATT).equals(GSXML.SERVICE_TYPE_OAI)))
+			{
+				test_s = (Element) services.item(i);
+				i++;
+			}
+			if (i == services.getLength())
+			{
+				// we have only found retrieve or oai services, so dont need descripitons anyway
+				return;
+			}
+			if (GSXML.getChildByTagName(test_s, GSXML.DISPLAY_TEXT_ELEM) != null)
+			{
+				// have got descriptions already, 
+				return;
+			}
+		}
+
+		// if get here, we need to get the service descriptions
+
+		// we will send all the requests in a single message
+		Element info_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		for (int i = 0; i < services.getLength(); i++)
+		{
+			Element c = (Element) services.item(i);
+			String name = c.getAttribute(GSXML.NAME_ATT);
+			String address = GSPath.appendLink(coll_name, name);
+			Element info_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, address, userContext);
+			//Element req_param_list = this.doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+			//req_param_list.appendChild(GSXML.createParameter(this.doc, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER));
+			//info_request.appendChild(req_param_list);
+			info_message.appendChild(info_request);
+
+		}
+
+		Element info_response = (Element) this.mr.process(info_message);
+
+		NodeList service_responses = info_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		// check that have same number of responses as collections
+		if (services.getLength() != service_responses.getLength())
+		{
+			logger.error(" didn't get a response for each service - somethings gone wrong!");
+			// for now, dont use the metadata
+		}
+		else
+		{
+			for (int i = 0; i < services.getLength(); i++)
+			{
+				Element c1 = (Element) services.item(i);
+				Element c2 = (Element) GSXML.getChildByTagName((Element) service_responses.item(i), GSXML.SERVICE_ELEM);
+				if (c1 != null && c2 != null && c1.getAttribute(GSXML.NAME_ATT).equals(c2.getAttribute(GSXML.NAME_ATT)))
+				{
+					//add the service data into the original response
+					GSXML.mergeElements(c1, c2);
+				}
+				else
+				{
+					logger.debug(" response does not correspond to request!");
+				}
+
+			}
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSException.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSException.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSException.java	(revision 31634)
@@ -0,0 +1,50 @@
+/*
+ *    GSException.java
+ *    a new exception for Greenstone
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.GSXML;
+
+public class GSException
+    extends Exception {
+
+    private String exception_type = null;
+    // need some better names for these
+    //public static int PERMANENT = 0;
+    //public static int TEMPORARY = 1;
+    //public static int OTHER = 2;
+
+    public GSException(String message) {
+	
+	super(message);
+	this.exception_type = GSXML.ERROR_TYPE_OTHER;
+    }
+    public GSException(String message, String type) {
+
+	super(message);
+	this.exception_type = type;
+    }
+
+    public String getType() {
+	return exception_type;
+    }
+	
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSHttpServletRequestWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSHttpServletRequestWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/GSHttpServletRequestWrapper.java	(revision 31634)
@@ -0,0 +1,64 @@
+package org.greenstone.gsdl3.core;
+
+import java.util.Map;
+import java.util.HashMap;
+
+import javax.servlet.ServletRequest;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+
+public class GSHttpServletRequestWrapper extends HttpServletRequestWrapper
+{
+    private HashMap<String, String[]> _newParams = new HashMap<String, String[]>();
+    
+    public GSHttpServletRequestWrapper(ServletRequest request)
+    {
+	super((HttpServletRequest) request);
+    }
+    
+    public void setParameter(String paramName, String[] paramValues)
+    {
+	_newParams.put(paramName, paramValues);
+    }
+    
+    public void setParameter(String paramName, String paramValue)
+    {
+	_newParams.put(paramName, new String[] { paramValue });
+    }
+    
+    public String getParameter(String paramName)
+    {
+	if (super.getParameter(paramName) != null)
+	    {
+		return super.getParameter(paramName);
+	    }
+	else
+	    {
+		if (_newParams.get(paramName) != null && _newParams.get(paramName)[0] != null)
+		    {
+			return _newParams.get(paramName)[0];
+		    }
+		return null;
+	    }
+    }
+    
+    public String[] getParameterValues(String paramName)
+    {
+	if (super.getParameterValues(paramName) != null)
+	    {
+		return super.getParameterValues(paramName);
+	    }
+	else
+	    {
+		return _newParams.get(paramName);
+	    }
+    }
+    
+    public Map<String, String[]> getParameterMap()
+    {
+	HashMap<String, String[]> returnMap = new HashMap<String, String[]>();
+	returnMap.putAll(super.getParameterMap());
+	returnMap.putAll(_newParams);
+	return returnMap;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/MessageRouter.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/MessageRouter.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/MessageRouter.java	(revision 31634)
@@ -0,0 +1,1399 @@
+/*
+ *    MessageRouter.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.collection.Collection;
+import org.greenstone.gsdl3.collection.ServiceCluster;
+import org.greenstone.gsdl3.comms.Communicator;
+import org.greenstone.gsdl3.comms.SOAPCommunicator;
+import org.greenstone.gsdl3.service.ServiceRack;
+import org.greenstone.gsdl3.util.CustomClassLoader;
+import org.greenstone.gsdl3.util.DisplayItemUtil;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * The hub of a Greenstone system.
+ * 
+ * Accepts XML requests (via process method of ModuleInterface) and routes them
+ * to the appropriate collection or service or external entity.
+ * 
+ * contains a map of module objects - may be services, collections, comms
+ * objects talking to other MessageRouters etc.
+ * 
+ * 
+ *      Since some service classes are moved into a separate directory in order
+ *      for them to be checked out from a different repository, we modify the
+ *      configureServices method to search some of the classes in other place if
+ *      they are not found in the service directory.
+ */
+public class MessageRouter implements ModuleInterface
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.MessageRouter.class.getName());
+
+	/** the (directory) name of the site */
+	protected String site_name = null;
+	/** site home - the home directory for the site */
+	protected String site_home = null;
+	/** the http address for this site */
+	protected String site_http_address = null;
+
+	protected String library_name = null;
+
+	/** map of names to Module objects */
+	protected HashMap<String, ModuleInterface> module_map = null;
+
+	/** the full description of this site */
+
+	// should these things be separated into local and remote??
+
+	/** the original xml config element */
+	public Element config_info = null;
+
+	/** list of collections that can be reached */
+	protected Element collection_list = null;
+	/** list of collections that are loaded but are private */
+	protected Element private_collection_list = null;
+	/** list of service clusters that can be reached */
+	protected Element cluster_list = null;
+	/** list of single services that can be reached */
+	protected Element service_list = null;
+	/** list of external sites that can be reached */
+	protected Element site_list = null;
+	/** list of metadata for the site */
+	protected Element metadata_list = null;
+  /** list of display items for all languages */
+  protected Element display_item_list = null;
+
+  /** a customised class loader so we can find resources in site resources folder*/
+  protected CustomClassLoader class_loader = null;
+  protected static final String DEFAULT_LANG = "en"; // hack for now, should be read from the coll cfg file? or site cfg file for cluster
+
+	//***************************************************************
+	// public methods
+	//***************************************************************
+
+	/** constructor */
+	public MessageRouter()
+	{
+	}
+
+	public void cleanUp()
+	{
+		cleanUpModuleMapEntire();
+	}
+
+	/** site_name must be set before configure is called */
+	public void setSiteName(String site_name)
+	{
+		this.site_name = site_name;
+	}
+
+	public String getSiteName()
+	{
+		return this.site_name;
+	}
+
+	/** library_name must be set before configure is called */
+	public void setLibraryName(String library_name)
+	{
+		this.library_name = library_name;
+	}
+
+	public String getLibraryName()
+	{
+		return this.library_name;
+	}
+
+	/**
+	 * configures the system
+	 * 
+	 * looks in site_home/collect for collections, reads config file
+	 * site_home/siteConfig.xml
+	 * 
+	 */
+	public boolean configure()
+	{
+
+		logger.info("configuring the Message Router");
+
+		if (this.site_name == null)
+		{
+			logger.error(" You must set site_name before calling configure");
+			return false;
+		}
+		this.site_home = GSFile.siteHome(GlobalProperties.getGSDL3Home(), this.site_name);
+		// set up the class loader
+		this.class_loader = new CustomClassLoader(this.getClass().getClassLoader(), GSFile.siteResourceDir(this.site_home));
+
+		String web_address = GlobalProperties.getGSDL3WebAddress();
+		if (web_address.equals("")) {
+		    this.site_http_address = "sites/"+this.site_name;
+		} else { 
+		    this.site_http_address = web_address + "/sites/" + this.site_name;
+		}
+		// are we behind a firewall?? - is there a better place to set up the proxy?
+		String host = GlobalProperties.getProperty("proxy.host");
+		String port = GlobalProperties.getProperty("proxy.port");
+		final String user = GlobalProperties.getProperty("proxy.user");
+		final String passwd = GlobalProperties.getProperty("proxy.password");
+
+		if (host != null && !host.equals("") && port != null && !port.equals(""))
+		{
+			System.setProperty("http.proxyType", "4");
+			System.setProperty("http.proxyHost", host);
+			System.setProperty("http.proxyPort", port);
+			System.setProperty("http.proxySet", "true");
+			// have we got a user/password?
+			if (user != null && !user.equals("") && passwd != null && !passwd.equals(""))
+			{
+				try
+				{
+					// set up the authenticator
+					Authenticator.setDefault(new Authenticator()
+					{
+						protected PasswordAuthentication getPasswordAuthentication()
+						{
+							return new PasswordAuthentication(user, new String(passwd).toCharArray());
+						}
+					});
+
+				}
+				catch (Exception e)
+				{
+					logger.error("MessageRouter Error: couldn't set up an authenticator the proxy");
+
+				}
+			}
+		}
+
+		this.module_map = new HashMap<String, ModuleInterface>();
+
+		// This stuff may be done at a reconfigure also
+		return configureLocalSite();
+
+	}
+
+	/**
+	 * Process an XML request - as a String
+	 * 
+	 * @param xml_in
+	 *            the request to process
+	 * @return the response - contains any error messages
+	 * @see String
+	 */
+	public String process(String xml_in)
+	{
+
+		Document doc = XMLConverter.getDOM(xml_in);
+
+		Node result = process(doc);
+		return XMLConverter.getString(result);
+	}
+
+	/**
+	 * Process an XML request - as a DOM Element
+	 * 
+	 * @param xml_in
+	 *            the message to process - should be <message>
+	 * @return the response - contains any error messages
+	 * @see Element
+	 */
+	public Node process(Node message_node)
+	{
+
+	  Document doc = XMLConverter.newDOM();
+		Element message = GSXML.nodeToElement(message_node);
+
+		// check that its a correct message tag
+		if (!message.getTagName().equals(GSXML.MESSAGE_ELEM))
+		{
+			logger.error(" Invalid message. GSDL message should start with <" + GSXML.MESSAGE_ELEM + ">, instead it starts with:" + message.getTagName() + ".");
+			return null;
+		}
+
+		NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+
+		Element mainResult = doc.createElement(GSXML.MESSAGE_ELEM);
+
+		// empty request
+		if (requests.getLength() == 0)
+		{
+			logger.error("empty request");
+			return mainResult;
+		}
+
+		// for now, just process each request one by one, and append the results to mainResult
+		// Note: if you add an element to another node in the same document, it
+		// gets removed from where it was. This changes the node list - you cant iterate over the node list in a normal manner if you are moving elements out of it
+		int num_requests = requests.getLength();
+		logger.info("**** num requests = "+num_requests);
+		for (int i = 0; i < num_requests; i++)
+		{
+			Node result = null;
+			Element req = (Element) requests.item(i);
+			if (req == null)
+			{
+				logger.warn("request " + i + " is null");
+				continue;
+			}
+			String path = req.getAttribute(GSXML.TO_ATT); // returns "" if no att of this name
+			if (path.equals(""))
+			{
+				// its a message for the message router
+				String type_att = req.getAttribute(GSXML.TYPE_ATT);
+				if (type_att.equals(GSXML.REQUEST_TYPE_MESSAGING))
+				{
+					// its a messaging request - modifies the requests/responses
+					result = modifyMessages(req, message, mainResult);
+				}
+				else
+				{
+					// standard request
+					result = processMessage(req);
+				}
+
+				if (result != null)
+				{
+					mainResult.appendChild(doc.importNode(result, true));
+				}
+			}
+			else
+			{
+				// The message needs to go to another module. The same message can 
+				// be passed to multiple modules  - they will be in a comma 
+				// separated list in the 'to' attribute
+				String[] modules = StringUtils.split(path, ",");
+
+				for (String this_mod : modules)
+				{
+					// why can't we do this outside the loop??
+					Element mess = doc.createElement(GSXML.MESSAGE_ELEM);
+					Element copied_request = (Element) doc.importNode(req, true);
+					mess.appendChild(copied_request);
+
+					// find the module to pass it on to
+					// need to put the request into a message element
+					String obj = GSPath.getFirstLink(this_mod);
+
+					if (this.module_map.containsKey(obj))
+					{
+						copied_request.setAttribute(GSXML.TO_ATT, this_mod);
+						result = this.module_map.get(obj).process(mess);
+
+						if (result != null)
+						{
+							// append the contents of the message to the mainResult - there will only be one response at this stage
+							Node res = GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
+							if (res != null)
+							{
+								mainResult.appendChild(doc.importNode(res, true));
+							}
+						}
+						else
+						{
+							// add in a place holder response
+							Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+							response.setAttribute(GSXML.FROM_ATT, this_mod);
+							mainResult.appendChild(response);
+							logger.error("MessageRouter Error: request had null result!");
+						}
+					}
+					else
+					{
+						logger.error("MessageRouter Error: request has illegal module name in:\n" + XMLConverter.getString(req));
+					}
+				}
+			}
+
+		} // for each request
+
+			logger.debug("MR returned response");
+		logger.debug(XMLConverter.getString(mainResult));
+
+		return mainResult;
+	}
+
+	public Element getCollectionList()
+	{
+		return collection_list;
+	}
+
+	public Element getPrivateCollectionList()
+	{
+		return private_collection_list;
+	}
+
+	public HashMap<String, ModuleInterface> getModuleMap()
+	{
+		return module_map;
+	}
+
+	// ********************************************************************
+	// auxiliary configure and cleanup methods
+	// *******************************************************************
+
+	/**
+	 * Calls clean up on all modules referenced in the module_map and removes
+	 * them .
+	 */
+	protected void cleanUpModuleMapEntire()
+	{
+		if (this.module_map != null)
+		{
+			Iterator<ModuleInterface> i = this.module_map.values().iterator();
+			while (i.hasNext())
+			{
+			    ModuleInterface i_next = i.next();
+			    
+			    i_next.cleanUp();
+			    i.remove();
+			}
+		}
+	}
+
+	/**
+	 * Goes through the children of list, and for each local/site-specific name
+	 * attribute, calls cleanUp on the module and removes it from the module_map
+	 * and removes it from the list
+	 */
+	protected void cleanUpModuleMapSubset(Element list, String remote_site)
+	{
+		logger.error(XMLConverter.getString(list));
+		NodeList elements = list.getChildNodes(); // we are assuming no extraneous nodes
+		for (int i = elements.getLength() - 1; i >= 0; i--)
+		{
+			Element item = (Element) elements.item(i);
+			String name = item.getAttribute(GSXML.NAME_ATT);
+			String potential_site_name = GSPath.getFirstLink(name);
+			if (remote_site != null)
+			{
+				if (remote_site.equals(potential_site_name))
+				{
+					list.removeChild(item);
+				}
+			}
+			else
+			{
+				if (name.equals(potential_site_name))
+				{// there was no site
+					list.removeChild(item);
+					ModuleInterface m = this.module_map.remove(name);
+					m.cleanUp(); // clean up any open files/connections etc 
+					m = null;
+				}
+			}
+		}
+		logger.error(XMLConverter.getString(list));
+	}
+
+	/**
+	 * removes all site modules from module_map, and any stored info about this
+	 * sites collections and services
+	 */
+	protected void cleanUpAllExternalSiteInfo()
+	{
+
+		NodeList site_nodes = this.site_list.getChildNodes();
+		for (int i = site_nodes.getLength() - 1; i >= 0; i--)
+		{
+			Element item = (Element) site_nodes.item(i);
+			String name = item.getAttribute(GSXML.NAME_ATT);
+			// will remove the node from site_list
+			deactivateModule(GSXML.SITE_ELEM, name);
+		}
+
+	}
+
+	/**
+	 * read thru own site config file - create services and connect to sites
+	 */
+	protected boolean configureLocalSite()
+	{
+
+		// this may be a reconfigure, so clean up the old moduleMap
+		cleanUpModuleMapEntire();
+
+		File configFile = new File(GSFile.siteConfigFile(this.site_home));
+
+		if (!configFile.exists())
+		{
+			logger.error(" site config file: " + configFile.getPath() + " not found!");
+			return false;
+		}
+
+		Document config_doc = XMLConverter.getDOM(configFile);
+		if (config_doc == null)
+		{
+			logger.error(" couldn't parse site config file: " + configFile.getPath());
+			return false;
+		}
+
+		this.config_info = config_doc.getDocumentElement();
+
+		// load up the services: serviceRackList
+		Document doc = XMLConverter.newDOM();
+		this.service_list = doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		Element service_rack_list_elem = (Element) GSXML.getChildByTagName(config_info, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+		configureServices(service_rack_list_elem);
+
+		// load up the service clusters
+		this.cluster_list = doc.createElement(GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+		Element cluster_list_elem = (Element) GSXML.getChildByTagName(config_info, GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+		configureClusters(cluster_list_elem);
+
+		// load up the collections
+		this.collection_list = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		this.private_collection_list = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		configureCollections();
+
+		// load up the external sites - this also adds their services/clusters/collections to the other lists - so must be done last
+		this.site_list = doc.createElement(GSXML.SITE_ELEM + GSXML.LIST_MODIFIER);
+		Element site_list_elem = (Element) GSXML.getChildByTagName(config_info, GSXML.SITE_ELEM + GSXML.LIST_MODIFIER);
+		configureExternalSites(site_list_elem);
+
+		// load up the site metadata
+		this.metadata_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		Element metadata_list_elem = (Element) GSXML.getChildByTagName(config_info, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		loadMetadata(metadata_list_elem);
+
+		// load up the displayItems
+		this.display_item_list = doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		Element display_item_list_elem = (Element) GSXML.getChildByTagName(config_info, GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		DisplayItemUtil.storeDisplayItems(this.display_item_list, display_item_list_elem);
+		return true;
+
+	}
+
+	protected boolean configureServices(Element service_rack_list)
+	{
+
+		// load up the individual services
+		logger.info("loading service modules...");
+
+		if (service_rack_list == null)
+		{
+			logger.info("... none to be loaded");
+			return true;
+		}
+
+		NodeList service_racks = service_rack_list.getElementsByTagName(GSXML.SERVICE_CLASS_ELEM);
+		if (service_racks.getLength() == 0)
+		{
+			logger.info("... none to be loaded");
+			return true;
+		}
+		Document doc = XMLConverter.newDOM();
+		Element service_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element service_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", new UserContext());
+		service_message.appendChild(service_request);
+
+		for (int i = 0; i < service_racks.getLength(); i++)
+		{
+			Element n = (Element) service_racks.item(i);
+			String service_name = n.getAttribute(GSXML.NAME_ATT);
+			logger.info("..." + service_name);
+
+			Class service_class = null;
+			try
+			{
+				service_class = Class.forName("org.greenstone.gsdl3.service." + service_name);
+			}
+			catch (ClassNotFoundException e)
+			{
+				try
+				{
+					//try the service_name alone in case the package name is already specified
+					service_class = Class.forName(service_name);
+				}
+				catch (ClassNotFoundException ae)
+				{
+					logger.info(ae.getMessage());
+				}
+			}
+			try
+			{
+				ServiceRack s = (ServiceRack) service_class.newInstance();
+				s.setSiteHome(this.site_home);
+				s.setSiteAddress(this.site_http_address);
+				s.setLibraryName(this.library_name);
+				s.setMessageRouter(this);
+				// pass the XML node to the service for service configuration
+				if (!s.configure(n, null))
+				{
+					logger.error("couldn't configure ServiceRack " + service_name);
+					continue;
+				}
+
+				// find out the supported services for this service module
+				Element service_response = (Element) s.process(service_message);
+				NodeList services = service_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+				if (services.getLength() == 0)
+				{
+					logger.error("MessageRouter configure error: serviceRack " + service_name + " has no services!");
+				}
+				else
+				{
+				  Document service_list_doc = this.service_list.getOwnerDocument();
+					for (int j = 0; j < services.getLength(); j++)
+					{
+						String service = ((Element) services.item(j)).getAttribute(GSXML.NAME_ATT);
+
+						this.module_map.put(service, s);
+
+						// add short info to service_list_ XML
+						this.service_list.appendChild(service_list_doc.importNode(services.item(j), true));
+					}
+				}
+			}
+			catch (Exception e)
+			{
+				logger.error("MessageRouter configure exception:  in ServiceRack class specification:  " + e.getMessage());
+				e.printStackTrace();
+			}
+		} // for each service module
+		return true;
+	}
+
+	protected boolean configureClusters(Element config_cluster_list)
+	{
+
+		// load up the service clusters
+		logger.info("loading service clusters ...");
+		if (config_cluster_list == null)
+		{
+			logger.info("... none to be loaded");
+			return true;
+		}
+		NodeList service_clusters = config_cluster_list.getElementsByTagName(GSXML.CLUSTER_ELEM);
+		if (service_clusters.getLength() == 0)
+		{
+			logger.info("... none to be loaded");
+			return true;
+		}
+
+		Document doc = this.cluster_list.getOwnerDocument();
+		for (int i = 0; i < service_clusters.getLength(); i++)
+		{
+			Element cluster = (Element) service_clusters.item(i);
+			String name = cluster.getAttribute(GSXML.NAME_ATT);
+			logger.info("..." + name);
+			ServiceCluster sc = new ServiceCluster();
+			sc.setSiteHome(this.site_home);
+			sc.setSiteAddress(this.site_http_address);
+			sc.setClusterName(name);
+			sc.setMessageRouter(this);
+			if (!sc.configure(cluster))
+			{
+				logger.error("couldn't configure ServiceCluster " + name);
+				continue;
+			}
+
+			this.module_map.put(name, sc); // this replaces the old one if there was one already present
+			//add short info to cluster list
+			Element e = doc.createElement(GSXML.CLUSTER_ELEM);
+			e.setAttribute(GSXML.NAME_ATT, name);
+			this.cluster_list.appendChild(e);
+
+		}
+		return true;
+	}
+
+	/**
+	 * looks through the collect directory and activates any collections it
+	 * finds. If this is a reconfigure, clean up must be done first before
+	 * calling this
+	 */
+	protected boolean configureCollections()
+	{
+
+		// read thru the collect directory and activate all the valid collections
+		File collectDir = new File(GSFile.collectDir(this.site_home));
+		if (collectDir.exists())
+		{
+			logger.info("Reading thru directory " + collectDir.getPath() + " to find collections.");
+			File[] contents = collectDir.listFiles();
+			for (int i = 0; i < contents.length; i++)
+			{
+				if (contents[i].isDirectory())
+				{
+
+					String colName = contents[i].getName();
+					if (!colName.startsWith("CVS") && !colName.startsWith(".svn"))
+					{
+						activateCollectionByName(colName);
+					}
+				}
+			}
+		} // collectDir
+		return true;
+	}
+
+	// testing whether a collection (or more generally, a module) is active
+	protected boolean pingModule(String name)
+	{
+		// module (including collection) would have been added to module_map 
+		// if activated, and removed from map if deactivated.
+		// The this.collection_list Element would contain the collection too, 
+		// but the following check seems to be more generally useful
+		return this.module_map.containsKey(name);
+	}
+
+	/**
+	 * creates and configures a new collection if this is done for a
+	 * reconfigure, the collection should be deactivated first.
+	 * 
+	 * @param col_name
+	 *            the name of the collection
+	 * @return true if collection created ok
+	 */
+	protected boolean activateCollectionByName(String col_name)
+	{
+
+		logger.info("Activating collection: " + col_name + ".");
+		Document doc = this.collection_list.getOwnerDocument();
+		// Look for the etc/collectionInit.xml file, and see what sort of Collection to load
+		Collection c = null;
+		File init_file = new File(GSFile.collectionInitFile(this.site_home, col_name));
+
+		if (init_file.exists())
+		{
+			Document init_doc = XMLConverter.getDOM(init_file);
+			if (init_doc != null)
+			{
+				Element init_elem = init_doc.getDocumentElement();
+				if (init_elem != null)
+				{
+					String coll_class_name = init_elem.getAttribute("class");
+					if (!coll_class_name.equals(""))
+					{
+						try
+						{
+							c = (Collection) Class.forName("org.greenstone.gsdl3.collection." + coll_class_name).newInstance();
+						}
+						catch (Exception e)
+						{
+							logger.info(" couldn't create a new collection, type " + coll_class_name + ", defaulting to class Collection");
+						}
+					}
+				}
+			}
+		}
+		if (c == null)
+		{ // we haven't found another classname to use
+			c = new Collection();
+		}
+
+		c.setCollectionName(col_name);
+		c.setSiteHome(this.site_home);
+		c.setSiteAddress(this.site_http_address);
+		c.setMessageRouter(this);
+		if (c.configure())
+		{
+			logger.info("have just configured collection " + col_name);
+			// add to list of collections
+			this.module_map.put(col_name, c);
+			Element e = doc.createElement(GSXML.COLLECTION_ELEM);
+			e.setAttribute(GSXML.NAME_ATT, col_name);
+
+			if (c.isPublic())
+			{
+				// only public collections will appear on the home page
+				// add short description_ to collection_list_
+				this.collection_list.appendChild(e);
+			}
+			else
+			{
+				this.private_collection_list.appendChild(e);
+			}
+			return true;
+		}
+		else
+		{
+			logger.error("Couldn't configure collection: " + col_name + ".");
+			return false;
+		}
+	}
+
+	/**
+	 * Goes through the siteList and activates each site found. If this is done
+	 * for a reconfigure, a clean up must be done first ****HOW???
+	 */
+	protected boolean configureExternalSites(Element config_site_list)
+	{
+
+		// load up the sites
+		logger.info("loading external sites...");
+		if (config_site_list == null)
+		{
+			logger.info("...none found");
+			return true;
+		}
+
+		NodeList sites = config_site_list.getElementsByTagName(GSXML.SITE_ELEM);
+		if (sites.getLength() == 0)
+		{
+			logger.info("...none found");
+			return true;
+		}
+
+		// this is a name to identify the current site in the Communicator
+		String local_site_name = config_site_list.getAttribute(GSXML.LOCAL_SITE_NAME_ATT);
+		if (local_site_name.equals(""))
+		{
+			local_site_name = site_name;
+		}
+
+		for (int i = 0; i < sites.getLength(); i++)
+		{
+			Element s = (Element) sites.item(i);
+			activateSite(s, local_site_name);
+		}
+		return true;
+	}
+
+	protected boolean activateSiteByName(String site_name)
+	{
+		logger.info("Activating site: " + site_name + ".");
+
+		File configFile = new File(GSFile.siteConfigFile(this.site_home));
+
+		if (!configFile.exists())
+		{
+			logger.error(" site config file: " + configFile.getPath() + " not found!");
+			return false;
+		}
+		Document config_doc = XMLConverter.getDOM(configFile);
+		if (config_doc == null)
+		{
+			logger.error(" couldn't parse site config file: " + configFile.getPath());
+			return false;
+		}
+		Element config_elem = config_doc.getDocumentElement();
+
+		Element config_site_list = (Element) GSXML.getChildByTagName(config_elem, GSXML.SITE_ELEM + GSXML.LIST_MODIFIER);
+		if (config_site_list == null)
+		{
+			logger.error("activateSite, no sites found");
+			return false;
+		}
+		// this is a name to identify the current site in the Communicator
+		String local_site_name = config_site_list.getAttribute("localSiteName");
+		if (local_site_name.equals(""))
+		{
+			local_site_name = site_name;
+		}
+
+		Element this_site_elem = GSXML.getNamedElement(config_site_list, GSXML.SITE_ELEM, GSXML.NAME_ATT, site_name);
+		if (this_site_elem == null)
+		{
+			logger.error("activateSite, site " + site_name + " not found");
+			return false;
+		}
+
+		return activateSite(this_site_elem, local_site_name);
+	}
+
+	protected boolean activateSite(Element site_elem, String local_site_name)
+	{
+
+		Communicator comm = null;
+		String type = site_elem.getAttribute(GSXML.TYPE_ATT);
+		String name = site_elem.getAttribute(GSXML.NAME_ATT);
+		if (type.equals(GSXML.COMM_TYPE_SOAP_JAVA))
+		{
+			logger.info("activating SOAP site " + name);
+			comm = new SOAPCommunicator();
+			if (comm.configure(site_elem))
+			{
+				comm.setLocalSiteName(local_site_name);
+
+				// add to map of modules
+				this.module_map.put(name, comm);
+				this.site_list.appendChild(this.site_list.getOwnerDocument().importNode(site_elem, true));
+				// need to get collection list and service
+				// list from here- if the site isn't up yet, the site will
+				// have to be added later
+				if (!getRemoteSiteInfo(comm, name))
+				{
+					logger.error(" couldn't get info from site");
+				}
+			}
+			else
+			{
+				logger.error(" couldn't configure site");
+				return false;
+			}
+
+		}
+		else
+		{
+			logger.error(" cant talk to server of type:" + type + ", so not making a connection to " + name);
+			return false;
+		}
+		return true;
+	}
+
+	/** Goes through the metadataList and loads each metadatum found */
+	protected boolean loadMetadata(Element config_metadata_list)
+	{
+	  
+		// load up the sites
+		logger.info("loading site metadata...");
+		if (config_metadata_list == null)
+		{
+			logger.info("...none found");
+			return true;
+		}
+
+		NodeList metadata = config_metadata_list.getElementsByTagName(GSXML.METADATA_ELEM);
+		if (metadata.getLength() == 0)
+		{
+			logger.info("...none found");
+			return true;
+		}
+
+		Document doc = this.metadata_list.getOwnerDocument();
+		for (int i = 0; i < metadata.getLength(); i++)
+		{
+			Element s = (Element) metadata.item(i);
+			this.metadata_list.appendChild(doc.importNode(s, true));
+		}
+		return true;
+	}
+
+	/**
+	 * get site info from external site
+	 * 
+	 * @param comm
+	 *            - the communicator object for the external site
+	 * @param site_name
+	 *            - the name of the external site
+	 * @return true if successful
+	 */
+	protected boolean getRemoteSiteInfo(Communicator comm, String site_name)
+	{
+
+		logger.info(" getting info from site:" + site_name);
+		Document doc = XMLConverter.newDOM();
+		Element info_request = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element req = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", new UserContext());
+		info_request.appendChild(req);
+
+		// process the message
+		Node info_response_node = comm.process(info_request);
+		Element info_response = GSXML.nodeToElement(info_response_node);
+
+		if (info_response == null)
+		{
+			return false;
+		}
+		// collection info
+		NodeList colls = info_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		if (colls.getLength() > 0)
+		{
+			for (int i = 0; i < colls.getLength(); i++)
+			{
+				Element e = (Element) colls.item(i);
+				String col_name = e.getAttribute(GSXML.NAME_ATT);
+				// add the info to own coll list - may want to keep
+				// this separate in future - so can distinguish own and
+				// other collections ??
+				e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(col_name, site_name));
+				this.collection_list.appendChild(doc.importNode(e, true));
+			}
+		}
+
+		// service info
+		NodeList services = info_response.getElementsByTagName(GSXML.SERVICE_ELEM);
+		if (services.getLength() > 0)
+		{
+			for (int i = 0; i < services.getLength(); i++)
+			{
+				Element e = (Element) services.item(i);
+				String serv_name = e.getAttribute(GSXML.NAME_ATT);
+				e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(serv_name, site_name));
+				this.service_list.appendChild(doc.importNode(e, true));
+			}
+		}
+
+		// serviceCluster info
+		NodeList clusters = info_response.getElementsByTagName(GSXML.CLUSTER_ELEM);
+		if (clusters.getLength() > 0)
+		{
+			for (int i = 0; i < clusters.getLength(); i++)
+			{
+				Element e = (Element) clusters.item(i);
+				String clus_name = e.getAttribute(GSXML.NAME_ATT);
+				e.setAttribute(GSXML.NAME_ATT, GSPath.prependLink(clus_name, site_name));
+				this.cluster_list.appendChild(doc.importNode(e, true));
+			}
+		}
+		return true;
+	}
+
+	protected boolean activateServiceClusterByName(String cluster_name)
+	{
+		return false;
+
+	}
+
+	protected boolean activateServiceRackByName(String module_name)
+	{
+		return false;
+	}
+
+	protected boolean deactivateModule(String type, String name)
+	{
+
+		logger.info("deactivating " + type + "  module: " + name);
+		if (this.module_map.containsKey(name))
+		{
+
+			logger.info("found the module");
+			ModuleInterface m = this.module_map.remove(name);
+			// also remove the xml bit from description list
+			if (type.equals(GSXML.COLLECTION_ELEM))
+			{
+				if (((Collection) m).isPublic())
+				{
+					Element this_col = GSXML.getNamedElement(this.collection_list, GSXML.COLLECTION_ELEM, GSXML.NAME_ATT, name);
+					if (this_col != null)
+					{
+						this.collection_list.removeChild(this_col);
+					}
+				}
+				else
+				{
+					// a private collection
+					Element this_col = GSXML.getNamedElement(this.private_collection_list, GSXML.COLLECTION_ELEM, GSXML.NAME_ATT, name);
+					if (this_col != null)
+					{
+						this.private_collection_list.removeChild(this_col);
+					}
+				}
+			}
+			else if (type.equals(GSXML.SERVICE_ELEM))
+			{
+				Element this_service = GSXML.getNamedElement(this.service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, name);
+				if (this_service != null)
+				{
+					this.service_list.removeChild(this_service);
+				}
+			}
+			else if (type.equals(GSXML.CLUSTER_ELEM))
+			{
+				Element this_cluster = GSXML.getNamedElement(this.cluster_list, GSXML.CLUSTER_ELEM, GSXML.NAME_ATT, name);
+				if (this_cluster != null)
+				{
+					this.cluster_list.removeChild(this_cluster);
+				}
+			}
+			else if (type.equals(GSXML.SITE_ELEM))
+			{
+				Element this_site = GSXML.getNamedElement(this.site_list, GSXML.SITE_ELEM, GSXML.NAME_ATT, name);
+				if (this_site != null)
+				{
+					this.site_list.removeChild(this_site);
+
+					// also remove this sites colls, services, clusters etc
+					cleanUpModuleMapSubset(this.collection_list, name);
+					cleanUpModuleMapSubset(this.cluster_list, name);
+					cleanUpModuleMapSubset(this.service_list, name);
+
+					// can remote collections be in the private coll list ??
+				}
+			}
+			else
+			{
+				logger.error("invalid module type: " + type + ", can't remove info about this module");
+			}
+
+			m.cleanUp(); // clean up any open files/connections etc - can cause trouble on windows
+			m = null;
+			return true;
+		}
+		// else not deactivated
+		logger.error(name + " module not found");
+		return false;
+
+	}
+
+	//*****************************************************************
+	// auxiliary process methods
+	//*****************************************************************
+
+	/**
+	 * handles requests made to the MessageRouter itself
+	 * 
+	 * @param req
+	 *            - the request Element- <request>
+	 * @return the result Element - should be <response>
+	 */
+	protected Element processMessage(Element req)
+	{
+
+		// message for self, should be type=describe/configure at this stage
+	        Document doc = XMLConverter.newDOM();
+		String type = req.getAttribute(GSXML.TYPE_ATT);
+		String lang = req.getAttribute(GSXML.LANG_ATT);
+		Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, "");
+		if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE))
+		{
+			response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_DESCRIBE);
+			// check the param list
+			Element param_list = (Element) GSXML.getChildByTagName(req, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (param_list == null)
+			{
+			  response.appendChild(doc.importNode(this.collection_list, true));
+				response.appendChild(doc.importNode(this.cluster_list, true));
+				response.appendChild(doc.importNode(this.site_list, true));
+				response.appendChild(doc.importNode(this.service_list, true));
+				response.appendChild(doc.importNode(this.metadata_list, true));
+				Element di_list = doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+				response.appendChild(di_list);
+				DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+				return response;
+			}
+
+			NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+
+			// go through the param list and see what components are wanted
+			for (int i = 0; i < params.getLength(); i++)
+			{
+
+				Element param = (Element) params.item(i);
+				// Identify the structure information desired
+				if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM))
+				{
+					String info = param.getAttribute(GSXML.VALUE_ATT);
+					if (info.equals(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER))
+					{
+						response.appendChild(doc.importNode(this.collection_list, true));
+					}
+					else if (info.equals(GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER))
+					{
+						response.appendChild(doc.importNode(this.cluster_list, true));
+					}
+					else if (info.equals(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER))
+					{
+						response.appendChild(doc.importNode(this.service_list, true));
+					}
+					else if (info.equals(GSXML.SITE_ELEM + GSXML.LIST_MODIFIER))
+					{
+						response.appendChild(doc.importNode(this.site_list, true));
+					}
+					else if (info.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER))
+					{
+						response.appendChild(doc.importNode(this.metadata_list, true));
+					}
+					else if (info.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+					{
+					  Element di_list = doc.createElement(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+					  response.appendChild(di_list);
+					  DisplayItemUtil.addLanguageSpecificDisplayItems(di_list, this.display_item_list, lang, DEFAULT_LANG, this.class_loader);
+					  
+					}
+				}
+			}
+			return response;
+
+		}
+
+		if (type.equals(GSXML.REQUEST_TYPE_SYSTEM))
+		{
+
+			// a list of system requests - should put any error messages
+			// or success messages into response
+			NodeList commands = req.getElementsByTagName(GSXML.SYSTEM_ELEM);
+			Element site_config_elem = null;
+			boolean success = false;
+
+			for (int i = 0; i < commands.getLength(); i++)
+			{
+				// all the commands should be Elements
+				Element elem = (Element) commands.item(i);
+				String action = elem.getAttribute(GSXML.TYPE_ATT);
+
+				if (action.equals(GSXML.SYSTEM_TYPE_PING))
+				{ // ?a=s&sa=ping or ?a=s&sa=ping(&st=collection)&sn=colname
+
+					String message = ""; // will be creating the same messages as in GS2's recept/pingaction.cpp
+					String module_name = elem.getAttribute(GSXML.SYSTEM_MODULE_NAME_ATT);
+					String module_type = elem.getAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT);
+
+					if (module_name.equals(""))
+					{ // server-level ping				
+						message = "Ping succeeded.";
+					}
+					else
+					{ // ping at collection level
+						if (pingModule(module_name))
+						{
+							message = "Ping for " + module_name + " succeeded.";
+						}
+						else
+						{
+							message = "Ping for " + module_name + " did not succeed.";
+						}
+					}
+					Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, message);
+					response.appendChild(s);
+				}
+				//else if (action.equals(GSXML.SYSTEM_TYPE_ISPERSISTENT)) {
+				//	Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM,  "Persistent: true.");
+				//	response.appendChild(s);
+				//}
+				else if (action.equals(GSXML.SYSTEM_TYPE_CONFIGURE))
+				{
+					String subset = elem.getAttribute(GSXML.SYSTEM_SUBSET_ATT);
+					if (subset.equals(""))
+					{
+						// need to reconfigure the MR
+						this.configureLocalSite();
+						Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, "MessageRouter reconfigured successfully");
+						response.appendChild(s);
+
+					}
+					else
+					{
+						// else it a specific request
+						if (subset.equals(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER))
+						{
+							// get rid of all the old collection stuff (not counting remote ones) before activating all the new ones
+							cleanUpModuleMapSubset(this.collection_list, null);
+							cleanUpModuleMapSubset(this.private_collection_list, null);
+							success = configureCollections();
+						}
+						else
+						{
+
+							// need the site config file
+							if (site_config_elem == null)
+							{
+
+								File configFile = new File(GSFile.siteConfigFile(this.site_home));
+								if (!configFile.exists())
+								{
+									logger.error(" site config file: " + configFile.getPath() + " not found!");
+									continue;
+								}
+								Document site_config_doc = XMLConverter.getDOM(configFile);
+								if (site_config_doc == null)
+								{
+									logger.error(" couldn't parse site config file: " + configFile.getPath());
+									continue;
+								}
+								site_config_elem = site_config_doc.getDocumentElement();
+							}
+							if (subset.equals(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER))
+							{
+								Element service_rack_list = (Element) GSXML.getChildByTagName(site_config_elem, GSXML.SERVICE_CLASS_ELEM + GSXML.LIST_MODIFIER);
+								cleanUpModuleMapSubset(this.service_list, null);
+								success = configureServices(service_rack_list);
+							}
+							else if (subset.equals(GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER))
+							{
+								Element cluster_list = (Element) GSXML.getChildByTagName(site_config_elem, GSXML.CLUSTER_ELEM + GSXML.LIST_MODIFIER);
+								cleanUpModuleMapSubset(this.cluster_list, null);
+								success = configureClusters(cluster_list);
+							}
+							else if (subset.equals(GSXML.SITE_ELEM + GSXML.LIST_MODIFIER))
+							{
+								Element site_list = (Element) GSXML.getChildByTagName(site_config_elem, GSXML.SITE_ELEM + GSXML.LIST_MODIFIER);
+								cleanUpAllExternalSiteInfo();
+								success = configureExternalSites(site_list);
+							}
+						}
+						String message = null;
+						if (success)
+						{
+							message = subset + "reconfigured successfully";
+						}
+						else
+						{
+							message = "Error in reconfiguring " + subset;
+						}
+						Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, message);
+						response.appendChild(s);
+					}
+
+				}
+				else
+				{
+					String module_name = elem.getAttribute(GSXML.SYSTEM_MODULE_NAME_ATT);
+					String module_type = elem.getAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT);
+
+					if (action.equals(GSXML.SYSTEM_TYPE_DEACTIVATE))
+					{
+						success = deactivateModule(module_type, module_name);
+						if (success)
+						{
+							Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, module_type + ": " + module_name + " deactivated", GSXML.SYSTEM_TYPE_DEACTIVATE, GSXML.SUCCESS);
+							response.appendChild(s);
+						}
+						else
+						{
+							Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, module_type + ": " + module_name + " could not be deactivated", GSXML.SYSTEM_TYPE_DEACTIVATE, GSXML.ERROR);
+							response.appendChild(s);
+						}
+
+					}
+					else if (action.equals(GSXML.SYSTEM_TYPE_ACTIVATE))
+					{
+						// we need to deactivate the module first, in case this is a 
+						// reconfigure
+						deactivateModule(module_type, module_name);
+						if (module_type.equals(GSXML.COLLECTION_ELEM))
+						{
+							success = activateCollectionByName(module_name);
+						}
+						else if (module_type.equals(GSXML.SITE_ELEM))
+						{
+							success = activateSiteByName(module_name);
+						}
+						else if (module_type.equals(GSXML.CLUSTER_ELEM))
+						{
+							success = activateServiceClusterByName(module_name);
+						}
+						if (success)
+						{
+							Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, module_type + ": " + module_name + " activated", GSXML.SYSTEM_TYPE_ACTIVATE, GSXML.SUCCESS);
+							response.appendChild(s);
+						}
+						else
+						{
+							Element s = GSXML.createTextElement(doc, GSXML.STATUS_ELEM, module_type + ": " + module_name + " could not be activated", GSXML.SYSTEM_TYPE_ACTIVATE, GSXML.ERROR);
+							response.appendChild(s);
+						}
+					}
+				} // else not a configure action
+			} // for all commands
+			return response;
+
+		} // system type request
+
+		// if get here something has gone wrong
+		String eol = System.getProperty("line.separator");
+
+		String mess = "Can't process request:" + eol + "  " + XMLConverter.getString(req);
+		logger.error(mess);
+		return null;
+
+	}
+
+	//* Used to copy nodes from one message to another. E.g. copy a response node to the next request. Not sure if this is actually used anywhere yet... */
+
+	protected Element modifyMessages(Element request, Element message, Element result)
+	{
+	  Document doc = XMLConverter.newDOM();
+		Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, "");
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_MESSAGING);
+
+		NodeList commands = request.getElementsByTagName("command");
+		if (commands == null)
+		{
+			logger.error("no commands, " + XMLConverter.getPrettyString(request));
+			return response;
+		}
+		for (int i = 0; i < commands.getLength(); i++)
+		{
+			Element action = (Element) commands.item(i);
+			String type = action.getAttribute(GSXML.TYPE_ATT);
+			if (type.equals("copyNode"))
+			{
+				// copies the from node as a child of to node
+				String from_path = action.getAttribute("from");
+				String to_path = action.getAttribute("to");
+				Element from_node = null;
+				String from_node_root = GSPath.getFirstLink(from_path);
+				if (from_node_root.startsWith(GSXML.REQUEST_ELEM))
+				{
+					from_node = message;
+				}
+				else if (from_node_root.startsWith(GSXML.RESPONSE_ELEM))
+				{
+					from_node = result;
+				}
+				if (from_node == null)
+				{
+					continue;
+				}
+				Element to_node = null;
+				String to_node_root = GSPath.getFirstLink(to_path);
+				if (to_node_root.startsWith(GSXML.REQUEST_ELEM))
+				{
+					to_node = message;
+				}
+				else if (to_node_root.startsWith(GSXML.RESPONSE_ELEM))
+				{
+					to_node = result;
+				}
+				if (to_node == null)
+				{
+					continue;
+				}
+				// now we know what node to copy where
+				Node orig_node = GSXML.getNodeByPathIndexed(from_node, from_path);
+				if (orig_node == null)
+				{
+					continue;
+				}
+				Node new_parent = GSXML.getNodeByPathIndexed(to_node, to_path);
+				if (new_parent == null)
+				{
+					continue;
+
+				}
+				new_parent.appendChild(to_node.getOwnerDocument().importNode(orig_node, true));
+			}
+
+			else if (type.equals("copyChildren"))
+			{
+
+			}
+		} // for each command
+		return response;
+	}
+
+	// ****************************************************
+	// other methods
+	// ****************************************************
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/ModuleInterface.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/ModuleInterface.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/ModuleInterface.java	(revision 31634)
@@ -0,0 +1,62 @@
+/*
+ *    ModuleInterface.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+   
+import org.w3c.dom.Node;
+
+/**
+ * interface for all modules in greenstone.
+ *
+ * all components talk via a process method - in its simplest form,
+ * it takes a String of XML, and returns a String of XML
+ *
+ * the more efficient process method uses DOM Nodes instead 
+ * of Strings - this avoids parsing the XML at each module
+ *
+ * @author Katherine Don
+ * @version $Revision$
+ */
+public interface ModuleInterface {
+ 
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xml_in the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+    abstract public String process(String xml_in);
+
+  /**
+   * Process an XML request - as a DOM model Node
+   *
+   * @param xml_in the request to process
+   * @return the response - contains any error messages
+   * @see org.w3c.dom.Node
+   */
+    abstract public Node process(Node xml_in);
+
+    /** 
+     * Do any clean up necessary for deactivating the module, eg
+     * close any open file handles (gdbm in particular) or windows
+     * holds locks on them.
+     */
+    abstract public void cleanUp();
+}    
+                                                                           
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIMessageRouter.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIMessageRouter.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIMessageRouter.java	(revision 31634)
@@ -0,0 +1,187 @@
+/*
+ *    OAIMessageRouter.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Iterator;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.collection.OAICollection;
+import org.greenstone.gsdl3.collection.ServiceCluster;
+import org.greenstone.gsdl3.comms.Communicator;
+import org.greenstone.gsdl3.comms.SOAPCommunicator;
+import org.greenstone.gsdl3.service.ServiceRack;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OAIXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * The hub of a Greenstone OAI server.
+ * 
+ * A simplified version of MessageRouter for OAIServer. Only loads up collections that have OAI services.
+ */
+public class OAIMessageRouter extends MessageRouter
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.MessageRouter.class.getName());
+
+  public Element oai_config = null;
+	//***************************************************************
+	// public methods
+	//***************************************************************
+
+	/** constructor */
+	public OAIMessageRouter()
+	{
+	}
+
+	/**
+	 * read thru own site config file - create services and connect to sites
+	 */
+	protected boolean configureLocalSite()
+	{
+	  
+	  // this may be a reconfigure, so clean up the old moduleMap
+		cleanUpModuleMapEntire();
+
+		// for oai, we don't do anything with the site config file. But we'll read it in and keep it in case need it later, eg for replace elements when retrieving metadata - which I don't think has been implemented
+		File configFile = new File(GSFile.siteConfigFile(this.site_home));
+
+		if (!configFile.exists())
+		{
+			logger.error(" site config file: " + configFile.getPath() + " not found!");
+			return false;
+		}
+
+		Document config_doc = XMLConverter.getDOM(configFile);
+		if (config_doc == null)
+		{
+			logger.error(" couldn't parse site config file: " + configFile.getPath());
+			return false;
+		}
+
+		this.config_info = config_doc.getDocumentElement();
+
+		// this is the receptionists OAIConfig.xml. Need to rethink how the MR gets this this if we ever talk to remote site, and whether it should be using it anyway
+		this.oai_config = OAIXML.getOAIConfigXML();
+		if (this.oai_config == null)
+		{
+		  logger.error("Couldn't load in OAIConfig.xml");
+		  return false;
+		}
+		Document doc = XMLConverter.newDOM();
+		// load up the collections
+		this.collection_list = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		configureCollections();
+
+		return true;
+
+	}
+
+
+	/**
+	 * creates and configures a new collection if this is done for a
+	 * reconfigure, the collection should be deactivated first.
+	 * 
+	 * @param col_name
+	 *            the name of the collection
+	 * @return true if collection created ok
+	 */
+	protected boolean activateCollectionByName(String col_name)
+	{
+
+		logger.info("Activating collection: " + col_name + ".");
+		Document doc = this.collection_list.getOwnerDocument();
+		// use our special OAICollection - this will only load in oai services
+		OAICollection c = new OAICollection();
+
+		c.setCollectionName(col_name);
+		c.setSiteHome(this.site_home);
+		c.setSiteAddress(this.site_http_address);
+		c.setMessageRouter(this);
+		if (!c.configure()) {
+		  logger.error("Couldn't configure collection: " + col_name + ".");
+		  return false;
+		}
+
+		logger.info("have just configured collection " + col_name);
+		if (!c.hasOAI()) {
+		  logger.info ("collection "+col_name+" has no OAI services. Not keeping it loaded");
+		  return false;
+		}
+		if (!c.configureOAI(this.oai_config)) {
+		  logger.info("couldn't configure the collection : "+col_name +" with the oai config info");
+		  return false;
+		}
+		// add to list of collections
+		this.module_map.put(col_name, c);
+		Element e = doc.createElement(GSXML.COLLECTION_ELEM);
+		e.setAttribute(GSXML.NAME_ATT, col_name);
+		e.setAttribute(OAIXML.LASTMODIFIED, "" + c.getLastmodified());
+		e.setAttribute(OAIXML.EARLIEST_DATESTAMP, "" + c.getEarliestDatestamp());
+		this.collection_list.appendChild(e);
+		return true;
+		
+	}
+
+
+
+	//*****************************************************************
+	// auxiliary process methods
+	//*****************************************************************
+
+	/**
+	 * handles requests made to the MessageRouter itself
+	 * 
+	 * @param req
+	 *            - the request Element- <request>
+	 * @return the result Element - should be <response>
+	 */
+  protected Element processMessage(Element req) {
+    Document doc = XMLConverter.newDOM();
+    String type = req.getAttribute(GSXML.TYPE_ATT);
+    Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+    response.setAttribute(GSXML.FROM_ATT, "");
+    
+    if (type.equals(OAIXML.OAI_SET_LIST))
+      {
+	logger.info("oaiSetList request received");
+	//this is the oai receptionist asking for a list of oai-support collections
+	response.setAttribute(GSXML.TYPE_ATT, OAIXML.OAI_SET_LIST);
+	response.appendChild(doc.importNode(this.collection_list, true));
+	return response;
+      }
+    return super.processMessage(req);
+  }
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIReceptionist.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIReceptionist.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/OAIReceptionist.java	(revision 31634)
@@ -0,0 +1,1082 @@
+/*
+ *    OAIReceptionist.java
+ *    Copyright (C) 2012 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.core;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.action.*;
+// XML classes
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+// other java classes
+import java.io.File;
+import java.util.*;
+
+import org.apache.log4j.*;
+
+/** a Receptionist, used for oai metadata response xml generation.
+ * This receptionist talks to the message router directly, 
+ * instead of via any action, hence no action map is needed.
+ * @see the basic Receptionist
+ */
+public class OAIReceptionist implements ModuleInterface {
+  
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.OAIReceptionist.class.getName());
+  
+  /** Instead of a config_params object, only a site_name is needed by oai receptionist. */
+  protected String site_name = null;
+  /** The unique  repository identifier */
+  protected String repository_id = null;
+      
+  /** the configure file of this receptionist passed from the oai servlet. */
+  protected Element oai_config = null;
+
+  /** contained in the OAIConfig.xml deciding whether the resumptionToken should be in use */
+  protected int resume_after = -1 ;
+  
+  /** the message router that the Receptionist and Actions will talk to */
+  protected ModuleInterface mr = null;
+  
+  // Some of the data/responses will not change while the servlet is running, so
+  // we can cache them
+  
+  /** A list of all the collections available to this OAI server */
+  protected Element collection_list = null;
+  /** a vector of the names, for convenience */
+  protected Vector<String> collection_name_list = null;
+  /** If this is true, then there are no OAI enabled collections, so can always return noRecordsMatch (after validating the request params) */
+  protected boolean noRecordsMatch = false;
+      
+  /** A set of all known 'sets' */
+  protected HashSet<String> set_set = null;
+
+  protected boolean has_super_colls = false;
+  /** a hash of super set-> collection list */
+  protected HashMap<String, Vector<String>> super_coll_map = null;
+  /** store the super coll elements for convenience */
+  HashMap<String, Element> super_coll_data = null;
+  /** store the metadata formats ??????*/
+  /** The identify response */
+  protected Element identify_response = null;
+  /** The list set response */
+  protected Element listsets_response = null;
+  /** the list metadata formats response */
+  protected Element listmetadataformats_response = null;
+
+  public OAIReceptionist() {
+
+  }
+  
+  public void cleanUp() {
+    if (this.mr != null) {
+      
+      this.mr.cleanUp();
+    }
+    OAIResumptionToken.saveTokensToFile();
+  }
+  
+  public void setSiteName(String site_name) {
+    this.site_name = site_name;
+  }
+  /** sets the message router  - it should already be created and
+   * configured in the init() of a servlet (OAIServer, for example) before being passed to the receptionist*/
+  public void setMessageRouter(ModuleInterface mr) {
+    this.mr = mr;
+  }
+  
+  /** configures the receptionist */
+  public boolean configure(Element config) {
+    
+    if (this.mr==null) {
+      logger.error(" message routers must be set before calling oai configure");
+      return false;
+    }
+    if (config == null) {
+      logger.error(" oai configure file is null");
+      return false;
+    }
+    oai_config = config;
+    resume_after = getResumeAfter();
+    
+    repository_id = getRepositoryIdentifier(); 
+    configureSuperSetInfo();
+    if (!configureSetInfo()) {
+      // there are no sets
+      logger.error("No sets (collections) available for OAI");
+      return false;
+    }
+
+    // load in tokens from OAIResumptionToken.xml, and then clear out any
+    // expired ones. 
+    OAIResumptionToken.init();
+    OAIResumptionToken.clearExpiredTokens();
+    
+    return true;
+  }
+
+  // assuming that sets are static. If collections change then the servlet 
+  // should be restarted.
+  private boolean configureSuperSetInfo() {
+    // do we have any super colls listed in web/WEB-INF/classes/OAIConfig.xml?
+    // Will be like 
+    // <oaiSuperSet>
+    //   <SetSpec>xxx</SetSpec>
+    //   <setName>xxx</SetName>
+    //   <SetDescription>xxx</setDescription>
+    // </oaiSuperSet>
+    // The super set is listed in OAIConfig, and collections themselves state
+    // whether they are part of the super set or not.
+    NodeList super_coll_list = this.oai_config.getElementsByTagName(OAIXML.OAI_SUPER_SET);
+    this.super_coll_data = new HashMap<String, Element>();
+    if (super_coll_list.getLength() > 0) {
+      this.has_super_colls = true;
+      for (int i=0; i<super_coll_list.getLength(); i++) {
+	Element super_coll = (Element)super_coll_list.item(i);
+	Element set_spec = (Element)GSXML.getChildByTagName(super_coll, OAIXML.SET_SPEC);
+	if (set_spec != null) {
+	  String name = GSXML.getNodeText(set_spec);
+	  if (!name.equals("")) {
+	    this.super_coll_data.put(name, super_coll);
+	    logger.info("adding in super coll "+name);
+	  }
+	}
+      }
+    
+      if (this.super_coll_data.size()==0) {
+	this.has_super_colls = false;
+      }
+    }
+    if (this.has_super_colls == true) {
+      this.super_coll_map = new HashMap<String, Vector<String>>();
+    }
+    return true;
+    
+  }
+  private boolean configureSetInfo() {
+    this.set_set = new HashSet<String>();
+
+    // First, we get a list of all the OAI enabled collections
+    // We get this by sending a listSets request to the MR
+    Document doc = XMLConverter.newDOM();
+    Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+    
+    Element request = GSXML.createBasicRequest(doc, OAIXML.OAI_SET_LIST, "", null);
+    message.appendChild(request);
+    Node msg_node = mr.process(message);
+    
+    if (msg_node == null) {
+      logger.error("returned msg_node from mr is null");
+      return false;
+    }
+    Element resp = (Element)GSXML.getChildByTagName(msg_node, GSXML.RESPONSE_ELEM);
+    Element coll_list = (Element)GSXML.getChildByTagName(resp, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+    if (coll_list == null) {
+      logger.error("coll_list is null");
+      return false;
+    }
+
+    this.collection_list = (Element)doc.importNode(coll_list, true);
+
+    // go through and store a list of collection names for convenience
+    // also create a 'to' attribute for the next request to the MR, which
+    // is a ListSets request to each collection
+    Node child = this.collection_list.getFirstChild();
+    if (child == null) {
+      logger.error("collection list has no children");
+      noRecordsMatch = true;
+      return false;
+    }
+    
+    this.collection_name_list = new Vector<String>();
+    StringBuffer to = new StringBuffer();
+    boolean first = true;
+    while (child != null) {
+      if (child.getNodeName().equals(GSXML.COLLECTION_ELEM)) {
+	String coll_id =((Element) child).getAttribute(GSXML.NAME_ATT);
+	this.collection_name_list.add(coll_id);
+	if (!first) {
+	  to.append(',');
+	}
+	first = false;
+	to.append(coll_id+"/"+OAIXML.LIST_SETS);
+      }
+      child = child.getNextSibling();
+    }
+    if (first) {
+      // we haven't found any collections
+      logger.error("found no collection elements in collectionList");
+      noRecordsMatch = true;
+      return false;
+    }
+    Document listsets_doc = XMLConverter.newDOM();
+    Element listsets_element = listsets_doc.createElement(OAIXML.LIST_SETS);
+    this.listsets_response = getMessage(listsets_doc, listsets_element);
+    
+    // Now, for each collection, get a list of all its sets
+    // might include subsets (classifiers) or super colls
+    // We'll reuse the first message, changing its type and to atts
+    request.setAttribute(GSXML.TYPE_ATT, "");
+    request.setAttribute(GSXML.TO_ATT, to.toString());
+    // send to MR
+    msg_node = mr.process(message);
+    logger.info("*** " + XMLConverter.getPrettyString(msg_node));
+    NodeList response_list =  ((Element)msg_node).getElementsByTagName(GSXML.RESPONSE_ELEM);
+    for (int c=0; c<response_list.getLength(); c++) {
+      // for each collection's response
+      Element response = (Element)response_list.item(c);
+      String coll_name = GSPath.getFirstLink(response.getAttribute(GSXML.FROM_ATT));
+      logger.info("*** coll from response "+coll_name);
+      NodeList set_list = response.getElementsByTagName(OAIXML.SET);
+      for (int j=0; j<set_list.getLength(); j++) {
+	// now check if it a super collection
+	Element set = (Element)set_list.item(j);
+	String set_spec = GSXML.getNodeText((Element)GSXML.getChildByTagName(set, OAIXML.SET_SPEC));
+	logger.info("*** set spec = "+set_spec);
+	// this may change if we add site name back in
+	// setSpecs will be collname or collname:subset or supercollname
+	if (set_spec.indexOf(":")==-1 && ! set_spec.equals(coll_name)) {
+	  // it must be a super coll spec
+	  logger.info("*** found super coll, "+set_spec);
+	  // check that it is a valid one from config
+	  if (this.has_super_colls == true && this.super_coll_data.containsKey(set_spec)) {
+	    Vector <String> subcolls = this.super_coll_map.get(set_spec);
+	    if (subcolls == null) {
+	      logger.info("*** its new!!");
+	      // not in there yet
+	      subcolls = new Vector<String>();
+	      this.set_set.add(set_spec);
+	      this.super_coll_map.put(set_spec, subcolls);
+	      // the first time a supercoll is mentioned, add into the set list
+	      logger.info("*** finding the set info "+XMLConverter.getPrettyString(this.super_coll_data.get(set_spec)));
+	      listsets_element.appendChild(GSXML.duplicateWithNewName(listsets_doc, this.super_coll_data.get(set_spec), OAIXML.SET, true));
+	    }
+	    // add this collection to the list for the super coll
+	    subcolls.add(coll_name);
+	  }
+	} else { // its either the coll itself or a subcoll
+	  // add in the set
+	  listsets_element.appendChild(listsets_doc.importNode(set, true));
+	  this.set_set.add(set_spec);
+	}
+      } // for each set in the collection
+    } // for each OAI enabled collection
+    return true;
+  }
+
+  protected void resetMessageRouter() {
+    // we just need to send a configure request to MR
+    Document doc = XMLConverter.newDOM();
+    Element mr_request_message = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_SYSTEM, "", null);
+    mr_request_message.appendChild(mr_request);
+    
+    Element system = doc.createElement(GSXML.SYSTEM_ELEM);
+    mr_request.appendChild(system);
+    system.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_CONFIGURE);
+
+    Element response = (Element) this.mr.process(mr_request_message);
+    logger.info("*** configure response = "+XMLConverter.getPrettyString(response));
+  }
+  /** process using strings - just calls process using Elements */
+  public String process(String xml_in) {
+    
+    Node message_node = XMLConverter.getDOM(xml_in);
+    Node page = process(message_node);
+    return XMLConverter.getString(page);
+  }
+
+  //Compose a message/response element used to send back to the OAIServer servlet. 
+  //This method is only used within OAIReceptionist
+  private Element getMessage(Document doc, Element e) {
+    Element msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+    msg.appendChild(response);
+    response.appendChild(e);
+    return msg;
+  }
+
+  /** process - produce xml data in response to a request
+   * if something goes wrong, it returns null -
+   */
+  public Node process(Node message_node) {
+    logger.info("*** OAIReceptionist received request");
+
+    Element message = GSXML.nodeToElement(message_node);
+    logger.info("*** " + XMLConverter.getString(message));
+
+    // check that its a correct message tag
+    if (!message.getTagName().equals(GSXML.MESSAGE_ELEM)) {
+      logger.error(" Invalid message. GSDL message should start with <"+GSXML.MESSAGE_ELEM+">, instead it starts with:"+message.getTagName()+".");
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "Internal messaging error");
+    }
+   
+    // get the request out of the message - assume that there is only one
+    Element request = (Element)GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+    if (request == null) {
+      logger.error(" message had no request!");
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "Internal messaging error");
+    }
+
+    // special case, reset=true for reloading the MR and recept data
+    String reset = request.getAttribute("reset");
+    if (!reset.equals("")) {
+      resetMessageRouter();
+      configureSetInfo();
+      return OAIXML.createResetResponse(true);
+    }
+
+    
+    //At this stage, the value of 'to' attribute of the request must be the 'verb'
+    //The only thing that the oai receptionist can be sure is that these verbs are valid, nothing else.
+    String verb = request.getAttribute(GSXML.TO_ATT);
+    if (verb.equals(OAIXML.IDENTIFY)) {
+      return doIdentify();
+    }
+    if (verb.equals(OAIXML.LIST_METADATA_FORMATS)) {
+      return doListMetadataFormats(request);
+    }
+    if (verb.equals(OAIXML.LIST_SETS)) {
+      // we have composed the list sets response on init
+      // Note this means that list sets never uses resumption tokens
+      return this.listsets_response; 
+    }
+    if (verb.equals(OAIXML.GET_RECORD)) {
+      return doGetRecord(request);
+    }
+    if (verb.equals(OAIXML.LIST_IDENTIFIERS)) {
+      return doListIdentifiersOrRecords(request,OAIXML.LIST_IDENTIFIERS , OAIXML.HEADER);
+    }
+    if (verb.equals(OAIXML.LIST_RECORDS)) {
+      return doListIdentifiersOrRecords(request, OAIXML.LIST_RECORDS, OAIXML.RECORD);
+    }
+    // should never get here as verbs were checked in OAIServer
+    return OAIXML.createErrorMessage(OAIXML.BAD_VERB, "Unexpected things happened");
+    
+  }
+
+
+  private int getResumeAfter() {
+    Element resume_after = (Element)GSXML.getChildByTagName(oai_config, OAIXML.RESUME_AFTER);
+    if(resume_after != null) return Integer.parseInt(GSXML.getNodeText(resume_after));
+    return -1;
+  }
+  private String getRepositoryIdentifier() {
+    Element ri = (Element)GSXML.getChildByTagName(oai_config, OAIXML.REPOSITORY_IDENTIFIER);
+    if (ri != null) { 
+      return GSXML.getNodeText(ri);
+    }
+    return "";
+  }
+
+
+  /** if the param_map contains strings other than those in valid_strs, return false;
+   *  otherwise true.
+   */
+  private boolean areAllParamsValid(HashMap<String, String> param_map, HashSet<String> valid_strs) {
+    ArrayList<String> param_list = new ArrayList<String>(param_map.keySet());
+    for(int i=0; i<param_list.size(); i++) {
+      logger.info("*** param, key =  "+param_list.get(i)+", value = "+param_map.get(param_list.get(i)));
+      if (valid_strs.contains(param_list.get(i)) == false) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private Element doListIdentifiersOrRecords(Element req, String verb, String record_type) {
+    // options: from, until, set, metadataPrefix, resumptionToken
+    // exceptions: badArgument, badResumptionToken, cannotDisseminateFormat, noRecordMatch, and noSetHierarchy
+    HashSet<String> valid_strs = new HashSet<String>();
+    valid_strs.add(OAIXML.FROM);
+    valid_strs.add(OAIXML.UNTIL);
+    valid_strs.add(OAIXML.SET);
+    valid_strs.add(OAIXML.METADATA_PREFIX);
+    valid_strs.add(OAIXML.RESUMPTION_TOKEN);
+    
+    Document result_doc = XMLConverter.newDOM();
+    Element result_element = result_doc.createElement(verb);
+    boolean result_token_needed = false; // does this result need to include a
+    // resumption token
+
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+
+    HashMap<String, String> param_map = GSXML.getParamMap(params); 
+    
+    // are all the params valid?
+    if (!areAllParamsValid(param_map, valid_strs)) {
+      logger.error("One of the params is invalid");
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "There was an invalid parameter");
+      // TODO, need to tell the user which one was invalid ??
+    }     
+    
+    // Do we have a resumption token??
+    String token = null;
+    String from = null;
+    String until = null;
+    boolean set_requested = false;
+    String set_spec_str = null;
+    String prefix_value = null;
+    int cursor = 0;
+    int current_cursor = 0;
+    String current_set = null;
+    long initial_time = 0;
+    
+    int total_size = -1; // we are only going to set this in resumption 
+    // token if it is easy to work out, i.e. not sending extra requests to
+    // MR just to calculate total size
+
+    if(param_map.containsKey(OAIXML.RESUMPTION_TOKEN)) {
+      // Is it an error to have other arguments? Do we need to check to make sure that resumptionToken is the only arg??
+      // validate resumptionToken
+      token = param_map.get(OAIXML.RESUMPTION_TOKEN);
+      logger.info("has resumptionToken " + token);
+      if(OAIResumptionToken.isValidToken(token) == false) {
+	logger.error("token is not valid");
+        return OAIXML.createErrorMessage(OAIXML.BAD_RESUMPTION_TOKEN, "");
+      }
+      result_token_needed = true; // we always need to send a token back if we have started with one. It may be empty if we are returning the end of the list
+      // initialise the request params from the stored token data
+      HashMap<String, String> token_data = OAIResumptionToken.getTokenData(token);
+      from = token_data.get(OAIXML.FROM);
+      until = token_data.get(OAIXML.UNTIL);
+      set_spec_str = token_data.get(OAIXML.SET);
+      if (set_spec_str != null) {
+	set_requested = true;
+      }
+      prefix_value = token_data.get(OAIXML.METADATA_PREFIX);
+      current_set = token_data.get(OAIResumptionToken.CURRENT_SET);
+      try {
+	cursor = Integer.parseInt(token_data.get(OAIXML.CURSOR));
+	cursor = cursor + resume_after; // increment cursor
+	current_cursor = Integer.parseInt(token_data.get(OAIResumptionToken.CURRENT_CURSOR));
+	initial_time = Long.parseLong(token_data.get(OAIResumptionToken.INITIAL_TIME));
+      } catch (NumberFormatException e) {
+	logger.error("tried to parse int from cursor data and failed");
+      }
+      
+      // check that the collections/sets haven't changed since the token was issued
+      if (collectionsChangedSinceTime(set_spec_str, initial_time)) {
+	logger.error("one of the collections in set "+set_spec_str+" has changed since token issued. Expiring the token");
+	OAIResumptionToken.expireToken(token);
+	return OAIXML.createErrorMessage(OAIXML.BAD_RESUMPTION_TOKEN, "Repository data has changed since this token was issued. Resend original request");
+      }
+    }
+    else {
+      // no resumption token, lets check the other params
+      // there must be a metadataPrefix
+      if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
+	logger.error("metadataPrefix param required");
+	return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "metadataPrefix param required");
+      }
+
+      //if there are any date params, check they're of the right format
+      Date from_date = null;
+      Date until_date = null;
+
+      from = param_map.get(OAIXML.FROM);
+      if(from != null) {	
+	from_date = OAIXML.getDate(from);
+	if(from_date == null) {
+	  logger.error("invalid date: " + from);
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid format for "+ OAIXML.FROM);
+	}
+      }
+      until = param_map.get(OAIXML.UNTIL);
+      if(until != null) {	
+	until_date = OAIXML.getDate(until);
+	if(until_date == null) {
+	  logger.error("invalid date: " + until);
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid format for "+ OAIXML.UNTIL);
+	} 
+      }    
+
+      if(from != null && until != null) { // check they are of the same date-time format (granularity)
+	if(from.length() != until.length()) {
+	  logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "The request has different granularities (date-time formats) for the From and Until date parameters.");
+	}
+
+	if(from_date.compareTo(until_date) > 0) { // from date can't be later than until date
+	    return OAIXML.createErrorMessage(OAIXML.NO_RECORDS_MATCH, "");
+	}
+      }
+      
+      if(until_date != null) {
+	  
+	  // Also call until_date.compareTo(earliestdatestamp) as the until date can't precede the earliest timestamp
+	  // Unfortunately, this test has to be done after the granularity test
+	  // compareTo() returns the value 0 if the argument Date is equal to this Date; a value less than 0 if this Date is before 
+	  // the Date argument; and a value greater than 0 if this Date is after the Date argument.    
+	  long earliestDatestamp = getEarliestDateStamp(collection_list);
+	  String earliestDatestamp_str = OAIXML.getTime(earliestDatestamp);
+	  Date earliestDatestamp_date = OAIXML.getDate(earliestDatestamp_str);
+	  
+	  if(until_date.compareTo(earliestDatestamp_date) < 0) {
+	      return OAIXML.createErrorMessage(OAIXML.NO_RECORDS_MATCH, "");
+	  }
+      }
+      
+  
+      // check the set arg is a set we know about
+      set_requested = param_map.containsKey(OAIXML.SET);
+      set_spec_str = null;
+      if(set_requested == true) {
+	set_spec_str = param_map.get(OAIXML.SET);
+	if (!this.set_set.contains(set_spec_str)) {
+	  // the set is not one we know about
+	  logger.error("requested set is not found in this repository");
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid set parameter");	
+	  
+	}
+      }
+      // Is the metadataPrefix arg one this repository supports?
+      prefix_value = param_map.get(OAIXML.METADATA_PREFIX);
+      if (repositorySupportsMetadataPrefix(prefix_value) == false) {
+	logger.error("requested metadataPrefix is not found in OAIConfig.xml");
+	return OAIXML.createErrorMessage(OAIXML.CANNOT_DISSEMINATE_FORMAT, "metadata format "+prefix_value+" not supported by this repository");
+      }
+      
+    } // else no resumption token, check other params    
+    
+    // Whew. Now we have validated the params, we can work on doing the actual
+    // request
+
+
+    Document doc = XMLConverter.newDOM();
+    Element mr_msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element mr_req = doc.createElement(GSXML.REQUEST_ELEM);
+    // TODO does this need a type???
+    mr_msg.appendChild(mr_req);
+
+    // copy in the from/until params if there
+    if (from != null) {
+      mr_req.appendChild(GSXML.createParameter(doc, OAIXML.FROM, from));
+    }
+    if (until != null) {
+      mr_req.appendChild(GSXML.createParameter(doc, OAIXML.UNTIL, until));
+    }
+    // add metadataPrefix
+    mr_req.appendChild(GSXML.createParameter(doc, OAIXML.METADATA_PREFIX, prefix_value));
+	
+    // do we have a set???
+    // if no set, we send to all collections in the collection list
+    // if super set, we send to all collections in super set list
+    // if a single collection, send to it
+    // if a subset, send to the collection
+    Vector<String> current_coll_list = getCollectionListForSet(set_spec_str);
+    boolean single_collection = false;
+    if (current_coll_list.size() == 1) {
+      single_collection = true;
+    }
+    if (set_spec_str != null && set_spec_str.indexOf(":") != -1) {
+      // we have a subset - add the set param back in
+      mr_req.appendChild(GSXML.createParameter(doc, OAIXML.SET, set_spec_str));
+    }
+
+    int num_collected_records = 0;
+    int start_point = current_cursor; // may not be 0 if we are using a resumption token
+    String resumption_collection = "";
+    boolean empty_result_token = false; // if we are sending the last part of a list, then the token value will be empty
+    
+    // iterate through the list of collections and send the request to each
+
+    int start_coll=0;
+    if (current_set != null) {
+      // we are resuming a previous request, need to locate the first collection
+      for (int i=0; i<current_coll_list.size(); i++) {
+	if (current_set.equals(current_coll_list.get(i))) {
+	  start_coll = i;
+	  break;
+	}
+      }
+    }
+    
+    for (int i=start_coll; i<current_coll_list.size(); i++) {
+      String current_coll = current_coll_list.get(i);
+      mr_req.setAttribute(GSXML.TO_ATT, current_coll+"/"+verb);
+
+      Element result = (Element)mr.process(mr_msg);
+      logger.info("*** " + verb+ " result for coll "+current_coll);
+      logger.info("*** " + XMLConverter.getPrettyString(result));
+      if (result == null) {
+	logger.info("message router returns null");
+	// do what??? carry on? fail??
+	return OAIXML.createErrorMessage("Internal service returns null", "");
+      }
+      Element res = (Element)GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
+      if(res == null) {
+        logger.info("response element in xml_result is null");
+        return OAIXML.createErrorMessage("Internal service returns null", "");
+      }
+      NodeList record_list = res.getElementsByTagName(record_type);
+      int num_records = record_list.getLength();
+      if(num_records == 0) {
+	logger.info("message router returns 0 records for coll "+current_coll);
+	continue; // try the next collection
+      }  
+      if (single_collection) {
+	total_size = num_records;
+      }
+      int records_to_add = (resume_after > 0 ? resume_after - num_collected_records : num_records);
+      if (records_to_add > (num_records-start_point)) {
+	records_to_add = num_records-start_point;
+      }
+      addRecordsToList(result_doc, result_element, record_list, start_point, records_to_add);
+      num_collected_records += records_to_add;
+      
+      // do we need to stop here, and do we need to issue a resumption token?
+      if (resume_after > 0 && num_collected_records == resume_after) {
+	// we have finished collecting records at the moment.
+	// but are we conincidentally at the end? or are there more to go?
+	if (records_to_add < (num_records - start_point)) {
+	  // we have added less than this collection had
+	  start_point += records_to_add;
+	  resumption_collection = current_coll;
+	  result_token_needed = true;
+	}
+	else {
+	  // we added all this collection had to offer
+	  // is there another collection in the list??
+	  if (i<current_coll_list.size()-1) {
+	    result_token_needed = true;
+	    start_point = 0;
+	    resumption_collection = current_coll_list.get(i+1);
+	  } 
+	  else {
+	    // we have finished one collection and there are no more collection
+	    // if we need to send a resumption token (in this case, only because we started with one, then it will be empty
+	    logger.info("*** at end of list, need empty result token");
+	    empty_result_token = true;
+	  }
+	}
+	break;
+      }
+      start_point = 0; // only the first one will have start non-zero, if we
+      // have a resumption token
+      
+    } // for each collection
+
+    if (num_collected_records ==0) {
+      // there were no matching results
+      return OAIXML.createErrorMessage(OAIXML.NO_RECORDS_MATCH, "");
+    }
+
+    if (num_collected_records < resume_after) {
+      // we have been through all collections, and there are no more
+      // if we need a result token - only because we started with one, so we need to send an empty one, then make sure everyone knows we are just sending an empty one
+      if (result_token_needed) {
+	empty_result_token = true;
+      }
+    }
+    
+    if (result_token_needed) {
+      // we need a resumption token
+      if (empty_result_token) {
+	logger.info("*** have empty result token");
+	token = "";
+      } else {
+	if (token != null) {
+	  // we had a token for this request, we can just update it
+	  token = OAIResumptionToken.updateToken(token, ""+cursor, resumption_collection, ""+start_point);
+	} else {
+	  // we are generating a new one
+	  token = OAIResumptionToken.createAndStoreResumptionToken(set_spec_str, prefix_value, from, until, ""+cursor, resumption_collection, ""+start_point );
+	}
+      }
+
+      // result token XML
+      long expiration_date = -1;
+      if (empty_result_token) {
+	// we know how many records in total as we have sent them all
+	total_size = cursor+num_collected_records;
+      } else {
+	// non-empty token, set the expiration date
+	expiration_date = OAIResumptionToken.getExpirationDate(token);
+      }
+      Element token_elem = OAIXML.createResumptionTokenElement(result_doc, token, total_size, cursor, expiration_date); 
+      // OAIXML.addToken(token_elem); // store it
+      result_element.appendChild(token_elem); // add to the result
+    }
+  
+
+    return getMessage(result_doc, result_element);
+  }
+
+  private Vector<String> getCollectionListForSet(String set) {
+    if (set == null) {
+      // no set requested, need the complete collection list
+      return this.collection_name_list;
+    }
+    if (has_super_colls && super_coll_map.containsKey(set)) {
+      return super_coll_map.get(set);
+    }
+
+    Vector<String> coll_list = new Vector<String>();
+    if (set.indexOf(":") != -1) {
+      String col_name = set.substring(0, set.indexOf(":"));
+      coll_list.add(col_name);
+    }
+    else {
+      coll_list.add(set);
+    }
+    return coll_list;
+  }
+  private void addRecordsToList(Document doc, Element result_element, NodeList
+				record_list, int start_point, int num_records) {
+    int end_point = start_point + num_records;
+    for (int i=start_point; i<end_point; i++) {
+      result_element.appendChild(doc.importNode(record_list.item(i), true));
+    }
+  }
+
+  private Element collectAll(Element result, Element msg, String verb, String elem_name) {
+    if(result == null) {
+      //in the first round, result is null
+      return msg;
+    }
+    Element res_in_result = (Element)GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
+    if(res_in_result == null) { // return the results of all other collections accumulated so far
+      return msg;
+    }
+    Element verb_elem = (Element)GSXML.getChildByTagName(res_in_result, verb);
+    if(msg == null) {
+      return result;
+    }
+
+    //e.g., get all <record> elements from the returned message. There may be none of
+    //such element, for example, the collection service returned an error message
+    NodeList elem_list = msg.getElementsByTagName(elem_name);
+    
+    for (int i=0; i<elem_list.getLength(); i++) {
+      verb_elem.appendChild(res_in_result.getOwnerDocument().importNode(elem_list.item(i), true));
+    }
+    return result;
+  }
+
+  
+  /** there are three possible exception conditions: bad argument, idDoesNotExist, and noMetadataFormat.
+   * The first one is handled here, and the last two are processed by OAIPMH.
+   */
+  private Element doListMetadataFormats(Element req) {
+    //if the verb is ListMetadataFormats, there could be only one parameter: identifier
+    //, or there is no parameter; otherwise it is an error
+    //logger.info("" + XMLConverter.getString(msg));
+    
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+    Element param = null;
+    Document lmf_doc = XMLConverter.newDOM();
+    if(params.getLength() == 0) {
+      //this is requesting metadata formats for the whole repository
+      //read the oaiConfig.xml file, return the metadata formats specified there.
+      if (this.listmetadataformats_response != null) {
+	// we have already created it
+	return this.listmetadataformats_response;
+      }
+
+      Element list_metadata_formats = lmf_doc.createElement(OAIXML.LIST_METADATA_FORMATS);
+      // get all the formats out of oai_config
+      NodeList formats = oai_config.getElementsByTagName(OAIXML.METADATA_FORMAT);
+      if (formats.getLength() ==0) {
+       	logger.error("OAIConfig.xml must contain the supported metadata formats");
+      	// TODO this is internal error, what to do???
+      	return getMessage(lmf_doc, list_metadata_formats);
+      }
+	
+      for(int i=0; i<formats.getLength(); i++) {
+	Element f = OAIXML.getMetadataFormatShort(lmf_doc, (Element)formats.item(i));
+	list_metadata_formats.appendChild(f);
+     }
+      this.listmetadataformats_response = getMessage(lmf_doc, list_metadata_formats);
+      return this.listmetadataformats_response;
+      
+    } 
+
+    if (params.getLength() > 1) {
+      //Bad argument. Can't be more than one parameters for ListMetadataFormats verb
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
+    } 
+    
+    // This is a request for the metadata of a particular item with an identifier
+    /**the request xml is in the form: <request>
+     *                                   <param name=.../>
+     *                                 </request>
+     *And there is a param element and one element only. (No paramList element in between).
+     */
+    param = (Element)params.item(0);
+    String param_name = param.getAttribute(GSXML.NAME_ATT);
+    String identifier = "";
+    if (!param_name.equals(OAIXML.IDENTIFIER)) {
+      //Bad argument
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
+    } 
+      
+    identifier = param.getAttribute(GSXML.VALUE_ATT);
+    // the identifier is in the form: <coll_name>:<OID>
+    // so it must contain at least one ':' characters 
+    // (the oid itself may contain : chars)
+    String[] strs = identifier.split(":", 2);
+    if(strs.length != 2) {
+      logger.error("identifier is not in the form coll:id" + identifier);
+      return OAIXML.createErrorMessage(OAIXML.ID_DOES_NOT_EXIST, "");
+    }
+        
+    // send request to message router
+    // get the names
+    String coll_name = strs[0];
+    String oid = strs[1];
+
+    Document msg_doc = XMLConverter.newDOM();
+    Element message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+    String verb = req.getAttribute(GSXML.TO_ATT);
+    String new_to = coll_name + "/" + verb;
+    Element request = GSXML.createBasicRequest(msg_doc, "oai???", new_to, null);
+    message.appendChild(request);
+    // add the id param
+    GSXML.addParameterToList(request, OAIXML.OID, oid);
+
+    //Now send the request to the message router to process
+    Node result_node = mr.process(message);
+    return GSXML.nodeToElement(result_node);
+  }
+  
+    
+
+
+  private void copyNamedElementfromConfig(Element to_elem, String element_name) {
+    Element original_element = (Element)GSXML.getChildByTagName(oai_config, element_name);
+    if(original_element != null) {
+      GSXML.copyNode(to_elem, original_element);
+    }
+  }
+
+
+  private Element doIdentify() {
+    //The validation for this verb has been done in OAIServer.validate(). So no bother here.
+    logger.info("");
+    if (this.identify_response != null) {
+      // we have already created it
+      return getMessage(this.identify_response.getOwnerDocument(), this.identify_response);
+    }
+    Document doc = XMLConverter.newDOM();
+    Element identify = doc.createElement(OAIXML.IDENTIFY);
+    //do the repository name
+    copyNamedElementfromConfig(identify, OAIXML.REPOSITORY_NAME);
+    //do the baseurl
+    copyNamedElementfromConfig(identify, OAIXML.BASE_URL);
+    //do the protocol version
+    copyNamedElementfromConfig(identify, OAIXML.PROTOCOL_VERSION);
+        
+    //There can be more than one admin email according to the OAI specification
+    NodeList admin_emails = GSXML.getChildrenByTagName(oai_config, OAIXML.ADMIN_EMAIL);
+    int num_admin = 0;
+    Element from_admin_email = null;  
+    if (admin_emails != null) {
+      num_admin = admin_emails.getLength();
+    }
+    for (int i=0; i<num_admin; i++) {
+      GSXML.copyNode(identify, admin_emails.item(i));
+    }
+
+    //do the earliestDatestamp
+    //send request to mr to search through the earliest datestamp amongst all oai collections in the repository.
+    //ask the message router for a list of oai collections
+    //NodeList oai_coll = getOAICollectionList();
+    long earliestDatestamp = getEarliestDateStamp(collection_list);
+    String earliestDatestamp_str = OAIXML.getTime(earliestDatestamp);
+    Element earliestDatestamp_elem = doc.createElement(OAIXML.EARLIEST_DATESTAMP);
+    GSXML.setNodeText(earliestDatestamp_elem, earliestDatestamp_str);
+    identify.appendChild(earliestDatestamp_elem);
+
+    //do the deletedRecord
+    copyNamedElementfromConfig(identify, OAIXML.DELETED_RECORD);
+    //do the granularity
+    copyNamedElementfromConfig(identify, OAIXML.GRANULARITY);
+     
+    // output the oai identifier
+    Element description = doc.createElement(OAIXML.DESCRIPTION);
+    identify.appendChild(description);
+    // TODO, make this a valid id
+    Element oaiIdentifier = OAIXML.createOAIIdentifierXML(doc, repository_id, "lucene-jdbm-demo", "ec159e");
+    description.appendChild(oaiIdentifier);
+
+    // if there are any oaiInfo metadata, add them in too.
+    Element info = (Element)GSXML.getChildByTagName(oai_config, OAIXML.OAI_INFO);
+    if (info != null) {
+      NodeList meta = GSXML.getChildrenByTagName(info, OAIXML.METADATA);
+      if (meta != null && meta.getLength() > 0) {
+	Element gsdl = OAIXML.createGSDLElement(doc);
+	description.appendChild(gsdl);
+	for (int m = 0; m<meta.getLength(); m++) {
+	  GSXML.copyNode(gsdl, meta.item(m));
+	}
+	
+      }
+    }
+    this.identify_response = identify;
+    return getMessage(doc, identify);
+  }
+  /** split the identifier into <collection + OID> as an array 
+      It has already been checked that the 'identifier' contains at least one ':'
+  */
+
+  /** validate if the specified metadata prefix value is supported by the repository
+   *  by checking it in the OAIConfig.xml
+   */
+  private boolean repositorySupportsMetadataPrefix(String prefix_value) {
+    NodeList prefix_list = oai_config.getElementsByTagName(OAIXML.METADATA_PREFIX);
+    
+    for(int i=0; i<prefix_list.getLength(); i++) {
+      if(prefix_value.equals(GSXML.getNodeText((Element)prefix_list.item(i)).trim() )) {
+        return true;
+      }
+    }
+    return false;
+  }
+  private Element doGetRecord(Element req){
+    logger.info("");
+    /** arguments:
+        identifier: required
+        metadataPrefix: required
+	*  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
+	*/ 
+    Document doc = XMLConverter.newDOM();
+    Element get_record = doc.createElement(OAIXML.GET_RECORD);
+
+    HashSet<String> valid_strs = new HashSet<String>();
+    valid_strs.add(OAIXML.IDENTIFIER);
+    valid_strs.add(OAIXML.METADATA_PREFIX);
+
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+    HashMap<String, String> param_map = GSXML.getParamMap(params);    
+    
+    if(!areAllParamsValid(param_map, valid_strs) ||
+       params.getLength() == 0 ||
+       param_map.containsKey(OAIXML.IDENTIFIER) == false ||
+       param_map.containsKey(OAIXML.METADATA_PREFIX) == false ) {
+      logger.error("must have the metadataPrefix/identifier parameter.");
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
+    }
+    
+    String prefix = param_map.get(OAIXML.METADATA_PREFIX);
+    String identifier = param_map.get(OAIXML.IDENTIFIER);
+    
+    // verify the metadata prefix
+    if (repositorySupportsMetadataPrefix(prefix) == false) {
+      logger.error("requested prefix is not found in OAIConfig.xml");
+      return OAIXML.createErrorMessage(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+
+    // get the names
+    String[] strs = identifier.split(":", 2);
+    if(strs == null || strs.length < 2) {
+      logger.error("identifier is not in the form coll:id" + identifier);
+      return OAIXML.createErrorMessage(OAIXML.ID_DOES_NOT_EXIST, "");
+    }    
+    //String name_of_site = strs[0];
+    String coll_name = strs[0];
+    String oid = strs[1];
+    
+    //re-organize the request element
+    // reset the 'to' attribute
+    String verb = req.getAttribute(GSXML.TO_ATT);
+    req.setAttribute(GSXML.TO_ATT, coll_name + "/" + verb);
+    // reset the identifier element
+    Element param = GSXML.getNamedElement(req, GSXML.PARAM_ELEM, GSXML.NAME_ATT, OAIXML.IDENTIFIER);
+    if (param != null) {
+      param.setAttribute(GSXML.NAME_ATT, OAIXML.OID);
+      param.setAttribute(GSXML.VALUE_ATT, oid);
+    }    
+
+    //Now send the request to the message router to process
+    Element msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    msg.appendChild(doc.importNode(req, true));
+    Node result_node = mr.process(msg);
+    return GSXML.nodeToElement(result_node);
+  }
+
+  // See OAIConfig.xml
+  // dynamically works out what the earliestDateStamp is, since it varies by collection
+  // returns this time in *milliseconds*.
+  protected long getEarliestDateStamp(Element oai_coll_list) {
+    // config earliest datstamp
+    long config_datestamp = 0;
+    Element config_datestamp_elem = (Element)GSXML.getChildByTagName(this.oai_config, OAIXML.EARLIEST_DATESTAMP);
+    if (config_datestamp_elem != null) {
+      String datest = GSXML.getNodeText(config_datestamp_elem);
+      config_datestamp = OAIXML.getTime(datest);
+      if (config_datestamp == -1) {
+	config_datestamp = 0;
+      }
+    }
+    //do the earliestDatestamp
+    long current_time = System.currentTimeMillis();
+    long earliestDatestamp = current_time;
+    NodeList oai_coll = oai_coll_list.getElementsByTagName(GSXML.COLLECTION_ELEM);
+    int oai_coll_size = oai_coll.getLength();
+    if (oai_coll_size == 0) {
+      logger.info("returned oai collection list is empty. Setting repository earliestDatestamp to be the earliest datestamp from OAIConfig.xml, or 1970-01-01 if not specified.");
+      return config_datestamp;
+    }
+    // the earliestDatestamp is now stored as a metadata element in the collection's buildConfig.xml file
+    // we get the earliestDatestamp among the collections
+    for(int i=0; i<oai_coll_size; i++) {
+      long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_DATESTAMP));
+      if (coll_earliestDatestamp == 0) {
+	// try last modified
+	coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.LAST_MODIFIED));
+      }
+      if (coll_earliestDatestamp > 0) {
+	earliestDatestamp = (earliestDatestamp > coll_earliestDatestamp)? coll_earliestDatestamp : earliestDatestamp;
+      }
+    }
+    if (earliestDatestamp == current_time) {
+      logger.info("no collection had a real datestamp, using value from OAIConfig");
+      return config_datestamp;
+    }
+    return earliestDatestamp; 
+  }
+
+  private boolean collectionsChangedSinceTime(String set_spec_str, long initial_time) {
+
+    // we need to look though all collections in the set to see if any have last modified dates > initial_time
+    Vector<String> set_coll_list = getCollectionListForSet(set_spec_str);
+
+    Node child = this.collection_list.getFirstChild();
+    while (child != null) {
+      if (child.getNodeName().equals(GSXML.COLLECTION_ELEM)) {
+	String coll_id =((Element) child).getAttribute(GSXML.NAME_ATT);
+	if (set_coll_list.contains(coll_id)) {
+	  long last_modified = Long.parseLong(((Element)child).getAttribute(OAIXML.LAST_MODIFIED));
+	  if (initial_time < last_modified) {
+	    return true;
+	  }
+	}
+      }
+      child = child.getNextSibling();
+    }
+    return false;
+ 
+  }
+
+}
+
+  
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Page.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Page.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Page.java	(revision 31634)
@@ -0,0 +1,136 @@
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashMap;
+
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.*;
+
+public class Page {
+
+	private Element page ;
+	private Element pageRequest ;
+	private Element pageResponse ;
+
+	private Document siteUi ;
+	private Document collectUi ;
+	private Document siteMetadata ;
+	private Document collectMetadata ;
+	
+	private Receptionist receptionist ;
+	
+	public Page(Element page, Receptionist receptionist) {
+		try {
+		this.page = page ;
+		this.receptionist = receptionist ;
+		
+		pageRequest = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+		pageResponse = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+		
+		String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), getSite()) ;
+		String collection = getCollection() ;
+				
+		File siteUiFile = new File(siteHome + File.separatorChar + "ui" + File.separatorChar + "ui.xml") ;	
+		//System.out.println("siteUi: " + siteUiFile.getAbsolutePath()) ;
+		siteUi = receptionist.converter.getDOM(siteUiFile, "utf-8");
+		if (siteUi == null) {
+			System.out.println(" could not parse site level ui file: " + siteUiFile.getPath());
+		}
+		
+		File siteMetadataFile = new File(siteHome + File.separatorChar + "metadata.xml") ;
+		//System.out.println("siteMetadata: " + siteMetadataFile.getAbsolutePath()) ;
+		siteMetadata = receptionist.converter.getDOM(siteMetadataFile, "utf-8");
+		if (siteMetadata == null) {
+			System.out.println(" could not parse site level metadata file: " + siteMetadataFile.getPath());
+		}
+		
+		if (!collection.equals("")) {
+			File collectUiFile = new File(GSFile.collectionBaseDir(siteHome, collection) + File.separatorChar + "ui" + File.separatorChar + "ui.xml") ;
+			collectUi = receptionist.converter.getDOM(collectUiFile, "utf-8");
+			if (collectUi == null) {
+				System.out.println(" could not parse collect level ui file: " + collectUiFile.getPath());
+			}
+			
+			File collectMetadataFile = new File(GSFile.collectionBaseDir(siteHome, collection) + File.separatorChar + "metadata.xml") ;
+			collectMetadata = receptionist.converter.getDOM(collectMetadataFile, "utf-8");
+			if (collectMetadata == null) {
+				System.out.println(" could not parse collect level metadata file: " + collectMetadataFile.getPath());
+			}
+		}
+		} catch (Exception e) {
+			e.printStackTrace() ;
+		}
+	}
+	
+	public Element getPage() {
+		return page ;
+	}
+	
+	public Element getPageRequest() {
+		return pageRequest ;
+	}
+	
+	public Element getPageResponse() {
+		return pageResponse ;
+	}
+	
+	public String getAction() {
+		return pageRequest.getAttribute("action") ;
+	}
+	
+	public String getSubaction() {
+		return pageRequest.getAttribute("subaction") ;
+	}
+	
+	public String getSiteHome() {
+		return GSFile.siteHome(GlobalProperties.getGSDL3Home(), getSite()) ;
+	}
+	
+	public String getCollectionHome() {
+		String collection = this.getCollection() ;
+		if (!collection.equals(""))
+			return GSFile.collectionBaseDir(getSiteHome(), collection) ;
+		else 
+			return null ;
+	}
+	
+	public Document getSiteUi() {
+		return siteUi ;
+	}
+	
+	public Document getCollectUi() {
+		return collectUi ;
+	}
+	
+	public Document getSiteMetadata() {
+		return siteMetadata ;
+	}
+	
+	public Document getCollectMetadata() {
+		return collectMetadata ;
+	}	
+	
+	public String getLanguage() {
+		String lang = pageRequest.getAttribute(GSXML.LANG_ATT);
+		return lang ;
+	}
+	
+	public String getSite() {
+		return (String) receptionist.config_params.get(GSConstants.SITE_NAME) ;
+	}
+	
+	public String getCollection() {
+		String collection = "" ;
+		Element request = (Element)GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+		
+		Element cgi_param_list = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		if (cgi_param_list != null) {
+			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+			collection = (String)params.get(GSParams.COLLECTION);
+			if (collection == null) collection = "";
+		}	
+		return collection ;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Receptionist.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Receptionist.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/Receptionist.java	(revision 31634)
@@ -0,0 +1,366 @@
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.action.Action;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * the most basic Receptionist, used for interface generation. Receives requests
+ * consisting of an xml representation of cgi args, and returns the page of
+ * data. The requests are processed by the appropriate action class
+ * 
+ * @see Action
+ */
+public class Receptionist implements ModuleInterface
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.Receptionist.class.getName());
+
+	/** the set up variables */
+	protected HashMap<String, Object> config_params = null;
+	/** container Document to create XML Nodes */
+	//protected Document doc = null;
+
+	/** a converter class to parse XML and create Docs */
+	protected XMLConverter converter = null;
+
+	/** the message router that the Receptionist and Actions will talk to */
+	protected ModuleInterface mr = null;
+
+	/** the list of actions */
+	protected HashMap<String, Action> action_map = null;
+
+	/** the list of params */
+	protected GSParams params = null;
+	protected Element language_list = null;
+
+	/** the list of interfaces this is based on */
+	protected ArrayList<String> base_interfaces = null;
+
+	public Receptionist()
+	{
+		this.converter = new XMLConverter();
+		//this.doc = XMLConverter.newDOM();
+		this.action_map = new HashMap<String, Action>();
+	}
+
+	public void cleanUp()
+	{
+		if (this.mr != null)
+		{
+			this.mr.cleanUp();
+		}
+	}
+
+	public void setParams(GSParams params)
+	{
+		this.params = params;
+	}
+
+	public void setConfigParams(HashMap<String, Object> params)
+	{
+		this.config_params = params;
+	}
+
+	public HashMap<String, Object> getConfigParams()
+	{
+		return this.config_params;
+	}
+
+	/**
+	 * sets the message router - it should already be created and configured
+	 * before being passed to the receptionist
+	 */
+	public void setMessageRouter(ModuleInterface m)
+	{
+		this.mr = m;
+	}
+
+	/**
+	 * gets the message router
+	 */
+	public ModuleInterface getMessageRouter()
+	{
+		return this.mr;
+	}
+
+	/** configures the receptionist */
+	public boolean configure()
+	{
+
+		if (this.config_params == null)
+		{
+			logger.error(" config variables must be set before calling configure");
+			return false;
+		}
+		if (this.mr == null)
+		{
+			logger.error(" message router must be set  before calling configure");
+			return false;
+		}
+
+		// find the config file containing a list of actions
+		File interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.INTERFACE_NAME))));
+		if (!interface_config_file.exists())
+		{
+			logger.error(" interface config file: " + interface_config_file.getPath() + " not found!");
+			return false;
+		}
+
+		Document config_doc = this.converter.getDOM(interface_config_file);
+		if (config_doc == null)
+		{
+			logger.error(" could not parse interface config file: " + interface_config_file.getPath());
+			return false;
+		}
+		Element config_elem = config_doc.getDocumentElement();
+		String base_interface = config_elem.getAttribute("baseInterface");
+		setUpBaseInterface(base_interface);
+		setUpInterfaceOptions(config_elem);
+
+		// load up the actions
+		Element action_list = (Element) GSXML.getChildByTagName(config_elem, GSXML.ACTION_ELEM + GSXML.LIST_MODIFIER);
+		NodeList actions = action_list.getElementsByTagName(GSXML.ACTION_ELEM);
+
+		for (int i = 0; i < actions.getLength(); i++)
+		{
+			Element action = (Element) actions.item(i);
+			String class_name = action.getAttribute("class");
+			String action_name = action.getAttribute("name");
+			Action ac = null;
+			try
+			{
+				ac = (Action) Class.forName("org.greenstone.gsdl3.action." + class_name).newInstance();
+			}
+			catch (Exception e)
+			{
+				logger.error(" couldn't load in action " + class_name);
+				e.printStackTrace();
+				continue;
+			}
+			ac.setConfigParams(this.config_params);
+			ac.setMessageRouter(this.mr);
+			ac.configure();
+			ac.addActionParameters(this.params);
+			this.action_map.put(action_name, ac);
+		}
+
+		this.language_list = (Element) GSXML.getChildByTagName(config_elem, "languageList");
+		if (language_list == null)
+		{
+			logger.error(" didn't find a language list in the config file!!");
+		}
+
+		return true;
+	}
+
+	public String process(String xml_in)
+	{
+
+		Node message_node = this.converter.getDOM(xml_in);
+		Node page = process(message_node);
+		return this.converter.getString(page);
+	}
+
+	/**
+	 * process - produce a page of data in response to a request if something
+	 * goes wrong, it returns null - TODO: return a suitable message to the user
+	 */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+
+		// get the request out of the message - assume that there is only one
+		Element request = (Element) GSXML.getChildByTagName(message, GSXML.REQUEST_ELEM);
+		if (request == null)
+		{
+			logger.error(" message had no request!");
+			return null;
+		}
+		// check the request type 
+		String type = request.getAttribute(GSXML.TYPE_ATT); // returns "" if no att of this name
+		if (type.equals(GSXML.REQUEST_TYPE_SECURITY))
+		{
+			return this.mr.process(message);
+		}
+
+		if (!type.equals(GSXML.REQUEST_TYPE_PAGE))
+		{
+			// now Receptionist forwards non-page requests straight to the MR, and returns the responses
+			logger.debug(" request type is not '" + GSXML.REQUEST_TYPE_PAGE + "', but it is '" + type + "', so forwarding the message to the MR!");
+			// process the whole message - mr needs <message> tags, and 
+			// in this case, there may be more than one request in the message
+			return this.mr.process(message);
+		}
+		// work out which action to pass to
+		String action = request.getAttribute(GSXML.ACTION_ATT);
+		if (action.equals(""))
+		{
+			logger.error(" no action specified in the request!");
+			return null;
+		}
+
+		// find the  appropriate action	
+		Action a = this.action_map.get(action);
+
+		String action_name = null;
+		if (a == null)
+		{ // not in the map yet
+			// try to load a new action
+			try
+			{
+				action_name = action.substring(0, 1).toUpperCase() + action.substring(1) + "Action";
+				Action ac = (Action) Class.forName("org.greenstone.gsdl3.action." + action_name).newInstance();
+				ac.setConfigParams(this.config_params);
+				ac.setMessageRouter(this.mr);
+				ac.configure();
+				ac.addActionParameters(this.params);
+				this.action_map.put(action, ac);
+				a = ac;
+			}
+			catch (Exception e)
+			{
+
+				logger.error(" a new action (" + action_name + ") was specified and it couldn't be created. Error message:" + e.getMessage());
+				return null;
+			}
+		}
+		// transform the request in some way -- does nothing!
+		preProcessRequest(request);
+		// set up the page
+		Document doc = XMLConverter.newDOM();
+		Element page = doc.createElement(GSXML.PAGE_ELEM);
+		page.setAttribute(GSXML.LANG_ATT, request.getAttribute(GSXML.LANG_ATT));
+		// just in case these namespaces end up in the page and we want to display the XML
+		GSXML.addGSNamespaces(page, false);
+		
+		//logger.info(a+" mesa=" + this.converter.getPrettyString(message));
+		// get the page data from the action
+
+		Node action_response = a.process(message);
+		boolean response_only = false;
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list != null)
+		{
+			Element param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, "ro");
+			if (param != null)
+			{
+				String value = param.getAttribute("value");
+				if (value.equals("1"))
+				{
+					response_only = true;
+				}
+			}
+		}
+		if (response_only)
+		{
+			// only the response from the action is sent back
+			return action_response;
+		}
+
+		// the request is part of the page
+		page.appendChild(GSXML.duplicateWithNewName(doc, request, GSXML.PAGE_REQUEST_ELEM, true));
+		// add the response too
+		Element page_response = GSXML.duplicateWithNewName(doc, (Element) GSXML.getChildByTagName(action_response, GSXML.RESPONSE_ELEM), GSXML.PAGE_RESPONSE_ELEM, true);
+		page.appendChild(page_response);
+
+		//logger.info(" raw page="+this.converter.getString(page));
+		// transform the result in some way
+		//Element resulting_page = postProcessPage(page);
+
+		Node resulting_page = postProcessPage(page);
+
+		//logger.debug("receptionist returned response");
+		//logger.error("receptionist returned response");
+		//logger.debug(this.converter.getString(resulting_page));
+		//logger.error(this.converter.getString(resulting_page));
+		//    logger.info("receptionist returned response");
+		//    logger.info(this.converter.getString(resulting_page));
+		return resulting_page;
+	}
+
+	protected boolean setUpBaseInterface(String base_interface)
+	{
+		if (base_interface == null || base_interface.equals(""))
+		{
+			// there was no base interface, the list remains null
+			return true;
+		}
+		// foreach base interface
+		while (!base_interface.equals(""))
+		{
+			// find the base interface config file
+			File base_interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), base_interface)));
+			if (!base_interface_config_file.exists())
+			{
+				logger.error(" base interface config file: " + base_interface_config_file.getPath() + " not found!");
+				return false;
+			}
+			// the interface name is valid, add it to the list
+			if (base_interfaces == null)
+			{
+				base_interfaces = new ArrayList<String>();
+			}
+			base_interfaces.add(base_interface);
+			this.config_params.put(GSConstants.BASE_INTERFACES, base_interfaces);
+			// now see if this has a base interface
+			Document config_doc = this.converter.getDOM(base_interface_config_file);
+			if (config_doc == null)
+			{
+				logger.error(" could not parse base interface config file: " + base_interface_config_file.getPath());
+				return false;
+			}
+			Element config_elem = config_doc.getDocumentElement();
+			base_interface = config_elem.getAttribute("baseInterface");
+		}
+		return true;
+	}
+
+	protected boolean setUpInterfaceOptions(Element config_elem)
+	{
+		Element option_list = (Element) GSXML.getChildByTagName(config_elem, "optionList");
+		if (option_list != null)
+		{
+			logger.debug("found an interface optionList");
+			// we set any options in the config params
+			NodeList options = option_list.getElementsByTagName("option");
+			for (int i = 0; i < options.getLength(); i++)
+			{
+				Element option = (Element) options.item(i);
+				String name = option.getAttribute(GSXML.NAME_ATT);
+				String value = option.getAttribute(GSXML.VALUE_ATT);
+				logger.debug("option: " + name + ", " + value);
+				if (!name.equals("") && !value.equals(""))
+				{
+					this.config_params.put(name, value);
+				}
+			}
+		}
+
+		return true;
+	}
+
+	protected void preProcessRequest(Element request)
+	{
+		return;
+	}
+
+	protected Node postProcessPage(Element page)
+	{
+		return page;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/TransformingReceptionist.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/TransformingReceptionist.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/TransformingReceptionist.java	(revision 31634)
@@ -0,0 +1,1296 @@
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.xerces.parsers.DOMParser;
+import org.greenstone.gsdl3.action.Action;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.util.XMLTransformer;
+import org.greenstone.gsdl3.util.XSLTUtil;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Comment;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+import org.xml.sax.InputSource;
+
+/**
+ * A receptionist that uses xslt to transform the page_data before returning it.
+ * . Receives requests consisting of an xml representation of cgi args, and
+ * returns the page of data - in html by default. The requests are processed by
+ * the appropriate action class
+ * 
+ * @see Action
+ */
+public class TransformingReceptionist extends Receptionist
+{
+	protected static final int CONFIG_PASS = 1;
+	protected static final int TEXT_PASS = 2;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.TransformingReceptionist.class.getName());
+
+	/** The preprocess.xsl file is in a fixed location */
+	static final String preprocess_xsl_filename = GlobalProperties.getGSDL3Home() + File.separatorChar + "interfaces" + File.separatorChar + "core" + File.separatorChar + "transform" + File.separatorChar + "preProcess.xsl";
+
+	/** the list of xslt to use for actions */
+	protected HashMap<String, String> xslt_map = null;
+
+	/** a transformer class to transform xml using xslt */
+	protected XMLTransformer transformer = null;
+
+	protected TransformerFactory transformerFactory = null;
+	protected DOMParser parser = null;
+
+	protected HashMap<String, ArrayList<String>> _metadataRequiredMap = new HashMap<String, ArrayList<String>>();
+
+	boolean _debug = true;
+
+	public TransformingReceptionist()
+	{
+		super();
+		this.xslt_map = new HashMap<String, String>();
+		this.transformer = new XMLTransformer();
+		try
+		{
+			transformerFactory = org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+			this.converter = new XMLConverter();
+			//transformerFactory.setURIResolver(new MyUriResolver()) ;
+
+			parser = new DOMParser();
+			parser.setFeature("http://xml.org/sax/features/validation", false);
+			// don't try and load external DTD - no need if we are not validating, and may cause connection errors if a proxy is not set up.
+			parser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+			// a performance test showed that having this on lead to increased 
+			// memory use for small-medium docs, and not much gain for large 
+			// docs.
+			// http://www.sosnoski.com/opensrc/xmlbench/conclusions.html
+			parser.setFeature("http://apache.org/xml/features/dom/defer-node-expansion", false);
+			parser.setFeature("http://apache.org/xml/features/continue-after-fatal-error", true);
+			// setting a handler for when fatal errors, errors or warnings happen during xml parsing
+			// call XMLConverter's getParseErrorMessage() to get the errorstring that can be rendered as web page
+			this.parser.setErrorHandler(new XMLConverter.ParseErrorHandler());
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	/** configures the receptionist - overwrite this to set up the xslt map */
+	public boolean configure()
+	{
+		if (this.config_params == null)
+		{
+			logger.error(" config variables must be set before calling configure");
+			return false;
+		}
+		if (this.mr == null)
+		{
+			logger.error(" message router must be set  before calling configure");
+			return false;
+		}
+
+		// find the config file containing a list of actions
+		File interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.INTERFACE_NAME))));
+		if (!interface_config_file.exists())
+		{
+			logger.error(" interface config file: " + interface_config_file.getPath() + " not found!");
+			return false;
+		}
+		Document config_doc = this.converter.getDOM(interface_config_file, "utf-8");
+		if (config_doc == null)
+		{
+			logger.error(" could not parse interface config file: " + interface_config_file.getPath());
+			return false;
+		}
+		Element config_elem = config_doc.getDocumentElement();
+		String base_interface = config_elem.getAttribute("baseInterface");
+		setUpBaseInterface(base_interface);
+		setUpInterfaceOptions(config_elem);
+
+		Element action_list = (Element) GSXML.getChildByTagName(config_elem, GSXML.ACTION_ELEM + GSXML.LIST_MODIFIER);
+		NodeList actions = action_list.getElementsByTagName(GSXML.ACTION_ELEM);
+
+		for (int i = 0; i < actions.getLength(); i++)
+		{
+			Element action = (Element) actions.item(i);
+			String class_name = action.getAttribute("class");
+			String action_name = action.getAttribute("name");
+			Action ac = null;
+			try
+			{
+				ac = (Action) Class.forName("org.greenstone.gsdl3.action." + class_name).newInstance();
+			}
+			catch (Exception e)
+			{
+				logger.error(" couldn't load in action " + class_name);
+				e.printStackTrace();
+				continue;
+			}
+			ac.setConfigParams(this.config_params);
+			ac.setMessageRouter(this.mr);
+			ac.configure();
+			ac.addActionParameters(this.params);
+			this.action_map.put(action_name, ac);
+
+			// now do the xslt map
+			String xslt = action.getAttribute("xslt");
+			if (!xslt.equals(""))
+			{
+				this.xslt_map.put(action_name, xslt);
+			}
+			NodeList subactions = action.getElementsByTagName(GSXML.SUBACTION_ELEM);
+			for (int j = 0; j < subactions.getLength(); j++)
+			{
+				Element subaction = (Element) subactions.item(j);
+				String subname = subaction.getAttribute(GSXML.NAME_ATT);
+				String subxslt = subaction.getAttribute("xslt");
+
+				String map_key = action_name + ":" + subname;
+				logger.debug("adding in to xslt map, " + map_key + "->" + subxslt);
+				this.xslt_map.put(map_key, subxslt);
+			}
+		}
+		Element lang_list = (Element) GSXML.getChildByTagName(config_elem, "languageList");
+		if (lang_list == null)
+		{
+			logger.error(" didn't find a language list in the config file!!");
+		}
+		else
+		{
+		  //this.language_list = (Element) this.doc.importNode(lang_list, true);
+		  this.language_list = lang_list;
+		}
+
+		getRequiredMetadataNamesFromXSLFiles();
+
+		return true;
+	}
+
+	protected void getRequiredMetadataNamesFromXSLFiles()
+	{
+		ArrayList<File> xslFiles = GSFile.getAllXSLFiles((String) this.config_params.get((String) this.config_params.get(GSConstants.SITE_NAME)));
+
+		HashMap<String, ArrayList<String>> includes = new HashMap<String, ArrayList<String>>();
+		HashMap<String, ArrayList<File>> files = new HashMap<String, ArrayList<File>>();
+		HashMap<String, ArrayList<String>> metaNames = new HashMap<String, ArrayList<String>>();
+
+		//First exploratory pass
+		for (File currentFile : xslFiles)
+		{
+
+		    String full_filename = currentFile.getPath();
+		    int sep_pos = full_filename.lastIndexOf(File.separator)+1;
+		    String local_filename = full_filename.substring(sep_pos);
+		    if (local_filename.startsWith(".")) {
+			logger.warn("Greenstone does not normally rely on 'dot' files for XSL transformations.\n Is the following file intended to be part of the digital library installation?\n XSL File being read in:\n    " + currentFile.getPath());
+		    }
+				
+			Document currentDoc = this.converter.getDOM(currentFile);
+			if (currentDoc == null)
+			{
+				// Can happen if an editor creates an auto-save temporary file 
+				// (such as #header.xsl#) that is not well formed XML
+				continue;
+			}
+
+			HashSet<String> extra_meta_names = new HashSet<String>();
+			GSXSLT.findExtraMetadataNames(currentDoc.getDocumentElement(), extra_meta_names);
+			ArrayList<String> names = new ArrayList<String>(extra_meta_names);			
+
+			metaNames.put(currentFile.getAbsolutePath(), names);
+
+			NodeList includeElems = currentDoc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "include");
+			NodeList importElems = currentDoc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "import");
+
+
+			ArrayList<String> includeAndImportList = new ArrayList<String>();
+			for (int i = 0; i < includeElems.getLength(); i++)
+			{
+				includeAndImportList.add(((Element) includeElems.item(i)).getAttribute(GSXML.HREF_ATT));
+			}
+			for (int i = 0; i < importElems.getLength(); i++)
+			{
+				includeAndImportList.add(((Element) importElems.item(i)).getAttribute(GSXML.HREF_ATT));
+			}
+			includes.put(currentFile.getAbsolutePath(), includeAndImportList);
+
+			String filename = currentFile.getName();
+			if (files.get(filename) == null)
+			{
+				ArrayList<File> fileList = new ArrayList<File>();
+				fileList.add(currentFile);
+				files.put(currentFile.getName(), fileList);
+			}
+			else
+			{
+				ArrayList<File> fileList = files.get(filename);
+				fileList.add(currentFile);
+			}
+		}
+
+		//Second pass
+		for (File currentFile : xslFiles)
+		{
+			ArrayList<File> filesToGet = new ArrayList<File>();
+			filesToGet.add(currentFile);
+
+			ArrayList<String> fullNameList = new ArrayList<String>();
+
+			while (filesToGet.size() > 0)
+			{
+				File currentFileTemp = filesToGet.remove(0);
+
+				//Add the names from this file
+				ArrayList<String> currentNames = metaNames.get(currentFileTemp.getAbsolutePath());
+				if (currentNames == null)
+				{
+					continue;
+				}
+
+				fullNameList.addAll(currentNames);
+
+				ArrayList<String> includedHrefs = includes.get(currentFileTemp.getAbsolutePath());
+
+				for (String href : includedHrefs)
+				{
+					int lastSepIndex = href.lastIndexOf("/");
+					if (lastSepIndex != -1)
+					{
+						href = href.substring(lastSepIndex + 1);
+					}
+
+					ArrayList<File> filesToAdd = files.get(href);
+					if (filesToAdd != null)
+					{
+						filesToGet.addAll(filesToAdd);
+					}
+				}
+			}
+
+			_metadataRequiredMap.put(currentFile.getAbsolutePath(), fullNameList);
+		}
+	}
+
+	protected void preProcessRequest(Element request)
+	{
+		String action = request.getAttribute(GSXML.ACTION_ATT);
+		String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+
+		String name = null;
+		if (!subaction.equals(""))
+		{
+			String key = action + ":" + subaction;
+			name = this.xslt_map.get(key);
+		}
+		// try the action by itself
+		if (name == null)
+		{
+			name = this.xslt_map.get(action);
+		}
+
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		String collection = "";
+
+		if (cgi_param_list != null)
+		{
+			// Don't waste time getting all the parameters
+			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+			collection = (String) params.get(GSParams.COLLECTION);
+			if (collection == null)
+			{
+				collection = "";
+			}
+		}
+
+		ArrayList<File> stylesheets = GSFile.getStylesheetFiles(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, name);
+
+		Document doc = XMLConverter.newDOM();
+		Element extraMetadataList = doc.createElement(GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
+		HashSet<String> name_set = new HashSet<String>();
+		for (File stylesheet : stylesheets)
+		{
+			ArrayList<String> requiredMetadata = _metadataRequiredMap.get(stylesheet.getAbsolutePath());
+
+			if (requiredMetadata != null)
+			{
+				for (String metadataString : requiredMetadata)
+				{
+				  if (!name_set.contains(metadataString)) {
+				      name_set.add(metadataString);
+					Element metadataElem = doc.createElement(GSXML.EXTRA_METADATA);
+					metadataElem.setAttribute(GSXML.NAME_ATT, metadataString);
+					extraMetadataList.appendChild(metadataElem);
+				    }
+				}
+			}
+	}
+		request.appendChild(request.getOwnerDocument().importNode(extraMetadataList, true));
+	}
+
+	protected Node postProcessPage(Element page)
+	{
+		// might need to add some data to the page
+		addExtraInfo(page);
+
+		
+		// transform the page using xslt
+
+		String currentInterface = (String) config_params.get(GSConstants.INTERFACE_NAME);
+
+		Element request = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+		String output = request.getAttribute(GSXML.OUTPUT_ATT);
+
+		boolean useClientXSLT = (Boolean) config_params.get(GSConstants.USE_CLIENT_SIDE_XSLT);
+		//logger.info("Client side transforms allowed? " + allowsClientXSLT);
+
+		if (useClientXSLT)
+		{
+		    // if not specified, output defaults to 'html', but this isn't what we want when useClientXSLT is on
+		    if (output.equals("html")) {
+			output = "xsltclient";
+		    }
+		}
+		Node transformed_page = transformPage(page,currentInterface,output);
+
+		if (useClientXSLT) {
+		    return transformed_page;
+		}
+		// if the user has specified they want only a part of the full page then subdivide it
+		boolean subdivide = false;
+		String excerptID = null;
+		String excerptTag = null;
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (cgi_param_list != null)
+		{
+			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+			if ((excerptID = (String) params.get(GSParams.EXCERPT_ID)) != null)
+			{
+				subdivide = true;
+			}
+			if ((excerptTag = (String) params.get(GSParams.EXCERPT_TAG)) != null)
+			{
+				subdivide = true;
+			}
+		}
+
+		if (subdivide)
+		{
+			Node subdivided_page = subdivide(transformed_page, excerptID, excerptTag);
+			if (subdivided_page != null)
+			{
+				return subdivided_page;
+			}
+			else return null;
+		}
+
+		return transformed_page;
+	}
+
+	protected Node subdivide(Node transformed_page, String excerptID, String excerptTag)
+	{
+		if (excerptID != null)
+		{
+			Node selectedElement = getNodeByIdRecursive(transformed_page, excerptID);
+			modifyNodesByTagRecursive(selectedElement, "a");
+			return selectedElement;
+		}
+		else if (excerptTag != null)
+		{
+			Node selectedElement = getNodeByTagRecursive(transformed_page, excerptTag);
+			return selectedElement;
+		}
+		return transformed_page;
+	}
+
+	protected Node getNodeByIdRecursive(Node parent, String id)
+	{
+		if (parent.hasAttributes() && ((Element) parent).getAttribute("id").equals(id))
+		{
+			return parent;
+		}
+
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node result = null;
+			if ((result = getNodeByIdRecursive(children.item(i), id)) != null)
+			{
+				return result;
+			}
+		}
+		return null;
+	}
+
+	protected Node getNodeByTagRecursive(Node parent, String tag)
+	{
+		if (parent.getNodeType() == Node.ELEMENT_NODE && ((Element) parent).getTagName().equals(tag))
+		{
+			return parent;
+		}
+
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node result = null;
+			if ((result = getNodeByTagRecursive(children.item(i), tag)) != null)
+			{
+				return result;
+			}
+		}
+		return null;
+	}
+
+	protected Node modifyNodesByTagRecursive(Node parent, String tag)
+	{
+		if (parent == null || (parent.getNodeType() == Node.ELEMENT_NODE && ((Element) parent).getTagName().equals(tag)))
+		{
+			return parent;
+		}
+
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node result = null;
+			if ((result = modifyNodesByTagRecursive(children.item(i), tag)) != null)
+			{
+				//TODO: DO SOMETHING HERE?
+			}
+		}
+		return null;
+	}
+
+        protected void replaceNodeWithInterfaceText(Document doc, String interface_name, String lang,
+						    Element elem, String attr_name, String attr_val)
+        {
+	    String pattern_str_3arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*\\)";
+	    String pattern_str_4arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*,\\s*(.+?)\\s*\\)$";
+    
+	    Pattern pattern3 = Pattern.compile(pattern_str_3arg);
+	    Matcher matcher3 = pattern3.matcher(attr_val);
+	    if (matcher3.find()) {
+		String dict_key = matcher3.group(1);
+		String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);
+		
+		Node parent_node = elem.getParentNode();
+
+		Text replacement_text_node = doc.createTextNode(dict_val);
+	        parent_node.replaceChild(replacement_text_node,elem);
+	    }	    
+	    else {
+		Pattern pattern4 = Pattern.compile(pattern_str_4arg);
+		Matcher matcher4 = pattern4.matcher(attr_val);
+		StringBuffer string_buffer4 = new StringBuffer();
+
+		if (matcher4.find()) {
+		    String dict_key = matcher4.group(1);
+		    String args     = matcher4.group(2);
+		    args = args.replaceAll("\\$","\\\\\\$");
+		    
+		    String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);
+
+		    matcher4.appendReplacement(string_buffer4, "js:getInterfaceTextSubstituteArgs('"+dict_val+"',string("+args+"))");
+		    matcher4.appendTail(string_buffer4);
+
+		    attr_val = string_buffer4.toString();
+		    elem.setAttribute(attr_name,attr_val);
+		}
+		else {
+		    logger.error("Failed to find match in attribute: " + attr_name + "=\"" + attr_val + "\"");
+		    attr_val = attr_val.replaceAll("util:getInterfaceText\\(.+?,.+?,\\s*(.+?)\\s*\\)","$1");
+		    elem.setAttribute(attr_name,attr_val);
+		}
+	    }
+	
+	}
+    
+        protected void resolveExtendedNamespaceAttributesXSLT(Document doc, String interface_name, String lang)
+        {
+	    String[] attr_list = new String[] {"select","test"};
+
+	    // http://stackoverflow.com/questions/13220520/javascript-replace-child-loop-issue
+	    // go through nodeList in reverse to avoid the 'skipping' problem, due to
+	    // replaceChild() calls removing items from the "live" nodeList
+	    
+	    NodeList nodeList = doc.getElementsByTagName("*");
+	    for (int i=nodeList.getLength()-1; i>=0; i--) {
+		Node node = nodeList.item(i);
+		if (node.getNodeType() == Node.ELEMENT_NODE) {
+		    Element elem = (Element)node;		
+		    for (String attr_name : attr_list) {
+			if (elem.hasAttribute(attr_name)) {
+			    String attr_val = elem.getAttribute(attr_name);
+			    
+			    if (attr_val.startsWith("util:getInterfaceText(")) {
+				// replace the node with dictionary lookup
+				replaceNodeWithInterfaceText(doc, interface_name,lang, elem,attr_name,attr_val);
+			    }							
+			    else if (attr_val.contains("util:")) {
+
+				attr_val = attr_val.replaceAll("util:getInterfaceStringsAsJavascript\\(.+?,.+?,\\s*(.+?)\\)","$1");
+
+				//attr_val = attr_val.replaceAll("util:escapeNewLinesAndQuotes\\(\\s*(.+?)\\s*\\)","'escapeNLandQ $1'");
+				//attr_val = attr_val.replaceAll("util:escapeNewLinesAndQuotes\\(\\s*(.+?)\\s*\\)","$1");					
+
+				// 'contains()' supported in XSLT 1.0, so OK to change any util:contains() into contains()
+				attr_val = attr_val.replaceAll("util:(contains\\(.+?\\))","$1");
+
+				elem.setAttribute(attr_name,attr_val);
+			    }
+
+			    if (attr_val.contains("java:")) {
+				if (attr_val.indexOf("getInterfaceTextSubstituteArgs")>=4) {
+				    
+				    attr_val = attr_val.replaceAll("java:.+?\\.(\\w+)\\((.*?)\\)$","js:$1($2)");
+				}
+				
+				elem.setAttribute(attr_name,attr_val);
+			    }
+			}
+		    }
+		    
+		}
+	    }
+	}
+
+
+        protected void resolveExtendedNamespaceAttributesXML(Document doc, String interface_name, String lang)
+        {
+	    String[] attr_list = new String[] {"src", "href"};
+
+	    // http://stackoverflow.com/questions/13220520/javascript-replace-child-loop-issue
+	    // go through nodeList in reverse to avoid the 'skipping' problem, due to
+	    // replaceChild() calls removing items from the "live" nodeList
+	    
+	    NodeList nodeList = doc.getElementsByTagName("*");
+	    for (int i=nodeList.getLength()-1; i>=0; i--) {
+		Node node = nodeList.item(i);
+		if (node.getNodeType() == Node.ELEMENT_NODE) {
+		    Element elem = (Element)node;
+		    for (String attr_name : attr_list) {
+			if (elem.hasAttribute(attr_name)) {
+			    String attr_val = elem.getAttribute(attr_name);
+
+			    if (attr_val.contains("util:getInterfaceText(")) {
+				String pattern_str_3arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*\\)";
+				Pattern pattern3 = Pattern.compile(pattern_str_3arg);
+				Matcher matcher3 = pattern3.matcher(attr_val);
+				
+				StringBuffer string_buffer3 = new StringBuffer();
+				
+				boolean found_match = false;
+				
+				while (matcher3.find()) {
+				    found_match = true;
+				    String dict_key = matcher3.group(1);
+				    String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);
+				    
+				    matcher3.appendReplacement(string_buffer3, dict_val);
+				}
+				matcher3.appendTail(string_buffer3);
+				
+				if (found_match) {
+				    attr_val = string_buffer3.toString();
+				    elem.setAttribute(attr_name,attr_val);				    
+				}
+				else {			    
+				    logger.error("Failed to find match in attribute: " + attr_name + "=\"" + attr_val + "\"");
+				    attr_val = attr_val.replaceAll("util:getInterfaceText\\(.+?,.+?,\\s*(.+?)\\s*\\)","$1");
+				    elem.setAttribute(attr_name,attr_val);
+				}
+			    }
+			    else if (attr_val.contains("util:")) {
+
+				logger.error("Encountered unexpected 'util:' prefix exension: " + attr_name + "=\"" + attr_val + "\"");
+			    }
+
+			    if (attr_val.contains("java:")) {
+				// make anything java: safe from the point of an XSLT without extensions
+				logger.error("Encountered unexpected 'java:' prefix exension: " + attr_name + "=\"" + attr_val + "\"");
+
+			    }
+			}
+		    }
+		    
+		}
+	    }
+	}
+
+    
+    
+	/**
+	 * overwrite this to add any extra info that might be needed in the page
+	 * before transformation
+	 */
+	protected void addExtraInfo(Element page)
+	{
+	}
+
+	/**
+	 * transform the page using xslt we need to get any format element out of
+	 * the page and add it to the xslt before transforming
+	 */
+        protected Node transformPage(Element page,String currentInterface,String output)
+	{
+		_debug = false;
+
+		Element request = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
+
+		//logger.info("Current output mode is: " + output + ", current interface name is: " + currentInterface);
+		
+		// DocType defaults in case the skin doesn't have an "xsl:output" element
+		String qualifiedName = "html";
+		String publicID = "-//W3C//DTD HTML 4.01 Transitional//EN";
+		String systemID = "http://www.w3.org/TR/html4/loose.dtd";
+
+		// We need to create an empty document with a predefined DocType,
+		// that will then be used for the transformation by the DOMResult
+		Document docWithDoctype = converter.newDOM(qualifiedName, publicID, systemID);
+
+		if (output.equals("xsltclient"))
+		{
+		        String baseURL = request.getAttribute(GSXML.BASE_URL);
+				
+			// If you're just getting the client-side transform page, why bother with the rest of this?
+			Element html = docWithDoctype.createElement("html");
+			Element img = docWithDoctype.createElement("img");
+			img.setAttribute("src", "loading.gif"); // Make it dynamic
+			img.setAttribute("alt", "Please wait...");
+			Text title_text = docWithDoctype.createTextNode("Please wait..."); // Make this language dependent
+			Element head = docWithDoctype.createElement("head");
+
+			// e.g., <base href="http://localhost:8383/greenstone3/" /><!-- [if lte IE 6]></base><![endif] -->
+			Element base = docWithDoctype.createElement("base");
+			base.setAttribute("href",baseURL);
+			Comment opt_end_base = docWithDoctype.createComment("[if lte IE 6]></base><![endif]");
+			
+			Element title = docWithDoctype.createElement("title");
+			title.appendChild(title_text);
+
+			Element body = docWithDoctype.createElement("body");
+
+			Element jquery_script = docWithDoctype.createElement("script");
+			jquery_script.setAttribute("src", "jquery-1.10-min.js");
+			jquery_script.setAttribute("type", "text/javascript");
+			Comment jquery_comment = docWithDoctype.createComment("jQuery");
+			jquery_script.appendChild(jquery_comment);
+
+			Element saxonce_script = docWithDoctype.createElement("script");
+			saxonce_script.setAttribute("src", "Saxonce/Saxonce.nocache.js");
+			saxonce_script.setAttribute("type", "text/javascript");
+			Comment saxonce_comment = docWithDoctype.createComment("SaxonCE");
+			saxonce_script.appendChild(saxonce_comment);
+
+			Element xsltutil_script = docWithDoctype.createElement("script");
+			xsltutil_script.setAttribute("src", "xslt-util.js");
+			xsltutil_script.setAttribute("type", "text/javascript");
+			Comment xsltutil_comment = docWithDoctype.createComment("JavaScript version of XSLTUtil.java");
+			xsltutil_script.appendChild(xsltutil_comment);
+
+			Element script = docWithDoctype.createElement("script");
+			Comment script_comment = docWithDoctype.createComment("Filler for browser");
+			script.setAttribute("src", "client-side-xslt.js");
+			script.setAttribute("type", "text/javascript");
+			script.appendChild(script_comment);
+			
+			Element pagevar = docWithDoctype.createElement("script");
+			Element style = docWithDoctype.createElement("style");
+			style.setAttribute("type", "text/css");
+			Text style_text = docWithDoctype.createTextNode("body { text-align: center; padding: 50px; font: 14pt Arial, sans-serif; font-weight: bold; }");
+			pagevar.setAttribute("type", "text/javascript");
+			Text page_var_text = docWithDoctype.createTextNode("var placeholder = true;");
+
+			html.appendChild(head);
+			head.appendChild(base); head.appendChild(opt_end_base);
+			head.appendChild(title);
+			head.appendChild(style);
+			style.appendChild(style_text);
+			html.appendChild(body);
+			head.appendChild(pagevar);
+			head.appendChild(jquery_script);
+			head.appendChild(saxonce_script);
+			head.appendChild(xsltutil_script);
+			head.appendChild(script);
+			pagevar.appendChild(page_var_text);
+
+			body.appendChild(img);
+			docWithDoctype.appendChild(html);
+
+			return (Node) docWithDoctype;
+		}
+
+		// Passing in the pretty string here means it needs to be generated even when not debugging; so use custom function to return blank when debug is off
+		logger.debug("page before transforming:");
+		logger.debug(this.converter.getPrettyStringLogger(page, logger));
+
+		String action = request.getAttribute(GSXML.ACTION_ATT);
+		String subaction = request.getAttribute(GSXML.SUBACTION_ATT);
+
+		// we should choose how to transform the data based on output, eg diff
+		// choice for html, and wml??
+		// for now, if output=xml, we don't transform the page, we just return 
+		// the page xml
+		Document theXML = null;
+
+		if (output.equals("xml") || (output.equals("json")) || output.equals("clientside"))
+		{
+			// Append some bits and pieces first...
+			theXML = converter.newDOM();
+			// Import into new document first!
+			Node newPage = theXML.importNode(page, true);
+			theXML.appendChild(newPage);
+			Element root = theXML.createElement("xsltparams");
+			newPage.appendChild(root);
+
+			Element libname = theXML.createElement("param");
+			libname.setAttribute("name", "library_name");
+			Text libnametext = theXML.createTextNode((String) config_params.get(GSConstants.LIBRARY_NAME));
+			libname.appendChild(libnametext);
+
+			Element intname = theXML.createElement("param");
+			intname.setAttribute("name", "interface_name");
+			Text intnametext = theXML.createTextNode((String) config_params.get(GSConstants.INTERFACE_NAME));
+			intname.appendChild(intnametext);
+
+			Element siteName = theXML.createElement("param");
+			siteName.setAttribute("name", "site_name");
+			Text siteNameText = theXML.createTextNode((String) config_params.get(GSConstants.SITE_NAME));
+			siteName.appendChild(siteNameText);
+
+			Element clientSideXSLTName = theXML.createElement("param");
+			clientSideXSLTName.setAttribute("name", "use_client_side_xslt");
+			Boolean useClientXSLT = (Boolean) config_params.get(GSConstants.USE_CLIENT_SIDE_XSLT);
+			Text clientSideXSLTNameText = theXML.createTextNode(useClientXSLT.toString());
+			clientSideXSLTName.appendChild(clientSideXSLTNameText);
+			
+			Element filepath = theXML.createElement("param");
+			filepath.setAttribute("name", "filepath");
+			Text filepathtext = theXML.createTextNode(GlobalProperties.getGSDL3Home());
+			filepath.appendChild(filepathtext);
+
+			root.appendChild(libname);
+			root.appendChild(intname);
+			root.appendChild(siteName);
+			root.appendChild(clientSideXSLTName);
+			root.appendChild(filepath);
+
+			if ((output.equals("xml")) || output.equals("json"))
+			{
+				// in the case of "json", calling method responsible for converting to JSON-string
+				return theXML.getDocumentElement();
+			}
+		}
+
+		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		String collection = "";
+		String inlineTemplate = "";
+		if (cgi_param_list != null)
+		{
+			// Don't waste time getting all the parameters
+			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+			collection = (String) params.get(GSParams.COLLECTION);
+			if (collection == null)
+			{
+				collection = "";
+			}
+
+			inlineTemplate = (String) params.get(GSParams.INLINE_TEMPLATE);
+			String debug_p = (String) params.get(GSParams.DEBUG);
+			if (debug_p != null && (debug_p.equals("on") || debug_p.equals("1") || debug_p.equals("true")))
+			{
+				String[] groups = new UserContext(request).getGroups();
+
+				boolean found = false;
+				for (String g : groups)
+				{
+					if (g.equals("administrator"))
+					{
+						found = true;
+						break;
+					}
+					if (!collection.equals("")) {
+					  if (g.equals("all-collections-editor")) {
+					    found = true;
+					    break;
+					  }
+					 
+					  if (g.equals(collection+"-collection-editor")) {
+					    found = true;
+					    break;
+					  }
+					}
+				}
+				if (found)
+				{
+					_debug = true;
+				}
+			}
+		}
+
+		config_params.put("collName", collection);
+
+		Document style_doc = getXSLTDocument(action, subaction, collection);
+		if (style_doc == null)
+		{
+		  // this getParseErrorMessage may have originally worked, but now it doesn't.
+		  // //String errorPage = this.converter.getParseErrorMessage();
+		  // 	if (errorPage != null)
+		  // 	{
+		  // 	  return XMLTransformer.constructErrorXHTMLPage("Cannot parse the xslt file\n");// + errorPage);
+		  // 	}
+			return page;
+		}
+
+		// put the page into a document - this is necessary for xslt to get
+		// the paths right if you have paths relative to the document root
+		// eg /page.
+		Document doc = XMLConverter.newDOM();
+		doc.appendChild(doc.importNode(page, true));
+		Element page_response = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_RESPONSE_ELEM);
+		Element format_elem = (Element) GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);
+
+		NodeList pageElems = doc.getElementsByTagName("page");
+		if (pageElems.getLength() > 0)
+		{
+			Element pageElem = (Element) pageElems.item(0);
+			String langAtt = pageElem.getAttribute(GSXML.LANG_ATT);
+
+			if (langAtt != null && langAtt.length() > 0)
+			{
+				config_params.put("lang", langAtt);
+			}
+		}
+
+		if (output.equals("formatelem"))
+		{
+			return format_elem;
+		}
+		if (format_elem != null)
+		{
+			//page_response.removeChild(format_elem);
+			logger.debug("format elem=" + this.converter.getPrettyStringLogger(format_elem, logger));
+			// need to transform the format info
+			String configStylesheet_file = GSFile.stylesheetFile(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, "config_format.xsl");
+			Document configStylesheet_doc = this.converter.getDOM(new File(configStylesheet_file));
+
+			if (configStylesheet_doc != null)
+			{
+				Document format_doc = XMLConverter.newDOM();
+				format_doc.appendChild(format_doc.importNode(format_elem, true));
+
+				if (_debug)
+				{
+					String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME));
+					GSXSLT.insertDebugElements(format_doc, GSFile.collectionConfigFile(siteHome, collection));
+				}
+
+				Node result = this.transformer.transform(configStylesheet_doc, format_doc, config_params); // Needs addressing <-
+
+				// Since we started creating documents with DocTypes, we can end up with 
+				// Document objects here. But we will be working with an Element instead, 
+				// so we grab the DocumentElement() of the Document object in such a case.
+				Element new_format;
+				if (result.getNodeType() == Node.DOCUMENT_NODE)
+				{
+					new_format = ((Document) result).getDocumentElement();
+				}
+				else
+				{
+					new_format = (Element) result;
+				}
+				logger.debug("new format elem=" + this.converter.getPrettyStringLogger(new_format, logger));
+				if (output.equals("newformat"))
+				{
+					return new_format;
+				}
+
+				// add extracted GSF statements in to the main stylesheet
+				if (_debug)
+				{
+					String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME));
+					GSXSLT.mergeStylesheetsDebug(style_doc, new_format, true, true, "OTHER1", GSFile.collectionConfigFile(siteHome, collection));
+				}
+				else
+				{
+					GSXSLT.mergeStylesheets(style_doc, new_format, true);
+				}
+				//System.out.println("added extracted GSF statements into the main stylesheet") ;
+
+				// add extracted GSF statements in to the debug test stylesheet
+				//GSXSLT.mergeStylesheets(oldStyle_doc, new_format);
+			}
+			else
+			{
+				logger.error(" couldn't parse the config_format stylesheet, adding the format info as is");
+				GSXSLT.mergeStylesheets(style_doc, format_elem, true);
+				//GSXSLT.mergeStylesheets(oldStyle_doc, format_elem);
+			}
+			logger.debug("the converted stylesheet is:");
+			logger.debug(this.converter.getPrettyStringLogger(style_doc.getDocumentElement(), logger));
+		}
+
+		//for debug purposes only
+		Document oldStyle_doc = style_doc;
+		Document preprocessingXsl;
+		try
+		{
+			preprocessingXsl = getDoc(preprocess_xsl_filename);
+			String errMsg = ((XMLConverter.ParseErrorHandler) parser.getErrorHandler()).getErrorMessage();
+			if (errMsg != null)
+			{
+				return XMLTransformer.constructErrorXHTMLPage("error loading preprocess xslt file: " + preprocess_xsl_filename + "\n" + errMsg);
+			}
+		}
+		catch (java.io.FileNotFoundException e)
+		{
+			return fileNotFoundErrorPage(e.getMessage());
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			System.out.println("error loading preprocess xslt");
+			return XMLTransformer.constructErrorXHTMLPage("error loading preprocess xslt\n" + e.getMessage());
+		}
+
+		Document libraryXsl = null;
+		try
+		{
+			libraryXsl = GSXSLT.mergedXSLTDocumentCascade("gslib.xsl", (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, _debug);
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			System.out.println("error loading gslib xslt");
+			return XMLTransformer.constructErrorXHTMLPage("error loading gslib xslt\n" + e.getMessage());
+		}
+
+		//   Combine the skin file and library variables/templates into one document. 
+		//   Please note: We dont just use xsl:import because the preprocessing stage  
+		//   needs to know what's available in the library.
+
+		Document skinAndLibraryXsl = null;
+		Document skinAndLibraryDoc = converter.newDOM();
+
+		// Applying the preprocessing XSLT - in its own block {} to allow use of non-unique variable names
+		{
+
+			skinAndLibraryXsl = converter.newDOM();
+			Element root = skinAndLibraryXsl.createElement("skinAndLibraryXsl");
+			skinAndLibraryXsl.appendChild(root);
+
+			Element s = skinAndLibraryXsl.createElement("skinXsl");
+			s.appendChild(skinAndLibraryXsl.importNode(style_doc.getDocumentElement(), true));
+			root.appendChild(s);
+
+			Element l = skinAndLibraryXsl.createElement("libraryXsl");
+			if (libraryXsl != null)
+			{
+				Element libraryXsl_el = libraryXsl.getDocumentElement();
+				l.appendChild(skinAndLibraryXsl.importNode(libraryXsl_el, true));
+			}
+			root.appendChild(l);
+
+			//System.out.println("Skin and Library XSL are now together") ;
+
+			//System.out.println("Pre-processing the skin file...") ;
+
+			//pre-process the skin style sheet
+			//In other words, apply the preProcess.xsl to 'skinAndLibraryXsl' in order to
+			//expand all GS-Lib statements into complete XSL statements and also to create
+			//a valid xsl style sheet document.
+
+			XMLTransformer preProcessor = new XMLTransformer();
+			// Perform the transformation, by passing in:
+			// preprocess-stylesheet, source-xsl (skinAndLibraryXsl), and the node that should 
+			// be in the result (skinAndLibraryDoc)
+			preProcessor.transform_withResultNode(preprocessingXsl, skinAndLibraryXsl, skinAndLibraryDoc);
+			//System.out.println("GS-Lib statements are now expanded") ;
+		}
+
+		// there is a thing called a URIResolver which you can set for a
+		// transformer or transformer factory. may be able to use this
+		// instead of this absoluteIncludepaths hack
+
+		//GSXSLT.absoluteIncludePaths(skinAndLibraryDoc, GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
+
+		//Same but for the debug version when we want the do the transformation like we use to do
+		//without any gslib elements.
+		GSXSLT.absoluteIncludePaths(oldStyle_doc, GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
+
+		//Send different stages of the skin xslt to the browser for debug purposes only
+		//using &o=skindoc or &o=skinandlib etc...
+		if (output.equals("skindoc"))
+		{
+			return converter.getDOM(getStringFromDocument(style_doc));
+		}
+		if (output.equals("skinandlib"))
+		{
+			return converter.getDOM(getStringFromDocument(skinAndLibraryXsl));
+		}
+		if (output.equals("skinandlibdoc"))
+		{
+
+			Node skinAndLib = converter.getDOM(getStringFromDocument(skinAndLibraryDoc));
+			return skinAndLib;			
+		}
+		if (output.equals("oldskindoc"))
+		{
+			return converter.getDOM(getStringFromDocument(oldStyle_doc));
+		}
+
+		// We now no longer create a document with doctype before the transformation
+		// We let the XMLTransformer do the work of first working out the doctype from any 
+		// that may be set in the (merged) stylesheets and then setting the doctype when transforming
+
+		/*
+		// Try to get the system and public ID from the current skin xsl document
+		// otherwise keep the default values.
+		Element root = skinAndLibraryDoc.getDocumentElement();
+		NodeList nodes = root.getElementsByTagName("xsl:output");
+		// If there is at least one "xsl:output" command in the final xsl then...
+		if (nodes.getLength() != 0)
+		{
+			// There should be only one element called xsl:output, 
+			// but if this is not the case get the last one
+			Element xsl_output = (Element) nodes.item(nodes.getLength() - 1);
+			if (xsl_output != null)
+			{
+				// Qualified name will always be html even for xhtml pages
+				//String attrValue = xsl_output.getAttribute("method");
+				//qualifiedName = attrValue.equals("") ? qualifiedName : attrValue;
+
+				String attrValue = xsl_output.getAttribute("doctype-system");
+				systemID = attrValue.equals("") ? systemID : attrValue;
+
+				attrValue = xsl_output.getAttribute("doctype-public");
+				publicID = attrValue.equals("") ? publicID : attrValue;
+			}
+		}
+
+		//System.out.println(converter.getPrettyString(docWithDoctype));
+		//System.out.println("Doctype vals: " + qualifiedName + " " + publicID + " " + systemID) ;
+
+		docWithDoctype = converter.newDOM(qualifiedName, publicID, systemID);
+		*/
+
+		//System.out.println("Generate final HTML from current skin") ;
+		//Transformation of the XML message from the receptionist to HTML with doctype
+
+		if (inlineTemplate != null)
+		{
+			try
+			{
+				Document inlineTemplateDoc = this.converter.getDOM("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<xsl:stylesheet version=\"1.0\" "+GSXML.ALL_NAMESPACES_ATTS +  ">" + inlineTemplate + "</xsl:stylesheet>", "UTF-8");
+
+				if (_debug)
+				{
+					GSXSLT.mergeStylesheetsDebug(skinAndLibraryDoc, inlineTemplateDoc.getDocumentElement(), true, true, "OTHER2", "INLINE");
+				}
+				else
+				{
+					GSXSLT.mergeStylesheets(skinAndLibraryDoc, inlineTemplateDoc.getDocumentElement(), true);
+				}
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+		}
+
+		if (_debug)
+		{
+			GSXSLT.inlineImportAndIncludeFilesDebug(skinAndLibraryDoc, null, _debug, this.getGSLibXSLFilename(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
+		}
+		else
+		{
+			GSXSLT.inlineImportAndIncludeFiles(skinAndLibraryDoc, null, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
+		}
+		skinAndLibraryDoc = (Document) performFormatPass(collection, skinAndLibraryDoc, doc, new UserContext(request), TEXT_PASS);
+		skinAndLibraryDoc = (Document) performFormatPass(collection, skinAndLibraryDoc, doc, new UserContext(request), CONFIG_PASS);
+
+		if (output.equals("xmlfinal"))
+		{
+			return doc;
+		}
+
+		if (output.equals("skinandlibdocfinal") || output.equals("clientside"))
+		{
+			if (output.equals("skinandlibdocfinal"))
+			{
+			        Node skinAndLibFinal = converter.getDOM(getStringFromDocument(skinAndLibraryDoc));
+				return skinAndLibFinal;
+			}
+			else
+			{
+			        // Go through and 'fix up' any 'util:...' or 'java:...' attributes the skinAndLibraryDoc has
+			        String lang = (String)config_params.get("lang");
+			        resolveExtendedNamespaceAttributesXSLT(skinAndLibraryDoc,currentInterface,lang); // test= and select= attributes
+				resolveExtendedNamespaceAttributesXML(skinAndLibraryDoc,currentInterface,lang);  // href= and src= attributes
+				Node skinAndLibFinal = converter.getDOM(getStringFromDocument(skinAndLibraryDoc));
+				
+				// Send XML and skinandlibdoc down the line together
+				Document finalDoc = converter.newDOM();
+				Node finalDocSkin = finalDoc.importNode(skinAndLibraryDoc.getDocumentElement(), true);
+				Node finalDocXML = finalDoc.importNode(theXML.getDocumentElement(), true);
+				Element root = finalDoc.createElement("skinlibfinalPlusXML");
+				root.appendChild(finalDocSkin);
+				root.appendChild(finalDocXML);
+				finalDoc.appendChild(root);
+				return (Node) finalDoc.getDocumentElement();
+			}
+
+
+
+			
+		}
+
+		// The transformer will now work out the resulting doctype from any set in the (merged) stylesheets and
+		// will set this in the output document it creates. So don't pass in any docWithDocType to the transformer
+		//Node finalResult = this.transformer.transform(skinAndLibraryDoc, doc, config_params, docWithDoctype);
+		Node finalResult = this.transformer.transform(skinAndLibraryDoc, doc, config_params);
+
+		if (_debug)
+		{
+			GSXSLT.fixTables((Document) finalResult);
+		}
+
+		return finalResult;
+
+		// The line below will do the transformation like we use to do before having Skin++ implemented,
+		// it will not contain any GS-Lib statements expanded, and the result will not contain any doctype.
+
+		//return (Element)this.transformer.transform(style_doc, doc, config_params);  
+		//return null; // For now - change later
+	}
+
+	protected Node performFormatPass(String collection, Document skinAndLibraryDoc, Document doc, UserContext userContext, int pass)
+	{
+		String formatFile;
+		if (pass == CONFIG_PASS)
+		{
+			formatFile = "config_format.xsl";
+		}
+		else
+		{
+			formatFile = "text_fragment_format.xsl";
+		}
+
+		String configStylesheet_file = GSFile.stylesheetFile(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, formatFile);
+		Document configStylesheet_doc = this.converter.getDOM(new File(configStylesheet_file));
+
+		if (configStylesheet_doc != null)
+		{
+			return this.transformer.transform(configStylesheet_doc, skinAndLibraryDoc, config_params);
+		}
+		return skinAndLibraryDoc;
+	}
+
+	// method to convert Document to a proper XML string for debug purposes only
+	protected String getStringFromDocument(Document doc)
+	{
+		String content = "";
+		try
+		{
+			DOMSource domSource = new DOMSource(doc);
+			StringWriter writer = new StringWriter();
+			StreamResult result = new StreamResult(writer);
+			TransformerFactory tf = TransformerFactory.newInstance();
+			Transformer transformer = tf.newTransformer();
+			transformer.transform(domSource, result);
+			content = writer.toString();
+			System.out.println("Change the & to &Amp; for proper debug display");
+			content = StringUtils.replace(content, "&", "&amp;");
+			writer.flush();
+		}
+		catch (TransformerException ex)
+		{
+			ex.printStackTrace();
+			return null;
+		}
+		return content;
+	}
+
+	protected synchronized Document getDoc(String docName) throws Exception
+	{
+		File xslt_file = new File(docName);
+
+		FileReader reader = new FileReader(xslt_file);
+		InputSource xml_source = new InputSource(reader);
+		this.parser.parse(xml_source);
+		Document doc = this.parser.getDocument();
+
+		return doc;
+	}
+
+	protected Document getXSLTDocument(String action, String subaction, String collection)
+	{
+		String name = null;
+		if (!subaction.equals(""))
+		{
+			String key = action + ":" + subaction;
+			name = this.xslt_map.get(key);
+		}
+		// try the action by itself
+		if (name == null)
+		{
+			name = this.xslt_map.get(action);
+		}
+		if (name == null)
+		{
+			// so we can reandomly create any named page
+			if (action.equals("p") && !subaction.equals(""))
+			{
+				// TODO: pages/ won't work for interface other than default!!
+				name = "pages/" + subaction + ".xsl";
+			}
+
+		}
+		
+		Document finalDoc = null;
+		if(name != null)
+		{
+			finalDoc = GSXSLT.mergedXSLTDocumentCascade(name, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, _debug);
+		}
+		return finalDoc;
+	}
+
+	// returns the path to the gslib.xsl file that is applicable for the current interface
+	protected String getGSLibXSLFilename()
+	{
+		return GSFile.xmlTransformDir(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.INTERFACE_NAME))) + File.separatorChar + "gslib.xsl";
+	}
+
+	// Call this when a FileNotFoundException could be thrown when loading an xsl (xml) file.
+	// Returns an error xhtml page indicating which xsl (or other xml) file is missing.
+	protected Document fileNotFoundErrorPage(String filenameMessage)
+	{
+		String errorMessage = "ERROR missing file: " + filenameMessage;
+		Element errPage = XMLTransformer.constructErrorXHTMLPage(errorMessage);
+		logger.error(errorMessage);
+		return errPage.getOwnerDocument();
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/URLFilter.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/URLFilter.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/core/URLFilter.java	(revision 31634)
@@ -0,0 +1,426 @@
+package org.greenstone.gsdl3.core;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpSession;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class URLFilter implements Filter
+{
+	private FilterConfig _filterConfig = null;
+	private static Logger _logger = Logger.getLogger(org.greenstone.gsdl3.core.URLFilter.class.getName());
+
+	//Restricted URLs
+	protected static final String SITECONFIG_URL = "sites/[^/]+/siteConfig.xml";
+	protected static final String USERS_DB_URL = "etc/usersDB/.*";
+	protected static final ArrayList<String> _restrictedURLs;
+	static
+	{
+		ArrayList<String> restrictedURLs = new ArrayList<String>();
+		restrictedURLs.add(SITECONFIG_URL);
+		restrictedURLs.add(USERS_DB_URL);
+		_restrictedURLs = restrictedURLs;
+	}
+
+	//Constants
+	protected static final String DOCUMENT_PATH = "document";
+	protected static final String COLLECTION_PATH = "collection";
+	protected static final String PAGE_PATH = "page";
+	protected static final String SYSTEM_PATH = "system";
+
+	protected static final String METADATA_RETRIEVAL_SERVICE = "DocumentMetadataRetrieve";
+	protected static final String ASSOCIATED_FILE_PATH = "/index/assoc/";
+	protected static final String COLLECTION_FILE_PATH = "/collect/";
+	protected static final String INTERFACE_PATH = "/interfaces/";
+
+	protected static final String SYSTEM_SUBACTION_CONFIGURE = "configure";
+	protected static final String SYSTEM_SUBACTION_RECONFIGURE = "reconfigure";
+	protected static final String SYSTEM_SUBACTION_ACTIVATE = "activate";
+	protected static final String SYSTEM_SUBACTION_DEACTIVATE = "deactivate";
+
+	public void init(FilterConfig filterConfig) throws ServletException
+	{
+		this._filterConfig = filterConfig;
+	}
+
+	public void destroy()
+	{
+		this._filterConfig = null;
+	}
+
+        @SuppressWarnings("deprecation")
+	public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException
+	{
+		if (request instanceof HttpServletRequest)
+		{
+			HttpServletRequest hRequest = ((HttpServletRequest) request);
+			HttpSession hSession = hRequest.getSession();
+			ServletContext context = hSession.getServletContext();
+
+			GSHttpServletRequestWrapper gRequest = new GSHttpServletRequestWrapper(hRequest);
+
+			String url = hRequest.getRequestURI().toString();
+
+			if (isURLRestricted(url))
+			{
+				response.getWriter().println("Access to this page is forbidden.");
+				return;
+			}
+
+			//If the user is trying to access a collection file we need to run a security check
+			if (url.contains(ASSOCIATED_FILE_PATH))
+			{
+				String dir = null;
+				int dirStart = url.indexOf(ASSOCIATED_FILE_PATH) + ASSOCIATED_FILE_PATH.length();
+				int dirEnd = -1;
+				if (dirStart < url.length() && url.indexOf("/", dirStart) != -1)
+				{
+					dirEnd = url.indexOf("/", dirStart);
+				}
+				if (dirEnd != -1)
+				{
+					dir = url.substring(dirStart, dirEnd);
+				}
+				if (dir == null)
+				{
+					return;
+				}
+
+				String collection = null;
+				int colStart = url.indexOf(COLLECTION_FILE_PATH) + COLLECTION_FILE_PATH.length();
+				int colEnd = -1;
+				if (colStart < url.length() && url.indexOf("/", colStart) != -1)
+				{
+					colEnd = url.indexOf("/", colStart);
+				}
+				if (colEnd != -1)
+				{
+					collection = url.substring(colStart, colEnd);
+				}
+				if (collection == null)
+				{
+					return;
+				}
+
+				MessageRouter gsRouter = (MessageRouter) context.getAttribute("GSRouter");
+				
+				if (gsRouter == null)
+				{
+					_logger.error("Receptionist is null, stopping filter");
+					return;
+				}
+
+				Document gsDoc = XMLConverter.newDOM();
+
+				Element metaMessage = gsDoc.createElement(GSXML.MESSAGE_ELEM);
+				Element metaRequest = GSXML.createBasicRequest(gsDoc, GSXML.REQUEST_TYPE_PROCESS, collection + "/" + METADATA_RETRIEVAL_SERVICE, new UserContext());
+				metaMessage.appendChild(metaRequest);
+
+				Element paramList = gsDoc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				metaRequest.appendChild(paramList);
+
+				Element param = gsDoc.createElement(GSXML.PARAM_ELEM);
+				paramList.appendChild(param);
+
+				param.setAttribute(GSXML.NAME_ATT, "metadata");
+				param.setAttribute(GSXML.VALUE_ATT, "contains");
+
+				Element docList = gsDoc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				metaRequest.appendChild(docList);
+
+				Element doc = gsDoc.createElement(GSXML.DOC_NODE_ELEM);
+				docList.appendChild(doc);
+
+				doc.setAttribute(GSXML.NODE_ID_ATT, dir);
+
+				Element metaResponse = (Element) gsRouter.process(metaMessage);
+
+				NodeList metadataList = metaResponse.getElementsByTagName(GSXML.METADATA_ELEM);
+				if (metadataList.getLength() == 0)
+				{
+					_logger.error("Could not find the document related to this url");
+				}
+				else
+				{
+					Element metadata = (Element) metadataList.item(0);
+					String document = metadata.getTextContent();
+
+					//Get the security info for this collection
+					Element securityMessage = gsDoc.createElement(GSXML.MESSAGE_ELEM);
+					Element securityRequest = GSXML.createBasicRequest(gsDoc, GSXML.REQUEST_TYPE_SECURITY, collection, new UserContext());
+					securityMessage.appendChild(securityRequest);
+					if (document != null && !document.equals(""))
+					{
+						securityRequest.setAttribute(GSXML.NODE_OID, document);
+					}
+
+					Element securityResponse = (Element) GSXML.getChildByTagName(gsRouter.process(securityMessage), GSXML.RESPONSE_ELEM);
+					ArrayList<String> groups = GSXML.getGroupsFromSecurityResponse(securityResponse);
+
+					if (!groups.contains(""))
+					{
+						boolean found = false;
+						for (String group : groups)
+						{
+							if (((HttpServletRequest) request).isUserInRole(group))
+							{
+								found = true;
+								break;
+							}
+						}
+
+						if (!found)
+						{
+							return;
+						}
+					}
+				}
+			}
+			else if (url.contains(INTERFACE_PATH))
+			{			    
+				String fileURL = url.replaceFirst(context.getContextPath(), "");
+				File requestedFile = new File(context.getRealPath(fileURL));
+
+				if (!requestedFile.exists())
+				{
+					int interfaceNameStart = fileURL.indexOf(INTERFACE_PATH) + INTERFACE_PATH.length();
+					int interfaceNameEnd = fileURL.indexOf("/", interfaceNameStart);
+					String interfaceName = fileURL.substring(interfaceNameStart, interfaceNameEnd);
+					String interfacesDir = fileURL.substring(0, interfaceNameStart);
+					File interfaceConfigFile = new File(context.getRealPath(interfacesDir + interfaceName + "/interfaceConfig.xml"));
+
+					if (interfaceConfigFile.exists())
+					{
+					  Document interfaceConfigDoc = XMLConverter.getDOM(interfaceConfigFile);
+
+						String baseInterface = interfaceConfigDoc.getDocumentElement().getAttribute("baseInterface");
+						if (baseInterface.length() > 0)
+						{
+							File baseInterfaceFile = new File(context.getRealPath(fileURL.replace("/" + interfaceName + "/", "/" + baseInterface + "/")));
+							if (baseInterfaceFile.exists())
+							{
+								ServletOutputStream out = response.getOutputStream();
+								out.write(FileUtils.readFileToByteArray(baseInterfaceFile));
+								out.flush();
+								out.close();
+								return;
+							}
+						}
+					}
+				}
+			}
+			else
+			{
+				//If we have a jsessionid on the end of our URL we want to ignore it
+				int index;
+				if ((index = url.indexOf(";jsessionid")) != -1)
+				{
+					url = url.substring(0, index);
+				}
+				String[] segments = url.split("/");
+				for (int i = 0; i < segments.length; i++)
+				{
+					String[] additionalParameters = null;
+					String[] defaultParamValues = null;
+
+					//COLLECTION
+					if (segments[i].equals(COLLECTION_PATH) && (i + 1) < segments.length)
+					{
+						gRequest.setParameter(GSParams.COLLECTION, segments[i + 1]);
+					}
+					//DOCUMENT
+					else if (segments[i].equals(DOCUMENT_PATH) && (i + 1) < segments.length)
+					{
+						gRequest.setParameter(GSParams.DOCUMENT, segments[i + 1]);
+
+						additionalParameters = new String[] { GSParams.ACTION };
+						defaultParamValues = new String[] { "d" };
+					}
+					//PAGE
+					else if (segments[i].equals(PAGE_PATH) && (i + 1) < segments.length)
+					{
+						gRequest.setParameter(GSParams.SUBACTION, segments[i + 1]);
+
+						additionalParameters = new String[] { GSParams.ACTION };
+						defaultParamValues = new String[] { "p" };
+					}
+					//SYSTEM
+					else if (segments[i].equals(SYSTEM_PATH) && (i + 1) < segments.length)
+					{
+						String sa = segments[i + 1];
+						if (sa.equals(SYSTEM_SUBACTION_CONFIGURE) || sa.equals(SYSTEM_SUBACTION_RECONFIGURE))
+						{
+							sa = "c";
+						}
+						else if (sa.equals(SYSTEM_SUBACTION_ACTIVATE))
+						{
+							sa = "a";
+						}
+						else if (sa.equals(SYSTEM_SUBACTION_DEACTIVATE))
+						{
+							sa = "d";
+						}
+
+						if (sa.equals("c") && (i + 2) < segments.length)
+						{
+							gRequest.setParameter(GSParams.SYSTEM_CLUSTER, segments[i + 2]);
+						}
+
+						if (sa.equals("a") && (i + 2) < segments.length)
+						{
+							gRequest.setParameter(GSParams.SYSTEM_MODULE_TYPE, "collection");
+							gRequest.setParameter(GSParams.SYSTEM_MODULE_NAME, segments[i + 2]);
+						}
+
+						if (sa.equals("d") && (i + 2) < segments.length)
+						{
+							gRequest.setParameter(GSParams.SYSTEM_CLUSTER, segments[i + 2]);
+						}
+
+						gRequest.setParameter(GSParams.SUBACTION, sa);
+
+						additionalParameters = new String[] { GSParams.ACTION };
+						defaultParamValues = new String[] { "s" };
+					}
+					//ADMIN
+					else if (segments[i].equals("admin") && (i + 1) < segments.length)
+					{
+						String pageName = segments[i + 1];
+
+						gRequest.setParameter("s1.authpage", pageName);
+
+						additionalParameters = new String[] { GSParams.ACTION, GSParams.REQUEST_TYPE, GSParams.SUBACTION, GSParams.SERVICE };
+						defaultParamValues = new String[] { "g", "r", "authen", "Authentication" };
+					}
+					//BROWSE
+					else if (segments[i].equals("browse") && (i + 1) < segments.length)
+					{
+						String cl = "";
+						for (int j = 1; (i + j) < segments.length; j++)
+						{
+							String currentSegment = segments[i + j].replace("CL", "").replace("cl", "");
+							if (currentSegment.contains("."))
+							{
+								String[] subsegments = currentSegment.split("\\.");
+								for (String subsegment : subsegments)
+								{
+									subsegment = subsegment.replace("CL", "").replace("cl", "");
+
+									if (cl.length() > 0)
+									{
+										cl += ".";
+									}
+
+									if (subsegment.length() > 0)
+									{
+										cl += subsegment;
+									}
+								}
+								continue;
+							}
+							if (!currentSegment.matches("^(CL|cl)?\\d+$"))
+							{
+								continue;
+							}
+
+							if (cl.length() > 0)
+							{
+								cl += ".";
+							}
+
+							cl += currentSegment;
+						}
+
+						gRequest.setParameter("cl", "CL" + cl);
+
+						additionalParameters = new String[] { GSParams.ACTION, GSParams.REQUEST_TYPE, GSParams.SERVICE };
+						defaultParamValues = new String[] { "b", "s", "ClassifierBrowse" };
+					}
+					//QUERY
+					else if (segments[i].equals("search"))
+					{
+						String serviceName = "";
+						if ((i + 1) < segments.length)
+						{
+							serviceName = segments[i + 1];
+							gRequest.setParameter("s", serviceName);
+
+							additionalParameters = new String[] { GSParams.ACTION, GSParams.SUBACTION, GSParams.REQUEST_TYPE };
+							defaultParamValues = new String[] { "q", "", "d" };
+						}
+						if ((i + 2) < segments.length)
+						{
+							if (serviceName.equals("TextQuery") || serviceName.equals("RawQuery"))
+							{
+
+								gRequest.setParameter("s1.query", segments[i + 2]);
+							}
+							else if (serviceName.equals("FieldQuery"))
+							{
+								gRequest.setParameter("s1.fqv", segments[i + 2]);
+							}
+							else if (serviceName.equals("AdvancedFieldQuery"))
+							{
+								gRequest.setParameter("s1.fqv", segments[i + 2]);
+							}
+						}
+					}
+					if (additionalParameters != null)
+					{
+						for (int j = 0; j < additionalParameters.length; j++)
+						{
+							if (gRequest.getParameter(additionalParameters[j]) == null)
+							{
+								gRequest.setParameter(additionalParameters[j], defaultParamValues[j]);
+							}
+						}
+					}
+				}
+			}
+
+			chain.doFilter(gRequest, response);
+		}
+		else
+		{
+			//Will this ever happen?
+			System.err.println("The request was not an HttpServletRequest");
+		}
+	}
+
+	private boolean isURLRestricted(String url)
+	{
+		for (String restrictedURL : _restrictedURLs)
+		{
+			if (url.matches(".*" + restrictedURL + ".*"))
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/DocumentStream.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/DocumentStream.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/DocumentStream.java	(revision 31634)
@@ -0,0 +1,55 @@
+/*
+ *    DocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import org.w3c.dom.Document; 
+
+// other java classes
+import java.io.Serializable;
+
+
+/**
+ * Document Stream represents a stream of XML documents
+ *
+ * Based on algorithms in ...
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * 
+ */
+public interface DocumentStream extends Cloneable, Serializable {
+  
+  /**
+   * Get the next document
+   * 
+   * @exception java.io.Exception when something goes wrong
+   * @return the next document
+   */
+  public Document nextDocument() throws Exception ;
+  /**
+   * Is there a next document?
+   * 
+   * @exception java.io.Exception when something goes wrong
+   * @return true if there is a next document
+   */
+  public boolean hasNextDocument() throws Exception ;
+  
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/FilesDocumentStream.java	(revision 31634)
@@ -0,0 +1,196 @@
+/*
+ *    DocumentStreamFromFiles.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import org.w3c.dom.Document; 
+import org.xml.sax.InputSource;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+
+// other java classes
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Vector;
+
+
+  
+/**
+ * DocumentStreamFromFiles creates a stream of documents from a directory or directories.
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see java.io.File
+ * @see DocumentStream
+ */
+public class FilesDocumentStream 
+  implements DocumentStream {
+  /** the files we've found so far */
+  Vector<String> files = new Vector<String>();
+  /** the directories we've found so far */
+  Vector<String> directories = new Vector<String>();
+
+  /**
+   * Filename constructor
+   * 
+   * @exception java.io.IOException file access do
+   * @param filename the filename of the file or directory
+   */
+  public FilesDocumentStream(String filename)  {
+    File file = new File(filename);
+    if (file.exists() && file.isDirectory())
+      directories.add(file.getAbsolutePath());
+    else
+      files.add(file.getAbsolutePath());
+  }
+  /**
+   * File constructor
+   * 
+   * @exception java.io.IOException file access do
+   * @param file a handle to the file
+   */
+  public FilesDocumentStream(File file) {
+    if (file.exists() && file.isDirectory())
+      directories.add(file.getAbsolutePath());
+    else
+      files.add(file.getAbsolutePath());
+  }
+
+  /** 
+   * A small inner class to filter out those files that don't 
+   * end in ".xml" or ".XML"
+   */
+  class XMLFilenameFilter implements FilenameFilter {
+    /** Tests if a specified file should be included in a file list. */
+    public boolean accept(File dir, String name) {
+      String target = ".xml";
+      if (name.length() < target.length())
+	return false;
+      String extension = name.substring(name.length() - 4, name.length());
+      //System.out.println(name + " ==>> " +  extension);
+      if (extension.equalsIgnoreCase(target)) {
+	return true;
+      } else {
+	return false;
+      }
+    }    
+  }
+
+  /**
+   * Opens a new directory if there are no files are queued
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   */
+  protected void expandIfNecessary() {
+    while (directories.size() > 0) {
+      String dirname = directories.elementAt(0);
+      directories.removeElement(dirname);
+      File dir = new File(dirname);
+      if (!dir.exists()) throw new Error ("error in expand: expecting a directory " + dirname);
+      String[] fileArray = dir.list();
+      for (int i=0;i<fileArray.length;i++){
+	File file = new File(dir, fileArray[i]);
+	if (file.exists() && file.isDirectory()) 
+	  directories.add(file.getPath());
+      }
+      
+      fileArray = dir.list(new XMLFilenameFilter());
+      for (int i=0;i<fileArray.length;i++){
+	File file = new File(dir, fileArray[i]);
+	if (file.exists() && !file.isDirectory())
+	  files.add(file.getPath());
+      }
+      if (files.size() > 0)
+	return;
+    }
+  }
+  
+  /**
+   * Returns the next document
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   * @return the next document
+   */
+  public Document nextDocument()
+    throws Exception {
+    
+    if (!hasNextDocument()) throw new Error("Doesn't have another Document");
+
+    String filename = files.elementAt(files.size() - 1);
+    files.removeElementAt(files.size() - 1);
+    File file = new File(filename);
+    
+    Reader reader = new FileReader(file);
+    InputSource xml_source = new InputSource(reader);
+
+    XMLUtil.getDOMParser().parse(xml_source);
+    Document doc = XMLUtil.getDOMParser().getDocument();
+    
+    return doc;
+  }
+    
+  /**
+   * Is there another document ?
+   * 
+   * @exception java.io.IOException when the underlying file access does
+   * @return the next document
+   */
+
+  public boolean hasNextDocument()  throws Exception {
+    expandIfNecessary();
+    return (files.size() > 0);
+  }
+  /**
+   * Tests...
+   *
+   * 
+   * @exception java.io.IOException when ...
+   * @param args the arguments ...
+   */
+
+  public static void main(String args[]) throws Exception
+  {
+   
+    StreamResult result = new StreamResult(System.out);
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    
+    FilesDocumentStream stream = new FilesDocumentStream(".");
+    while (stream.hasNextDocument()) {
+      Document document = stream.nextDocument();
+      Transformer transformer = transformerFactory.newTransformer();
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+      System.out.println();
+      System.out.println();
+    }
+
+  } 
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/GeneratedDocumentStream.java	(revision 31634)
@@ -0,0 +1,148 @@
+/*
+ *    GenerateDocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+
+
+// other java classes
+
+/**
+ * GeneratedDocumentStream generates an unbounded stream of XML documents
+ * Each containing a single string.
+ *
+ * Based as a wrapper around an Increment object
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see DocumentStream
+ * @see Increment
+ * 
+ */
+public class GeneratedDocumentStream implements DocumentStream {
+  
+  /** The next string */
+  protected String next = "";
+  /** The underlying Increment object */
+  protected Increment incrementer = new IncrementRange();
+
+  /** The default constructor. Generates ASCII-only documents */
+  public GeneratedDocumentStream() {
+
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param incrementer the incrementer to use (controls the character set to use)
+   */
+  public GeneratedDocumentStream(Increment incrementer) {
+    this.incrementer = incrementer;
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param next the string for the first document (must suit the incrementor)
+   */
+  public GeneratedDocumentStream(String next) {
+    this.next = next;
+  }
+  
+  /** 
+   * Custom constructor 
+   *
+   * @param incrementer the incrementer to use (controls the character set to use)
+   * @param next the string for the first document (must suit the incrementor)
+   */
+  public GeneratedDocumentStream(Increment incrementer, String next) {
+    this.incrementer = incrementer;
+    this.next = next;
+  }
+  /** 
+   * Custom constructor 
+   *
+   * @param first the first letter in the alphabet
+   * @param last the last letter in the alphabet
+   */
+  public GeneratedDocumentStream(char first, char last) {
+    this.incrementer = new IncrementRange(first,last);
+  }
+
+
+  /**
+   * Generates the nest document in the sequence
+   * 
+   * @exception java.io.Exception when something goes wrong in the XML stuff
+   * @return the next Document
+   * @see org.w3c.dom.Document
+   */
+  public Document nextDocument() {
+
+
+   DocumentImpl doc = new DocumentImpl(true);
+    
+   ElementImpl root = new ElementImpl(doc,"document");
+   TextImpl text = new TextImpl(doc,next);
+    
+   doc.appendChild(root);
+   root.appendChild(text);
+ 
+   next = incrementer.incrementString(next);
+   return doc;
+   
+  }
+  /**
+   * Are there more documents ?
+   * 
+   * @return yes
+   */
+  public boolean hasNextDocument(){
+    return true;
+  }
+  /**
+   * Generates an unbounded stream of documents
+   */
+  public static void main(String args[]) throws Exception
+  {
+    
+    StreamResult result = new StreamResult(System.out);
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    
+    GeneratedDocumentStream stream = new GeneratedDocumentStream();
+    while (stream.hasNextDocument()) {
+      Document document = stream.nextDocument();
+      Transformer transformer = transformerFactory.newTransformer();
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+    }
+  }
+  
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/Increment.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/Increment.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/Increment.java	(revision 31634)
@@ -0,0 +1,111 @@
+/*
+ *    Increment.java
+ *    Copyright (C) 2000-2001 Stuart Yeates 
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+// the package we're in
+package org.greenstone.gsdl3.selfContained;
+
+// java standard library classes used
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.Writer;
+
+
+/**
+ * Class Increment is an abstract class for incrementing streams and strings
+ *
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ */
+
+abstract public class Increment implements Cloneable, Serializable
+{
+  /**
+   * Increment a string of characters
+   * 
+   * @exception java.io.Exception if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  abstract String incrementString(String str);
+
+  /**
+   * Increment a stream of characters, reading from the reader and writing the incremented characters to the writer
+   * 
+   * @exception java.io.Exception if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  abstract long incrementStream(Reader rdr, Writer wtr);
+
+  /**
+   * Print to standard out a verbose message about the 
+   * results of incrementing this string
+   * 
+   * @param s the string to increment
+   * @see #incrementString(String)
+   */
+  final void checkStr(String s) {
+    try {
+      System.out.println("incrementing: \"" + s +
+			 "\" to \"" + incrementString(s) +
+			 "\"  " + stringToOctal(s) +
+			 " ===> " + stringToHex(incrementString(s)));
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+  }
+
+  /**
+   * Converts a unicode string to Hex
+   * 
+   * @exception java.io.IOException when ...
+   * @param s the string to convert
+   * @return a string in hex
+   * @see Integer#toHexString(int)
+   */
+  static final String stringToHex(String s) {
+    StringBuffer buffer = new StringBuffer();
+    for (int i=0;i<s.length();i++) {
+      buffer.append(" 0x");
+      buffer.append(Integer.toHexString(s.charAt(i)));
+    } 
+    return buffer.toString();
+  }
+
+  /**
+   * Converts a unicode string to Octal
+   * 
+   * @param s the string
+   * @return the string in octal
+   * @see Integer#toOctalString(int)
+   */
+  static final String stringToOctal(String s) {
+    StringBuffer buffer = new StringBuffer();
+    for (int i=0;i<s.length();i++) {
+      buffer.append(" 0x");
+      buffer.append(Integer.toOctalString(s.charAt(i)));
+    } 
+    return buffer.toString();
+  }
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/IncrementRange.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/IncrementRange.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/IncrementRange.java	(revision 31634)
@@ -0,0 +1,240 @@
+/*
+ *    IncrementRange.java
+ *    Copyright (C) 2000-2001 Stuart Yeates 
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+// the package we're in
+package org.greenstone.gsdl3.selfContained;
+
+// java standard library classes used
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.Writer;
+
+/**
+ * Class IncrementRange increments strings and streams over a limited range of Characters.
+ *
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see Increment
+ * 
+ */
+final public class IncrementRange
+  extends Increment 
+  implements Cloneable, Serializable
+{
+  /**
+   * Create an IncrementRange which counts between first and last inclusive
+   * 
+   * @param first the first character in the range
+   * @param last the last character in the range
+   * @see Increment
+   */
+  IncrementRange(char first, char last){
+    this.first = first;
+    this.last = last;
+  }
+  /**
+   * Create a default IncrementRange
+   * 
+   * @see Increment
+   */
+  IncrementRange(){
+  }
+
+  /**
+   * Create a IncrementRange over all Unicode characters
+   * 
+   * @see Increment
+   */
+  IncrementRange(boolean b){
+    if (true) {
+      this.first = Character.MIN_VALUE;
+      this.last = Character.MAX_VALUE;
+    }
+  }  
+  /** the last character in out range */
+  private char first = ' ';
+  /** the last character in out range */
+  private char last = 'z';
+
+  /**
+   * Increment a stream of characters, reading from the reader and writing the incremented characters to the writer
+   * 
+   * @exception java.io.Error if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  long incrementStream(Reader rdr, Writer wtr) {
+    try {
+      long count = 0; // the return value
+    
+      int i = rdr.read();
+      boolean carry = true;
+    
+      while (i != -1) {
+	char c = (char) i;
+      
+	// check the validity of the character we're about to increment
+	if (c < first || c > last) 
+	  throw new Error("Character '" + c + "' is ouside range");
+      
+	if (c == last) {
+	  if (carry == true) {
+	    wtr.write(first);
+	    carry = true;
+	  } else {
+	    wtr.write(c);
+	    carry = false;
+	  }
+	} else {
+	  if (carry == true) {
+	    wtr.write(++c);
+	    carry = false;
+	  } else {
+	    wtr.write(c);
+	    carry = false;
+	  } 
+	}
+	count++;
+
+	// read the next character
+	i = rdr.read();
+      }
+      if (carry == true) {
+	wtr.write(first);
+	count++;
+      }
+      return count;
+    } catch (IOException e) {
+      System.err.println("IOException in incrementStream()");
+      throw new Error(e.toString());
+    }
+  }
+
+  /**
+   * Increment a string of characters
+   * 
+   * @exception java.lang.Error if any of the characters in the string are outside the range of first-last
+   * @param str the string to be incremented
+   * @return the incremented string
+   * @see Increment
+   */
+  public final String incrementString(String str)  {
+    StringBuffer buffer = new StringBuffer();
+    if (str.equals("")) {
+      buffer.append(first);
+      return buffer.toString();
+    } else {
+      boolean carry = true;
+      for (int i=0;i<str.length();i++) {
+	char c = str.charAt(i);
+	if (c < first || c > last) 
+	  throw new Error("Character '" + c + "' is ouside range");
+
+	if (c == last) {
+	  if (carry == true) {
+	    buffer.append(first);
+	    carry = true;
+	  } else {
+	    buffer.append(c);
+	    carry = false;
+	  }
+	} else {
+	  if (carry == true) {
+	    buffer.append(++c);
+	    carry = false;
+	  } else {
+	    buffer.append(c);
+	    carry = false;
+	  } 
+	}
+      }
+      if (carry == true)
+	buffer.append(first);
+    }
+    return buffer.toString();
+  }
+
+
+  /**
+   * Test the incrementing of strings (never returns)
+   */
+  public static void testStringIncrement() throws IOException
+  {
+    IncrementRange n = new IncrementRange ();
+    try {
+      String tmp = "";
+      System.out.println(stringToHex(tmp));
+      while (true) {
+	//checkStr(tmp);
+	tmp = n.incrementString(tmp);
+	n.checkStr(tmp);
+      }
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+
+  }
+
+  /**
+   * Test the incrementing of streams (never returns)
+   *
+   */
+  public static void testStreamIncrement() 
+  {
+    IncrementRange n = new IncrementRange ();
+    try {
+      long counter = 0;
+      String tmp = "";
+      System.out.println(stringToHex(tmp));
+      while (true) {
+	//checkStr(tmp);
+	String tmp2 = n.incrementString(tmp);
+	StringWriter wtr = new StringWriter();
+	n.incrementStream(new StringReader(tmp),wtr);
+	if (!tmp2.equals(wtr.toString()))
+	  throw new Exception("Error: \"" + tmp2 + 
+			      "\" and \"" + tmp2 + 
+			      "\"");
+	counter++;
+	if (counter % 1000000 == 0)
+	  System.out.println(counter + " " + tmp);
+
+	tmp = tmp2;
+      }
+    } catch (Exception e) {
+      System.out.println("Caught exception: " + e);
+    }
+  }
+
+  /**
+   * Tests stream incrementing 
+   *
+   * 
+   */
+  public static void main(String args[]) 
+  {
+    testStreamIncrement();
+  } 
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/QueryDocumentStream.java	(revision 31634)
@@ -0,0 +1,281 @@
+/*
+ *    QueryDocumentStream.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMResult;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import org.w3c.dom.Document;
+
+// other java classes
+import java.io.StringReader;
+
+/**
+ * QueryDocumentStream takes another document stream and removes 
+ * from the stream all documents which fail to match a XSLT query.
+ * The default query returns only those documents containing the 
+ * string "the".
+ *
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * @see DocumentStream
+ * 
+ */
+public class QueryDocumentStream implements DocumentStream {
+  
+  /** The default XSLT query string. Is true (returns a non-empty document) when the input document contains the string "the". */
+  public static final String DEFAULT_QUERY_STRING =   
+    "<xsl:stylesheet version=\"1.0\" " +
+    "                    xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" +
+    "   <xsl:output omit-xml-declaration=\"yes\"/>" +
+    
+    "   <xsl:template match=\"/*\">" +
+    "      <xsl:if test=\"contains(.,'the')\">" +
+    "         <xsl:copy-of select=\".\"/>" +
+    "      </xsl:if>" + 
+    "   </xsl:template>" +
+    "</xsl:stylesheet>";
+  /** The first part of the default quesy for a string other than "the" */
+  public static final String DEFAULT_QUERY_STRING_1_1 =   
+    "<xsl:stylesheet version=\"1.0\" " +
+    "                    xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">" +
+    "   <xsl:output omit-xml-declaration=\"yes\"/>" +
+    
+    "   <xsl:template match=\"/*\">" +
+    "      <xsl:if test=\"contains(.,'";
+
+  /** The second part of the default quesy for a string other than "the" */
+  public static final String DEFAULT_QUERY_STRING_1_2 =   
+    "')\">" +
+    "         <xsl:copy-of select=\".\"/>" +
+    "      </xsl:if>" + 
+    "   </xsl:template>" +
+    "</xsl:stylesheet>";
+
+
+  /** The query */
+  protected StreamSource query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+
+  /** The underlying DocumentStream object */
+  protected DocumentStream stream = null;
+
+  /** A cached Document */
+  protected Document cached = null;
+
+  /** The transformer factory */
+  TransformerFactory transformerFactory  = null;
+
+  /** The transformer */
+  Transformer transformer  = null;
+
+  /** Uses the default query (looking for the) over the default DocumentStream (all ascii strings) */
+  public QueryDocumentStream() {
+    this.query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+    this.stream = new GeneratedDocumentStream();
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+  /** Looks for the given string in the stream of documents  */
+  public QueryDocumentStream(DocumentStream stream, String query) {
+    String fullQuery = DEFAULT_QUERY_STRING_1_1 + query + DEFAULT_QUERY_STRING_1_2;
+    this.query = new StreamSource(new StringReader(fullQuery));
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(this.query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+
+  /** Applies the stylesheet to the stream.  */
+  public QueryDocumentStream(DocumentStream stream, StreamSource source) {
+    this.query = source;
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  
+
+  /** The default constructor. Searches for documents containing "the" */
+  public QueryDocumentStream(DocumentStream stream) {
+    this.query = new StreamSource(new StringReader(DEFAULT_QUERY_STRING));
+    this.stream = stream;
+
+    try {
+      this.transformerFactory 
+	= org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+      
+      this.transformer
+	= transformerFactory.newTransformer(query);
+    } catch (TransformerConfigurationException e) {
+      System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+    }
+  }
+  /**
+   * Generates the nest document in the sequence
+   * 
+   * @exception java.io.Exception when something goes wrong in the XML stuff
+   * @return the next Document
+   * @see org.w3c.dom.Document
+   */
+  public Document nextDocument() throws Exception {
+
+    if (hasNextDocument() == false)
+      throw new Error("no more docs");
+    Document result = cached;
+    cached = null;
+    return result;
+  }
+  /**
+   * Are there more documents ?
+   * 
+   * @return true if there are more docs
+   */
+  public boolean hasNextDocument() throws Exception {
+    lookInStream();
+    return (cached != null);
+  }
+  /**
+   *
+   * 
+   * @exception java.io.IOException when ...
+   * @param arg2 ... 
+   * @param arg1 ...
+   * @return ...
+   */
+  protected boolean lookInStream() throws Exception {
+
+    while (stream.hasNextDocument() && cached == null) {
+      
+      Document candidate = stream.nextDocument();
+      
+      DOMSource input = new DOMSource(candidate);
+      DOMResult output = new DOMResult();
+      
+      try {
+	
+	transformer.transform(input, output);
+      
+	// if the transformed document is not empty don't cache it
+	if (output.getNode().getFirstChild() != null)
+	  cached = (Document) output.getNode();
+      
+      } catch (TransformerConfigurationException e) {
+	System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+      } catch (TransformerException e) {
+	System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	cached = null;
+	return false;
+      } catch (Exception e) {
+	System.err.println("Exception: " + e.getMessage());
+	cached = null;
+	return false;
+      }
+      
+      if (false) { // debugging info
+	
+	try {
+	  StreamResult result = new StreamResult(System.out);
+	  Transformer transformer = transformerFactory.newTransformer();
+	  
+	  Source source = new DOMSource(candidate);
+	  transformer.transform(source,result);
+	  
+	  System.out.print(" ==> ");
+	  
+	  if (cached != null) {
+	    Source source2 = new DOMSource(cached);
+	    transformer.transform(source2,result);
+	  } 
+	  System.out.println();
+	  System.out.println("======================================================");
+	  
+	} catch (TransformerConfigurationException e) {
+	  System.err.println("XMLTransformer: couldn't create transformer object: "+e.getMessage());
+	} catch (TransformerException e) {
+	  System.err.println("XMLTransformer: couldn't transform the source: " + e.getMessage());
+	  cached = null;
+	  return false;
+	} catch (Exception e) {
+	  System.err.println("Exception: " + e.getMessage());
+	  cached = null;
+	  return false;
+	}
+	
+      }
+    }
+    return cached == null;    
+  }
+
+  /**
+   * Generates an unbounded stream of documents
+   */
+  public static void main(String args[]) throws Exception
+  {
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    Transformer transformer = transformerFactory.newTransformer();
+    
+    GeneratedDocumentStream gStream = new GeneratedDocumentStream();
+    QueryDocumentStream qStream = new QueryDocumentStream(gStream);
+    while (qStream.hasNextDocument()) {
+      Document document = qStream.nextDocument();
+      
+      StreamResult result = new StreamResult(System.out);
+      
+      Source source = new DOMSource(document);
+      transformer.transform(source,result);
+      
+      System.out.println();
+      System.out.println("======================================================");
+    }
+  }
+  
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/XMLUtil.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/XMLUtil.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/XMLUtil.java	(revision 31634)
@@ -0,0 +1,96 @@
+/*
+ *    XMLUtil.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.selfContained;
+
+// XML classes
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.xerces.parsers.DOMParser;
+
+// other java classes
+import java.io.IOException;
+
+
+/**
+ * Class XMLUtil holds static global XML configuration objects. 
+ * 
+ * @author <a href="http://www.cs.waikato.ac.nz/~say1/">stuart yeates</a> (<a href="mailto:s.yeates@cs.waikato.ac.nz">s
+.yeates@cs.waikato.ac.nz</a>) at the <a href="http://www.nzdl.org">New Zealand Digital Library</a>
+ * @version $Revision$
+ * 
+ */
+class XMLUtil {
+
+  /** the singleton instance */
+  protected static XMLUtil instance = null;
+
+
+  /** JAXP parser factory */ 
+  protected DocumentBuilderFactory doc_build_fact=null;
+
+  /** JAXP parser factory */ 
+  static public DocumentBuilderFactory getDocumentBuilderFactory() {
+    if (instance == null) 
+      init();
+    return instance.doc_build_fact;
+  }
+
+  /** JAXP parser */ 
+  protected DocumentBuilder doc_builder=null;
+
+  /** JAXP parser */ 
+  static public DocumentBuilder getDocumentBuilder() {
+    if (instance == null) 
+      init();
+    return instance.doc_builder;
+   }
+
+  /** xerces parser */
+  protected DOMParser parser = null;
+
+  /** xerces parser */
+  static public DOMParser getDOMParser() {
+    if (instance == null) 
+      init();
+    return instance.parser;
+  }
+
+  /** initalise the statics */
+  static protected void init() {
+    instance = new XMLUtil();
+    try {
+      instance.doc_build_fact = DocumentBuilderFactory.newInstance();
+      
+      instance.doc_builder = instance.doc_build_fact.newDocumentBuilder(); 
+      instance.parser = new DOMParser();
+    } catch (Exception e) {
+      System.out.println("XMLConverter:exception "+e.getMessage());
+    }
+  }
+
+  /**
+   * Checks that the objects can be instantiated
+   *
+   */
+  public static void main(String args[]) throws IOException
+  {
+    XMLUtil.getDOMParser();
+  }
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/go.dtd
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/go.dtd	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/selfContained/go.dtd	(revision 31634)
@@ -0,0 +1,103 @@
+<!-- P is the Paragraph of HTML -->
+<!ELEMENT P (#PCDATA)>
+
+<!-- Each Go file consists of one or several GoGame -->
+<!ELEMENT Go (GoGame*)>
+
+<!-- Each GoGame consists of the game Information and the game tree -->
+
+<!ELEMENT GoGame (Information,Nodes)>
+<!ATTLIST GoGame name CDATA #IMPLIED>
+<!ELEMENT Nodes (((Node|Black|White),(Node|Variation|Black|White|Comment)*)?)>
+
+<!ELEMENT Information (Application?|BoardSize?|BlackPlayer?|BlackRank?|
+	WhitePlayer?|WhiteRank?|Date?|Time?|Komi?|Result?|Handicap?|User?|
+	Copyright?|Annotation?|BlackTeam?|WhiteTeam?|Round?|Rules?|Source?)*>
+<!ELEMENT Application (#PCDATA)>
+<!ATTLIST Application format CDATA #IMPLIED>
+<!ELEMENT BoardSize (#PCDATA)>
+<!ATTLIST BoardSize format CDATA #IMPLIED>
+<!ELEMENT BlackPlayer (#PCDATA)>
+<!ATTLIST BlackPlayer format CDATA #IMPLIED>
+<!ELEMENT WhitePlayer (#PCDATA)>
+<!ATTLIST WhitePlayer format CDATA #IMPLIED>
+<!ELEMENT BlackRank (#PCDATA)>
+<!ATTLIST BlackRank format CDATA #IMPLIED>
+<!ELEMENT WhiteRank (#PCDATA)>
+<!ATTLIST WhiteRank format CDATA #IMPLIED>
+<!ELEMENT Date (#PCDATA)>
+<!ATTLIST Date format CDATA #IMPLIED>
+<!ELEMENT Time (#PCDATA)>
+<!ATTLIST Time format CDATA #IMPLIED>
+<!ELEMENT Komi (#PCDATA)>
+<!ATTLIST Komi format CDATA #IMPLIED>
+<!ELEMENT Handicap (#PCDATA)>
+<!ATTLIST Handicap format CDATA #IMPLIED>
+<!ELEMENT Result (#PCDATA)>
+<!ATTLIST Result format CDATA #IMPLIED>
+<!ELEMENT User (#PCDATA)>
+<!ELEMENT Copyright (P+)>
+<!ELEMENT Annotation (#PCDATA)>
+<!ELEMENT BlackTeam (#PCDATA)>
+<!ELEMENT WhiteTeam (#PCDATA)>
+<!ELEMENT Round (#PCDATA)>
+<!ELEMENT Rules (#PCDATA)>
+<!ATTLIST Rules format CDATA #IMPLIED>
+<!ELEMENT Source (#PCDATA)>
+<!ATTLIST Source format CDATA #IMPLIED>
+
+<!-- Game representation in Nodes -->
+
+<!ELEMENT Node ((SGF|Comment|Black|White|AddBlack|AddWhite|Mark|Delete)*)>
+<!ATTLIST Node name CDATA #IMPLIED>
+
+<!-- Possible content of a node (unordered) -->
+
+<!-- Alternative form of specifying the position as tag content. -->
+<!-- The board position is either like D10 or 4,10 (for larger boards). -->
+<!ELEMENT at (#PCDATA)>
+
+<!ELEMENT Comment (P+)>
+<!ELEMENT Black ((at)*)>
+<!ATTLIST Black
+	at CDATA #IMPLIED
+	number CDATA #IMPLIED
+	timeleft CDATA #IMPLIED
+	annotate CDATA #IMPLIED>
+<!ELEMENT White ((at)*)>
+<!ATTLIST White
+	at CDATA #IMPLIED
+	number CDATA #IMPLIED
+	timeleft CDATA #IMPLIED
+	annotate CDATA #IMPLIED>
+<!ELEMENT AddBlack ((at)*)>
+<!ATTLIST AddBlack
+	at CDATA #IMPLIED>
+<!ELEMENT AddWhite ((at)*)>
+<!ATTLIST AddWhite
+	at CDATA #IMPLIED>
+<!ELEMENT Delete ((at)*)>
+<!ATTLIST Delete
+	at CDATA #IMPLIED>
+<!ELEMENT Mark ((at)*)>
+<!ATTLIST Mark 
+	label CDATA #IMPLIED
+	at CDATA #IMPLIED
+	type (square|circle|triangle) #IMPLIED
+	territory (black|white) #IMPLIED>
+<!ELEMENT BlackToPlay (#PCDATA)>
+<!ELEMENT WhiteToPlay (#PCDATA)>
+<!ELEMENT Line (#PCDATA)>
+<!ATTLIST Line 
+	format CDATA #IMPLIED
+	type CDATA #IMPLIED>
+
+<!-- This is to take care of SGF tags, which are not translated -->
+<!ELEMENT SGF (Arg*)>
+<!ATTLIST SGF
+	type CDATA #REQUIRED>
+<!ELEMENT Arg (#PCDATA)>
+
+<!-- Note that the Variation are ALTERNATIVES to the current node -->
+<!-- This is different from SGF and makes the file easier to read by humans -->
+<!ELEMENT Variation ((Node|Variation|White|Black|Comment)*)>
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractBrowse.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractBrowse.java	(revision 31634)
@@ -0,0 +1,842 @@
+/*
+ *    AbstractBrowse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.AbstractBasicDocument;
+import org.greenstone.gsdl3.util.BasicDocument;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * Partially implements a generic classifier service
+ * 
+ */
+public abstract class AbstractBrowse extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractBrowse.class.getName());
+
+	// the browsing services
+	private static final String CLASSIFIER_SERVICE = "ClassifierBrowse";
+	private static final String CLASSIFIER_METADATA_SERVICE = "ClassifierBrowseMetadataRetrieve";
+
+	// do we want to keep info request?
+
+	protected static final String STRUCT_PARAM = "structure";
+	protected static final String INFO_PARAM = "info";
+
+	protected static final String STRUCT_ANCESTORS = "ancestors";
+	protected static final String STRUCT_PARENT = "parent";
+	protected static final String STRUCT_SIBS = "siblings";
+	protected static final String STRUCT_CHILDREN = "children";
+	protected static final String STRUCT_DESCENDS = "descendants";
+	protected static final String STRUCT_ENTIRE = "entire";
+
+	protected static final String INFO_NUM_SIBS = "numSiblings";
+	protected static final String INFO_NUM_CHILDREN = "numChildren";
+	protected static final String INFO_SIB_POS = "siblingPosition";
+
+	protected AbstractBasicDocument gs_doc = null;
+
+	protected Element config_info = null; // the xml from the config file
+
+	protected MacroResolver macro_resolver = null;
+
+	/**
+	 * the default document type - use if all documents are the same type
+	 */
+	protected String default_document_type = null;
+
+	/** constructor */
+	protected AbstractBrowse()
+	{
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring AbstractBrowse...");
+		this.config_info = info;
+		if (macro_resolver != null)
+		{
+		  macro_resolver.setSiteDetails(this.site_http_address, this.cluster_name, this.getLibraryName());
+		  // set up the macro resolver
+		  Element replacement_elem = (Element) GSXML.getChildByTagName(extra_info, GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER);
+		  if (replacement_elem != null)
+		    {
+		      macro_resolver.addMacros(replacement_elem);
+		    }
+		  
+		  // look for any refs to global replace lists
+		  NodeList replace_refs_elems = extra_info.getElementsByTagName(GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER + GSXML.REF_MODIFIER);
+		  for (int i = 0; i < replace_refs_elems.getLength(); i++)
+		    {
+		      String id = ((Element) replace_refs_elems.item(i)).getAttribute("id");
+		      if (!id.equals(""))
+			{
+			  Element replace_list = GSXML.getNamedElement(this.router.config_info, GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER, "id", id);
+			  if (replace_list != null)
+			    {
+			      macro_resolver.addMacros(replace_list);
+			    }
+			}
+		    }
+		}
+
+		
+
+		// check that there are classifiers specified
+		Element class_list = (Element) GSXML.getChildByTagName(info, GSXML.CLASSIFIER_ELEM + GSXML.LIST_MODIFIER);
+		if (class_list == null)
+		{
+			// no classifiers specified
+			return false;
+		}
+
+		// get the display and format elements from the coll config file for
+		// the classifiers
+		extractExtraClassifierInfo(info, extra_info);
+
+		// short_service_info_ - the browse one
+		Element cb_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+		cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+		this.short_service_info.appendChild(cb_service);
+
+		// metadata retrieval for the browsing  
+		Element cbmr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+		this.short_service_info.appendChild(cbmr_service);
+
+		// the format info
+		Element cb_format_info = this.desc_doc.createElement(GSXML.FORMAT_ELEM);
+		boolean format_found = false;
+
+		// try the default format first
+		Element def_format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+		if (def_format != null)
+		{
+			cb_format_info.appendChild(GSXML.duplicateWithNewName(this.desc_doc, def_format, GSXML.DEFAULT_ELEM, true));
+			format_found = true;
+		}
+
+		// add in to the description a simplified list of classifiers
+		NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+		for (int i = 0; i < classifiers.getLength(); i++)
+		{
+			Element cl = (Element) classifiers.item(i);
+
+			Element new_cl = (Element) this.desc_doc.importNode(cl, false); // just import this node, not the children
+
+			// get the format info out, and put inside a classifier element
+			Element format_cl = (Element) new_cl.cloneNode(false);
+			Element format = (Element) GSXML.getChildByTagName(cl, GSXML.FORMAT_ELEM);
+			if (format != null)
+			{
+				//copy all the children
+				NodeList elems = format.getChildNodes();
+				for (int j = 0; j < elems.getLength(); j++)
+				{
+					format_cl.appendChild(this.desc_doc.importNode(elems.item(j), true));
+				}
+				cb_format_info.appendChild(format_cl);
+				format_found = true;
+			}
+
+		}
+
+		if (format_found)
+		{
+			this.format_info_map.put(CLASSIFIER_SERVICE, cb_format_info);
+		}
+
+		// look for document display format - is there a default doc type??
+		String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+		Element display_format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (display_format != null)
+		{
+			// check for docType option.
+			Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+			if (doc_type_opt != null)
+			{
+				String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+				if (!value.equals(""))
+				{
+					this.default_document_type = value;
+				}
+			}
+		}
+
+		// Base line for document (might be overriden by sub-classes)
+		gs_doc = new BasicDocument(this.default_document_type);
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+
+		if (service_id.equals(CLASSIFIER_SERVICE))
+		{
+
+			Element class_list = (Element) GSXML.getChildByTagName(this.config_info, GSXML.CLASSIFIER_ELEM + GSXML.LIST_MODIFIER);
+			if (class_list == null)
+			{
+				// no classifiers specified
+				return null;
+			}
+
+			Element cb_service = doc.createElement(GSXML.SERVICE_ELEM);
+			cb_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_BROWSE);
+			cb_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_SERVICE);
+			cb_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(CLASSIFIER_SERVICE + ".name", lang)));
+			cb_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(CLASSIFIER_SERVICE + ".description", lang)));
+
+			Element cl_list = doc.createElement(GSXML.CLASSIFIER_ELEM + GSXML.LIST_MODIFIER);
+			cb_service.appendChild(cl_list);
+			NodeList classifiers = class_list.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+			for (int i = 0; i < classifiers.getLength(); i++)
+			{
+				Element cl = (Element) classifiers.item(i);
+				Element new_cl = (Element) doc.importNode(cl, false); // just import this node, not the children
+				cl_list.appendChild(new_cl);
+
+				// have we got a displayItem for the classifier title?
+				String text = getDisplayText(cl, GSXML.DISPLAY_TEXT_NAME, lang, "en", "metadata_names");
+				//get the classify title  from the database
+				String class_id = cl.getAttribute(GSXML.NAME_ATT);
+				String content = getMetadata(class_id, "Title");
+				if (text == null || text.equals("")) {
+				
+				  // no display element was specified, use the metadata name
+				  if (!content.equals("")) {
+				    
+				    text = getTextString(content + ".buttonname", lang, "metadata_names");
+				    if (text == null) {
+				      
+				      text = content;
+				    }
+				  }
+				  
+				  if (text == null) {
+				    text=class_id;
+				  }
+				}
+				Element cl_name = GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, text);
+				new_cl.appendChild(cl_name);
+
+				// description
+
+				String meta_name = getTextString(content, lang, "metadata_names");
+				if (meta_name == null)
+				{
+					meta_name = content;
+				}
+				String[] array = { meta_name };
+				String description = getTextString("ClassifierBrowse.classifier_help", lang, array);
+				Element cl_desc = GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, description);
+				new_cl.appendChild(cl_desc);
+
+			}
+			return cb_service;
+		}
+
+		// these ones are probably never called, but put them here just in case
+
+		if (service_id.equals(CLASSIFIER_METADATA_SERVICE))
+		{
+
+			Element cbmr_service = doc.createElement(GSXML.SERVICE_ELEM);
+			cbmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			cbmr_service.setAttribute(GSXML.NAME_ATT, CLASSIFIER_METADATA_SERVICE);
+			return cbmr_service;
+		}
+
+		return null;
+	}
+
+	/**
+	 * this looks for any classifier specific display or format info from
+	 * extra_info and adds it in to the correct place in info
+	 */
+	static boolean extractExtraClassifierInfo(Element info, Element extra_info)
+	{
+		if (extra_info == null)
+		{
+			return false;
+		}
+
+		Document owner = info.getOwnerDocument();
+		// so far we have display and format elements that we need for classifiers
+		NodeList classifiers = info.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+		Element config_browse = (Element) GSXML.getChildByTagName(extra_info, GSXML.BROWSE_ELEM);
+
+		for (int i = 0; i < classifiers.getLength(); i++)
+		{
+			Element cl = (Element) classifiers.item(i);
+			String name = cl.getAttribute(GSXML.NAME_ATT);
+
+			//Element node_extra = GSXML.getNamedElement(config_browse,
+			//					       GSXML.CLASSIFIER_ELEM,
+			//					       GSXML.NAME_ATT,
+			//					       name);
+			//now use the position to get the node - CL1 
+			// assumes the same number of classifiers in collectionCOnfig as in buildConfig
+			//            int position = Integer.parseInt(name.substring(2));
+
+			Element node_extra = null;
+			NodeList cls = config_browse.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+			//if (position >0 && position <= cls.getLength()) {
+			//    node_extra  = (Element) cls.item((position -1)); 
+			//}
+			if (i < cls.getLength())
+			{
+				node_extra = (Element) cls.item(i);
+			}
+
+			if (node_extra == null)
+			{
+				logger.error("GS2REtrieve: haven't found extra info for classifier named " + name);
+				continue;
+			}
+
+			// get the display elements if any - displayName
+			NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+			if (display_names != null)
+			{
+				Element display = owner.createElement(GSXML.DISPLAY_ELEM);
+				for (int j = 0; j < display_names.getLength(); j++)
+				{
+					Element e = (Element) display_names.item(j);
+					cl.appendChild(owner.importNode(e, true));
+				}
+			}
+
+			// get the format element if any 
+			Element format = (Element) GSXML.getChildByTagName(node_extra, GSXML.FORMAT_ELEM);
+			if (format != null)
+			{ // append to index info
+				cl.appendChild(owner.importNode(format, true));
+			}
+		} // for each classifier
+
+		// now check for default format info
+		Element default_format = (Element) GSXML.getChildByTagName(config_browse, GSXML.FORMAT_ELEM);
+		if (default_format != null)
+		{ // append to  info
+			info.appendChild(owner.importNode(default_format, true));
+		}
+
+		return true;
+	}
+
+	protected Element processClassifierBrowse(Element request)
+	{
+
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		Element query_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (query_node_list == null)
+		{
+			logger.error(" ClassifierBrowse request specified no classifier nodes.\n");
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error(" ClassifierBrowse request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		// the type of info required
+		boolean want_structure = false;
+		boolean want_info = false;
+
+		ArrayList<String> info_types = new ArrayList<String>();
+		// The document structure information desired
+		boolean want_ancestors = false;
+		boolean want_parent = false;
+		boolean want_siblings = false;
+		boolean want_children = false;
+		boolean want_descendants = false;
+
+		boolean want_entire_structure = false;
+		// Process the request parameters
+		NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+		for (int i = 0; i < params.getLength(); i++)
+		{
+
+			Element param = (Element) params.item(i);
+			String p_name = param.getAttribute(GSXML.NAME_ATT);
+			String p_value = GSXML.getValue(param);
+			// Identify the structure information desired
+			if (p_name.equals(STRUCT_PARAM))
+			{
+				want_structure = true;
+
+				// This is NOT locale sensitive
+				if (p_value.equals(STRUCT_ANCESTORS))
+					want_ancestors = true;
+				else if (p_value.equals(STRUCT_PARENT))
+					want_parent = true;
+				else if (p_value.equals(STRUCT_SIBS))
+					want_siblings = true;
+				else if (p_value.equals(STRUCT_CHILDREN))
+					want_children = true;
+				else if (p_value.equals(STRUCT_DESCENDS))
+					want_descendants = true;
+				else if (p_value.equals(STRUCT_ENTIRE))
+					want_entire_structure = true;
+				else
+					logger.error("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+			}
+			else if (p_name.equals(INFO_PARAM))
+			{
+				want_info = true;
+				info_types.add(p_value);
+			}
+		}
+
+		// Make sure there is no repeated information
+		if (want_ancestors)
+			want_parent = false;
+		if (want_descendants)
+			want_children = false;
+
+		// the node list to hold the results
+		Element node_list = result_doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(node_list);
+
+		// Get the nodes
+		String[] node_ids = GSXML.getAttributeValuesFromList(query_node_list, GSXML.NODE_ID_ATT);
+		for (int i = 0; i < node_ids.length; i++)
+		{
+			// Add the document to the list
+			Element node = result_doc.createElement(GSXML.CLASS_NODE_ELEM);
+			node_list.appendChild(node);
+
+			String node_id = node_ids[i];
+			node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+
+			if (idNeedsTranslating(node_id))
+			{
+				node_id = translateId(node_id);
+				if (node_id == null)
+				{
+					continue;
+				}
+				node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+			}
+
+			if (want_info)
+			{
+				Element node_info_elem = result_doc.createElement("nodeStructureInfo");
+				node.appendChild(node_info_elem);
+
+				for (int j = 0; j < info_types.size(); j++)
+				{
+					String info_type = info_types.get(j);
+					String info_value = getStructureInfo(node_id, info_type);
+					if (info_value != null)
+					{
+						Element info_elem = result_doc.createElement("info");
+						info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+						info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+						node_info_elem.appendChild(info_elem);
+					}
+				}
+			}
+
+			if (want_structure)
+			{
+				// all structure info goes into a nodeStructure elem
+				Element structure_elem = result_doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+				node.appendChild(structure_elem);
+
+				if (want_entire_structure)
+				{
+					String root_id = getRootId(node_id);
+					Element root_node = createClassifierNode(result_doc, root_id); //, true, false);
+					addDescendants(root_node, root_id, true);
+					structure_elem.appendChild(root_node);
+					continue; // with the next document, we dont need to do any more here
+				}
+
+				// Add the requested structure information
+				Element base_node = createClassifierNode(result_doc, node_id); //, false, false);
+
+				//Ancestors: continually add parent nodes until the root is reached
+				Element top_node = base_node; // the top node so far
+				if (want_ancestors)
+				{
+					String current_id = node_id;
+					while (true)
+					{
+						String parent_id = getParentId(current_id);
+						//Element parent = getParent(current_id);
+						if (parent_id == null)
+							break; // no parent
+						Element parent_node = createClassifierNode(result_doc, parent_id);
+						parent_node.appendChild(top_node);
+						current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+						top_node = parent_node;
+					}
+				}
+				// Parent: get the parent of the selected node
+				else if (want_parent)
+				{
+					String parent_id = getParentId(node_id);
+					if (parent_id != null)
+					{
+					  Element parent_node = createClassifierNode(result_doc, parent_id);
+						parent_node.appendChild(base_node);
+						top_node = parent_node;
+					}
+				}
+
+				// now the top node is the root of the structure
+				structure_elem.appendChild(top_node);
+
+				//Siblings: get the other descendants of the selected node's parent
+				if (want_siblings)
+				{
+
+					Element parent_node = (Element) base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+					String parent_id = getParentId(node_id);
+					// add siblings, - returns a pointer to the new current node 
+					base_node = addSiblings(parent_node, parent_id, node_id);
+				}
+
+				// Children: get the descendants, but only one level deep
+				if (want_children)
+				{
+					addDescendants(base_node, node_id, false);
+				}
+				// Descendants: recursively get every descendant 
+				else if (want_descendants)
+				{
+					addDescendants(base_node, node_id, true);
+				}
+
+				NodeList classifierElements = result.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+				for (int j = 0; j < classifierElements.getLength(); j++)
+				{
+					Element current = (Element) classifierElements.item(j);
+					Node parentNode = current.getParentNode();
+
+					if (parentNode == null)
+					{
+						continue;
+					}
+
+					Element parent = (Element) parentNode;
+					String childType = parent.getAttribute(GSXML.CHILD_TYPE_ATT);
+					if (childType == null || childType.length() == 0)
+					{
+						continue;
+					}
+
+					current.setAttribute(GSXML.CLASSIFIER_STYLE_ATT, childType);
+				}
+			} // if want structure
+		} // for each doc
+		return result;
+	}
+
+	protected Element processClassifierBrowseMetadataRetrieve(Element request)
+	{
+
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		result.setAttribute(GSXML.FROM_ATT, CLASSIFIER_METADATA_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("AbstractBrowse, ClassifierBrowseMetadataRetrieve Error: missing paramList.\n");
+			return result; // Return the empty result
+		}
+
+		// The metadata information required
+		ArrayList<String> metadata_names_list = new ArrayList<String>();
+		boolean all_metadata = false;
+		// Process the request parameters
+		Element param = GSXML.getFirstElementChild(param_list);//(Element) param_list.getFirstChild();
+		while (param != null)
+		{
+			// Identify the metadata information desired
+			if (param.getAttribute(GSXML.NAME_ATT).equals("metadata"))
+			{
+				String metadata = GSXML.getValue(param);
+				if (metadata.equals("all"))
+				{
+					all_metadata = true;
+					break;
+				}
+				metadata_names_list.add(metadata);
+			}
+			param = (Element) param.getNextSibling();
+		}
+
+		Element node_list = result_doc.createElement(GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(node_list);
+
+		// Get the nodes
+		Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (request_node_list == null)
+		{
+			logger.error(" ClassifierBrowseMetadataRetrieve request had no " + GSXML.CLASS_NODE_ELEM + GSXML.LIST_MODIFIER);
+			return result;
+		}
+
+		NodeList request_nodes = request_node_list.getChildNodes();
+		for (int i = 0; i < request_nodes.getLength(); i++)
+		{
+			Element request_node = (Element) request_nodes.item(i);
+			String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+
+			// Add the document to the results list
+			Element new_node = (Element) result_doc.importNode(request_node, false);
+			node_list.appendChild(new_node);
+
+			if (idNeedsTranslating(node_id))
+			{
+				node_id = translateId(node_id);
+			}
+			if (node_id == null)
+			{
+				continue;
+			}
+
+			Element metadata_list = getMetadataList(result_doc, node_id, all_metadata, metadata_names_list);
+			new_node.appendChild(metadata_list);
+		}
+
+		return result;
+	}
+
+	/** Creates a classifier node */
+  protected Element createClassifierNode(Document doc, String node_id)
+	{
+		Element node = doc.createElement(GSXML.CLASS_NODE_ELEM);
+		node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+		node.setAttribute(GSXML.CHILD_TYPE_ATT, getChildType(node_id));
+		return node;
+	}
+
+	/**
+	 * create an element to go into the structure. A node element has the form
+	 * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+	 */
+  protected Element createDocNode(Document doc, String node_id)
+	{
+	  return this.gs_doc.createDocNode(doc, node_id);
+	}
+
+	/**
+	 * returns the node type of the specified node. should be one of
+	 * GSXML.NODE_TYPE_LEAF, GSXML.NODE_TYPE_INTERNAL, GSXML.NODE_TYPE_ROOT
+	 */
+	protected String getNodeType(String node_id, String doc_type)
+	{
+	  return this.gs_doc.getNodeType(node_id, doc_type);
+	}
+
+	/**
+	 * adds all the children of doc_id the the doc element, and if
+	 * recursive=true, adds all their children as well
+	 */
+	protected void addDescendants(Element node, String node_id, boolean recursive)
+	{
+		ArrayList<String> child_ids = getChildrenIds(node_id);
+		String[] offsets = getMDOffsets(node_id);
+		if (child_ids == null)
+			return;
+		Document doc = node.getOwnerDocument();
+		if (offsets != null && child_ids.size() != offsets.length) {
+		  offsets = null; // something is wrong, they should be the same length
+		}
+		for (int i = 0; i < child_ids.size(); i++)
+		{
+			String child_id = child_ids.get(i);
+			Element child_elem;
+			if (isDocumentId(child_id))
+			{
+			  child_elem = createDocNode(doc, child_id);
+			  if (offsets != null) {
+			    child_elem.setAttribute("mdoffset", offsets[i]);
+			  }
+			}
+			else
+			{
+			  child_elem = createClassifierNode(doc, child_id);
+			}
+			node.appendChild(child_elem);
+			if (recursive)
+			{
+				addDescendants(child_elem, child_id, recursive);
+			}
+		}
+	}
+
+	/**
+	 * adds all the siblings of current_id to the parent element. returns the
+	 * new current element
+	 */
+	protected Element addSiblings(Element parent_node, String parent_id, String current_id)
+	{
+		Element current_node = GSXML.getFirstElementChild(parent_node);//(Element)parent_node.getFirstChild();
+		if (current_node == null)
+		{
+			// create a sensible error message
+			logger.error(" there should be a first child.");
+			return null;
+		}
+		// remove the current child,- will add it in later in its correct place
+		parent_node.removeChild(current_node);
+
+		// add in all the siblings,- might be classifier/document nodes
+		addDescendants(parent_node, parent_id, false);
+
+		// find the node that is now the current node
+		// this assumes that the new node that was created is the same as 
+		// the old one that was removed - we may want to replace the new one 
+		// with the old one.
+		Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+		return new_current;
+
+	}
+
+	/**
+	 * returns true if oid ends in .fc (firstchild), .lc (lastchild), .pr
+	 * (parent), .ns (next sibling), .ps (prev sibling), .rt (root) .ss
+	 * (specified sibling), false otherwise
+	 */
+	protected boolean idNeedsTranslating(String node_id)
+	{
+		return OID.needsTranslating(node_id);
+	}
+
+	/** returns the list of sibling ids, including the specified node_id */
+	protected ArrayList<String> getSiblingIds(String node_id)
+	{
+	  return this.gs_doc.getSiblingIds(node_id);
+	}
+
+	/** if id ends in .fc, .pc etc, then translate it to the correct id */
+	abstract protected String translateId(String node_id);
+
+	/** Gets the type of list a classifier is (e.g. VList or HList) */
+	abstract protected String getChildType(String node_id);
+
+	/**
+	 * returns the document type of the doc that the specified node belongs to.
+	 * should be one of GSXML.DOC_TYPE_SIMPLE, GSXML.DOC_TYPE_PAGED,
+	 * GSXML.DOC_TYPE_HIERARCHY
+	 */
+  protected String getDocType(String node_id) {
+    return this.gs_doc.getDocType(node_id);
+  }
+
+	/**
+	 * returns the id of the root node of the document containing node node_id.
+	 * . may be the same as node_id
+	 */
+  protected String getRootId(String node_id) {
+    return this.gs_doc.getRootId(node_id);
+  }
+
+	/** returns a list of the child ids in order, null if no children */
+  protected ArrayList<String> getChildrenIds(String node_id) {
+    return this.gs_doc.getChildrenIds(node_id);
+  }
+
+  protected String[] getMDOffsets(String node_id) {
+    
+    String offset_str =  this.gs_doc.getMetadata(node_id, "mdoffset");
+    if (offset_str.equals("")) {
+      return null;
+    }
+    return offset_str.split(";"); //, -1); // use -1 limit to get trailing empty strings
+  }
+	/** returns the node id of the parent node, null if no parent */
+  protected String getParentId(String node_id) {
+    return this.gs_doc.getParentId(node_id);
+  }
+
+	/**
+	 * returns true if the id refers to a document (rather than a classifier
+	 * node)
+	 */
+	abstract protected boolean isDocumentId(String node_id);
+
+	/**
+	 * get the metadata for the classifier node node_id returns a metadataList
+	 * element: <metadataList><metadata
+	 * name="xxx">value</metadata></metadataList> if all_metadata is true,
+	 * returns all available metadata, otherwise just returns requested metadata
+	 */
+  abstract protected Element getMetadataList(Document doc, String node_id, boolean all_metadata, ArrayList<String> metadata_names);
+
+	/**
+	 * get the particular metadata (identified by the metadata name) for the
+	 * classifier node node_id
+	 * 
+	 */
+	abstract protected String getMetadata(String node_id, String metadata_name);
+
+	/**
+	 * returns the structural information asked for. info_type may be one of
+	 * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS, INFO_DOC_TYPE
+	 */
+  protected String getStructureInfo(String node_id, String info_type) {
+    return this.gs_doc.getStructureInfo(node_id, info_type);
+  }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractDocumentRetrieve.java	(revision 31634)
@@ -0,0 +1,790 @@
+/*
+ *    AbstractDocumentRetrieve.java
+ *    a base class for retrieval services
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.AbstractBasicDocument;
+import org.greenstone.gsdl3.util.BasicDocument;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+import org.apache.log4j.*;
+
+/**
+ * Abstract class for Document Retrieval Services
+ */
+
+public abstract class AbstractDocumentRetrieve extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractDocumentRetrieve.class.getName());
+
+	// the services on offer
+	protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
+	protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+	protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
+
+	protected static final String STRUCT_PARAM = "structure";
+	protected static final String INFO_PARAM = "info";
+
+	protected static final String STRUCT_ANCESTORS = "ancestors";
+	protected static final String STRUCT_PARENT = "parent";
+	protected static final String STRUCT_SIBS = "siblings";
+	protected static final String STRUCT_CHILDREN = "children";
+	protected static final String STRUCT_DESCENDS = "descendants";
+	protected static final String STRUCT_ENTIRE = "entire";
+
+
+	protected AbstractBasicDocument gs_doc = null;
+
+	// means the id is not a greenstone id and needs translating
+	//	protected static final String EXTID_PARAM = "ext";
+
+	protected Element config_info = null; // the xml from the config file
+
+	protected String default_document_type = null;
+	protected MacroResolver macro_resolver = null;
+
+	/** does this class provide the service?? */
+	protected boolean does_metadata = true;
+	protected boolean does_content = true;
+	protected boolean does_structure = true;
+
+	protected Element highlightedNode = null;
+	/** constructor */
+	public AbstractDocumentRetrieve()
+	{
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring AbstractDocumentRetrieve...");
+		this.config_info = info;
+
+		// set up short_service_info_ - for now just has name and type
+		if (does_structure)
+		{
+			Element dsr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+			this.short_service_info.appendChild(dsr_service);
+		}
+
+		if (does_metadata)
+		{
+			Element dmr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+			this.short_service_info.appendChild(dmr_service);
+		}
+
+		if (does_content)
+		{
+			Element dcr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+			this.short_service_info.appendChild(dcr_service);
+		}
+
+		// look for document display format
+		String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+		Element display_format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (display_format != null)
+		{
+			this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.desc_doc.importNode(display_format, true));
+			// should we keep a copy?
+			// check for docType option.
+			Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+			if (doc_type_opt != null)
+			{
+				String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+				if (!value.equals(""))
+				{
+					this.default_document_type = value;
+				}
+			}
+		}
+
+		if (macro_resolver != null)
+		{
+			macro_resolver.setSiteDetails(this.site_http_address, this.cluster_name, this.getLibraryName());
+			// set up the macro resolver
+			Element replacement_elem = (Element) GSXML.getChildByTagName(extra_info, GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER);
+			if (replacement_elem != null)
+			{
+				macro_resolver.addMacros(replacement_elem);
+			}
+			// look for any refs to global replace lists
+			NodeList replace_refs_elems = extra_info.getElementsByTagName(GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER + GSXML.REF_MODIFIER);
+			for (int i = 0; i < replace_refs_elems.getLength(); i++)
+			{
+				String id = ((Element) replace_refs_elems.item(i)).getAttribute("id");
+				if (!id.equals(""))
+				{
+					Element replace_list = GSXML.getNamedElement(this.router.config_info, GSXML.REPLACE_ELEM + GSXML.LIST_MODIFIER, "id", id);
+					if (replace_list != null)
+					{
+						macro_resolver.addMacros(replace_list);
+					}
+				}
+			}
+		}
+
+		// Base line for document (might be overriden by sub-classes)
+		gs_doc = new BasicDocument(this.default_document_type);
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+
+		// these ones are probably never called, but put them here just in case
+		Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+		service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		service_elem.setAttribute(GSXML.NAME_ATT, service_id);
+		return service_elem;
+	}
+
+	protected Element processDocumentMetadataRetrieve(Element request)
+	{
+
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		if (!does_metadata)
+		{
+			// shouldn't get here
+			return result;
+		}
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, "DocumentMetadataRetrieve: missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// The metadata information required
+		ArrayList<String> metadata_names_list = new ArrayList<String>();
+		boolean all_metadata = false;
+		// Process the request parameters
+		Element param = GSXML.getFirstElementChild(param_list);//(Element) param_list.getFirstChild();
+		while (param != null)
+		{
+			// Identify the metadata information desired
+			if (param.getAttribute(GSXML.NAME_ATT).equals("metadata"))
+			{
+				String metadata = GSXML.getValue(param);
+				if (metadata.equals("all"))
+				{
+					all_metadata = true;
+				}
+				metadata_names_list.add(metadata);
+			}
+
+			param = (Element) param.getNextSibling();
+		}
+
+		// check that there has been some metadata specified
+		if (!all_metadata && metadata_names_list.size() == 0)
+		{
+			GSXML.addError(result, "DocumentMetadataRetrieve: no metadata names found in the " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the documents
+		Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (request_node_list == null)
+		{
+			GSXML.addError(result, "DocumentMetadataRetrieve: missing " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// copy the request doc node list to the response
+		Element response_node_list = (Element) result_doc.importNode(request_node_list, true);
+		result.appendChild(response_node_list);
+
+		// use the copied list so that we add the metadata into the copy
+		// are we just adding metadata for the top level nodes? or can we accept a hierarchy here???
+		NodeList doc_nodes = GSXML.getChildrenByTagName(response_node_list, GSXML.DOC_NODE_ELEM);
+		if (doc_nodes.getLength() == 0)
+		{
+			GSXML.addError(result, "DocumentMetadataRetrieve: no " + GSXML.DOC_NODE_ELEM + " found in the " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Whew, now we have checked (almost) all the syntax of the request, now we can process it.
+		for (int i = 0; i < doc_nodes.getLength(); i++)
+		{
+			Element doc_node = (Element) doc_nodes.item(i);
+			String node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+			boolean is_href_id = false;
+			if (node_id.equals(""))
+			{
+				node_id = getGreenstoneIdFromHref(doc_node);
+				if (node_id == null)
+				{
+					// **** TODO, is this good enough???
+					doc_node.setAttribute("external_link", "true");
+					continue;
+				}
+
+			}
+
+			// may have modifiers .rt, .1.ss etc
+			if (idNeedsTranslating(node_id))
+			{
+				node_id = translateId(node_id);
+			}
+
+			if (node_id == null)
+			{
+				continue;
+			}
+			try
+			{
+			  Element metadata_list = getMetadataList(result_doc, node_id, all_metadata, metadata_names_list, lang);
+				if (metadata_list != null)
+				{
+					doc_node.appendChild(metadata_list);
+				}
+			}
+			catch (GSException e)
+			{
+				GSXML.addError(result, e.getMessage(), e.getType());
+				if (e.getType().equals(GSXML.ERROR_TYPE_SYSTEM))
+				{
+					// there is no point trying any others
+					return result;
+				}
+			}
+
+		} // for each doc node
+
+		return result;
+	}
+
+	protected Element processDocumentStructureRetrieve(Element request)
+	{
+
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		if (!does_structure)
+		{
+			// shouldn't get here
+			return result;
+		}
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, "DocumentStructureRetrieve: missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// get the documents of the request
+		Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (query_doc_list == null)
+		{
+			GSXML.addError(result, "DocumentStructureRetrieve: missing " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// copy the doc_list to the response
+		Element response_node_list = (Element) result_doc.importNode(query_doc_list, true);
+		result.appendChild(response_node_list);
+
+		// check that we have some doc nodes specified
+		NodeList node_list = GSXML.getChildrenByTagName(response_node_list, GSXML.DOC_NODE_ELEM);
+		if (node_list.getLength() == 0)
+		{
+			GSXML.addError(result, "DocumentStructureRetrieve: no " + GSXML.DOC_NODE_ELEM + " found in the " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// the type of info required
+		boolean want_structure = false;
+		boolean want_info = false;
+
+		ArrayList<String> info_types = new ArrayList<String>();
+		// The document structure information desired
+		boolean want_ancestors = false;
+		boolean want_parent = false;
+		boolean want_siblings = false;
+		boolean want_children = false;
+		boolean want_descendants = false;
+
+		boolean want_entire_structure = false;
+		// Process the request parameters
+		NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+		for (int i = 0; i < params.getLength(); i++)
+		{
+
+			Element param = (Element) params.item(i);
+			String p_name = param.getAttribute(GSXML.NAME_ATT);
+			String p_value = GSXML.getValue(param);
+			// Identify the structure information desired
+			if (p_name.equals(STRUCT_PARAM))
+			{
+				want_structure = true;
+
+				// This is NOT locale sensitive
+				if (p_value.equals(STRUCT_ANCESTORS))
+					want_ancestors = true;
+				else if (p_value.equals(STRUCT_PARENT))
+					want_parent = true;
+				else if (p_value.equals(STRUCT_SIBS))
+					want_siblings = true;
+				else if (p_value.equals(STRUCT_CHILDREN))
+					want_children = true;
+				else if (p_value.equals(STRUCT_DESCENDS))
+					want_descendants = true;
+				else if (p_value.equals(STRUCT_ENTIRE))
+					want_entire_structure = true;
+				else
+					logger.error("AbstractDocumentRetrieve Warning: Unknown value \"" + p_value + "\".");
+			}
+			else if (p_name.equals(INFO_PARAM))
+			{
+				want_info = true;
+				info_types.add(p_value);
+			}
+		}
+
+		// Make sure there is no repeated information
+		if (want_ancestors)
+			want_parent = false;
+		if (want_descendants)
+			want_children = false;
+
+		// for each of the doc nodes, get the required info
+		// these nodes are part of result_doc - can use that to create Elements
+		for (int i = 0; i < node_list.getLength(); i++)
+		{
+			Element doc_node = (Element) node_list.item(i);
+
+			String doc_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+			boolean is_href_id = false;
+			if (doc_id.equals(""))
+			{
+				doc_id = getGreenstoneIdFromHref(doc_node);
+				if (doc_id == null)
+				{
+					// **** TODO, is this good enough???
+					doc_node.setAttribute("external_link", "true");
+					continue;
+				}
+				doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+			}
+
+			if (idNeedsTranslating(doc_id))
+			{
+				doc_id = translateId(doc_id);
+				doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+			}
+
+			if (doc_id == null)
+			{
+				continue;
+			}
+
+			if (want_info)
+			{
+			  Element node_info_elem = result_doc.createElement("nodeStructureInfo");
+				doc_node.appendChild(node_info_elem);
+
+				for (int j = 0; j < info_types.size(); j++)
+				{
+					String info_type = info_types.get(j);
+					String info_value = getStructureInfo(doc_id, info_type);
+					if (info_value != null)
+					{
+						Element info_elem = result_doc.createElement("info");
+						info_elem.setAttribute(GSXML.NAME_ATT, info_type);
+						info_elem.setAttribute(GSXML.VALUE_ATT, info_value);
+						node_info_elem.appendChild(info_elem);
+					}
+				}
+			}
+
+			if (want_structure)
+			{
+				// all structure info goes into a nodeStructure elem
+				Element structure_elem = result_doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+				doc_node.appendChild(structure_elem);
+
+				if (want_entire_structure)
+				{
+					String root_id = getRootId(doc_id);
+					Element root_node = createDocNode(result_doc, root_id); //, true, false);
+					addDescendants(root_node, root_id, true);
+					structure_elem.appendChild(root_node);
+					continue; // with the next document, we dont need to do any more here
+				}
+
+				// Add the requested structure information
+				Element base_node = createDocNode(result_doc, doc_id); //, false, false);
+
+				//Ancestors: continually add parent nodes until the root is reached
+				Element top_node = base_node; // the top node so far
+				if (want_ancestors)
+				{
+					String current_id = doc_id;
+					while (true)
+					{
+						String parent_id = getParentId(current_id);
+						//Element parent = getParent(current_id);
+						if (parent_id == null)
+							break; // no parent
+						Element parent_node = createDocNode(result_doc, parent_id);
+						parent_node.appendChild(top_node);
+						current_id = parent_id;//.getAttribute(GSXML.NODE_ID_ATT);
+						top_node = parent_node;
+					}
+				}
+				// Parent: get the parent of the selected node
+				else if (want_parent)
+				{
+					String parent_id = getParentId(doc_id);
+					if (parent_id != null)
+					{
+					  Element parent_node = createDocNode(result_doc, parent_id);
+						parent_node.appendChild(base_node);
+						top_node = parent_node;
+					}
+				}
+
+				// now the top node is the root of the structure
+				structure_elem.appendChild(top_node);
+
+				//Siblings: get the other descendants of the selected node's parent
+				if (want_siblings)
+				{
+					String parent_id = getParentId(doc_id);
+					if (parent_id != null)
+					{
+						// if parent == current id, then we are at the top 
+						// and can't get siblings
+						Element parent_node = (Element) base_node.getParentNode(); // this may be the structure element if there has been no request for parents or ancestors
+
+						// add siblings, - returns a pointer to the new current node 
+						base_node = addSiblings(parent_node, parent_id, doc_id);
+					}
+
+				}
+
+				// Children: get the descendants, but only one level deep
+				if (want_children)
+				{
+					addDescendants(base_node, doc_id, false);
+				}
+				// Descendants: recursively get every descendant 
+				else if (want_descendants)
+				{
+					addDescendants(base_node, doc_id, true);
+				}
+			} // if want structure
+
+		} // for each doc
+		return result;
+	}
+
+	/** Retrieve the content of a document */
+	protected Element processDocumentContentRetrieve(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		//import highlighted node to result
+		highlightedNode = (Element) GSXML.getChildByTagName(request, GSXML.NODE_CONTENT_ELEM);
+		
+		if (!does_content)
+		{
+			// shouldn't get here
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		// Get the request content 
+		Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (query_doc_list == null)
+		{
+			logger.error("Error: DocumentContentRetrieve request specified no doc nodes.\n");
+			return result;
+		}
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+
+		// copy the request doc node list to the response
+		Element response_node_list = (Element) result_doc.importNode(query_doc_list, true);
+		result.appendChild(response_node_list);
+
+		NodeList doc_nodes = GSXML.getChildrenByTagName(response_node_list, GSXML.DOC_NODE_ELEM);
+		if (doc_nodes.getLength() == 0)
+		{
+			GSXML.addError(result, "DocumentContentRetrieve: no " + GSXML.DOC_NODE_ELEM + " found in the " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		//Element doc_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		//result.appendChild(doc_list);
+
+		// set up the retrieval??
+
+		// Get the documents
+		//String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, GSXML.NODE_ID_ATT);
+		//String[] is_externals = GSXML.getAttributeValuesFromList(query_doc_list, "externalURL");
+
+		for (int i = 0; i < doc_nodes.getLength(); i++)
+		{
+			Element doc_node = (Element) doc_nodes.item(i);
+			String node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+			
+			if (node_id.equals(""))
+			{
+				node_id = getGreenstoneIdFromHref(doc_node);
+				if (node_id == null)
+				{
+					// **** TODO, is this good enough???
+					doc_node.setAttribute("external_link", "true");
+					continue;
+				}
+				doc_node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+			}
+
+			// may have modifiers .rt, .1.ss etc
+			if (idNeedsTranslating(node_id))
+			{
+				node_id = translateId(node_id);
+			}
+
+			if (node_id == null)
+			{
+				continue;
+			}
+			try
+			{
+			  Element node_content = getNodeContent(result_doc, node_id, lang);
+				if (node_content != null)
+				{
+				  doc_node.appendChild(node_content);
+				}
+			}
+			catch (GSException e)
+			{
+				GSXML.addError(result, e.getMessage());
+				return result;
+			}
+		} // for each node
+		return result;
+	} // processDocumentContentRetrieve
+
+	/**
+	 * create an element to go into the structure. A node element has the form
+	 * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+	 */
+  protected Element createDocNode(Document doc, String node_id)
+	{
+	  return this.gs_doc.createDocNode(doc, node_id);
+	}
+
+	/**
+	 * adds all the children of doc_id the the doc element, and if
+	 * recursive=true, adds all their children as well
+	 */
+	protected void addDescendants(Element doc, String doc_id, boolean recursive)
+	{
+	  this.gs_doc.addDescendants(doc, doc_id, recursive);
+	}
+
+	/**
+	 * adds all the siblings of current_id to the parent element. returns the
+	 * new current element
+	 */
+	protected Element addSiblings(Element parent_node, String parent_id, String current_id)
+	{
+	  return this.gs_doc.addSiblings(parent_node, parent_id, current_id);
+	}
+
+	/**
+	 * returns true if oid ends in .fc (firstchild), .lc (lastchild), .pr
+	 * (parent), .ns (next sibling), .ps (prev sibling), .rt (root) .ss
+	 * (specified sibling), false otherwise
+	 */
+	protected boolean idNeedsTranslating(String id)
+	{
+		return OID.needsTranslating(id);
+	}
+
+	/** returns the list of sibling ids, including the specified node_id */
+	protected ArrayList<String> getSiblingIds(String node_id)
+	{
+		String parent_id = getParentId(node_id);
+		if (parent_id == null)
+		{
+			return null;
+		}
+		return getChildrenIds(parent_id);
+
+	}
+
+	/**
+	 * returns the node type of the specified node. should be one of
+	 * GSXML.NODE_TYPE_LEAF, GSXML.NODE_TYPE_INTERNAL, GSXML.NODE_TYPE_ROOT
+	 */
+	protected String getNodeType(String node_id, String doc_type)
+	{
+	return this.gs_doc.getNodeType(node_id, doc_type);
+	}
+
+	/**
+	 * if id ends in .fc, .pc etc, then translate it to the correct id 
+	 * default implementation: just remove the suffix
+	 */
+	protected String translateId(String id)
+	{
+		return id.substring(0, id.length());
+	}
+
+	/**
+	 * if an id is not a greenstone id (an external id) then translate it to a
+	 * greenstone one default implementation: return the id
+	 */
+	protected String translateExternalId(String id)
+	{
+		return id;
+	}
+
+	protected String getGreenstoneIdFromHref(Element doc_node)
+	{
+		String node_id = doc_node.getAttribute(GSXML.HREF_ID_ATT);
+		node_id = translateExternalId(node_id);
+		if (node_id == null)
+		{
+			return node_id;
+		}
+		// check for id modifiers
+		String id_mods = doc_node.getAttribute(GSXML.ID_MOD_ATT);
+		if (!id_mods.equals(""))
+		{
+			node_id = node_id + id_mods;
+		}
+		return node_id;
+	}
+
+	/**
+	 * returns the document type of the doc that the specified node belongs to.
+	 * should be one of GSXML.DOC_TYPE_SIMPLE, GSXML.DOC_TYPE_PAGED,
+	 * GSXML.DOC_TYPE_HIERARCHY
+	 */
+	protected String getDocType(String node_id)
+	{
+		return this.gs_doc.getDocType(node_id);
+	}
+
+	/**
+	 * returns the id of the root node of the document containing node node_id.
+	 * may be the same as node_id 
+	 */
+	protected String getRootId(String node_id)
+	{
+		return  this.gs_doc.getRootId(node_id);
+	}
+
+	/**
+	 * returns a list of the child ids in order, null if no children 
+	 */
+	protected ArrayList<String> getChildrenIds(String node_id)
+	{
+	  return this.gs_doc.getChildrenIds(node_id);
+	}
+
+	/**
+	 * returns the node id of the parent node, null if no parent 
+	 */
+	protected String getParentId(String node_id)
+	{
+	  return this.gs_doc.getParentId(node_id);
+	}
+
+	/**
+	 * get the metadata for the doc node doc_id returns a metadataList element:
+	 * <metadataList><metadata name="xxx">value</metadata></metadataList>
+	 */
+  abstract protected Element getMetadataList(Document doc, String doc_id, boolean all_metadata, ArrayList<String> metadata_names, String lang) throws GSException;
+
+	/**
+	 * returns the content of a node should return a nodeContent element:
+	 * <nodeContent>text content or other elements</nodeContent> can return
+	 */
+  abstract protected Element getNodeContent(Document doc, String doc_id, String lang) throws GSException;
+
+	/**
+	 * returns the structural information asked for. info_type may be one of
+	 * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS
+	 */
+  protected String getStructureInfo(String doc_id, String info_type) {
+
+    return this.gs_doc.getStructureInfo(doc_id, info_type);
+  }
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2DocumentRetrieve.java	(revision 31634)
@@ -0,0 +1,365 @@
+/*
+ *    AbstractGS2DocumentRetrieve.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.BasicDocumentDatabase;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSConstants;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.DBInfo;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+
+import org.apache.log4j.*;
+
+// Apache Commons
+import org.apache.commons.lang3.*;
+
+/**
+ * Implements the generic retrieval and classifier services for GS2 collections.
+ * 
+ */
+
+public abstract class AbstractGS2DocumentRetrieve extends AbstractDocumentRetrieve
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractGS2DocumentRetrieve.class.getName());
+
+	//    protected static final String EXTLINK_PARAM = "ext"; here or in base??    
+	protected String index_stem = null;
+
+	protected SimpleCollectionDatabase coll_db = null;
+  BasicDocumentDatabase gs_doc_db = null;
+	/** constructor */
+	protected AbstractGS2DocumentRetrieve()
+	{
+		this.macro_resolver = new GS2MacroResolver();
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+		this.coll_db.closeDatabase();
+		this.gs_doc_db.cleanUp();
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring AbstractGS2DocumentRetrieve...");
+		//this.config_info = info;
+
+		// the index stem is either specified in the config file or is  the collection name
+		Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+		if (index_stem_elem != null)
+		{
+			this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (this.index_stem == null || this.index_stem.equals(""))
+		{
+			logger.error("AbstractGS2DocumentRetrieve.configure(): indexStem element not found, stem will default to collection name");
+			this.index_stem = this.cluster_name;
+		}
+
+		// find out what kind of database we have
+		Element database_type_elem = (Element) GSXML.getChildByTagName(info, GSXML.DATABASE_TYPE_ELEM);
+		String database_type = null;
+		if (database_type_elem != null)
+		{
+			database_type = database_type_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (database_type == null || database_type.equals(""))
+		{
+			database_type = "gdbm"; // the default
+		}
+		coll_db = new SimpleCollectionDatabase(database_type);
+		if (!coll_db.databaseOK())
+		{
+			logger.error("Couldn't create the collection database of type " + database_type);
+			return false;
+		}
+
+		// Open database for querying
+		String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, this.index_stem, database_type);
+		if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ))
+		{
+			logger.error("Could not open collection database!");
+			return false;
+		}
+
+		gs_doc_db = new BasicDocumentDatabase(database_type, this.site_home, this.cluster_name, this.index_stem);
+		if (!gs_doc_db.isValid())
+		{
+			logger.error("Failed to open Document Database.");
+			return false;
+		}
+		this.gs_doc = gs_doc_db;
+
+		// we need to set the database for our GS2 macro resolver
+		GS2MacroResolver gs2_macro_resolver = (GS2MacroResolver) this.macro_resolver;
+		gs2_macro_resolver.setDB(this.coll_db);
+		// set the class loader in case we have collection specific properties files
+		gs2_macro_resolver.setClassLoader(this.class_loader);
+		return true;
+	}
+
+	/** if id ends in .fc, .pc etc, then translate it to the correct id */
+	protected String translateId(String node_id)
+	{
+		return OID.translateOID(this.coll_db, node_id); //return this.coll_db.translateOID(node_id);
+	}
+
+	/**
+	 * if an id is not a greenstone id (an external id) then translate it to a
+	 * greenstone one
+	 */
+	protected String translateExternalId(String node_id)
+	{
+		return this.coll_db.externalId2OID(node_id);
+	}
+
+	/**
+	 * returns the id of the root node of the document containing node node_id.
+	 * . may be the same as node_id
+	 */
+	protected String getRootId(String node_id)
+	{
+	  return this.gs_doc.getRootId(node_id);
+	}
+
+
+
+	/**
+	 * get the metadata for the classifier node node_id returns a metadataList
+	 * element: <metadataList><metadata
+	 * name="xxx">value</metadata></metadataList>
+	 */
+  protected Element getMetadataList(Document doc, String node_id, boolean all_metadata, ArrayList<String> metadata_names, String lang) throws GSException
+	{
+		Element metadata_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return null;
+		}
+		
+		if (all_metadata) // this will get all metadata for current node
+		{
+			// return everything out of the database
+			Set<String> keys = info.getKeys();
+			Iterator<String> it = keys.iterator();
+			while (it.hasNext())
+			{
+				String key = it.next();
+				//String value = info.getInfo(key);
+				Vector<String> values = info.getMultiInfo(key);
+				for (int i = 0; i < values.size(); i++)
+				{
+				  GSXML.addMetadata(metadata_list, key, this.macro_resolver.resolve(values.elementAt(i), lang, MacroResolver.SCOPE_META, node_id));
+				}
+			}
+
+		}
+		// now we go through the list of names. If we have specified 
+		// all_metadata, then here we only get the ones like 
+		// parent_Title, that are not the current node.
+		for (int i = 0; i < metadata_names.size(); i++)
+		  {
+		    String meta_name = metadata_names.get(i);
+		    
+		    if (!all_metadata || meta_name.indexOf(GSConstants.META_RELATION_SEP)!=-1) {
+		    Vector <String> values = getMetadata(node_id, info, meta_name, lang);
+		    if (values != null) {
+		      for (int j = 0; j < values.size(); j++)
+			{
+			  // some of these may be parent/ancestor. does resolve need a different id???
+			  GSXML.addMetadata(metadata_list, meta_name,  this.macro_resolver.resolve(values.elementAt(j), lang, MacroResolver.SCOPE_META, node_id));
+			}
+		    }
+		    }
+		  }
+		
+		return metadata_list;
+	}
+
+  protected Vector<String> getMetadata(String node_id, DBInfo info, String metadata, String lang) {
+
+    DBInfo current_info = info;
+    
+    int index = metadata.indexOf(GSConstants.META_RELATION_SEP);
+    if (index == -1) {
+      // metadata is for this node
+      return info.getMultiInfo(metadata);
+    }
+    // we need to get metadata for one or more different nodes
+    // we have a relation root, parent, ancestors, siblings, children, descendents
+    String relation = metadata.substring(0, index);
+    String relation_id="";
+    metadata = metadata.substring(index + 1);
+    if (relation.equals("root")) {
+      relation_id = OID.getTop(node_id);
+      if (relation_id.equals(node_id)) {
+	// use the current node info
+	return info.getMultiInfo(metadata);
+      } else {
+	return getMetaValuesForOID(relation_id, metadata);
+      }
+    }
+    if (relation.equals("parent")) {
+      relation_id = OID.getParent(node_id);
+      if (relation_id.equals(node_id)) {
+	// no parent
+	return null;
+      }
+      return getMetaValuesForOID(relation_id, metadata);
+    }
+
+    if (relation.equals("ancestors")) {
+      if (OID.isTop(node_id)) {
+	return null;
+      }
+      String current_id = node_id;
+      relation_id = OID.getParent(current_id);
+      Vector<String> values = new Vector<String>();
+      while (!relation_id.equals(current_id)) {
+	
+	Vector<String> more_values = getMetaValuesForOID(relation_id, metadata);
+	if (more_values != null) {
+	  values.addAll(0, more_values);
+	}
+	current_id = relation_id;
+	relation_id = OID.getParent(current_id);
+      }
+      return values;
+    }
+    if (relation.equals("siblings")) {
+      String parent_id = OID.getParent(node_id);
+      if (parent_id.equals(node_id)) {
+	// no parent, therefore no siblings
+	return null;
+      }
+      // siblings is the same as asking for children of the parent
+      node_id = parent_id;
+      relation = "children";
+      current_info = this.coll_db.getInfo(parent_id);
+      if (current_info == null) {
+	return null;
+      }
+    }
+    if (relation.equals("children")) {
+      Vector<String> values = new Vector<String>();
+      String contains = current_info.getInfo("contains");
+      contains = StringUtils.replace(contains, "\"", node_id);
+      String[] children = contains.split(";"); 
+      for (int i = 0; i < children.length; i++) {
+	
+	String child_id = children[i];
+	Vector<String> more_values = getMetaValuesForOID(child_id, metadata);
+	if (more_values != null) {
+	  values.addAll(more_values);
+	}
+      }
+      return values;
+    } 
+    if (relation.equals("descendents")) {
+      return null;
+    }
+    // unknown relation
+    logger.error("asked for relation "+relation+" and don't understand it, so getting the full metadata name "+relation+GSConstants.META_RELATION_SEP+metadata);
+    // we assume that maybe the metadata has an _ in the name, so get the original name
+    return info.getMultiInfo(relation+GSConstants.META_RELATION_SEP+metadata);
+
+  }
+      
+
+    protected Vector<String> getMetaValuesForOID(String oid, String metadata) {
+      DBInfo info = this.coll_db.getInfo(oid);
+      if (info == null) {
+	return null;
+      }
+      
+      Vector<String> values = info.getMultiInfo(metadata);
+      // lets look through the values and look for [xxx] things. We need to look up metadata for them.
+      if (values == null) { return values; }
+      
+      for (int j = 0; j < values.size(); j++) {
+	String val = values.elementAt(j);
+	if (val.contains("[")) {
+	  // look for metadata refs
+	  String [] metas = StringUtils.substringsBetween(val, "[", "]");
+	  for (int i=0; i<metas.length; i++) {
+	    String meta = metas[i];
+	    String meta_val = info.getInfo(meta);
+	    if (!meta_val.equals("")) {
+	      val = StringUtils.replace(val,"["+meta+"]",meta_val);
+	    }
+	  }
+	  values.set(j,val);
+	}
+      }
+      return values;
+    }
+	protected int getNumChildren(String node_id)
+	{
+	  return this.gs_doc.getNumChildren(node_id);
+	}
+
+
+	/**
+	 * returns the content of a node should return a nodeContent element:
+	 * <nodeContent>text content or other elements</nodeContent>
+	 */
+  abstract protected Element getNodeContent(Document doc, String doc_id, String lang) throws GSException;
+
+
+	/**
+	 * needs to get info from collection database - if the calling code gets it
+	 * already it may pay to pass it in instead
+	 */
+	protected String resolveTextMacros(String doc_content, String doc_id, String lang)
+	{
+		// resolve any collection specific macros
+		doc_content = macro_resolver.resolve(doc_content, lang, MacroResolver.SCOPE_TEXT, doc_id);
+		return doc_content;
+	}
+
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2FieldSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2FieldSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2FieldSearch.java	(revision 31634)
@@ -0,0 +1,958 @@
+/*
+ *    AbstractGS2FieldSearch.java
+ *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.FacetWrapper;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+abstract public class AbstractGS2FieldSearch extends AbstractGS2TextSearch
+{
+	// extra services offered by mgpp collections
+	protected static final String FIELD_QUERY_SERVICE = "FieldQuery";
+	protected static final String ADVANCED_FIELD_QUERY_SERVICE = "AdvancedFieldQuery";
+	protected static final String RAW_QUERY_SERVICE = "RawQuery";
+
+	// extra parameters used
+	protected static final String LEVEL_PARAM = "level";
+	protected static final String RANK_PARAM = "sortBy";
+	protected static final String RANK_PARAM_RANK = "1";
+	protected static final String RANK_PARAM_NONE = "0";
+	protected static final String SIMPLE_FIELD_PARAM = "simpleField";
+	protected static final String ADVANCED_FIELD_PARAM = "complexField";
+
+	protected static final String RAW_PARAM = "rawquery";
+
+	// more params for field query
+	protected static final String FIELD_QUERY_PARAM = "fqv";
+	protected static final String FIELD_STEM_PARAM = "fqs";
+	protected static final String FIELD_CASE_PARAM = "fqc";
+	protected static final String FIELD_ACCENT_PARAM = "fqa";
+	protected static final String FIELD_FIELD_PARAM = "fqf";
+	protected static final String FIELD_COMBINE_PARAM = "fqk";
+	protected static final String FIELD_COMBINE_PARAM_AND = "0";
+	protected static final String FIELD_COMBINE_PARAM_OR = "1";
+	protected static final String FIELD_COMBINE_PARAM_NOT = "2";
+
+	// some stuff for config files
+	protected static final String SEARCH_TYPE_ELEM = "searchType";
+	protected static final String SEARCH_TYPE_PLAIN = "plain";
+	protected static final String SEARCH_TYPE_SIMPLE_FORM = "simpleform";
+	protected static final String SEARCH_TYPE_ADVANCED_FORM = "advancedform";
+	protected static final String SEARCH_TYPE_RAW = "raw";
+
+	protected static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+	protected static final String DEFAULT_DB_LEVEL_ELEM = "defaultDBLevel";
+	protected static final String LEVEL_ELEM = "level";
+	protected static final String FIELD_ATT = "field";
+
+	protected static final int TEXT_QUERY = 0;
+	protected static final int SIMPLE_QUERY = 1;
+	protected static final int ADVANCED_QUERY = 2;
+	protected static final int RAW_QUERY = 3;
+
+	protected String AND_OPERATOR = "&";
+	protected String OR_OPERATOR = "|";
+	protected String NOT_OPERATOR = "!";
+
+	// the default level for searching
+	protected String default_level = null;
+	// default level for collection db
+	protected String default_db_level = null;
+        // metadata for service, such as does_paging = true
+        protected Element service_metadata_list = null;
+	// which search services will we offer??
+	protected boolean plain_search = false;
+	protected boolean simple_form_search = false;
+	protected boolean advanced_form_search = false;
+	protected boolean raw_search = false;
+	//Parameters to get full field with highlighted terms
+	protected String hldocOID = null;
+	//index Field for highlighting
+	protected String indexField = null;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractGS2FieldSearch.class.getName());
+
+	/** constructor */
+	public AbstractGS2FieldSearch()
+	{
+	  super();
+	  paramDefaults.put(RANK_PARAM, RANK_PARAM_RANK);
+	  paramDefaults.put(FIELD_COMBINE_PARAM, FIELD_COMBINE_PARAM_AND);
+	  // paramDefaults.put(FIELD_CASE_PARAM, BOOLEAN_PARAM_ON);
+	  // paramDefaults.put(FIELD_STEM_PARAM, BOOLEAN_PARAM_OFF);
+	  // paramDefaults.put(FIELD_ACCENT_PARAM, BOOLEAN_PARAM_ON);
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+		
+		// Get the default level out of <defaultLevel> (buildConfig.xml)
+		Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+		if (def != null)
+		{
+			this.default_level = def.getAttribute(GSXML.SHORTNAME_ATT);
+		}
+		if (this.default_level == null || this.default_level.equals(""))
+		{
+			logger.error("default level not specified!, assuming Doc");
+			this.default_level = "Doc";
+		}
+
+		// Get the default DB level
+		def = (Element) GSXML.getChildByTagName(info, DEFAULT_DB_LEVEL_ELEM);
+		if (def != null)
+		{
+			this.default_db_level = def.getAttribute(GSXML.SHORTNAME_ATT);
+		}
+		if (this.default_db_level == null || this.default_db_level.equals(""))
+		{
+			logger.error("default database level (defaultDBLevel) not specified!, assuming Sec");
+			this.default_db_level = "Sec";
+		}
+
+		// get stuff from from extra info (which is collectionConfig.xml)
+		if (extra_info != null)
+		{
+
+			// the search element
+			Element config_search = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+
+			NodeList search_types = config_search.getElementsByTagName(SEARCH_TYPE_ELEM);
+			if (search_types == null)
+			{
+				// none specified, assume plain only
+				this.plain_search = true;
+			}
+			else
+			{
+				for (int i = 0; i < search_types.getLength(); i++)
+				{
+					Element t = (Element) search_types.item(i);
+
+					String type_name = t.getAttribute(GSXML.NAME_ATT);
+					if (type_name.equals(SEARCH_TYPE_PLAIN))
+					{
+						this.plain_search = true;
+					}
+					else if (type_name.equals(SEARCH_TYPE_SIMPLE_FORM))
+					{
+						this.simple_form_search = true;
+					}
+					else if (type_name.equals(SEARCH_TYPE_ADVANCED_FORM))
+					{
+						this.advanced_form_search = true;
+					}
+					else if (type_name.equals(SEARCH_TYPE_RAW))
+					{
+						this.raw_search = true;
+					}
+				}
+			}
+
+			// AbstractGS2TextSearch has set up the TextQuery service, but we may not want it
+			if (!this.plain_search)
+			{
+				// need to remove the TextQuery service
+				Element tq_service = GSXML.getNamedElement(short_service_info, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, QUERY_SERVICE);
+				short_service_info.removeChild(tq_service);
+			}
+
+			Document owner = info.getOwnerDocument();
+
+			NodeList levels = info.getElementsByTagName(GSXML.LEVEL_ELEM);
+
+			for (int i = 0; i < levels.getLength(); i++)
+			{
+				Element lev = (Element) levels.item(i);
+				String name = lev.getAttribute(GSXML.NAME_ATT);
+				Element node_extra = GSXML.getNamedElement(config_search, GSXML.LEVEL_ELEM, GSXML.NAME_ATT, name);
+				if (node_extra == null)
+				{
+					logger.error("haven't found extra info for level named " + name);
+					continue;
+				}
+
+				// get the display elements if any - displayName
+				NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+				if (display_names != null)
+				{
+					for (int j = 0; j < display_names.getLength(); j++)
+					{
+						Element e = (Element) display_names.item(j);
+						lev.appendChild(owner.importNode(e, true));
+					}
+				}
+			} // for each level
+		}
+		else
+		{
+			// for soem reason we don't have the collectionConfig file. assume plain only
+			this.plain_search = true;
+		}
+
+		// the format info is the same for all services
+		Element format_info = (Element) format_info_map.get(QUERY_SERVICE);
+
+		// set up the extra services which are available for this collection
+		if (this.simple_form_search)
+		{
+			// set up short_service_info_ - for now just has id and type - name will be added in on the fly
+			Element fq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			fq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			fq_service.setAttribute(GSXML.NAME_ATT, FIELD_QUERY_SERVICE);
+			this.short_service_info.appendChild(fq_service);
+
+			if (format_info != null)
+			{
+				this.format_info_map.put(FIELD_QUERY_SERVICE, format_info);
+			}
+		}
+
+		if (this.advanced_form_search)
+		{
+			Element afq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			afq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			afq_service.setAttribute(GSXML.NAME_ATT, ADVANCED_FIELD_QUERY_SERVICE);
+			this.short_service_info.appendChild(afq_service);
+
+			if (format_info != null)
+			{
+				this.format_info_map.put(ADVANCED_FIELD_QUERY_SERVICE, format_info);
+			}
+		}
+
+		if (this.raw_search)
+		{
+			Element rq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			rq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			rq_service.setAttribute(GSXML.NAME_ATT, RAW_QUERY_SERVICE);
+			this.short_service_info.appendChild(rq_service);
+
+			if (format_info != null)
+			{
+				this.format_info_map.put(RAW_QUERY_SERVICE, format_info);
+			}
+		}
+
+		return true;
+	}
+
+  // if field case/stem/accent defaults are not specified explicitly, we should use case/stem/accent defaults
+  protected void getSearchParamDefaults(Element search_elem) {
+    
+    boolean found_f_stem = false;
+    boolean found_f_case = false;
+    boolean found_f_accent = false;
+    NodeList param_defaults_list = GSXML.getChildrenByTagName(search_elem, GSXML.PARAM_DEFAULT_ELEM);
+    for (int i=0; i<param_defaults_list.getLength(); i++) {
+      Element paramdef = (Element)param_defaults_list.item(i);
+      String name = paramdef.getAttribute(GSXML.NAME_ATT);
+      String val = paramdef.getAttribute(GSXML.VALUE_ATT);
+      if (!name.equals("") && !val.equals("")) {
+	paramDefaults.put(name, val);
+	if (name.equals(FIELD_STEM_PARAM)) {
+	  found_f_stem = true;
+	} else if (name.equals(FIELD_CASE_PARAM)) {
+	  found_f_case = true;
+	} else if (name.equals(FIELD_ACCENT_PARAM)) {
+	  found_f_accent = true;
+	} 
+	if (!found_f_stem) {
+	  paramDefaults.put (FIELD_STEM_PARAM, paramDefaults.get(STEM_PARAM));
+	}
+	if (!found_f_case) {
+	  paramDefaults.put (FIELD_CASE_PARAM, paramDefaults.get(CASE_PARAM));
+	}
+	if (!found_f_accent) {
+	  paramDefaults.put (FIELD_ACCENT_PARAM, paramDefaults.get(ACCENT_PARAM));
+	}
+
+      }
+    }
+  }
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		// should we check that the service is actually on offer? presumably we wont get asked for services that we haven't advertised previously.
+
+		if (!service_id.equals(FIELD_QUERY_SERVICE) && !service_id.equals(ADVANCED_FIELD_QUERY_SERVICE) && !service_id.equals(RAW_QUERY_SERVICE))
+		{
+		  return super.getServiceDescription(doc, service_id, lang, subset);
+		}
+
+		Element service = doc.createElement(GSXML.SERVICE_ELEM);
+		service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		service.setAttribute(GSXML.NAME_ATT, service_id);
+		if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+		{
+			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service_id + ".name", lang)));
+			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service_id + ".submit", lang)));
+			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service_id + ".description", lang)));
+
+		}
+		if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+		{
+			Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			service.appendChild(param_list);
+			if (service_id.equals(FIELD_QUERY_SERVICE))
+			{
+
+				addCustomQueryParams(param_list, lang);
+				if (!default_index_subcollection.equals(""))
+				{
+					createParameter(INDEX_SUBCOLLECTION_PARAM, param_list, lang);
+				}
+				if (!default_index_language.equals(""))
+				{
+					createParameter(INDEX_LANGUAGE_PARAM, param_list, lang);
+				}
+
+				if (does_chunking) {
+				  createParameter(MAXDOCS_PARAM, param_list, lang);
+				}
+				if (does_paging)
+				{
+					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
+					createParameter(START_PAGE_PARAM, param_list, lang);
+				}
+
+				// create a multi param for the fields etc
+				// text box, field
+				Element multiparam = null;
+				Element param = null;
+				multiparam = GSXML.createParameterDescription(doc, SIMPLE_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+				multiparam.setAttribute("occurs", "4");
+				param_list.appendChild(multiparam);
+
+				// the components
+
+				createParameter(FIELD_QUERY_PARAM, multiparam, lang);
+				createParameter(FIELD_FIELD_PARAM, multiparam, lang);
+
+			}
+			else if (service_id.equals(ADVANCED_FIELD_QUERY_SERVICE))
+			{
+				addCustomQueryParamsAdvField(param_list, lang);
+				if (!default_index_subcollection.equals(""))
+				{
+					createParameter(INDEX_SUBCOLLECTION_PARAM, param_list, lang);
+				}
+				if (!default_index_language.equals(""))
+				{
+					createParameter(INDEX_LANGUAGE_PARAM, param_list, lang);
+				}
+
+				// create a multi param for the fields etc
+				// text box, stem, case, field
+
+				Element multiparam = null;
+				Element param = null;
+
+				multiparam = GSXML.createParameterDescription(doc, ADVANCED_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
+				multiparam.setAttribute("occurs", "4");
+				param_list.appendChild(multiparam);
+
+				createParameter(FIELD_COMBINE_PARAM, multiparam, lang);
+				createParameter(FIELD_QUERY_PARAM, multiparam, lang);
+				if (this.does_case)
+				{
+					createParameter(FIELD_CASE_PARAM, multiparam, lang);
+				}
+				if (this.does_stem)
+				{
+					createParameter(FIELD_STEM_PARAM, multiparam, lang);
+				}
+				if (this.does_accent)
+				{
+					createParameter(FIELD_ACCENT_PARAM, multiparam, lang);
+				}
+				if (does_chunking) {
+				  createParameter(MAXDOCS_PARAM, param_list, lang);
+				}
+				if (does_paging)
+				{
+					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
+					createParameter(START_PAGE_PARAM, param_list, lang);
+				}
+				createParameter(FIELD_FIELD_PARAM, multiparam, lang);
+
+			}
+			else if (service_id.equals(RAW_QUERY_SERVICE))
+			{
+				if (does_chunking)
+				{
+					createParameter(MAXDOCS_PARAM, param_list, lang);
+				}
+				if (does_paging)
+				{
+					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
+					createParameter(START_PAGE_PARAM, param_list, lang);
+				}
+				createParameter(RAW_PARAM, param_list, lang);
+				service.appendChild(param_list);
+			}
+		}
+		if (subset == null || subset.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER)) {
+		  if (service_metadata_list == null) {
+		    Document ml_doc = XMLConverter.newDOM();
+		    service_metadata_list = ml_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		    if (does_paging) {
+		      service_metadata_list.appendChild(GSXML.createMetadataElement(ml_doc, "does_paging", "true"));
+		    }
+		  }
+		  service.appendChild(doc.importNode(service_metadata_list, true));
+		}
+		return service;
+
+	}
+
+	/** add in the level params to TextQuery */
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		createParameter(LEVEL_PARAM, param_list, lang);
+		super.addCustomQueryParams(param_list, lang);
+	}
+	/** add in the level params to TextQuery */
+	protected void addCustomQueryParamsAdvField(Element param_list, String lang)
+	{
+		createParameter(LEVEL_PARAM, param_list, lang);
+	}
+
+	/** create a param and add to the list */
+	protected void createParameter(String name, Element param_list, String lang)
+	{
+	  Document doc = param_list.getOwnerDocument();
+		Element param = null;
+		String param_default = paramDefaults.get(name);
+		if (name.equals(LEVEL_PARAM))
+		{
+			ArrayList<String> level_ids = new ArrayList<String>();
+			ArrayList<String> level_names = new ArrayList<String>();
+			getLevelData(level_ids, level_names, lang);
+			if (level_ids.size() > 1)
+			{
+				// the first one is the default
+				//param = GSXML.createParameterDescription2(doc, LEVEL_PARAM, getTextString("param."+LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)level_ids.get(0), level_ids, level_names);
+				param = GSXML.createParameterDescription2(doc, LEVEL_PARAM, getTextString("param." + LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, this.default_level, level_ids, level_names);
+			}
+			else
+			{
+				// we need to set the level, but hidden, in case there is an invalid level saved
+				//param = GSXML.createParameterDescription(doc, LEVEL_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, (String)level_ids.get(0), null, null);
+				param = GSXML.createParameterDescription(doc, LEVEL_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, this.default_level, null, null);
+			}
+		}
+		else if (name.equals(RANK_PARAM))
+		{
+			String[] vals1 = { RANK_PARAM_RANK, RANK_PARAM_NONE };
+			String[] vals1_texts = { getTextString("param." + RANK_PARAM + "." + RANK_PARAM_RANK, lang), getTextString("param." + RANK_PARAM + "." + RANK_PARAM_NONE, lang) };
+
+			param = GSXML.createParameterDescription(doc, RANK_PARAM, getTextString("param." + RANK_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals1, vals1_texts);
+
+		}
+		else if (name.equals(FIELD_QUERY_PARAM))
+		{
+			param = GSXML.createParameterDescription(doc, FIELD_QUERY_PARAM, getTextString("param." + FIELD_QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+
+		}
+		else if (name.equals(FIELD_CASE_PARAM) || name.equals(FIELD_STEM_PARAM) || name.equals(FIELD_ACCENT_PARAM))
+		{
+			String[] bool_ops = { "0", "1" };
+			String[] bool_texts = { getTextString("param.boolean.off", lang), getTextString("param.boolean.on", lang) };
+			param = GSXML.createParameterDescription(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_BOOLEAN, param_default, bool_ops, bool_texts);
+
+		}
+		else if (name.equals(FIELD_FIELD_PARAM))
+		{
+			ArrayList<String> fields = new ArrayList<String>();
+			ArrayList<String> field_names = new ArrayList<String>();
+			getIndexData(fields, field_names, lang);
+			// the field list -  read from config file
+
+			// Fix for http://trac.greenstone.org/ticket/245 "java crash, index out of bounds"
+			// org.greenstone.gsdl3.service.AbstractGS2FieldSearch.createParameter(AbstractGS2FieldSearch.java:362)
+			// Changed from:
+			// param = GSXML.createParameterDescription2(doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)fields.get(0), fields, field_names );
+			String default_value = (fields.size() > 0) ? fields.get(0) : null;
+			// don't want to access element 0 if fields.size()==0, and
+			// GSXML.createParameterDescription2 checks for default_value==null condition
+			param = GSXML.createParameterDescription2(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, default_value, fields, field_names);
+
+		}
+		else if (name.equals(FIELD_COMBINE_PARAM))
+		{
+
+			String[] vals = { FIELD_COMBINE_PARAM_AND, FIELD_COMBINE_PARAM_OR, FIELD_COMBINE_PARAM_NOT };
+			String[] val_texts = { getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_AND, lang), getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_OR, lang), getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_NOT, lang) };
+
+			param = GSXML.createParameterDescription(doc, FIELD_COMBINE_PARAM, "", GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals, val_texts);
+			param.setAttribute(GSXML.PARAM_IGNORE_POS_ATT, "0");
+		}
+
+		if (param != null)
+		{
+			param_list.appendChild(param);
+		}
+		else
+		{
+			super.createParameter(name, param_list, lang);
+		}
+	}
+
+	// should cache some of this
+	protected void getLevelData(ArrayList<String> level_ids, ArrayList<String> level_names, String lang)
+	{
+		Element level_list = (Element) GSXML.getChildByTagName(this.config_info, LEVEL_ELEM + GSXML.LIST_MODIFIER);
+		NodeList levels = level_list.getElementsByTagName(LEVEL_ELEM);
+		for (int i = 0; i < levels.getLength(); i++)
+		{
+			Element level = (Element) levels.item(i);
+			String shortname = level.getAttribute(GSXML.SHORTNAME_ATT);
+			if (shortname.equals(""))
+			{
+				continue;
+			}
+			level_ids.add(shortname);
+			String display_name = getDisplayText(level, GSXML.DISPLAY_TEXT_NAME, lang, "en");
+			if (display_name.equals(""))
+			{
+				// we'll use the name, and look it up in the dictionary
+				display_name = level.getAttribute(GSXML.NAME_ATT);
+				if (display_name.equals(""))
+				{
+					display_name = shortname;
+				}
+				else
+				{
+					display_name = getTextString("level." + display_name, lang);
+				}
+			}
+			level_names.add(display_name);
+		}
+	}
+
+	// the following three functions are needed so the base class can 
+	// call the process+SERVICE_NAME methods
+	/** process a text query */
+	protected Element processTextQuery(Element request)
+	{
+		return processAnyQuery(request, TEXT_QUERY);
+	}
+
+	/** process a field query */
+	protected Element processFieldQuery(Element request)
+	{
+		return processAnyQuery(request, SIMPLE_QUERY);
+	}
+
+	/** process an advanced field query */
+	protected Element processAdvancedFieldQuery(Element request)
+	{
+		return processAnyQuery(request, ADVANCED_QUERY);
+	}
+
+	protected Element processRawQuery(Element request)
+	{
+		return processAnyQuery(request, RAW_QUERY);
+	}
+
+	/** process a query */
+	protected Element processAnyQuery(Element request, int query_type)
+	{
+				
+		String service_name = null;
+		String empty_query_test_param = null;
+		// set up the type specific bits
+		switch (query_type)
+		{
+		case TEXT_QUERY:
+			service_name = QUERY_SERVICE;
+			empty_query_test_param = QUERY_PARAM;
+			break;
+		case SIMPLE_QUERY:
+			service_name = FIELD_QUERY_SERVICE;
+			empty_query_test_param = FIELD_QUERY_PARAM;
+			break;
+		case ADVANCED_QUERY:
+			service_name = ADVANCED_FIELD_QUERY_SERVICE;
+			empty_query_test_param = FIELD_QUERY_PARAM;
+			break;
+		case RAW_QUERY:
+			service_name = RAW_QUERY_SERVICE;
+			empty_query_test_param = RAW_PARAM;
+			break;
+		default:
+			// should never get here
+			logger.error("wrong query type!!");
+			return null;
+		}
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, service_name);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("TextQuery request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		// Process the request parameters
+		
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		//Save as variable to make it accessable from GS2SolrSearch
+		hldocOID = (String) params.get("hldocOID");
+		//System.out.println("@@@@@Extracted hldocOID is " + hldocOID);
+		// Make sure a query has been specified
+		String query = (String) params.get(empty_query_test_param);
+		if (query == null || query.equals(""))
+		{
+			return result; // Return the empty result
+		}
+
+		// If a field hasn't been specified, use the default - for textQuery
+		String field = (String) params.get(INDEX_PARAM);
+		//Save as variable to make it accessable from GS2SolrSearch
+		
+		if (field == null)
+		{
+			field = default_index;
+		}
+		indexField = field;
+		// set up the appropriate query system
+		if (!setUpQueryer(params))
+		{
+			return result;
+		}
+
+		// if field search, create the query string
+		switch (query_type)
+		{
+		case TEXT_QUERY:
+			query = addFieldInfo(query, field);
+			break;
+		case SIMPLE_QUERY:
+			query = parseFieldQueryParams(params);
+			break;
+		case ADVANCED_QUERY:
+			query = parseAdvancedFieldQueryParams(params);
+			break;
+		}
+		
+		// run the query
+		Object query_result = runQuery(query);
+		
+		// We want highlighted text to be returned right now!
+		if (hldocOID != null && does_full_field_highlighting)
+		{
+			Element nodeContentElement = result_doc.createElement(GSXML.NODE_CONTENT_ELEM);
+			nodeContentElement.setTextContent((String) query_result);
+			result.appendChild(nodeContentElement);
+			return result;
+		}
+		
+			
+		// build up the response
+
+		// Create a metadata list to store information about the query results
+		// should we be using metadataList? or something else?
+		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(metadata_list);
+
+		// Add a metadata element specifying the number of matching documents
+		long totalDocs = numDocsMatched(query_result);
+
+		GSXML.addMetadata(metadata_list, "numDocsMatched", "" + totalDocs);
+
+		// Create a document list to store the matching documents, and add them
+		String[] docs = getDocIDs(query_result);
+		String[] doc_ranks = getDocRanks(query_result);
+
+		// add a metadata item to specify docs returned
+		if (does_chunking) // this means we have a max docs param, and might ask for only a subset of results
+		{
+		  logger.error("does_chunking = true");
+		        int docs_returned = docs.length;
+			String maxdocs_str = (String) params.get(MAXDOCS_PARAM);
+			if (maxdocs_str != null)
+			{
+				int maxdocs = Integer.parseInt(maxdocs_str);
+				if (maxdocs > 0) { // maxdocs==-1 means return all
+				  docs_returned = (maxdocs < (int) totalDocs ? maxdocs : (int) totalDocs);
+				}
+			}
+			GSXML.addMetadata(metadata_list, "numDocsReturned", "" + docs_returned);
+		}
+		
+		Map<String, Map<String, List<String>>> hlResults = null;
+		if (does_highlight_snippets)
+		{
+			hlResults = getHighlightSnippets(query_result);
+		}
+		
+		// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+		GSXML.addMetadata(metadata_list, "query", query);
+		if (docs.length > 0)
+		{
+			Element document_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			result.appendChild(document_list);
+			Element snippet_list = result_doc.createElement(GSXML.HL_SNIPPET_ELEM + GSXML.LIST_MODIFIER);
+			result.appendChild(snippet_list);
+			for (int d = 0; d < docs.length; d++)
+			{
+				String doc_id = internalNum2OID(docs[d]);
+				Element doc_node = createDocNode(result_doc, doc_id, doc_ranks[d]);
+				if (hlResults != null && hlResults.get(docs[d]) != null && hlResults.get(docs[d]).get(indexField) != null) {
+					for (String snippet : hlResults.get(docs[d]).get(indexField)){
+						//remove html tags
+						snippet = snippet.replaceAll("\\<.*?>", "");
+						//remove truncated tags
+						snippet = snippet.replaceAll(".*>", "");
+						snippet = snippet.replaceAll("\\<.*", "");
+						//remove unwanted symbols at start of line
+						snippet = snippet.replaceAll("^[ .,Â»):;-â]+", "");
+						//highlighting tags transformation
+						snippet = snippet.replaceAll("&lt;", "<");
+						snippet = snippet.replaceAll("&gt;", ">");
+						Element snippet_node = result_doc.createElement(GSXML.HL_SNIPPET_ELEM);
+						snippet_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+						snippet_node.setTextContent(snippet);
+						snippet_list.appendChild(snippet_node);
+					}
+				}
+				document_list.appendChild(doc_node);
+			}
+		}
+
+		// Create a term list to store the term information, and add it
+		Element term_list = result_doc.createElement(GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(term_list);
+		addTermInfo(term_list, params, query_result);
+
+		if(does_faceting)
+		{
+			ArrayList<FacetWrapper> facets = getFacets(query_result);
+			if(facets != null)
+			{
+				Element facet_list = result_doc.createElement(GSXML.FACET_ELEM + GSXML.LIST_MODIFIER);
+				result.appendChild(facet_list);
+
+				for(FacetWrapper currentFacet : facets)
+				{
+					Element facet_elem = result_doc.createElement(GSXML.FACET_ELEM);
+					facet_elem.setAttribute(GSXML.NAME_ATT, currentFacet.getName());
+					facet_list.appendChild(facet_elem);
+					
+					HashMap<String, Long> countMap = currentFacet.getCounts();
+					
+					for(String countName : countMap.keySet())
+					{
+						long countValue = countMap.get(countName);
+						if(countValue > 0)
+						{
+							Element count_elem = result_doc.createElement(GSXML.COUNT_ELEM);
+							count_elem.setAttribute(GSXML.NAME_ATT, countName);
+							count_elem.setTextContent("" + countValue);
+
+							facet_elem.appendChild(count_elem);
+						}
+					}
+				}
+			}
+		}
+
+		
+		return result;
+
+	}
+
+	/** methods to handle actually doing the query */
+	/** do any initialisation of the query object */
+	abstract protected boolean setUpQueryer(HashMap<String, Serializable> params);
+
+	/** do the query */
+	abstract protected Object runQuery(String query);
+
+	/** get the total number of docs that match */
+	abstract protected long numDocsMatched(Object query_result);
+
+	/** get the list of doc ids */
+	abstract protected String[] getDocIDs(Object query_result);
+
+	/** get the list of doc ranks */
+	abstract protected String[] getDocRanks(Object query_result);
+	
+	/** get the list of facets */
+	abstract protected ArrayList<FacetWrapper> getFacets(Object query_result);
+	
+	/** get the map of highlighting snippets */
+	abstract protected Map<String, Map<String, List<String>>> getHighlightSnippets(Object query_result);
+
+	/** add in term info if available */
+	abstract protected boolean addTermInfo(Element term_list, HashMap<String, Serializable> params, Object query_result);
+
+	/**
+	 * combines all the field params into a single query - for simple field
+	 * query
+	 */
+	/** We assume the combination (AND/OR) is done by the match param */
+	protected String parseFieldQueryParams(HashMap<String, Serializable> params)
+	{
+
+		StringBuffer final_query = new StringBuffer(256);
+		String text_line = (String) params.get(FIELD_QUERY_PARAM);
+		String[] texts = text_line.split(",", -1);
+		String field_line = (String) params.get(FIELD_FIELD_PARAM);
+		String[] fields = field_line.split(",", -1);
+
+		for (int i = 0; i < texts.length; i++)
+		{
+			String q = texts[i].trim();
+			if (!q.equals(""))
+			{
+				final_query.append(" " + addFieldInfo(q, fields[i]));
+			}
+		}
+
+		return final_query.toString();
+	}
+
+	abstract protected String addFieldInfo(String query, String field);
+
+	/**
+	 * combines all the field params into a single query - for advanced field
+	 * query
+	 */
+	protected String parseAdvancedFieldQueryParams(HashMap<String, Serializable> params)
+	{
+
+		StringBuffer final_query = new StringBuffer(256);
+		String text_line = (String) params.get(FIELD_QUERY_PARAM);
+		String[] texts = text_line.split(",", -1);
+		String field_line = (String) params.get(FIELD_FIELD_PARAM);
+		String[] fields = field_line.split(",", -1);
+		String[] cases = null;
+		String[] stems = null;
+		String[] accents = null;
+		if (does_case)
+		{
+			String case_line = (String) params.get(FIELD_CASE_PARAM);
+			if (case_line != null)
+				cases = case_line.split(",", -1);
+		}
+		if (does_stem)
+		{
+			String stem_line = (String) params.get(FIELD_STEM_PARAM);
+			if (stem_line != null)
+				stems = stem_line.split(",", -1);
+		}
+		if (does_accent)
+		{
+			String accent_line = (String) params.get(FIELD_ACCENT_PARAM);
+			if (accent_line != null)
+				accents = accent_line.split(",", -1);
+		}
+		String combine_line = (String) params.get(FIELD_COMBINE_PARAM);
+		String[] combines = combine_line.split(",", -1);
+		String combine = "";
+		for (int i = 0; i < texts.length; i++)
+		{
+			if (i == 0)
+			{// assume first one is blank
+				combine = "";
+			}
+			else
+			{
+				String x = combines[i];
+				if (x.equals(FIELD_COMBINE_PARAM_AND))
+				{
+					combine = AND_OPERATOR;
+				}
+				else if (x.equals(FIELD_COMBINE_PARAM_OR))
+				{
+					combine = OR_OPERATOR;
+				}
+				else if (x.equals(FIELD_COMBINE_PARAM_NOT))
+				{
+					combine = NOT_OPERATOR;
+				}
+
+			}
+
+			String q = texts[i].trim();
+			boolean modified = false;
+			if (!q.equals(""))
+			{
+				String c = null;
+				String s = null;
+				String a = null;
+				if (does_case)
+				{
+					modified = true;
+					c = cases[i];
+				}
+				if (does_stem)
+				{
+					modified = true;
+					s = stems[i];
+				}
+				if (does_accent)
+				{
+					modified = true;
+					a = accents[i];
+				}
+				if (modified)
+				{
+					q = addStemOptions(q, s, c, a);
+				}
+				addQueryElem(final_query, q, fields[i], combine);
+			}
+		}
+		return final_query.toString();
+	}
+
+	abstract protected void addQueryElem(StringBuffer final_query, String query, String field, String combine);
+
+	abstract protected String addStemOptions(String query, String stem, String casef, String accent);
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2TextSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2TextSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractGS2TextSearch.java	(revision 31634)
@@ -0,0 +1,376 @@
+/*
+ *    AbstractGS2TextSearch.java
+ *    Copyright (C) 2011 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.BasicDocumentDatabase;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public abstract class AbstractGS2TextSearch extends AbstractTextSearch
+{
+	protected static final String EQUIV_TERM_ELEM = "equivTerm";
+
+	protected static final String STEM_ATT = "stem";
+	protected static final String NUM_DOCS_MATCH_ATT = "numDocsMatch";
+	protected static final String FREQ_ATT = "freq";
+
+	// Elements used in the config file that are specific to this class
+	protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+	protected static final String INDEX_STEM_ELEM = "indexStem";
+	protected static final String INDEX_ELEM = "index";
+	protected static final String DEFAULT_INDEX_SUBCOLLECTION_ELEM = "defaultIndexSubcollection";
+	protected static final String DEFAULT_INDEX_LANGUAGE_ELEM = "defaultIndexLanguage";
+	protected static final String INDEX_SUBCOLLECTION_ELEM = "indexSubcollection";
+	protected static final String INDEX_LANGUAGE_ELEM = "indexLanguage";
+
+	// Some indexing options
+	protected static final String STEMINDEX_OPTION = "stemIndexes";
+	protected static final String MAXNUMERIC_OPTION = "maxnumeric";
+
+	/** the stem used for the index files */
+	protected String index_stem = null;
+
+	// stem indexes available
+	protected boolean does_case = false;
+	protected boolean does_stem = false;
+	protected boolean does_accent = false;
+
+	// maxnumeric - 
+	protected int maxnumeric = 4;
+
+	BasicDocumentDatabase gs_doc_db = null;
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractGS2TextSearch.class.getName());
+
+	/** constructor */
+	public AbstractGS2TextSearch()
+	{
+
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+		this.gs_doc_db.cleanUp();
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		// find out what kind of database we have
+		Element database_type_elem = (Element) GSXML.getChildByTagName(info, GSXML.DATABASE_TYPE_ELEM);
+		String database_type = null;
+		if (database_type_elem != null)
+		{
+			database_type = database_type_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (database_type == null || database_type.equals(""))
+		{
+			database_type = "gdbm"; // the default
+		}
+
+		// the index stem is either the collection name or is specified in the config file
+		Element index_stem_elem = (Element) GSXML.getChildByTagName(info, INDEX_STEM_ELEM);
+		if (index_stem_elem != null)
+		{
+			this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (this.index_stem == null || this.index_stem.equals(""))
+		{
+			logger.warn("indexStem element not found, stem will default to collection name");
+			this.index_stem = this.cluster_name;
+		}
+
+		// replaces default AbstractSearch version with one tied to database
+		gs_doc_db = new BasicDocumentDatabase(database_type, this.site_home, this.cluster_name, this.index_stem);
+		if (!gs_doc_db.isValid())
+		{
+			logger.error("Failed to open Document Database.");
+			return false;
+		}
+		this.gs_doc = gs_doc_db;
+
+		// do we support any of the extended features?
+		//does_chunking = true;
+
+		// Get the default index out of <defaultIndex> (buildConfig.xml)
+		Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+		if (def != null)
+		{
+			this.default_index = def.getAttribute(GSXML.SHORTNAME_ATT);
+		} // otherwise will be "", and the first one will be the default
+
+		//get the default indexSubcollection out of <defaultIndexSubcollection> (buildConfig.xml)
+		Element defSub = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_SUBCOLLECTION_ELEM);
+		if (defSub != null)
+		{
+			this.default_index_subcollection = defSub.getAttribute(GSXML.SHORTNAME_ATT);
+		}
+
+		//get the default indexLanguage out of <defaultIndexLanguage> (buildConfig.xml)
+		Element defLang = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_LANGUAGE_ELEM);
+		if (defLang != null)
+		{
+			this.default_index_language = defLang.getAttribute(GSXML.SHORTNAME_ATT);
+		} //concate defaultIndex + defaultIndexSubcollection + defaultIndexLanguage
+
+		// get index options
+		Element index_option_list = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_OPTION_ELEM + GSXML.LIST_MODIFIER);
+		if (index_option_list != null)
+		{
+			NodeList options = index_option_list.getElementsByTagName(GSXML.INDEX_OPTION_ELEM);
+			for (int i = 0; i < options.getLength(); i++)
+			{
+				Element opt = (Element) options.item(i);
+				String name = opt.getAttribute(GSXML.NAME_ATT);
+				String value = opt.getAttribute(GSXML.VALUE_ATT);
+				if (name.equals(MAXNUMERIC_OPTION))
+				{
+					int maxnum = Integer.parseInt(value);
+					if (4 <= maxnum && maxnum < 512)
+					{
+						maxnumeric = maxnum;
+					}
+				}
+				else if (name.equals(STEMINDEX_OPTION))
+				{
+					int stemindex = Integer.parseInt(value);
+					if ((stemindex & 1) != 0)
+					{
+						does_case = true;
+					}
+					if ((stemindex & 2) != 0)
+					{
+						does_stem = true;
+					}
+					if ((stemindex & 4) != 0)
+					{
+						does_accent = true;
+					}
+				}
+			}
+		}
+
+		// get display info from extra info
+		if (extra_info != null)
+		{
+			Document owner = info.getOwnerDocument();
+			Element config_search = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+
+			// so far we have index and indexSubcollection specific display elements, and global format elements 
+
+			NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+			for (int i = 0; i < indexes.getLength(); i++)
+			{
+				Element ind = (Element) indexes.item(i);
+				String name = ind.getAttribute(GSXML.NAME_ATT);
+				Element node_extra = GSXML.getNamedElement(config_search, GSXML.INDEX_ELEM, GSXML.NAME_ATT, name);
+				if (node_extra == null)
+				{
+					logger.error("haven't found extra info for index named " + name);
+					continue;
+				}
+
+				// get the display elements if any - displayName
+				NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+				if (display_names != null)
+				{
+					for (int j = 0; j < display_names.getLength(); j++)
+					{
+						Element e = (Element) display_names.item(j);
+						ind.appendChild(owner.importNode(e, true));
+					}
+				}
+			} // for each index
+
+			NodeList indexSubcollections = info.getElementsByTagName(INDEX_SUBCOLLECTION_ELEM); // buildConfig.xml
+
+			for (int i = 0; i < indexSubcollections.getLength(); i++)
+			{
+				Element indexSubcollection = (Element) indexSubcollections.item(i);
+				String name = indexSubcollection.getAttribute(GSXML.NAME_ATT);
+				Element node_extra = GSXML.getNamedElement(config_search, INDEX_SUBCOLLECTION_ELEM, GSXML.NAME_ATT, name); // collectionConfig.xml
+				if (node_extra == null)
+				{
+					logger.error("haven't found extra info for indexSubCollection named " + name);
+					continue;
+				}
+
+				// get the display elements if any - displayName
+				NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+				if (display_names != null)
+				{
+					for (int j = 0; j < display_names.getLength(); j++)
+					{
+						Element e = (Element) display_names.item(j);
+						indexSubcollection.appendChild(owner.importNode(e, true));
+					}
+				}
+			} // for each indexSubCollection
+		}
+		return true;
+	}
+
+	protected void getIndexData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang)
+	{
+		// the index info -
+		Element index_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_ELEM + GSXML.LIST_MODIFIER);
+		NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+		int len = indexes.getLength();
+		// now add even if there is only one
+		for (int i = 0; i < len; i++)
+		{
+			Element index = (Element) indexes.item(i);
+			String shortname = index.getAttribute(GSXML.SHORTNAME_ATT);
+			if (shortname.equals(""))
+			{
+				continue;
+			}
+			index_ids.add(shortname);
+			String display_name = getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en", "metadata_names");
+			if (display_name.equals(""))
+			{
+				display_name = index.getAttribute(GSXML.NAME_ATT);
+				if (display_name.equals(""))
+				{
+					display_name = shortname;
+				}
+				String d_name = getTextString(display_name+".buttonname", lang, "metadata_names");
+				if (d_name == null) {
+				  d_name = getTextString(cleanUpMetadata(display_name)+".buttonname", lang, "metadata_names");
+				}
+				if (d_name == null) {
+				  d_name = getTextString(display_name, lang, "metadata_names");
+				}
+				if (d_name != null) {
+
+				  display_name = d_name;
+				}
+			}
+			index_names.add(display_name);
+		}
+	}
+
+  protected String cleanUpMetadata(String meta) {
+    // remove namespace, and only take the first item if there is a list of them.
+    String[] parts = meta.split("[,;]");
+    String cleaned = parts[0];
+    if (cleaned.lastIndexOf('.') != -1) {
+      cleaned = cleaned.substring(cleaned.lastIndexOf('.')+1);
+    }
+    return cleaned;
+    
+  }
+	protected void getIndexSubcollectionData(ArrayList<String> index_sub_ids, ArrayList<String> index_sub_names, String lang)
+	{
+		// the index info -
+		Element index_sub_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_SUBCOLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		NodeList index_subs = index_sub_list.getElementsByTagName(INDEX_SUBCOLLECTION_ELEM);
+		int len = index_subs.getLength();
+		// now add even if there is only one
+		for (int i = 0; i < len; i++)
+		{
+			Element indexsub = (Element) index_subs.item(i);
+			String shortname = indexsub.getAttribute(GSXML.SHORTNAME_ATT);
+			if (shortname.equals(""))
+			{
+				continue;
+			}
+			index_sub_ids.add(shortname);
+			String display_name = getDisplayText(indexsub, GSXML.DISPLAY_TEXT_NAME, lang, "en");
+			if (display_name.equals(""))
+			{
+				display_name = indexsub.getAttribute(GSXML.NAME_ATT);
+				if (display_name.equals(""))
+				{
+					display_name = shortname;
+				}
+			}
+			index_sub_names.add(display_name);
+		}
+	}
+
+	protected void getIndexLanguageData(ArrayList<String> index_lang_ids, ArrayList<String> index_lang_names, String lang)
+	{
+		// the index info -
+		Element index_lang_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_LANGUAGE_ELEM + GSXML.LIST_MODIFIER);
+		NodeList index_langs = index_lang_list.getElementsByTagName(INDEX_LANGUAGE_ELEM);
+		int len = index_langs.getLength();
+		// now add even if there is only one
+		for (int i = 0; i < len; i++)
+		{
+			Element indexlang = (Element) index_langs.item(i);
+			String shortname = indexlang.getAttribute(GSXML.SHORTNAME_ATT);
+			if (shortname.equals(""))
+			{
+				continue;
+			}
+			index_lang_ids.add(shortname);
+			String display_name = getDisplayText(indexlang, GSXML.DISPLAY_TEXT_NAME, lang, "en", "metadata_names");
+			if (display_name.equals(""))
+			{
+				display_name = indexlang.getAttribute(GSXML.NAME_ATT);
+				if (display_name.equals(""))
+				{
+					display_name = shortname;
+				}
+			}
+			index_lang_names.add(display_name);
+		}
+
+	}
+
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		if (this.does_case)
+		{
+		  createParameter(CASE_PARAM, param_list, lang);//, case_default);
+		}
+		if (this.does_stem)
+		{
+		  createParameter(STEM_PARAM, param_list, lang);//, stem_default);
+		}
+		if (this.does_accent)
+		{
+		  createParameter(ACCENT_PARAM, param_list, lang);//, accent_default);
+		}
+		createParameter(MATCH_PARAM, param_list, lang);
+	}
+
+	/** convert indexer internal id to Greenstone oid */
+	protected String internalNum2OID(long docnum)
+	{
+		return this.gs_doc_db.internalNum2OID(docnum);
+	}
+
+	protected String internalNum2OID(String docnum)
+	{
+		return this.gs_doc_db.internalNum2OID(docnum);
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractSearch.java	(revision 31634)
@@ -0,0 +1,411 @@
+/*
+ *    AbstractSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.AbstractBasicDocument;
+import org.greenstone.gsdl3.util.BasicDocument;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+/**
+ * Partially implements a generic search service
+ * 
+ */
+
+public abstract class AbstractSearch extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractSearch.class.getName());
+
+	// the search service
+	protected String QUERY_SERVICE = null; // set by subclass 
+
+	// compulsory params
+	protected static final String INDEX_PARAM = "index";
+	protected static final String QUERY_PARAM = "query";
+	protected static final String RAW_PARAM = "rawquery";
+
+	// optional standard params - some of these have to be implemented
+	protected static final String MAXDOCS_PARAM = "maxDocs";
+	protected static final String HITS_PER_PAGE_PARAM = "hitsPerPage";
+	protected static final String START_PAGE_PARAM = "startPage";
+
+	protected AbstractBasicDocument gs_doc = null;
+
+	/** can more than one index be searched at the same time? */
+	protected boolean does_multi_index_search = false;
+	/** does this service support paging of results? */
+	protected boolean does_paging = false;
+	/** does this service support asking for a subset of results? */
+	protected boolean does_chunking = false;
+	/** does this service support faceting search results */
+	protected boolean does_faceting = false;
+	/** does this service support highlighting snippets results */
+	protected boolean does_highlight_snippets = false;
+	protected boolean does_full_field_highlighting = false;
+	/**
+	 * the default document type - use if all documents are the same type
+	 */
+	protected String default_document_type = null;
+	/**
+	 * the default index, or comma separated list if more than one is the
+	 * default (with start and end commas, eg ,TI,SU,). Should be set by
+	 * configure()
+	 */
+	protected String default_index = "";
+
+  protected Element service_metadata_list = null;
+  protected HashMap<String, String> paramDefaults = null;
+
+	public AbstractSearch()
+	{
+	  paramDefaults = new HashMap<String, String>();
+	}
+
+	/**
+	 * Sets up the short service info for service by QUERY_SERVICE (e.g.
+	 * TextQuery or AudioQuery) If other services will be provided, should be
+	 * added in the subclass configure also looks for search format info, and
+	 * document format info
+	 */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring AbstractSearch...");
+
+		this.config_info = info;
+
+		// set up short_service_info_ 
+		// => for now just has id and type. the name (lang dependent)
+		//    will be added in if the list is requested.
+
+		Element tq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		tq_service.setAttribute(GSXML.NAME_ATT, QUERY_SERVICE);
+		this.short_service_info.appendChild(tq_service);
+
+		// load up paging defaults
+		if (does_chunking) {
+		  paramDefaults.put(MAXDOCS_PARAM, "100");
+		}
+		if (does_paging) {
+		  paramDefaults.put(HITS_PER_PAGE_PARAM, "20");
+		  paramDefaults.put(START_PAGE_PARAM, "1");
+		}
+
+		// load up any search param defaults
+		Element search_elem = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+		if (search_elem != null) {
+		  getSearchParamDefaults(search_elem);
+		}
+		// add some format info to service map if there is any 
+		// => lookin extra info first look in buildConfig
+
+		Element format = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+
+		if (format == null)
+		{
+		  // try to find a format element inside <search> that contains a gsf:template. Note what if we have only xsl:templates??
+			
+			NodeList format_elems = null;
+			if (search_elem != null)
+			{
+				format_elems = search_elem.getElementsByTagName(GSXML.FORMAT_ELEM);
+			}
+			for (int i = 0; i < format_elems.getLength(); i++)
+			{
+				format = (Element) format_elems.item(i);
+				if (format.getElementsByTagName("gsf:template").getLength() != 0)
+				{
+					break;
+				}
+			}
+		}//end of if(format==null)
+			//
+		if (format != null)
+		{
+			this.format_info_map.put(QUERY_SERVICE, this.desc_doc.importNode(format, true));
+		}
+
+		// look for document display format - for documentType
+		String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+		Element display_format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (display_format != null)
+		{
+			// check for docType option.
+			Element doc_type_opt = GSXML.getNamedElement(display_format, "gsf:option", GSXML.NAME_ATT, "documentType");
+			if (doc_type_opt != null)
+			{
+				String value = doc_type_opt.getAttribute(GSXML.VALUE_ATT);
+				if (!value.equals(""))
+				{
+					this.default_document_type = value;
+				}
+			}
+		}
+
+		// Base line for document (might be overriden by sub-classes)
+		gs_doc = new BasicDocument(this.default_document_type);
+
+		return true;
+	}
+
+  protected void getSearchParamDefaults(Element search_elem) {
+
+    NodeList param_defaults_list = GSXML.getChildrenByTagName(search_elem, GSXML.PARAM_DEFAULT_ELEM);
+    for (int i=0; i<param_defaults_list.getLength(); i++) {
+      Element paramdef = (Element)param_defaults_list.item(i);
+      String name = paramdef.getAttribute(GSXML.NAME_ATT);
+      String val = paramdef.getAttribute(GSXML.VALUE_ATT);
+      if (!name.equals("") && !val.equals("")) {
+	paramDefaults.put(name, val);
+      }
+    }
+  }
+	/**
+	 * returns a basic description for QUERY_SERVICE. If a subclass provides
+	 * other services they need to provide their own descriptions
+	 */
+  protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+		if (!service.equals(QUERY_SERVICE))
+		{
+			return null;
+		}
+
+		Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+		tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		tq_service.setAttribute(GSXML.NAME_ATT, QUERY_SERVICE);
+		if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+		{
+			tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getServiceName(QUERY_SERVICE, lang)));
+			tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getServiceSubmit(QUERY_SERVICE, lang)));
+			tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getServiceDescription(QUERY_SERVICE, lang)));
+		}
+		if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+		{
+			Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			addCustomQueryParams(param_list, lang);
+			addStandardQueryParams(param_list, lang);
+			tq_service.appendChild(param_list);
+		}
+		if (subset == null || subset.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER)) {
+
+		  if (service_metadata_list == null) {
+		    Document ml_doc = XMLConverter.newDOM();
+		    service_metadata_list = ml_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		    if (does_paging) {
+		      service_metadata_list.appendChild(GSXML.createMetadataElement(ml_doc, "does_paging", "true"));
+		    }
+		  }
+		  tq_service.appendChild(doc.importNode(service_metadata_list, true));
+		}
+		return tq_service;
+
+	}
+
+	// perhaps these should be changed to search down the class hierarchy for 
+	// values - do can just put the info in the resource bundle to use it
+	/** returns the default name for the TextQuery service */
+	protected String getServiceName(String service_id, String lang)
+	{
+		return getTextString(service_id + ".name", lang);
+	}
+
+	/** returns the default description for the TextQuery service */
+	protected String getServiceDescription(String service_id, String lang)
+	{
+		return getTextString(service_id + ".description", lang);
+	}
+
+	/** returns the default submit button text for the TextQuery service */
+	protected String getServiceSubmit(String service_id, String lang)
+	{
+		return getTextString(service_id + ".submit", lang);
+
+	}
+
+	/** adds the standard query params into the service description */
+	protected void addStandardQueryParams(Element param_list, String lang)
+	{
+		// this test is not so good. here we are using absence of default index
+		// to determine whether we have indexes or not. But in other places,
+		// absence of default index just means to use the first one as default.
+		if (!default_index.equals(""))
+		{
+			createParameter(INDEX_PARAM, param_list, lang);
+		}
+		if (does_chunking)
+		{
+			createParameter(MAXDOCS_PARAM, param_list, lang);
+		}
+		if (does_paging)
+		{
+			createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
+			createParameter(START_PAGE_PARAM, param_list, lang);
+		}
+		createParameter(QUERY_PARAM, param_list, lang);
+	}
+
+	/**
+	 * adds any service specific query params into the service default
+	 * implementation: add nothing. subclasses may need to override this to add
+	 * in their specific parameters
+	 */
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		// default behaviour, do nothing
+	}
+
+	protected void createParameter(String name, Element param_list, String lang)
+	{
+		createParameter(name, param_list, lang, null);
+	}
+
+	protected void createParameter(String name, Element param_list, String lang, String default_value)
+	{
+		// at this level, not interested in boolean return type
+		createParameterChain(name, param_list, lang, default_value);
+	}
+
+	/**
+	 * default implementations for the standard parameters plus some other
+	 * common ones index, maxDocs, hitsPerPage, startPage
+	 */
+
+	protected boolean createParameterChain(String name, Element param_list, String lang, String default_value)
+	{
+	  Document doc = param_list.getOwnerDocument();
+		Element param = null;
+		String param_default = default_value;
+		if (default_value == null) {
+		  // have we got a stored up default? will be null if not there
+		  param_default = paramDefaults.get(name);
+		}
+		if (name.equals(QUERY_PARAM) || name.equals(RAW_PARAM))
+		{
+			param = GSXML.createParameterDescription(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_STRING, param_default, null, null);
+			param_list.appendChild(param);
+			return true;
+		}
+		else if (name.equals(INDEX_PARAM))
+		{
+			// should we make these class fields?
+			ArrayList<String> index_ids = new ArrayList<String>();
+			ArrayList<String> index_names = new ArrayList<String>();
+			getIndexData(index_ids, index_names, lang);
+			String param_type = GSXML.PARAM_TYPE_ENUM_SINGLE;
+			if (does_multi_index_search)
+			{
+				param_type = GSXML.PARAM_TYPE_ENUM_MULTI;
+			}
+			if (param_default == null)
+			{
+				param_default = this.default_index;
+			}
+			param = GSXML.createParameterDescription2(doc, INDEX_PARAM, getTextString("param." + INDEX_PARAM, lang), param_type, param_default, index_ids, index_names);
+			param_list.appendChild(param);
+			return true;
+		}
+		else if (name.equals(MAXDOCS_PARAM))
+		{
+			param = GSXML.createParameterDescription(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_INTEGER, param_default, null, null);
+			param_list.appendChild(param);
+			return true;
+		}
+		else if (name.equals(HITS_PER_PAGE_PARAM))
+		{
+			param = GSXML.createParameterDescription(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_INTEGER, param_default, null, null);
+			param_list.appendChild(param);
+			return true;
+		}
+		else if (name.equals(START_PAGE_PARAM))
+		{
+			// start page - set to 1 for the search page
+			param = GSXML.createParameterDescription(doc, START_PAGE_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, param_default, null, null);
+			param_list.appendChild(param);
+			return true;
+		}
+
+		// Get to there then none of the above params matched
+		// => return false so the chain can continue
+		return false;
+	}
+
+	/**
+	 * create an element to go into the search results list. A node element has
+	 * the form <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'
+	 * rank='0.23'/>
+	 */
+  protected Element createDocNode(Document doc, String node_id, String rank)
+	{
+	  return this.gs_doc.createDocNode(doc, node_id, rank);
+	}
+
+	/**
+	 * returns the document type of the doc that the specified node belongs to.
+	 * should be one of GSXML.DOC_TYPE_SIMPLE, GSXML.DOC_TYPE_PAGED,
+	 * GSXML.DOC_TYPE_HIERARCHY
+	 */
+	protected String getDocType(String node_id)
+	{
+		return this.gs_doc.getDocType(node_id);
+	}
+
+	/**
+	 * returns the node type of the specified node. should be one of
+	 * GSXML.NODE_TYPE_LEAF, GSXML.NODE_TYPE_INTERNAL, GSXML.NODE_TYPE_ROOT
+	 */
+	protected String getNodeType(String node_id, String doc_type)
+	{
+		return this.gs_doc.getNodeType(node_id, doc_type);
+	}
+
+	/** returns true if the node has child nodes */
+	protected boolean hasChildren(String node_id)
+	{
+		return this.gs_doc.hasChildren(node_id);
+	}
+
+	/** returns true if the node has a parent */
+	protected boolean hasParent(String node_id)
+	{
+		return this.gs_doc.hasParent(node_id);
+	}
+
+	/**
+	 * get the details about the indexes available must be implemented by
+	 * subclass there must be at least one index
+	 */
+	abstract protected void getIndexData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang);
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractTextSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractTextSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/AbstractTextSearch.java	(revision 31634)
@@ -0,0 +1,191 @@
+/*
+ *    AbstractTextSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSXML;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Partially implements a generic search service
+ * 
+ */
+
+public abstract class AbstractTextSearch extends AbstractSearch
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractTextSearch.class.getName());
+
+	// optional standard params - some of these have to be implemented
+	protected static final String INDEX_SUBCOLLECTION_PARAM = "indexSubcollection";
+	protected static final String INDEX_LANGUAGE_PARAM = "indexLanguage";
+
+	protected static final String INDEX_SUBCOLLECTION_ELEM = "indexSubcollection";
+	protected static final String INDEX_LANGUAGE_ELEM = "indexLanguage";
+
+	// some other common params that may be used
+	protected static final String CASE_PARAM = "case";
+	protected static final String STEM_PARAM = "stem";
+	protected static final String ACCENT_PARAM = "accent";
+
+	protected static final String BOOLEAN_PARAM_ON = "1";
+	protected static final String BOOLEAN_PARAM_OFF = "0";
+	protected static final String MATCH_PARAM = "matchMode";
+	protected static final String MATCH_PARAM_ALL = "all";
+	protected static final String MATCH_PARAM_SOME = "some";
+
+	protected String default_index_subcollection = "";
+
+	protected String default_index_language = "";
+
+	public AbstractTextSearch()
+	{
+	  super();
+		// the search service
+		QUERY_SERVICE = "TextQuery";
+		paramDefaults.put(CASE_PARAM, BOOLEAN_PARAM_ON);
+		paramDefaults.put(STEM_PARAM, BOOLEAN_PARAM_OFF);
+		paramDefaults.put(ACCENT_PARAM, BOOLEAN_PARAM_ON);
+		paramDefaults.put(MATCH_PARAM, MATCH_PARAM_SOME);
+	}
+
+	/** adds the standard query params into the service description */
+	protected void addStandardQueryParams(Element param_list, String lang)
+	{
+		if (!default_index_subcollection.equals(""))
+		{
+			createParameter(INDEX_SUBCOLLECTION_PARAM, param_list, lang);
+		}
+		if (!default_index_language.equals(""))
+		{
+			createParameter(INDEX_LANGUAGE_PARAM, param_list, lang);
+		}
+
+		super.addStandardQueryParams(param_list, lang);
+	}
+
+	/**
+	 * Top up createParameterChain with TextQuery specific params: case, stem
+	 * ...
+	 */
+	protected boolean createParameterChain(String name, Element param_list, String lang, String default_value)
+	{
+	  Document doc = param_list.getOwnerDocument();
+		Element param = null;
+		String param_default = default_value;
+		if (default_value == null) {
+		  // have we got a stored up default? will be null if not there
+		  param_default = paramDefaults.get(name);
+		}
+
+		if (super.createParameterChain(name, param_list, lang, default_value))
+		{
+			// found a match, so can stop here
+			return true;
+		}
+		// otherwise look to see if it is a text specific parameter
+		if (name.equals(INDEX_SUBCOLLECTION_PARAM))
+		{
+			Element index_sub_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_SUBCOLLECTION_ELEM + GSXML.LIST_MODIFIER);
+			if (index_sub_list == null)
+				return true; // processed, just not a very interesting result
+			ArrayList<String> index_sub_ids = new ArrayList<String>();
+			ArrayList<String> index_sub_names = new ArrayList<String>();
+			getIndexSubcollectionData(index_sub_ids, index_sub_names, lang);
+			String param_type = GSXML.PARAM_TYPE_ENUM_SINGLE;
+			if (does_multi_index_search)
+			{
+				param_type = GSXML.PARAM_TYPE_ENUM_MULTI;
+			}
+			if (param_default == null)
+			{
+				param_default = this.default_index_subcollection;
+			}
+			param = GSXML.createParameterDescription2(doc, INDEX_SUBCOLLECTION_PARAM, getTextString("param." + INDEX_SUBCOLLECTION_PARAM, lang), param_type, param_default, index_sub_ids, index_sub_names);
+			param_list.appendChild(param);
+			return true;
+		}
+		else if (name.equals(INDEX_LANGUAGE_PARAM))
+		{
+			Element index_lang_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_LANGUAGE_ELEM + GSXML.LIST_MODIFIER);
+			if (index_lang_list == null)
+				return true; // processed, just not a very interesting result
+			ArrayList<String> index_lang_ids = new ArrayList<String>();
+			ArrayList<String> index_lang_names = new ArrayList<String>();
+			getIndexLanguageData(index_lang_ids, index_lang_names, lang);
+			String param_type = GSXML.PARAM_TYPE_ENUM_SINGLE;
+			if (does_multi_index_search)
+			{
+				param_type = GSXML.PARAM_TYPE_ENUM_MULTI;
+			}
+			if (param_default == null)
+			{
+				param_default = this.default_index_language;
+			}
+			param = GSXML.createParameterDescription2(doc, INDEX_LANGUAGE_PARAM, getTextString("param." + INDEX_LANGUAGE_PARAM, lang), param_type, param_default, index_lang_ids, index_lang_names);
+			param_list.appendChild(param);
+			return true;
+		}
+	else if (name.equals(CASE_PARAM) || name.equals(STEM_PARAM) || name.equals(ACCENT_PARAM)) {
+	    String[] bool_ops = {"0", "1"};
+	    String[] bool_texts = {getTextString("param.boolean.off", lang),getTextString("param.boolean.on", lang)}; 
+	    param = GSXML.createParameterDescription(doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_BOOLEAN, param_default, bool_ops, bool_texts);
+	    param_list.appendChild(param);
+	    return true;
+	} else if (name.equals(MATCH_PARAM)) {
+	  
+	    String[] vals = {MATCH_PARAM_SOME, MATCH_PARAM_ALL };
+	    String[] val_texts = {getTextString("param."+MATCH_PARAM+"."+MATCH_PARAM_SOME, lang), getTextString("param."+MATCH_PARAM+"."+MATCH_PARAM_ALL, lang)}; 
+	    param = GSXML.createParameterDescription(doc, MATCH_PARAM, getTextString("param."+MATCH_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals, val_texts);
+	    param_list.appendChild(param);
+	    return true;
+	}
+		// Get to here then none of the above params matched
+		// => return false so the chain can continue
+		return false;
+
+	}
+
+	/**
+	 * do the actual query must be implemented by subclass
+	 */
+	abstract protected Element processTextQuery(Element request);
+
+	/**
+	 * get the details about the indexexSubcollections available might be
+	 * implemented by subclass
+	 */
+	protected void getIndexSubcollectionData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang)
+	{
+	}
+
+	/**
+	 * get the details about the indexes available might be implemented by
+	 * subclass
+	 */
+	protected void getIndexLanguageData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang)
+	{
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveIO.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveIO.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveIO.java	(revision 31634)
@@ -0,0 +1,351 @@
+/*
+ *    ArchiveIO.java
+ *    Used to retrieve information about collection archives
+ *
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.DBHelper;
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.util.GSDocumentModel;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.apache.log4j.*;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Vector;
+
+public class ArchiveIO extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.ArchiveIO.class.getName());
+	GSDocumentModel _GSDM = null;
+	
+	/********************************************************************
+	 * The list of services the Archive retrieval service rack supports *
+	 *******************************************************************/
+	protected static final String ARCHIVE_GET_DOCUMENT_FILE_PATH = "ArchiveGetDocumentFilePath";
+	protected static final String ARCHIVE_GET_ASSOCIATED_IMPORT_FILES = "ArchiveGetAssociatedImportFiles";
+	protected static final String ARCHIVE_GET_SOURCE_FILE_OID = "ArchiveGetSourceFileOID";
+	protected static final String ARCHIVE_CHECK_DOCUMENT_OR_SECTION_EXISTS = "ArchiveCheckDocumentOrSectionExists";
+	protected static final String ARCHIVE_WRITE_ENTRY_TO_DATABASE = "ArchiveWriteEntryToDatabase";
+	protected static final String ARCHIVE_REMOVE_ENTRY_FROM_DATABASE = "ArchiveRemoveEntryFromDatabase";
+	/*******************************************************************/
+	
+	String[] _services = { ARCHIVE_GET_DOCUMENT_FILE_PATH, ARCHIVE_GET_ASSOCIATED_IMPORT_FILES, ARCHIVE_GET_SOURCE_FILE_OID, ARCHIVE_CHECK_DOCUMENT_OR_SECTION_EXISTS, ARCHIVE_WRITE_ENTRY_TO_DATABASE, ARCHIVE_REMOVE_ENTRY_FROM_DATABASE };
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring DocXMLUtil...");
+		this.config_info = info;
+
+		for (int i = 0; i < _services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, _services[i]);
+			this.short_service_info.appendChild(service);
+		}
+		
+		_GSDM = new GSDocumentModel(this.site_home, this.router);
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < _services.length; i++)
+		{
+			if (service_id.equals(_services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, _services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+	
+	/************
+	 * Services *
+	 ***********/
+
+	protected Element processArchiveGetDocumentFilePath(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_GET_DOCUMENT_FILE_PATH);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_GET_DOCUMENT_FILE_PATH + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String oid = (String) params.get(GSXML.NODE_ID_ATT);
+		String collection = (String) params.get(GSXML.COLLECTION_ATT);
+
+		String filePath = _GSDM.archiveGetDocumentFilePath(oid, collection, userContext);
+		
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, "docfilepath", filePath)); //TODO: Replace "docfilepath" with a constant 
+		result.appendChild(metadataList);
+
+		return result;
+	}
+
+	protected Element processArchiveGetSourceFileOID(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_GET_SOURCE_FILE_OID);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_GET_SOURCE_FILE_OID + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String srcFile = (String) params.get("sourcefile"); //TODO: Replace with a constant
+		String collection = (String) params.get(GSXML.COLLECTION_ATT);
+
+		String oid = _GSDM.archiveGetSourceFileOID(srcFile, collection, userContext);
+		if(_GSDM.checkError(result, ARCHIVE_GET_SOURCE_FILE_OID))
+		{
+			return result;
+		}
+
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, GSXML.NODE_ID_ATT, oid));
+		result.appendChild(metadataList);
+
+		return result;
+	}
+
+	protected Element processArchiveCheckDocumentOrSectionExists(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_CHECK_DOCUMENT_OR_SECTION_EXISTS);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_CHECK_DOCUMENT_OR_SECTION_EXISTS + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String oid = (String) params.get(GSXML.NODE_ID_ATT);
+		String collection = (String) params.get(GSXML.COLLECTION_ATT);
+		
+		boolean exists = _GSDM.archiveCheckDocumentOrSectionExists(oid, collection, userContext);
+		if(_GSDM.checkError(result, ARCHIVE_CHECK_DOCUMENT_OR_SECTION_EXISTS))
+		{
+			return result;
+		}
+		
+		result.setAttribute("check", exists ? "true" : "false");
+
+		return result;
+	}
+
+	protected Element processArchiveWriteEntryToDatabase(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_WRITE_ENTRY_TO_DATABASE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_WRITE_ENTRY_TO_DATABASE + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, ArrayList<String>> info = new HashMap<String, ArrayList<String>>();
+		String collection = null;
+		String oid = null;
+
+		NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+		for (int i = 0; i < params.getLength(); i++)
+		{
+			Element currentParam = (Element) params.item(i);
+			String type = currentParam.getAttribute(GSXML.TYPE_ATT);
+			String name = currentParam.getAttribute(GSXML.NAME_ATT);
+			String value = currentParam.getAttribute(GSXML.VALUE_ATT);
+			if (type != null && type.equals("entry"))
+			{
+				if(info.get(name) != null)
+				{
+					info.get(name).add(value);
+				}
+				else
+				{
+					ArrayList<String> values = new ArrayList<String>();
+					values.add(value);
+					info.put(name, values);
+				}
+			}
+			else
+			{
+				if (name.equals(GSXML.COLLECTION_ATT))
+				{
+					collection = value;
+				}
+				else if (name.equals(GSXML.NODE_ID_ATT))
+				{
+					oid = value;
+				}
+			}
+		}
+		
+		_GSDM.archiveWriteEntryToDatabase(oid, collection, info, userContext);
+		_GSDM.checkError(result, ARCHIVE_WRITE_ENTRY_TO_DATABASE);
+
+		return result;
+	}
+	
+	protected Element processArchiveRemoveEntryFromDatabase(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_REMOVE_ENTRY_FROM_DATABASE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_REMOVE_ENTRY_FROM_DATABASE + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		String collection = null;
+		String oid = null;
+
+		NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+		for (int i = 0; i < params.getLength(); i++)
+		{
+			Element currentParam = (Element) params.item(i);
+			String name = currentParam.getAttribute(GSXML.NAME_ATT);
+			String value = currentParam.getAttribute(GSXML.VALUE_ATT);
+	
+			if (name.equals(GSXML.COLLECTION_ATT))
+			{
+				collection = value;
+			}
+			else if (name.equals(GSXML.NODE_ID_ATT))
+			{
+				oid = value;
+			}
+		}
+		
+		_GSDM.archiveRemoveEntryFromDatabase(oid, collection, userContext);
+		_GSDM.checkError(result, ARCHIVE_REMOVE_ENTRY_FROM_DATABASE);
+
+		return result;
+	}
+
+	protected Element processArchiveGetAssociatedImportFiles(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ARCHIVE_GET_ASSOCIATED_IMPORT_FILES);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, ARCHIVE_GET_ASSOCIATED_IMPORT_FILES + ": Missing " + GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String oid = (String) params.get(GSXML.NODE_ID_ATT);
+		String collection = (String) params.get(GSXML.COLLECTION_ATT);
+
+		ArrayList<String> assocFiles = _GSDM.archiveGetAssociatedImportFiles(oid, collection, userContext);
+		if(_GSDM.checkError(result, ARCHIVE_GET_ASSOCIATED_IMPORT_FILES))
+		{
+			return result;
+		}
+		
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, "srcfile", assocFiles.get(0)));
+
+		for (int i = 1; i < assocFiles.size(); i++)
+		{
+			metadataList.appendChild(GSXML.createMetadataElement(result_doc, "assocfile", assocFiles.get(i)));
+		}
+
+		result.appendChild(metadataList);
+
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ArchiveRetrieve.java	(revision 31634)
@@ -0,0 +1,351 @@
+/*
+*    ArchiveRetrieve.java
+*    a base class for retrieval services
+
+*    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+*
+*    This program is free software; you can redistribute it and/or modify
+*    it under the terms of the GNU General Public License as published by
+*    the Free Software Foundation; either version 2 of the License, or
+*    (at your option) any later version.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU General Public License for more details.
+*
+*    You should have received a copy of the GNU General Public License
+*    along with this program; if not, write to the Free Software
+*    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.DBHelper;
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+
+import org.apache.log4j.*;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Vector;
+
+public class ArchiveRetrieve extends ServiceRack 
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.ArchiveRetrieve.class.getName());
+
+	protected static final String DOCUMENT_FILE_PATH_RETRIEVE_SERVICE = "DocumentFilePathRetrieve";
+	protected static final String ASSOCIATED_IMPORT_FILES_RETRIEVE_SERVICE = "AssociatedImportFilesRetrieve";
+	protected static final String SOURCE_FILE_OID_RETRIEVE = "SourceFileOIDRetrieve";
+	
+	protected SimpleCollectionDatabase coll_db = null;
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring ArchiveRetrieve...");
+		this.config_info = info;
+		
+		Element documentFilePathRetrieveService = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		documentFilePathRetrieveService.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		documentFilePathRetrieveService.setAttribute(GSXML.NAME_ATT, DOCUMENT_FILE_PATH_RETRIEVE_SERVICE);
+		this.short_service_info.appendChild(documentFilePathRetrieveService);
+		
+		Element associatedImportFilesRetrieveService = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		associatedImportFilesRetrieveService.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		associatedImportFilesRetrieveService.setAttribute(GSXML.NAME_ATT, ASSOCIATED_IMPORT_FILES_RETRIEVE_SERVICE);
+		this.short_service_info.appendChild(associatedImportFilesRetrieveService);
+		
+		Element sourceFileDocIDRetrieveService = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		sourceFileDocIDRetrieveService.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		sourceFileDocIDRetrieveService.setAttribute(GSXML.NAME_ATT, SOURCE_FILE_OID_RETRIEVE);
+		this.short_service_info.appendChild(sourceFileDocIDRetrieveService);
+		
+		return true;
+	}
+	
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset) 
+	{
+		if (service_id.equals(DOCUMENT_FILE_PATH_RETRIEVE_SERVICE)) 
+		{
+			Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+			service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service_elem.setAttribute(GSXML.NAME_ATT, DOCUMENT_FILE_PATH_RETRIEVE_SERVICE);
+			return service_elem;
+		}
+		else if (service_id.equals(ASSOCIATED_IMPORT_FILES_RETRIEVE_SERVICE)) 
+		{
+			Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+			service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service_elem.setAttribute(GSXML.NAME_ATT, ASSOCIATED_IMPORT_FILES_RETRIEVE_SERVICE);
+			return service_elem;
+		}
+		else if (service_id.equals(SOURCE_FILE_OID_RETRIEVE))
+		{
+			Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+			service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service_elem.setAttribute(GSXML.NAME_ATT, SOURCE_FILE_OID_RETRIEVE);
+			return service_elem;
+		}
+		return null;
+	}
+	
+	protected Element processDocumentFilePathRetrieve(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_FILE_PATH_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		
+		UserContext userContext = new UserContext(request);
+		
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		if (param_list == null) {
+			GSXML.addError(result, "DocumentFilePathRetrieve: missing "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;  
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		
+		String oid = (String) params.get("oid");
+		String collection = (String) params.get("c");
+		
+		String assocFilePath = getAssocFilePathFromDocID(oid, collection, userContext);
+		
+		String docFilePath = this.site_home + File.separatorChar + 
+			"collect" + File.separatorChar +
+			collection + File.separatorChar + 
+			"archives" + File.separatorChar + 
+			assocFilePath + File.separatorChar + 
+			"doc.xml";
+
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, "docfilepath", docFilePath));
+		result.appendChild(metadataList);
+		
+		return result;
+	}
+
+
+     /** @function processSourceFileOIDRetrieveService(Element)
+     *  @brief
+     *  @param request
+     *  @return Element
+     */
+	protected Element processSourceFileOIDRetrieveService(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, SOURCE_FILE_OID_RETRIEVE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		
+		UserContext userContext = new UserContext(request);
+		
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		if (param_list == null) 
+		{
+			GSXML.addError(result, "DocumentFilePathRetrieve: missing "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;  
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		
+		String srcFile = (String) params.get("srcfile");
+		String collection = (String) params.get("c");
+		
+		//Find out what kind of database we have
+		String databaseType = getDatabaseTypeFromCollection(collection, userContext);
+		if (databaseType == null || databaseType.equals("")) 
+		{
+			databaseType = "gdbm"; // the default
+		}
+
+		coll_db = new SimpleCollectionDatabase(databaseType);
+		if (!coll_db.databaseOK()) 
+		{
+			logger.error("Couldn't create the collection database of type "+databaseType);
+			return null;
+		}
+
+	// Moved to ensure that the appropriate FlatDatabaseWrapper
+	// has been initialised during the SimpleCollectionDatabase
+	// call above. That way we can easily retrieve the database
+	// extension from the DBHelper [jmt12]
+	String dbExt = DBHelper.getDBExtFromDBType(databaseType);
+	if (null == dbExt || dbExt.equals("")) {
+	   // assume gdbm
+	   logger.warn("Could not recognise database type \"" + databaseType + "\", defaulting to GDBM and extension \".gdb\"");
+	   dbExt = ".gdb";
+	}
+
+		coll_db.openDatabase
+		(
+			this.site_home + File.separatorChar + 
+			"collect" + File.separatorChar +
+			collection + File.separatorChar + 
+			"archives" + File.separatorChar + 
+			"archiveinf-src" + dbExt, 
+			SimpleCollectionDatabase.READ
+		);
+		
+		DBInfo info = coll_db.getInfo(srcFile);
+		
+		if (info == null)
+		{
+			return result;
+		}
+		
+		String oid = info.getInfo("oid");
+		
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, "oid", oid));
+		result.appendChild(metadataList);
+		
+		return result;
+	}
+	/** processSourceFileOIDRetrieveService(Element) **/
+
+	protected Element processAssociatedImportFilesRetrieve(Element request)
+	{
+		//Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, ASSOCIATED_IMPORT_FILES_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		
+		UserContext userContext = new UserContext(request);
+		
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		if (param_list == null) 
+		{
+			GSXML.addError(result, "AssociatedImportFilesRetrieve: missing "+ GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER, GSXML.ERROR_TYPE_SYNTAX);
+			return result;  
+		}
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		
+		String oid = (String) params.get("oid");
+		String collection = (String) params.get("c");
+		
+		String databaseType = getDatabaseTypeFromCollection(collection, userContext);
+		if (databaseType == null || databaseType.equals("")) 
+		{
+			databaseType = "gdbm"; // the default
+		}
+		
+		coll_db = new SimpleCollectionDatabase(databaseType);
+		if (!coll_db.databaseOK()) 
+		{
+			logger.error("Couldn't create the collection database of type "+databaseType);
+			return null;
+		}
+
+	// Moved to ensure that the appropriate FlatDatabaseWrapper
+	// has been initialised during the SimpleCollectionDatabase
+	// call above. That way we can easily retrieve the database
+	// extension from the DBHelper.
+	String dbExt = DBHelper.getDBExtFromDBType(databaseType);
+	if (null == dbExt || dbExt.equals(""))
+	{
+	    // assume gdbm
+	    logger.warn("Could not recognise database type \"" + databaseType + "\", defaulting to GDBM and extension \".gdb\"");
+	    dbExt = ".gdb";
+	}
+		
+		coll_db.openDatabase
+		(
+			this.site_home + File.separatorChar + 
+			"collect" + File.separatorChar +
+			collection + File.separatorChar + 
+			"archives" + File.separatorChar + 
+			"archiveinf-doc" + dbExt, 
+			SimpleCollectionDatabase.READ
+		);
+		
+		DBInfo info = coll_db.getInfo(oid);
+		
+		if (info == null)
+		{
+			return result;
+		}
+		
+		String srcFile = info.getInfo("src-file");
+		Vector<String> data = info.getMultiInfo("assoc-file");
+		
+		Element metadataList = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		metadataList.appendChild(GSXML.createMetadataElement(result_doc, "srcfile", srcFile));
+		
+		for (int i = 0; i < data.size(); i++)
+		{
+		  metadataList.appendChild(GSXML.createMetadataElement(result_doc, "assocfile", data.get(i)));
+		}
+		
+		result.appendChild(metadataList);
+		
+		return result;
+	}
+	
+	
+	public String getAssocFilePathFromDocID(String oid, String collection, UserContext userContext)
+	{
+	  Document doc = XMLConverter.newDOM();
+		Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_query_request = GSXML.createBasicRequest (doc, GSXML.REQUEST_TYPE_PAGE, collection + "/DocumentMetadataRetrieve", userContext);
+		mr_query_message.appendChild(mr_query_request);
+		
+		Element paramList = doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		paramList.appendChild(GSXML.createMetadataElement(doc, "metadata", "assocfilepath"));
+		
+		mr_query_request.appendChild(paramList);
+
+		Element docListElem = doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+		Element docElem = doc.createElement(GSXML.DOC_NODE_ELEM);
+		docElem.setAttribute(GSXML.NODE_ID_ATT, oid);
+		docListElem.appendChild(docElem);
+		mr_query_request.appendChild(docListElem);
+
+		Element response = (Element) this.router.process(mr_query_message);
+		
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+		path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
+		path = GSPath.appendLink(path, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		Element metadataListElem = (Element) GSXML.getNodeByPath(response, path);
+		Element metadataElem = (Element) metadataListElem.getFirstChild();
+		
+		return metadataElem.getFirstChild().getNodeValue();
+	}
+	
+	public String getDatabaseTypeFromCollection(String collection, UserContext userContext)
+	{
+		//Find out what kind of database we have
+	  Document doc = XMLConverter.newDOM();
+		Element dbTypeMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element dbTypeRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, collection, userContext);
+		dbTypeMessage.appendChild(dbTypeRequest);
+		Element dbTypeResponse = (Element)this.router.process(dbTypeMessage);
+		
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.COLLECTION_ELEM);
+		Element collectionElem = (Element) GSXML.getNodeByPath(dbTypeResponse, path);
+		
+		if (collectionElem != null)
+		{
+			return collectionElem.getAttribute(GSXML.DB_TYPE_ATT);
+		}
+		return null;
+	}
+}   
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/Authentication.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/Authentication.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/Authentication.java	(revision 31634)
@@ -0,0 +1,1179 @@
+package org.greenstone.gsdl3.service;
+
+import java.io.File;
+import java.io.Serializable;
+import java.math.BigInteger;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.UUID;
+import java.util.Vector;
+import java.util.regex.Pattern;
+
+import net.tanesha.recaptcha.ReCaptchaImpl;
+import net.tanesha.recaptcha.ReCaptchaResponse;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.greenstone.gsdl3.util.DerbyWrapper;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserQueryResult;
+import org.greenstone.gsdl3.util.UserTermInfo;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class Authentication extends ServiceRack
+{
+	//Some useful constants
+	protected static final int USERNAME_MIN_LENGTH = 2;
+	protected static final int USERNAME_MAX_LENGTH = 30;
+	protected static final int PASSWORD_MIN_LENGTH = 3;
+	protected static final int PASSWORD_MAX_LENGTH = 64;
+
+	//Error codes
+	protected static final int NO_ERROR = 0;
+	protected static final int ERROR_REQUEST_HAS_NO_PARAM_LIST = -1;
+	protected static final int ERROR_NOT_LOGGED_IN = -2;
+	protected static final int ERROR_ADMIN_NOT_LOGGED_IN = -3;
+	protected static final int ERROR_COULD_NOT_GET_USER_INFO = -4;
+	protected static final int ERROR_USERNAME_NOT_SPECIFIED = -5;
+	protected static final int ERROR_REQUESTED_USER_NOT_FOUND = -6;
+	protected static final int ERROR_SQL_EXCEPTION = -7;
+	protected static final int ERROR_INVALID_USERNAME = -8;
+	protected static final int ERROR_PASSWORD_NOT_ENTERED = -9;
+	protected static final int ERROR_PASSWORD_TOO_SHORT = -10;
+	protected static final int ERROR_PASSWORD_TOO_LONG = -11;
+	protected static final int ERROR_PASSWORD_USES_ILLEGAL_CHARACTERS = -12;
+	protected static final int ERROR_INCORRECT_PASSWORD = -13;
+	protected static final int ERROR_USER_ALREADY_EXISTS = -14;
+	protected static final int ERROR_ADDING_USER = -15;
+	protected static final int ERROR_REMOVING_USER = -16;
+	protected static final int ERROR_CAPTCHA_DOES_NOT_MATCH = -17;
+	protected static final int ERROR_CAPTCHA_MISSING = -18;
+	protected static final int ERROR_NOT_AUTHORISED = -19;
+
+	protected static final HashMap<Integer, String> _errorMessageMap;
+	static
+	{
+		//Corresponding error messages
+		HashMap<Integer, String> errorMessageMap = new HashMap<Integer, String>();
+		errorMessageMap.put(ERROR_REQUEST_HAS_NO_PARAM_LIST, "The list of parameters for this request was empty.");
+		errorMessageMap.put(ERROR_NOT_LOGGED_IN, "You must be logged in to access this page.");
+		errorMessageMap.put(ERROR_ADMIN_NOT_LOGGED_IN, "You must be logged in as an administrator to access this page.");
+		errorMessageMap.put(ERROR_COULD_NOT_GET_USER_INFO, "There was a error getting the user information.");
+		errorMessageMap.put(ERROR_USERNAME_NOT_SPECIFIED, "No username was specified.");
+		errorMessageMap.put(ERROR_REQUESTED_USER_NOT_FOUND, "The requested user was not found in the database.");
+		errorMessageMap.put(ERROR_SQL_EXCEPTION, "There was an SQL exception while accessing the database.");
+		errorMessageMap.put(ERROR_INVALID_USERNAME, "The username specified was invalid.");
+		errorMessageMap.put(ERROR_PASSWORD_NOT_ENTERED, "No password was entered.");
+		errorMessageMap.put(ERROR_PASSWORD_TOO_SHORT, "The password you entered was too short (minimum of 3 characters).");
+		errorMessageMap.put(ERROR_PASSWORD_TOO_LONG, "The password you entered was too long (maximum of 64 characters).");
+		errorMessageMap.put(ERROR_PASSWORD_USES_ILLEGAL_CHARACTERS, "The password you entered contains illegal characters.");
+		errorMessageMap.put(ERROR_INCORRECT_PASSWORD, "The password specified was incorrect.");
+		errorMessageMap.put(ERROR_USER_ALREADY_EXISTS, "This user already exists and therefore cannot be added.");
+		errorMessageMap.put(ERROR_ADDING_USER, "There was an error adding this user to the database.");
+		errorMessageMap.put(ERROR_REMOVING_USER, "There was an error removing this user from the database.");
+		errorMessageMap.put(ERROR_CAPTCHA_DOES_NOT_MATCH, "The words you entered did not match the image, please try again.");
+		errorMessageMap.put(ERROR_CAPTCHA_MISSING, "The information from the captcha is missing.");
+		errorMessageMap.put(ERROR_NOT_AUTHORISED, "You are not authorised to access this page.");
+
+		_errorMessageMap = errorMessageMap;
+	}
+
+	//Admin-required operations
+	protected static final String LIST_USERS = "ListUsers";
+	protected static final String PERFORM_ADD = "PerformAdd";
+	protected static final String PERFORM_EDIT = "PerformEdit";
+	protected static final String ADD_USER = "AddUser";
+	protected static final String EDIT_USER = "EditUser";
+	protected static final String PERFORM_DELETE_USER = "PerformDeleteUser";
+
+	protected static final ArrayList<String> _adminOpList;
+	static
+	{
+		ArrayList<String> opList = new ArrayList<String>();
+		opList.add(LIST_USERS);
+		opList.add(PERFORM_ADD);
+		opList.add(PERFORM_EDIT);
+		opList.add(EDIT_USER);
+		opList.add(PERFORM_DELETE_USER);
+
+		_adminOpList = opList;
+	}
+
+	//User-required operations
+	protected static final String ACCOUNT_SETTINGS = "AccountSettings";
+	protected static final String PERFORM_ACCOUNT_EDIT = "PerformAccEdit";
+	protected static final String PERFORM_RESET_PASSWORD = "PerformResetPassword";
+	protected static final String PERFORM_CHANGE_PASSWORD = "PerformChangePassword";
+	protected static final String PERFORM_RETRIEVE_PASSWORD = "PerformRetrievePassword";
+	protected static final ArrayList<String> _userOpList;
+	static
+	{
+		ArrayList<String> opList = new ArrayList<String>();
+		opList.add(ACCOUNT_SETTINGS);
+		opList.add(PERFORM_ACCOUNT_EDIT);
+		opList.add(PERFORM_RESET_PASSWORD);
+		opList.addAll(_adminOpList);
+		_userOpList = opList;
+	}
+
+	//Other operations
+	protected static final String REGISTER = "Register";
+	protected static final String PERFORM_REGISTER = "PerformRegister";
+	protected static final String LOGIN = "Login";
+
+	//the services on offer
+	protected static final String AUTHENTICATION_SERVICE = "Authentication";
+	protected static final String GET_USER_INFORMATION_SERVICE = "GetUserInformation";
+	protected static final String CHANGE_USER_EDIT_MODE_SERVICE = "ChangeUserEditMode";
+	protected static final String REMOTE_AUTHENTICATION_SERVICE = "RemoteAuthentication";
+
+	protected static boolean _derbyWrapperDoneForcedShutdown = false;
+
+	protected String _recaptchaPrivateKey = null;
+	protected String _recaptchaPublicKey = null;
+
+	/** constructor */
+	public Authentication()
+	{
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+
+		if (!_derbyWrapperDoneForcedShutdown)
+		{
+
+			// This boolean is used to ensure we always shutdown the derby server, even if it is never
+			// used by the Authentication server.  This is because the Tomcat greenstone3.xml
+			// config file also specifies a connection to the database, which can result in the
+			// server being initialized when the servlet is first accessed.  Note also, 
+			// Authentication is a ServiceRack, meaning cleanUp() is called for each service
+			// supported, however we only need to shutdown the Derby server once.  Again
+			// this boolean variable helps achieve this.
+
+			logger.info("Authentication Service performing forced shutdown of Derby Server ...");
+
+			DerbyWrapper.shutdownDatabaseServer();
+			_derbyWrapperDoneForcedShutdown = true;
+		}
+	}
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		logger.info("Configuring Authentication...");
+		this.config_info = info;
+
+		// set up Authentication service info - for now just has name and type
+		Element authentication_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		authentication_service.setAttribute(GSXML.TYPE_ATT, "authen");
+		authentication_service.setAttribute(GSXML.NAME_ATT, AUTHENTICATION_SERVICE);
+		this.short_service_info.appendChild(authentication_service);
+
+		Element getUserInformation_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		getUserInformation_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		getUserInformation_service.setAttribute(GSXML.NAME_ATT, GET_USER_INFORMATION_SERVICE);
+		this.short_service_info.appendChild(getUserInformation_service);
+
+		Element changeEditMode_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		changeEditMode_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		changeEditMode_service.setAttribute(GSXML.NAME_ATT, CHANGE_USER_EDIT_MODE_SERVICE);
+		this.short_service_info.appendChild(changeEditMode_service);
+		
+		Element remoteAuthentication_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		remoteAuthentication_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		remoteAuthentication_service.setAttribute(GSXML.NAME_ATT, REMOTE_AUTHENTICATION_SERVICE);
+		this.short_service_info.appendChild(remoteAuthentication_service);
+		
+
+		DerbyWrapper.createDatabaseIfNeeded();
+
+		NodeList recaptchaElems = info.getElementsByTagName("recaptcha");
+		for (int i = 0; i < recaptchaElems.getLength(); i++)
+		{
+			Element currentElem = (Element) recaptchaElems.item(i);
+			if (currentElem.getAttribute(GSXML.NAME_ATT) != null && currentElem.getAttribute(GSXML.NAME_ATT).equals("public_key"))
+			{
+				if (currentElem.getAttribute(GSXML.VALUE_ATT) != null)
+				{
+					_recaptchaPublicKey = currentElem.getAttribute(GSXML.VALUE_ATT);
+				}
+			}
+			else if (currentElem.getAttribute(GSXML.NAME_ATT) != null && currentElem.getAttribute(GSXML.NAME_ATT).equals("private_key"))
+			{
+				if (currentElem.getAttribute(GSXML.VALUE_ATT) != null)
+				{
+					_recaptchaPrivateKey = currentElem.getAttribute(GSXML.VALUE_ATT);
+				}
+			}
+		}
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+
+		Element authen_service = doc.createElement(GSXML.SERVICE_ELEM);
+
+		if (service_id.equals(AUTHENTICATION_SERVICE))
+		{
+			authen_service.setAttribute(GSXML.TYPE_ATT, "authen");
+			authen_service.setAttribute(GSXML.NAME_ATT, AUTHENTICATION_SERVICE);
+		}
+		else if (service_id.equals(GET_USER_INFORMATION_SERVICE))
+		{
+			authen_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+			authen_service.setAttribute(GSXML.NAME_ATT, GET_USER_INFORMATION_SERVICE);
+		}
+		else if (service_id.equals(CHANGE_USER_EDIT_MODE_SERVICE))
+		{
+			authen_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+			authen_service.setAttribute(GSXML.NAME_ATT, CHANGE_USER_EDIT_MODE_SERVICE);
+		}
+		else if (service_id.equals(REMOTE_AUTHENTICATION_SERVICE))
+		{
+			authen_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+			authen_service.setAttribute(GSXML.NAME_ATT, REMOTE_AUTHENTICATION_SERVICE);
+		}		
+		else
+		{
+			return null;
+		}
+
+		if (service_id.equals(AUTHENTICATION_SERVICE) && (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER)))
+		{
+			authen_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getServiceName(service_id, lang)));
+			authen_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getServiceDescription(service_id, lang)));
+		}
+		return authen_service;
+	}
+
+	protected String getServiceName(String service_id, String lang)
+	{
+		return getTextString(service_id + ".name", lang);
+	}
+
+	protected String getServiceSubmit(String service_id, String lang)
+	{
+		return getTextString(service_id + ".submit", lang);
+	}
+
+	protected String getServiceDescription(String service_id, String lang)
+	{
+		return getTextString(service_id + ".description", lang);
+	}
+
+	protected Element processChangeUserEditMode(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		result.setAttribute(GSXML.FROM_ATT, CHANGE_USER_EDIT_MODE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (paramList == null)
+		{
+			GSXML.addError(result, _errorMessageMap.get(ERROR_REQUEST_HAS_NO_PARAM_LIST));
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(paramList, true);
+
+		String username = (String) params.get("username");
+		String editMode = (String) params.get("enabled");
+
+		if (!editMode.toLowerCase().equals("true") && !editMode.toLowerCase().equals("false"))
+		{
+			editMode = "false";
+		}
+
+		DerbyWrapper dw = openDatabase();
+		dw.addUserData(username, "USER_EDIT_ENABLED", editMode);
+		dw.closeDatabase();
+
+		return result;
+	}
+
+	/**
+	 * This method replaces the gliserver.pl code for authenticating a user against the derby database
+	 * gliserver.pl needed to instantiate its own JVM to access the derby DB, but the GS3 already has
+	 * the Derby DB open and 2 JVMs are not allowed concurrent access to an open embedded Derby DB.
+	 * Gliserver.pl now goes through this method (via ServletRealmCheck.java), thereby using the same 
+	 * connection to the DerbyDB. This method reproduces the same behaviour as gliserver.pl used to,
+	 * by returning the user_groups on successful authentication, else returns the specific 
+	 * "Authentication failed" messages that glisever.pl would produce.
+	 * http://remote-host-name:8383/greenstone3/library?a=s&sa=authenticated-ping&excerptid=gs_content&un=admin&pw=<PW>&col=demo
+	*/
+	protected Element processRemoteAuthentication(Element request) {
+		//logger.info("*** Authentication::processRemoteAuthentication");	
+		
+		String message = "";
+		
+		Element system = (Element) GSXML.getChildByTagName(request, GSXML.REQUEST_TYPE_SYSTEM);		
+		String username = system.hasAttribute("username") ? system.getAttribute("username") : "";
+		String password = system.hasAttribute("password") ? system.getAttribute("password") : "";
+		
+		
+		// If we're not editing a collection then the user doesn't need to be in a particular group
+		String collection = system.hasAttribute("collection") ? system.getAttribute("collection") : "";
+				
+		
+		if(username.equals("") || password.equals("")) {
+			message = "Authentication failed: no (username or) password specified.";
+			//logger.error("*** Remote login failed. No username or pwd provided");
+		}		
+		else {		
+			String storedPassword = retrieveDataForUser(username, "password");
+			if(storedPassword != null && (password.equals(storedPassword) || hashPassword(password).equals(storedPassword))) {
+				
+				// gliserver.pl used to return the groups when authentication succeeded
+				String groups = retrieveDataForUser(username, "groups"); //comma-separated list
+				
+				if(collection.equals("")) {
+					message = groups;
+				} else { 					
+					
+					if(groups.indexOf("all-collections-editor") != -1) { // Does this user have access to all collections?
+						message = groups;
+					} else if(groups.indexOf("personal-collections-editor") != -1 && collection.startsWith(username+"-")) { // Does this user have access to personal collections, and is this one?
+						message = groups;
+					} else if(groups.indexOf(collection+"-collection-editor") != -1) { //  Does this user have access to this collection?
+						message = groups;
+					}
+					else {
+						message = "Authentication failed: user is not in the required group.";
+						//logger.error("*** Remote login failed. Groups did not match for the collection specified");
+					}
+				}
+				
+			} else {
+				
+				if(storedPassword == null) {
+					message = "Authentication failed: no account for user '" + username + "'";
+					//logger.error("*** Remote login failed. User not found or password not set for user.");
+				} else {
+					message = "Authentication failed: incorrect password.";
+					//logger.error("*** Remote login failed. Password did not match for user");
+				}
+			}
+		}
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, REMOTE_AUTHENTICATION_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);		
+		Element s = GSXML.createTextElement(result_doc, GSXML.STATUS_ELEM, message);
+		result.appendChild(s);
+		return result;
+	}
+	
+	protected Element processGetUserInformation(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		result.setAttribute(GSXML.FROM_ATT, GET_USER_INFORMATION_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (paramList == null)
+		{
+			GSXML.addError(result, _errorMessageMap.get(ERROR_REQUEST_HAS_NO_PARAM_LIST));
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(paramList, true);
+
+		String username = (String) params.get("username");
+
+		if (username == null)
+		{
+			GSXML.addError(result, _errorMessageMap.get(ERROR_USERNAME_NOT_SPECIFIED));
+			return result;
+		}
+
+		DerbyWrapper derbyWrapper = openDatabase();
+
+		UserQueryResult userQueryResult = derbyWrapper.findUser(username);
+		String editEnabled = derbyWrapper.getUserData(username, "USER_EDIT_ENABLED");
+
+		Vector<UserTermInfo> terms = userQueryResult.getUserTerms();
+
+		if (terms.size() == 0)
+		{
+			GSXML.addError(result, _errorMessageMap.get(ERROR_REQUESTED_USER_NOT_FOUND));
+			return result;
+		}
+
+		UserTermInfo userInfo = terms.get(0);
+		Element userInfoList = result_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(userInfoList);
+
+		Element usernameField = GSXML.createParameter(result_doc, "username", userInfo.username);
+		Element passwordField = GSXML.createParameter(result_doc, "password", userInfo.password);
+		Element groupsField = GSXML.createParameter(result_doc, "groups", userInfo.groups);
+		Element accountStatusField = GSXML.createParameter(result_doc, "accountstatus", userInfo.accountstatus);
+		Element commentField = GSXML.createParameter(result_doc, "comment", userInfo.comment);
+
+		if (editEnabled != null)
+		{
+			Element editEnabledElem = GSXML.createParameter(result_doc, "editEnabled", editEnabled);
+			userInfoList.appendChild(editEnabledElem);
+		}
+
+		userInfoList.appendChild(usernameField);
+		userInfoList.appendChild(passwordField);
+		userInfoList.appendChild(groupsField);
+		userInfoList.appendChild(accountStatusField);
+		userInfoList.appendChild(commentField);
+
+		derbyWrapper.closeDatabase();
+
+		return result;
+	}
+
+	protected Element processAuthentication(Element request)
+	{
+		checkAdminUserExists();
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, AUTHENTICATION_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		// Create an Authentication node put into the result
+		Element authenNode = result_doc.createElement(GSXML.AUTHEN_NODE_ELEM);
+		result.appendChild(authenNode);
+		result.appendChild(getCollectList(result_doc, this.site_home + File.separatorChar + "collect"));
+
+		// Create a service node added into the Authentication node
+		Element serviceNode = result_doc.createElement(GSXML.SERVICE_ELEM);
+		authenNode.appendChild(serviceNode);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			serviceNode.setAttribute("operation", LOGIN);
+			GSXML.addError(result, _errorMessageMap.get(ERROR_REQUEST_HAS_NO_PARAM_LIST));
+			return result; // Return the empty result
+		}
+		HashMap<String, Serializable> paramMap = GSXML.extractParams(param_list, false);
+		String op = (String) paramMap.get("authpage");
+		serviceNode.setAttribute("operation", op);
+
+		String username = null;
+		String groups = null;
+
+		Element userInformation = (Element) GSXML.getChildByTagName(request, GSXML.USER_INFORMATION_ELEM);
+		if (userInformation == null && _userOpList.contains(op))
+		{
+			serviceNode.setAttribute("operation", LOGIN);
+			GSXML.addError(result, _errorMessageMap.get(ERROR_NOT_LOGGED_IN));
+			return result;
+		}
+
+		if (userInformation != null)
+		{
+			username = userInformation.getAttribute(GSXML.USERNAME_ATT);
+			groups = userInformation.getAttribute(GSXML.GROUPS_ATT);
+		}
+
+		if (username == null && _userOpList.contains(op))
+		{
+			serviceNode.setAttribute("operation", LOGIN);
+			GSXML.addError(result, _errorMessageMap.get(ERROR_NOT_LOGGED_IN));
+			return result;
+		}
+
+		if (_adminOpList.contains(op) && (groups == null || !groups.matches(".*\\badministrator\\b.*")))
+		{
+			serviceNode.setAttribute("operation", LOGIN);
+			GSXML.addError(result, _errorMessageMap.get(ERROR_ADMIN_NOT_LOGGED_IN));
+			return result;
+		}
+
+		if (op.equals(LIST_USERS))
+		{
+			int error = addUserInformationToNode(null, serviceNode);
+			if (error != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+		}
+		else if (op.equals(PERFORM_ADD))
+		{
+			String newUsername = (String) paramMap.get("username");
+			String newPassword = (String) paramMap.get("password");
+			String newGroups = (String) paramMap.get("groups");
+			String newStatus = (String) paramMap.get("status");
+			String newComment = (String) paramMap.get("comment");
+			String newEmail = (String) paramMap.get("email");
+
+			//Check the given user name
+			int error;
+			if ((error = checkUsername(newUsername)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			//Check the given password
+			if ((error = checkPassword(newPassword)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			newPassword = hashPassword(newPassword);
+
+			error = addUser(newUsername, newPassword, newGroups, newStatus, newComment, newEmail);
+			if (error != NO_ERROR)
+			{
+				serviceNode.setAttribute("operation", ADD_USER);
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+			else
+			{
+				addUserInformationToNode(null, serviceNode);
+				serviceNode.setAttribute("operation", LIST_USERS);
+			}
+		}
+		else if (op.equals(PERFORM_REGISTER))
+		{
+			String newUsername = (String) paramMap.get("username");
+			String newPassword = (String) paramMap.get("password");
+			String newEmail = (String) paramMap.get("email");
+
+			//Check the given user name
+			int error;
+			if ((error = checkUsername(newUsername)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			//Check the given password
+			if ((error = checkPassword(newPassword)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			newPassword = hashPassword(newPassword);
+
+			if (_recaptchaPrivateKey != null && _recaptchaPrivateKey.length() > 0)
+			{
+				ReCaptchaImpl reCaptcha = new ReCaptchaImpl();
+				reCaptcha.setPrivateKey(_recaptchaPrivateKey);
+
+				try
+				{
+					//If this line throws an exception then we'll assume the user has a firewall that is too restrictive
+					//(or that they're not connected to the Internet) to allow access to google services.
+					//In this situation we won't use the recaptcha test.
+					reCaptcha.checkAnswer(request.getAttribute("remoteAddress"), "", "");
+
+					String challenge = (String) paramMap.get("recaptcha_challenge_field");
+					String uResponse = (String) paramMap.get("recaptcha_response_field");
+
+					if (challenge == null || uResponse == null)
+					{
+						serviceNode.setAttribute("operation", REGISTER);
+						GSXML.addError(result, _errorMessageMap.get(ERROR_CAPTCHA_MISSING));
+						return result;
+					}
+
+					ReCaptchaResponse reCaptchaResponse = reCaptcha.checkAnswer(request.getAttribute("remoteAddress"), challenge, uResponse);
+
+					if (!reCaptchaResponse.isValid())
+					{
+						serviceNode.setAttribute("operation", REGISTER);
+						GSXML.addError(result, _errorMessageMap.get(ERROR_CAPTCHA_DOES_NOT_MATCH));
+						return result;
+					}
+				}
+				catch (Exception ex)
+				{
+				}
+			}
+
+			error = addUser(newUsername, newPassword, "", "true", "", newEmail);
+			if (error != NO_ERROR)
+			{
+				serviceNode.setAttribute("operation", REGISTER);
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+		}
+		else if (op.equals(PERFORM_EDIT))
+		{
+			String previousUsername = (String) paramMap.get("prevUsername");
+			String newUsername = (String) paramMap.get("newUsername");
+			String newPassword = (String) paramMap.get("password");
+			String newGroups = (String) paramMap.get("groups");
+			String newStatus = (String) paramMap.get("status");
+			String newComment = (String) paramMap.get("comment");
+			String newEmail = (String) paramMap.get("newEmail");
+
+			//Check the given user name
+			int error;
+			if ((error = checkUsername(newUsername)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			if (newPassword == null)
+			{
+				newPassword = retrieveDataForUser(previousUsername, "password");
+			}
+			else
+			{
+				//Check the given password
+				if ((error = checkPassword(newPassword)) != NO_ERROR)
+				{
+					GSXML.addError(result, _errorMessageMap.get(error));
+					return result;
+				}
+
+				newPassword = hashPassword(newPassword);
+			}
+
+			error = removeUser(previousUsername);
+			if (error != NO_ERROR)
+			{
+				if (error == ERROR_USERNAME_NOT_SPECIFIED)
+				{
+					addUserInformationToNode(null, serviceNode);
+					serviceNode.setAttribute("operation", LIST_USERS);
+				}
+				else
+				{
+					serviceNode.setAttribute("operation", EDIT_USER);
+					GSXML.addError(result, _errorMessageMap.get(error));
+				}
+				return result;
+			}
+
+			error = addUser(newUsername, newPassword, newGroups, newStatus, newComment, newEmail);
+			if (error != NO_ERROR)
+			{
+				serviceNode.setAttribute("operation", EDIT_USER);
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+			else
+			{
+				addUserInformationToNode(null, serviceNode);
+				serviceNode.setAttribute("operation", LIST_USERS);
+			}
+		}
+		else if (op.equals(PERFORM_ACCOUNT_EDIT))
+		{
+			String previousUsername = (String) paramMap.get("prevUsername");
+			String newUsername = (String) paramMap.get("newUsername");
+			String oldPassword = (String) paramMap.get("oldPassword");
+			String newPassword = (String) paramMap.get("newPassword");
+			String newEmail = (String) paramMap.get("newEmail");
+
+			//Make sure the user name does not already exist
+			if (!previousUsername.equals(newUsername) && checkUserExists(newUsername))
+			{
+				addUserInformationToNode(previousUsername, serviceNode);
+				serviceNode.setAttribute("operation", ACCOUNT_SETTINGS);
+				GSXML.addError(result, _errorMessageMap.get(ERROR_USER_ALREADY_EXISTS));
+				return result;
+			}
+
+			String prevPassword = retrieveDataForUser(previousUsername, "password");
+
+			if (newPassword != null)
+			{
+				oldPassword = hashPassword(oldPassword);
+
+				if (oldPassword == null || !oldPassword.equals(prevPassword))
+				{
+					addUserInformationToNode(previousUsername, serviceNode);
+					serviceNode.setAttribute("operation", ACCOUNT_SETTINGS);
+					GSXML.addError(result, _errorMessageMap.get(ERROR_INCORRECT_PASSWORD), "Incorrect Password");
+					return result;
+				}
+
+				//Check the given password
+				int error;
+				if ((error = checkPassword(newPassword)) != NO_ERROR)
+				{
+					GSXML.addError(result, _errorMessageMap.get(error));
+					return result;
+				}
+
+				newPassword = hashPassword(newPassword);
+			}
+			else
+			{
+				newPassword = prevPassword;
+			}
+
+			//Check the given user name
+			int error;
+			if ((error = checkUsername(newUsername)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			String prevGroups = retrieveDataForUser(previousUsername, "groups");
+			String prevStatus = retrieveDataForUser(previousUsername, "status");
+			String prevComment = retrieveDataForUser(previousUsername, "comment");
+
+			error = removeUser(previousUsername);
+			if (error != NO_ERROR)
+			{
+				if (error == ERROR_USERNAME_NOT_SPECIFIED)
+				{
+					addUserInformationToNode(null, serviceNode);
+					serviceNode.setAttribute("operation", LIST_USERS);
+				}
+				else
+				{
+					addUserInformationToNode(previousUsername, serviceNode);
+					serviceNode.setAttribute("operation", ACCOUNT_SETTINGS);
+					GSXML.addError(result, _errorMessageMap.get(error));
+				}
+				return result;
+			}
+
+			error = addUser(newUsername, newPassword, prevGroups, prevStatus, prevComment, newEmail);
+			if (error != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+
+			addUserInformationToNode(null, serviceNode);
+			serviceNode.setAttribute("operation", LIST_USERS);
+		}
+		else if (op.equals(PERFORM_RETRIEVE_PASSWORD))
+		{
+
+		}
+		else if (op.equals(PERFORM_CHANGE_PASSWORD))
+		{
+			serviceNode.setAttribute("operation", PERFORM_CHANGE_PASSWORD);
+			String user_name = (String) paramMap.get("username");
+			String oldPassword = (String) paramMap.get("oldPassword");
+			String newPassword = (String) paramMap.get("newPassword");
+			if (user_name == null || oldPassword == null || newPassword == null)
+			{
+				GSXML.addError(result, _errorMessageMap.get("missing compulsory parameters: username, oldPassword, or newPassword"));
+				return result;
+			}
+
+			String prevPassword = retrieveDataForUser(user_name, "password");
+			if (!hashPassword(oldPassword).equals(prevPassword))
+			{
+				addUserInformationToNode(user_name, serviceNode);
+				GSXML.addError(result, _errorMessageMap.get(ERROR_INCORRECT_PASSWORD), "Incorrect Password");
+				return result;
+			}
+
+			//Check the given password
+			int error;
+			if ((error = checkPassword(newPassword)) != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+				return result;
+			}
+
+			DerbyWrapper derbyWrapper = openDatabase();
+			String chpa_groups = retrieveDataForUser(user_name, "groups");
+			String chpa_comment = "password_changed_by_user";
+			String info = derbyWrapper.modifyUserInfo(user_name, hashPassword(newPassword), chpa_groups, null, chpa_comment, null);
+			derbyWrapper.closeDatabase();
+			if (info != "succeed")
+			{//see DerbyWrapper.modifyUserInfo
+				GSXML.addError(result, _errorMessageMap.get(info));
+				return result;
+			}
+		}
+		else if (op.equals(EDIT_USER))
+		{
+			String editUsername = (String) paramMap.get("username");
+			int error = addUserInformationToNode(editUsername, serviceNode);
+			if (error != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+		}
+		else if (op.equals(ACCOUNT_SETTINGS))
+		{
+			String editUsername = (String) paramMap.get("username");
+
+			if (editUsername == null)
+			{
+				serviceNode.setAttribute("operation", "");
+				GSXML.addError(result, _errorMessageMap.get(ERROR_USERNAME_NOT_SPECIFIED));
+				return result;
+			}
+
+			if (!editUsername.equals(username))
+			{
+				serviceNode.setAttribute("operation", LOGIN);
+				GSXML.addError(result, _errorMessageMap.get(ERROR_NOT_AUTHORISED));
+				return result;
+			}
+			int error = addUserInformationToNode(editUsername, serviceNode);
+			if (error != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+		}
+		else if (op.equals(PERFORM_RESET_PASSWORD))
+		{
+			String passwordResetUser = (String) paramMap.get("username");
+
+			String newPassword = UUID.randomUUID().toString();
+			newPassword = newPassword.substring(0, newPassword.indexOf("-"));
+
+			String email = retrieveDataForUser(passwordResetUser, "email");
+			String from = "admin@greenstone.org";
+			String host = request.getAttribute("remoteAddress");
+
+			//TODO: FINISH THIS
+		}
+		else if (op.equals(REGISTER))
+		{
+			if (_recaptchaPrivateKey != null && _recaptchaPrivateKey.length() > 0)
+			{
+				try
+				{
+					ReCaptchaImpl reCaptcha = new ReCaptchaImpl();
+					reCaptcha.setPrivateKey(_recaptchaPrivateKey);
+					reCaptcha.checkAnswer(request.getAttribute("remoteAddress"), "", "");
+				}
+				catch (Exception ex)
+				{
+					return result;
+				}
+			}
+
+			if (_recaptchaPublicKey != null && _recaptchaPrivateKey != null)
+			{
+				Element recaptchaElem = result_doc.createElement("recaptcha");
+				recaptchaElem.setAttribute("publicKey", _recaptchaPublicKey);
+				recaptchaElem.setAttribute("privateKey", _recaptchaPrivateKey);
+				result.appendChild(recaptchaElem);
+			}
+		}
+		else if (op.equals(PERFORM_DELETE_USER))
+		{
+			String usernameToDelete = (String) paramMap.get("username");
+			int error = removeUser(usernameToDelete);
+			if (error != NO_ERROR)
+			{
+				GSXML.addError(result, _errorMessageMap.get(error));
+			}
+			addUserInformationToNode(null, serviceNode);
+			serviceNode.setAttribute("operation", LIST_USERS);
+		}
+
+		return result;
+	}
+
+	public int checkUsernameAndPassword(String username, String password)
+	{
+		int uResult = checkUsername(username);
+		int pResult = checkPassword(password);
+
+		return (uResult != NO_ERROR ? uResult : (pResult != NO_ERROR ? pResult : NO_ERROR));
+	}
+
+	public int checkUsername(String username)
+	{
+		//Check the given user name
+		if ((username == null) || (username.length() < USERNAME_MIN_LENGTH) || (username.length() > USERNAME_MAX_LENGTH) || (!(Pattern.matches("[a-zA-Z0-9//_//.]+", username))))
+		{
+			return ERROR_INVALID_USERNAME;
+		}
+		return NO_ERROR;
+	}
+
+	public int checkPassword(String password)
+	{
+		//Check the given password
+		if (password == null)
+		{
+			return ERROR_PASSWORD_NOT_ENTERED;
+		}
+		else if (password.length() < PASSWORD_MIN_LENGTH)
+		{
+			return ERROR_PASSWORD_TOO_SHORT;
+		}
+		else if (password.length() > PASSWORD_MAX_LENGTH)
+		{
+			return ERROR_PASSWORD_TOO_LONG;
+		}
+		else if (!(Pattern.matches("[\\p{ASCII}]+", password)))
+		{
+			return ERROR_PASSWORD_USES_ILLEGAL_CHARACTERS;
+		}
+		return NO_ERROR;
+	}
+
+	public static String hashPassword(String password)
+	{
+		return DigestUtils.sha1Hex(password);
+	}
+
+	// This method can also be used for printing out the password in hex (in case
+	// the password used the UTF-8 Charset), or the hex values in any unicode string.
+	// From http://stackoverflow.com/questions/923863/converting-a-string-to-hexadecimal-in-java
+	public static String toHex(String arg)
+	{
+		try
+		{
+			return String.format("%x", new BigInteger(arg.getBytes("US-ASCII"))); // set to same charset as used by hashPassword
+		}
+		catch (Exception e)
+		{ // UnsupportedEncodingException
+			e.printStackTrace();
+		}
+		return "Unable to print";
+	}
+
+	private void checkAdminUserExists()
+	{
+		DerbyWrapper derbyWrapper = openDatabase();
+		UserQueryResult userQueryResult = derbyWrapper.findUser(null, null);
+		derbyWrapper.closeDatabase();
+
+		if (userQueryResult != null)
+		{
+			Vector userInfo = userQueryResult.users;
+
+			boolean adminFound = false;
+			for (int i = 0; i < userQueryResult.getSize(); i++)
+			{
+				if (((UserTermInfo) userInfo.get(i)).groups != null && ((UserTermInfo) userInfo.get(i)).groups.matches(".*\\badministrator\\b.*"))
+				{
+					adminFound = true;
+				}
+			}
+
+			if (!adminFound)
+			{
+				addUser("admin", "admin", "administrator", "true", "Change the password for this account as soon as possible", "");
+			}
+		}
+	}
+
+	private DerbyWrapper openDatabase()
+	{
+		// check the usersDb database, if it isn't existing, check the etc dir, create the etc dir if it isn't existing, then create the  user database and add a "admin" user
+		String usersDB_dir = GlobalProperties.getGSDL3Home() + File.separatorChar + "etc" + File.separatorChar + "usersDB";
+		DerbyWrapper derbyWrapper = new DerbyWrapper(usersDB_dir);
+		return derbyWrapper;
+	}
+
+	private int addUserInformationToNode(String username, Element serviceNode)
+	{
+		DerbyWrapper derbyWrapper = openDatabase();
+		UserQueryResult userQueryResult = derbyWrapper.findUser(username, null);
+		derbyWrapper.closeDatabase();
+
+		if (userQueryResult != null)
+		{
+		  Element user_node = getUserNodeList(serviceNode.getOwnerDocument(), userQueryResult);
+			serviceNode.appendChild(user_node);
+			return NO_ERROR;
+		}
+
+		return ERROR_COULD_NOT_GET_USER_INFO;
+	}
+
+	private int removeUser(String username)
+	{
+		if (username == null)
+		{
+			return ERROR_USERNAME_NOT_SPECIFIED;
+		}
+
+		DerbyWrapper derbyWrapper = openDatabase();
+		boolean success = derbyWrapper.deleteUser(username);
+		derbyWrapper.closeDatabase();
+
+		if (success)
+		{
+			return NO_ERROR;
+		}
+
+		return ERROR_REMOVING_USER;
+	}
+
+	private int addUser(String newUsername, String newPassword, String newGroups, String newStatus, String newComment, String newEmail)
+	{
+		newGroups = newGroups.replaceAll(" ", "");
+
+		//Check if the user already exists
+		DerbyWrapper derbyWrapper = openDatabase();
+		UserQueryResult userQueryResult = derbyWrapper.findUser(newUsername, null);
+
+		if (userQueryResult != null)
+		{
+			derbyWrapper.closeDatabase();
+			return ERROR_USER_ALREADY_EXISTS;
+		}
+		else
+		{
+			boolean success = derbyWrapper.addUser(newUsername, newPassword, newGroups, newStatus, newComment, newEmail);
+			derbyWrapper.closeDatabase();
+
+			if (!success)
+			{
+				return ERROR_ADDING_USER;
+			}
+		}
+
+		return NO_ERROR;
+	}
+
+	private boolean checkUserExists(String username)
+	{
+		boolean check_status = false;
+
+		DerbyWrapper derbyWrapper = openDatabase();
+		try
+		{
+			UserQueryResult result = derbyWrapper.findUser(username);
+
+			if (result != null)
+			{
+				check_status = true;
+			}
+
+		}
+		catch (Exception ex)
+		{
+			// some error occurred accessing the database
+			ex.printStackTrace();
+		}
+		derbyWrapper.closeDatabase();
+
+		return check_status;
+	}
+
+	private String retrieveDataForUser(String username, String dataType)
+	{
+		openDatabase();
+
+		String data = null;
+
+		try
+		{
+			DerbyWrapper derbyWrapper = openDatabase();
+			UserQueryResult result = derbyWrapper.findUser(username);
+			derbyWrapper.closeDatabase();
+			Vector userInfo = result.users;
+
+			for (int i = 0; i < result.getSize(); i++)
+			{
+				if (dataType.equals("password"))
+				{
+					data = ((UserTermInfo) userInfo.get(i)).password;
+					break;
+				}
+				else if (dataType.equals("groups"))
+				{
+					data = ((UserTermInfo) userInfo.get(i)).groups;
+					break;
+				}
+				else if (dataType.equals("status"))
+				{
+					data = ((UserTermInfo) userInfo.get(i)).accountstatus;
+					break;
+				}
+				else if (dataType.equals("comment"))
+				{
+					data = ((UserTermInfo) userInfo.get(i)).comment;
+					break;
+				}
+				else if (dataType.equals("email"))
+				{
+					data = ((UserTermInfo) userInfo.get(i)).email;
+					break;
+				}
+			}
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+
+		return data;
+	}
+
+  private Element getUserNodeList(Document doc, UserQueryResult userQueryResult)
+	{
+		Element user_list_node = doc.createElement(GSXML.USER_NODE_ELEM + GSXML.LIST_MODIFIER);
+
+		Vector userInfo = userQueryResult.users;
+
+		for (int i = 0; i < userQueryResult.getSize(); i++)
+		{
+			Element user_node = doc.createElement(GSXML.USER_NODE_ELEM);
+			String username = ((UserTermInfo) userInfo.get(i)).username;
+			String groups = ((UserTermInfo) userInfo.get(i)).groups;
+			String accountstatus = ((UserTermInfo) userInfo.get(i)).accountstatus;
+			String comment = ((UserTermInfo) userInfo.get(i)).comment;
+			String email = ((UserTermInfo) userInfo.get(i)).email;
+			user_node.setAttribute("username", username);
+			user_node.setAttribute("groups", groups);
+			user_node.setAttribute("status", accountstatus);
+			user_node.setAttribute("comment", comment);
+			user_node.setAttribute("email", email);
+
+			user_list_node.appendChild(user_node);
+		}
+		return user_list_node;
+	}
+
+  private Element getCollectList(Document doc, String collect)
+	{
+		Element collect_list_node = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		File[] collect_dir = (new File(collect)).listFiles();
+		if (collect_dir != null && collect_dir.length > 0)
+		{
+			for (int i = 0; i < collect_dir.length; i++)
+			{
+				if (collect_dir[i].isDirectory() && (!collect_dir[i].getName().startsWith(".svn")))
+				{
+					Element collect_node = doc.createElement(GSXML.COLLECTION_ELEM);
+					collect_node.setAttribute(GSXML.NAME_ATT, collect_dir[i].getName());
+					collect_list_node.appendChild(collect_node);
+				}
+			}
+		}
+		return collect_list_node;
+	}
+
+	// main() method - calls hashPassword() on any String argument, printing this to stdout
+	// This main() is invoked by gliserver.pl perl code to encrypt passwords identically to Java code.
+	public static void main(String[] args)
+	{
+		if (args.length < 1)
+		{
+			System.err.println("Usage: Authentication <string to encrypt>");
+			System.exit(-1);
+		}
+		// just hash the first argument
+		String hash = Authentication.hashPassword(args[0]);
+		System.out.println(hash);
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/BerryBasket.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/BerryBasket.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/BerryBasket.java	(revision 31634)
@@ -0,0 +1,694 @@
+/*
+ *    BerryBasket.java
+ *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+import java.sql.Statement;
+import java.util.Hashtable;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.Properties;
+import java.util.Date;
+
+import javax.mail.*;
+import javax.mail.internet.*;
+
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import javax.swing.Timer;
+
+import org.apache.log4j.*;
+
+public class BerryBasket extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.BerryBasket.class.getName());
+
+	// the services on offer
+	// these strings must match what is found in the properties file
+	protected static final String ADD_ITEM_SERVICE = "AddItem";
+	protected static final String DISPLAY_ITEMS_SERVICE = "DisplayList";
+	protected static final String ITEM_NUM_SERVICE = "ItemNum";
+	protected static final String DELETE_ITEMS_SERVICE = "DeleteItems";
+	protected static final String SEND_MAIL_SERVICE = "SendMail";
+	protected static final String DELETE_ITEM_SERVICE = "DeleteItem";
+
+	protected static final String ITEM_PARAM = "item";
+	protected static final String delimiter = "|";
+	protected static final int delay = 1800000;
+
+	protected Hashtable<String, Hashtable<String, Hashtable<String, Item>>> userMap = null;
+	protected Hashtable<String, UserTimer> timerMap = null;
+	protected String username = "";
+	protected String password = "";
+
+	/** constructor */
+	public BerryBasket()
+	{
+		userMap = new Hashtable<String, Hashtable<String, Hashtable<String, Item>>>();
+		timerMap = new Hashtable<String, UserTimer>();
+	}
+
+	private Hashtable<String, Hashtable<String, Item>> updateDocMap(Element request)
+	{
+
+		String id = request.getAttribute("uid");
+
+		if (userMap.containsKey(id))
+		{
+			if (timerMap.containsKey(id))
+			{
+				UserTimer timer = timerMap.get(id);
+				timer.restart();
+			}
+			return userMap.get(id);
+		}
+		else
+		{
+			UserTimer timer = new UserTimer(delay, id);
+			timerMap.put(id, timer);
+			timer.start();
+			Hashtable<String, Hashtable<String, Item>> newDocs = new Hashtable<String, Hashtable<String, Item>>();
+			userMap.put(id, newDocs);
+			return newDocs;
+		}
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		logger.info("Configuring BerryBasket...");
+		this.config_info = info;
+
+		// set up short_service_info_ - for now just has name and type
+		Element add_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		add_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		add_service.setAttribute(GSXML.NAME_ATT, ADD_ITEM_SERVICE);
+		this.short_service_info.appendChild(add_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element disp_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		disp_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		disp_service.setAttribute(GSXML.NAME_ATT, DISPLAY_ITEMS_SERVICE);
+		this.short_service_info.appendChild(disp_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element num_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		num_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		num_service.setAttribute(GSXML.NAME_ATT, ITEM_NUM_SERVICE);
+		this.short_service_info.appendChild(num_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element delete_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		delete_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		delete_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEMS_SERVICE);
+		this.short_service_info.appendChild(delete_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element deleteone_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		deleteone_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		deleteone_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEM_SERVICE);
+		this.short_service_info.appendChild(deleteone_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element mail_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		mail_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		mail_service.setAttribute(GSXML.NAME_ATT, SEND_MAIL_SERVICE);
+		this.short_service_info.appendChild(mail_service);
+
+		return true;
+
+	}
+
+	/** returns a specific service description */
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+
+		if (service_id.equals(ADD_ITEM_SERVICE))
+		{
+			Element add_service = doc.createElement(GSXML.SERVICE_ELEM);
+			add_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			add_service.setAttribute(GSXML.NAME_ATT, ADD_ITEM_SERVICE);
+			return add_service;
+		}
+		if (service_id.equals(DISPLAY_ITEMS_SERVICE))
+		{
+
+			Element disp_service = doc.createElement(GSXML.SERVICE_ELEM);
+			disp_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			disp_service.setAttribute(GSXML.NAME_ATT, DISPLAY_ITEMS_SERVICE);
+			return disp_service;
+		}
+
+		if (service_id.equals(ITEM_NUM_SERVICE))
+		{
+
+			Element num_service = doc.createElement(GSXML.SERVICE_ELEM);
+			num_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			num_service.setAttribute(GSXML.NAME_ATT, ITEM_NUM_SERVICE);
+			return num_service;
+		}
+
+		if (service_id.equals(DELETE_ITEMS_SERVICE))
+		{
+
+			Element del_service = doc.createElement(GSXML.SERVICE_ELEM);
+			del_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			del_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEMS_SERVICE);
+			return del_service;
+		}
+
+		if (service_id.equals(DELETE_ITEM_SERVICE))
+		{
+
+			Element delone_service = doc.createElement(GSXML.SERVICE_ELEM);
+			delone_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			delone_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEM_SERVICE);
+			return delone_service;
+		}
+
+		if (service_id.equals(SEND_MAIL_SERVICE))
+		{
+
+			Element mail_service = doc.createElement(GSXML.SERVICE_ELEM);
+			mail_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			mail_service.setAttribute(GSXML.NAME_ATT, SEND_MAIL_SERVICE);
+			return mail_service;
+		}
+
+		return null;
+	}
+
+	protected Element processAddItem(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("BerryBasket Error: AddItem request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String item = (String) params.get("item");
+		String collection = "";
+		int pos = item.indexOf(":");
+		if (pos != -1)
+		{
+			collection = item.substring(0, pos);
+			item = item.substring(pos + 1);
+		}
+
+		if (docsMap.containsKey(collection))
+		{
+			Hashtable<String, Item> items = docsMap.get(collection);
+			if (!items.containsKey(item))
+			{
+				Item newItem = generateItem(collection, item);
+				items.put(item, newItem);
+				result.appendChild(newItem.wrapIntoElement(result_doc));
+			}
+		}
+		else
+		{
+			Hashtable<String, Item> items = new Hashtable<String, Item>();
+			Item newItem = generateItem(collection, item);
+			items.put(item, newItem);
+			docsMap.put(collection, items);
+			result.appendChild(newItem.wrapIntoElement(result_doc));
+		}
+
+		return result;
+	}
+
+	private Item generateItem(String collection, String id)
+	{
+
+		Item item = new Item(collection, id);
+		String to = GSPath.appendLink(collection, "DocumentMetadataRetrieve");
+		ArrayList<String> tmp = new ArrayList<String>();
+		tmp.add(id);
+		
+		UserContext userContext = new UserContext();
+		userContext.setLanguage("en");
+		userContext.setUserID("dumy");
+		Element response = getDocumentMetadata(to, userContext, tmp.iterator());
+		Element doc_node = (Element) response.getElementsByTagName(GSXML.DOC_NODE_ELEM).item(0);
+
+		String node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+		Element metadata_list = (Element) doc_node.getElementsByTagName(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER).item(0);
+
+		//assign title metadata if any
+		Element metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "Title");
+		if (metadata != null)
+		{
+			item.title = GSXML.getNodeText(metadata).trim();
+		}
+		//assign date metadata if any
+		metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "Date");
+		if (metadata != null)
+		{
+			item.date = GSXML.getNodeText(metadata).trim();
+		}
+
+		//assign root title metadata if any
+		metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "root_Title");
+		if (metadata != null)
+		{
+			String rootTitle = GSXML.getNodeText(metadata).trim();
+			if (!rootTitle.equals(item.title) && !rootTitle.equals(""))
+			{
+				item.rootTitle = rootTitle;
+			}
+
+		}
+
+		return item;
+	}
+
+	protected Element processDeleteItems(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			logger.error("BerryBasket Error: DeleteItem request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String param = (String) params.get("items");
+
+		if (param == null)
+			return result;
+
+		String[] items = param.split("\\|");
+
+		for (int i = 0; i < items.length; i++)
+		{
+			String item = items[i];
+			if (item.trim().length() == 0)
+				continue;
+			String collection = "";
+			int pos = item.indexOf(":");
+			if (pos != -1)
+			{
+				collection = item.substring(0, pos);
+				item = item.substring(pos + 1);
+			}
+
+			if (docsMap.containsKey(collection))
+			{
+				Hashtable itemMap = docsMap.get(collection);
+				if (itemMap.containsKey(item))
+				{
+					itemMap.remove(item);
+				}
+				if (itemMap.size() == 0)
+				{
+					docsMap.remove(collection);
+				}
+			}
+
+		}
+
+		return result;
+	}
+
+	protected Element processDeleteItem(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			logger.error("BerryBasket Error: DeleteItem request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String param = (String) params.get("item");
+
+		if (param == null)
+			return result;
+
+		String item = param;
+
+		String collection = "";
+		int pos = item.indexOf(":");
+
+		if (pos != -1)
+		{
+			collection = item.substring(0, pos);
+			item = item.substring(pos + 1);
+		}
+
+		if (docsMap.containsKey(collection))
+		{
+			Hashtable itemMap = docsMap.get(collection);
+			if (itemMap.containsKey(item))
+			{
+				itemMap.remove(item);
+			}
+			if (itemMap.size() == 0)
+			{
+				docsMap.remove(collection);
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processItemNum(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		int size = 0;
+		String ids = "";
+		Iterator<String> keys = docsMap.keySet().iterator();
+
+		while (keys.hasNext())
+		{
+			Hashtable items = docsMap.get(keys.next());
+			size += items.size();
+			Iterator values = items.values().iterator();
+			while (values.hasNext())
+			{
+				Item item = (Item) values.next();
+				result.appendChild(item.wrapIntoElement(result_doc));
+			}
+		}
+
+		Element selement = result_doc.createElement("size");
+		selement.setAttribute("value", size + "");
+		result.appendChild(selement);
+
+		return result;
+	}
+
+	private Element getDocumentMetadata(String to, UserContext userContext, Iterator<String> ids)
+	{
+
+		// Build a request to obtain some document metadata
+	  Document doc = XMLConverter.newDOM();
+		Element dm_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element dm_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		dm_message.appendChild(dm_request);
+
+		// Create a parameter list to specify the required metadata information
+		HashSet<String> meta_names = new HashSet<String>();
+		meta_names.add("Title"); // the default
+		meta_names.add("root_Title");
+		meta_names.add("Date");
+
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		Element param = null;
+		Iterator<String> i = meta_names.iterator();
+		while (i.hasNext())
+		{
+			String name = i.next();
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param_list.appendChild(param);
+			param.setAttribute(GSXML.NAME_ATT, "metadata");
+			param.setAttribute(GSXML.VALUE_ATT, name);
+
+		}
+
+		dm_request.appendChild(param_list);
+
+		// create the doc node list for the metadata request
+		Element dm_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		dm_request.appendChild(dm_doc_list);
+
+		while (ids.hasNext())
+		{
+			// Add the documentNode to the list
+			Element dm_doc_node = doc.createElement(GSXML.DOC_NODE_ELEM);
+			dm_doc_list.appendChild(dm_doc_node);
+			dm_doc_node.setAttribute(GSXML.NODE_ID_ATT, ids.next());
+		}
+
+		return (Element) this.router.process(dm_message);
+
+	}
+
+	protected Element processDisplayList(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		Iterator<String> keys = docsMap.keySet().iterator();
+
+		while (keys.hasNext())
+		{
+			String collection = keys.next();
+			Hashtable items = docsMap.get(collection);
+			Iterator itemItr = items.values().iterator();
+
+			Element collectionNode = result_doc.createElement("berryList");
+			collectionNode.setAttribute("name", collection);
+			result.appendChild(collectionNode);
+
+			while (itemItr.hasNext())
+			{
+				Item item = (Item) itemItr.next();
+				Element itemElement = result_doc.createElement("item");
+
+				collectionNode.appendChild(itemElement);
+				itemElement.setAttribute("name", item.docid);
+				itemElement.setAttribute("collection", item.collection);
+				itemElement.setAttribute("title", item.title);
+				itemElement.setAttribute("date", item.date);
+				itemElement.setAttribute("root_title", item.rootTitle);
+			}
+		}
+
+		return result;
+
+	}
+
+	public Element processSendMail(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			logger.error("BerryBasket Error: SendMail request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String to = (String) params.get("address");
+		String subject = (String) params.get("subject");
+		String content = (String) params.get("content");
+		String cc = (String) params.get("cc");
+		String bcc = (String) params.get("bcc");
+
+		String mailhost = GlobalProperties.getProperty("mail.smtp.host");
+		username = GlobalProperties.getProperty("mail.smtp.username");
+		password = GlobalProperties.getProperty("mail.smtp.password");
+		String from = GlobalProperties.getProperty("mail.from");
+
+		String mailer = "msgsend";
+
+		try
+		{
+
+			Properties props = System.getProperties();
+
+			//Setup smtp host and from address
+			// XXX - could use Session.getTransport() and Transport.connect()
+			// XXX - assume we're using SMTP
+			if (mailhost != null && !mailhost.trim().equals(""))
+			{
+				props.put("mail.smtp.host", mailhost);
+			}
+			else
+			{
+				props.put("mail.smtp.host", "localhost");
+			}
+			if (from != null && !from.trim().equals(""))
+			{
+				props.put("mail.from", from);
+			}
+
+			//setup username and password to the smtp server
+			if (username == null || username.trim().equals(""))
+				username = "";
+			if (password == null || password.trim().equals(""))
+				password = "";
+			Authenticator auth = new Authenticator()
+			{
+				protected PasswordAuthentication getPasswordAuthentication()
+				{
+					return new PasswordAuthentication(username, password);
+				}
+			};
+
+			Session session = Session.getInstance(props, auth);
+
+			Message msg = new MimeMessage(session);
+			msg.setFrom();
+			msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to, false));
+			if (cc != null)
+			{
+				msg.setRecipients(Message.RecipientType.CC, InternetAddress.parse(cc, false));
+			}
+			if (bcc != null)
+			{
+				msg.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(bcc, false));
+			}
+			msg.setSubject(subject);
+			msg.setText(content.replaceAll("-------", "&"));
+			msg.setHeader("X-Mailer", mailer);
+			msg.setSentDate(new Date());
+
+			// send the thing off
+			Transport.send(msg);
+
+			logger.info("\nMail was sent successfully.");
+			result.appendChild(result_doc.createTextNode("Mail was sent successfully."));
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			result.appendChild(result_doc.createTextNode(e.getMessage()));
+		}
+
+		return result;
+	}
+
+	protected class Item
+	{
+		public String collection;
+		public String docid;
+		public String title = "";
+		public String query = "";
+		public String date = "";
+		public String rootTitle = "";
+
+		public Item(String coll, String id)
+		{
+			this.collection = coll;
+			this.docid = id;
+		}
+
+		public boolean equals(Object o)
+		{
+			if (!(o instanceof Item))
+			{
+				return false;
+			}
+			Item item = (Item) o;
+			String id = collection + ":" + docid;
+			String idin = item.collection + ":" + item.docid;
+			return id.equals(idin);
+
+		}
+
+		public String toString()
+		{
+
+			return collection + ":" + docid + ":" + "[" + ((!rootTitle.equals("")) ? (rootTitle + ":") : "") + title + "]";
+		}
+
+		public Element wrapIntoElement(Document doc)
+		{
+			Element itemElement = doc.createElement("item");
+			itemElement.setAttribute("name", docid);
+			itemElement.setAttribute("collection", collection);
+			itemElement.setAttribute("title", title);
+			itemElement.setAttribute("date", date);
+			itemElement.setAttribute("root_title", rootTitle);
+			return itemElement;
+		}
+	}
+
+	private class UserTimer extends Timer implements ActionListener
+	{
+		String id = "";
+
+		public UserTimer(int delay, String id)
+		{
+			super(delay, (ActionListener) null);
+			addActionListener(this);
+			this.id = id;
+		}
+
+		public void actionPerformed(ActionEvent e)
+		{
+			userMap.remove(id);
+			timerMap.remove(id);
+			stop();
+		}
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CollectionGroups.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CollectionGroups.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CollectionGroups.java	(revision 31634)
@@ -0,0 +1,527 @@
+package org.greenstone.gsdl3.service;
+
+import java.io.File;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.Set;
+
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
+import java.util.HashSet;
+import java.util.Iterator;
+
+public class CollectionGroups extends ServiceRack {
+
+	private Element hierarchy = null;
+	private Element groupDesc = null;
+
+	private static final String GROUP_CONTENT = "GroupCurrentContent";
+	private static final String UNIQUE_COLLECTIONS = "UniqueCollections";
+	private static final String COLLECTIONS_HIERARCHY = "CollectionsHierarchy";
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.BerryBasket.class.getName());
+
+	@Override
+	protected Element getServiceDescription(Document doc, String service, String lang, String subset) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	String[] _services = { GROUP_CONTENT, UNIQUE_COLLECTIONS, COLLECTIONS_HIERARCHY };
+
+	public boolean configure(Element info, Element extra_info) {
+		if (!super.configure(info, extra_info)) {
+			return false;
+		}
+
+		logger.info("Configuring CollectionGroups...");
+		this.config_info = info;
+
+		for (int i = 0; i < _services.length; i++) {
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_GROUPINFO);
+			service.setAttribute(GSXML.NAME_ATT, _services[i]);
+			this.short_service_info.appendChild(service);
+		}
+		// Load group configuration from file
+		return readGroupConfiguration();
+	}
+
+	protected Element processGroupCurrentContent(Element request) {
+
+		Document doc = XMLConverter.newDOM();
+		UserContext userContext = new UserContext(request);
+
+	  // Prepare basic response
+		Element result = GSXML.createBasicResponse(doc, GSXML.SERVICE_TYPE_GROUPINFO);
+		
+		
+		// Get param list from request
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		// Set default group path (main page)
+		String groupPath = "";
+		// If param list not null and group param exists extract value and
+		// override groupPath variable
+		if (paramList != null) {
+			Element paramGroup = GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSXML.GROUP_ELEM);
+			if (paramGroup != null) {
+				groupPath = paramGroup.getAttribute(GSXML.VALUE_ATT);
+			}
+		}
+		//Remove leading, ending / and dupliclated /
+		groupPath = groupPath.replaceAll("(/+)", "/");
+		groupPath = groupPath.replaceAll("(^/+)|(/+$)", "");
+
+		Element mrCollectionList = getAvailableCollectionList(userContext);
+		if (mrCollectionList == null){
+			result.appendChild(doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER));
+			return result;
+		}
+		//Get current groups and collections
+		Element groupContent = getRawCurrentContent(groupPath);
+	
+		//Get ungrouped collection list
+		Element ungroupedCollections = getUngroupedCollections(mrCollectionList);
+		
+		// If groupContent is empty return empty collection list
+		if (groupContent == null) {
+			result.appendChild(doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER));
+			return result;
+		}
+		NodeList currentContent = groupContent.getChildNodes();
+		if (currentContent != null && currentContent.getLength() > 0) {
+			// Create CollectionList element in response
+			Element result_collection_list = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+			result.appendChild(result_collection_list);
+			Element result_group_list = doc.createElement(GSXML.GROUP_ELEM + GSXML.LIST_MODIFIER);
+			result.appendChild(result_group_list);
+			// Iterate over collections from current view
+			// i represents current position in groupConfig.xml
+			int i;
+			for (i=0; i < currentContent.getLength(); i++) {
+				//logger.warn("NODE CuR Content" + GSXML.xmlNodeToString(currentContent.item(i)));
+				
+				if (currentContent.item(i).getNodeName() == GSXML.COLLECTION_ELEM) {
+					
+					Element collection = (Element) currentContent.item(i);
+					String collection_name = collection.getAttribute(GSXML.NAME_ATT);
+					// Check whether collection from current view exists in message router response
+					Element checkedCollection = GSXML.getNamedElement(mrCollectionList, GSXML.COLLECTION_ELEM,GSXML.NAME_ATT, collection_name);
+					if (checkedCollection != null) {
+						//Set position value
+						checkedCollection.setAttribute(GSXML.POSITION_ATT, String.valueOf(i));
+						// Add collection to response
+						result_collection_list.appendChild(doc.importNode(checkedCollection, true));
+					}
+					
+					//Iterate over groups in current view
+				} else if (currentContent.item(i).getNodeName() == GSXML.GROUP_ELEM) {
+					Element currentGroup = (Element) currentContent.item(i);
+					String currentGroupName = currentGroup.getAttribute(GSXML.NAME_ATT);
+					Element groupDescription = getGroupDescription(currentGroupName);
+					
+					groupDescription.setAttribute(GSXML.POSITION_ATT, String.valueOf(i));
+					result_group_list.appendChild(doc.importNode(groupDescription, true));
+				}
+
+			}
+			//Add ungrouped collections if groupPath /+ or "" or null
+			if (groupPath.isEmpty()) {
+				
+				//Add each ungrouped collection to the collection list in loop
+				NodeList ungroupedCollectionNodes = GSXML.getChildrenByTagName(ungroupedCollections, GSXML.COLLECTION_ATT);
+				if (ungroupedCollectionNodes != null) {
+					for (int j = 0; j < ungroupedCollectionNodes.getLength(); j++) {
+						//logger.warn("UNGROUPED COLL ELEM" + GSXML.xmlNodeToString(ungroupedCollections).intern());
+						Element ungroupedCollection = (Element) doc.importNode(ungroupedCollectionNodes.item(j), true);
+						ungroupedCollection.setAttribute(GSXML.POSITION_ATT, String.valueOf(i++));
+						result_collection_list.appendChild(ungroupedCollection);
+					}
+				}
+				
+			} else {
+				Element groupDescription = getPathInfo(groupPath);
+				if (groupContent != null){
+					result.appendChild(doc.importNode(groupDescription, true));
+				}
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processUniqueCollections(Element request) {
+		Document doc = XMLConverter.newDOM();
+		UserContext userContext = new UserContext(request);
+		// Prepare basic response
+		Element result = GSXML.createBasicResponse(doc, GSXML.SERVICE_TYPE_GROUPINFO);
+		Element resultCollections = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(resultCollections);
+		Element mrCollectionList = getAvailableCollectionList(userContext);
+		//Collections from message router check
+		if (mrCollectionList == null){
+			logger.error("mrCollectionList is null!");
+			return result;
+		}
+		//paramList check
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (paramList == null) {
+			logger.error("UniqueCollections request had no paramList.");
+			return result;
+		}
+		//Add collections from request
+		Set<String> uniq_colls = new HashSet<String>();
+		Element collParam = GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSXML.COLLECTION_ELEM);
+		if (collParam != null){
+			String colls = GSXML.getValue(collParam);
+			if (!colls.isEmpty())
+			{
+				String[] _colls = colls.split(",");
+				for (int i=0;i < _colls.length;i++){
+					uniq_colls.add(_colls[i]);
+				}
+			}
+		}
+		//groupParam check
+		Element groupParam = GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSXML.GROUP_ELEM);
+		if (groupParam == null) {
+			logger.error("UniqueCollections request had no groupParam.");
+			return result;
+		}
+		//Add collections from groups
+		String[] groups = null;
+		groups = GSXML.getValue(groupParam).split(",");
+		for (int i = 0; i < groups.length; i++) {
+			Element groupContent = getRawCurrentContent(groups[i]);
+			//If group exists
+			if (groupContent != null) {
+				NodeList collectionNodes = GSXML.getChildrenByTagName(groupContent, GSXML.COLLECTION_ELEM);
+				for (int j = 0; j < collectionNodes.getLength(); j++) {
+					String collName = ((Element) collectionNodes.item(j)).getAttribute(GSXML.NAME_ATT);
+					uniq_colls.add(collName);
+				}
+			}
+	
+		}
+		//Fill result collectionList 
+		for (Iterator<String> iterator = uniq_colls.iterator(); iterator.hasNext();) {
+			String collectionName = (String) iterator.next();
+			Element checkedCollection = GSXML.getNamedElement(mrCollectionList, GSXML.COLLECTION_ELEM,GSXML.NAME_ATT, collectionName);
+			if (checkedCollection != null){
+				resultCollections.appendChild(doc.importNode(checkedCollection, true));
+			}
+		}
+		return result;
+
+	}
+	
+	protected Element processCollectionsHierarchy(Element request){
+		
+		Document doc = XMLConverter.newDOM();
+		UserContext userContext = new UserContext(request);
+		// Prepare basic response
+		Element result = GSXML.createBasicResponse(doc, GSXML.SERVICE_TYPE_GROUPINFO);
+		String currentPath = "";
+		Element currentContent = getRawCurrentContent(currentPath);
+		if (currentContent == null){
+			return result;
+		}
+		Element searchableCollectionList = getSearchableCollectionList(userContext);
+		if (searchableCollectionList == null){
+			return result;
+		}
+		
+		//Get ungrouped collection list
+		sanitizeCurrentContent(currentContent, searchableCollectionList);
+		addGroupInfo(currentContent, currentPath);
+		addUngroupedCollections(currentContent, searchableCollectionList);
+		addAllOption(currentContent);
+		result.appendChild(doc.importNode(currentContent, true));
+		return result;
+	}
+	
+	private void addAllOption(Element currentContent) {
+		if (currentContent == null){
+			return;
+		}
+		Document doc = currentContent.getOwnerDocument();
+		Element allOption = doc.createElement(GSXML.COLLECTION_ELEM);
+		allOption.setAttribute(GSXML.NAME_ATT, "all");
+		if (currentContent.hasChildNodes()){
+			currentContent.insertBefore(allOption,currentContent.getFirstChild());	
+		} else 
+			currentContent.appendChild(allOption);
+		
+		
+	}
+
+	private void addGroupInfo(Element currentContent, String groupPath) {
+		NodeList groups = GSXML.getChildrenByTagName(currentContent,GSXML.GROUP_ELEM);
+		for (int i=0;i<groups.getLength();i++){
+			Element group = (Element) groups.item(i);
+			String name = group.getAttribute(GSXML.NAME_ATT);
+			String newPath = groupPath + "/" + name; 
+			group.setAttribute(GSXML.PATH_ATT, newPath);
+			Element groupDescription = getGroupDescription(name);
+			Element titleEl = (Element) GSXML.getChildByTagName(groupDescription, GSXML.TITLE_ELEM);
+			String title = titleEl.getTextContent();
+			group.setAttribute(GSXML.TITLE_ELEM, title	);
+			addGroupInfo(group, newPath);
+		}
+		
+	}
+
+	private Element getRawCurrentContent(String path) {
+
+		if (path == null) {
+			 path = "";
+		}
+
+		Document doc = XMLConverter.newDOM();
+		path = path.replaceAll("(/+)", "/");
+		path = path.replaceAll("(^/+)|(/+$)", "");
+		
+		String[] pathSteps = path.split("/");
+
+		
+		Element currentContent = (Element) hierarchy.cloneNode(true);
+		// Get the current view
+		for (int i = 0; i < pathSteps.length; i++) {
+			if (!pathSteps[i].isEmpty()) {
+				currentContent = GSXML.getNamedElement(currentContent, GSXML.GROUP_ELEM, GSXML.NAME_ATT, pathSteps[i]);
+				if (currentContent == null){
+					break;
+				}
+			}
+		}
+		if (currentContent == null || !currentContent.hasChildNodes()) {
+			// Return to the main page
+			return null;
+		}
+		return currentContent;
+	}
+	
+	private void sanitizeCurrentContent(Element currentContent, Element checkedCollectionList){
+		if (currentContent == null){
+			return;
+		}
+		NodeList nodes = currentContent.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		for (int i = 0; i < nodes.getLength(); i++) {
+			Element element = (Element) nodes.item(i);
+			String name = element.getAttribute(GSXML.NAME_ATT);
+			Element checkedCollection = GSXML.getNamedElement(checkedCollectionList, GSXML.COLLECTION_ELEM, GSXML.NAME_ATT, name);
+			if (checkedCollection == null) {
+				element.getParentNode().removeChild(element);
+				i--;
+			}
+		}
+	}
+	
+	private void addUngroupedCollections(Element currentContent, Element availableCollections){
+		if (currentContent == null){
+			return;
+		}
+		Document doc = currentContent.getOwnerDocument();
+		NodeList collectionList = availableCollections.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		for (int i = 0; i < collectionList.getLength();i++){
+			Element collection = (Element) collectionList.item(i);
+			String name = collection.getAttribute(GSXML.NAME_ATT);
+			NodeList foundCollection = GSXML.getNamedElements(currentContent, GSXML.COLLECTION_ELEM, GSXML.NAME_ATT, name);
+			if (foundCollection.getLength() == 0){
+				Element ungroupedCollection = doc.createElement(GSXML.COLLECTION_ELEM);
+				ungroupedCollection.setAttribute(GSXML.NAME_ATT, name);
+				currentContent.appendChild(ungroupedCollection);
+			}
+		}
+		
+	}
+
+	private boolean readGroupConfiguration() {
+
+		File configFile = new File(GSFile.groupConfigFile(site_home));
+		//
+		if (!configFile.exists()) {
+			logger.info("Groups config file " + configFile.getPath() + " does not exist.");
+			return false;
+		}
+		// Try to read and catch exception if it fails
+		Document doc = XMLConverter.getDOM(configFile);
+		Element content = doc.getDocumentElement();
+		
+		// XPath to find empty text nodes.
+		try {
+			XPathFactory xpathFactory = XPathFactory.newInstance();
+			XPathExpression xpathExp = xpathFactory.newXPath().compile("//text()[normalize-space(.) = '']");  
+			NodeList emptyTextNodes = (NodeList) xpathExp.evaluate(doc, XPathConstants.NODESET);
+			for (int i = 0; i < emptyTextNodes.getLength(); i++) {
+			    Node emptyTextNode = emptyTextNodes.item(i);
+			    emptyTextNode.getParentNode().removeChild(emptyTextNode);
+			}
+			
+		} catch (XPathExpressionException e) {
+			logger.error("Error occurred while trying to remove emtpy nodes from groupConfig.xml");
+			e.printStackTrace();
+			return false;
+		}  
+		
+		hierarchy = (Element) GSXML.getChildByTagName(content, GSXML.HIERARCHY_ELEM);
+		groupDesc = (Element) GSXML.getChildByTagName(content, GSXML.GROUP_DESC_ELEM);
+		if (hierarchy == null || groupDesc == null){
+			logger.error("Error processing groups configuration. Check groupConfig.xml");
+			return false;
+		}
+		verifyGroupDescription();
+		return true;
+	}
+
+	private void verifyGroupDescription() {
+		Document doc = groupDesc.getOwnerDocument();
+		NodeList groups = hierarchy.getElementsByTagName(GSXML.GROUP_ELEM);
+		for (int i = 0; i < groups.getLength(); i++) {
+			Element group = (Element) groups.item(i); 
+			String name = group.getAttribute(GSXML.NAME_ATT);
+			Element foundDescription = GSXML.getNamedElement(groupDesc, GSXML.GROUP_ELEM, GSXML.NAME_ATT, name);
+			if (foundDescription == null){
+				Element defaultDescription = doc.createElement(GSXML.GROUP_ELEM); 
+				defaultDescription.setAttribute(GSXML.NAME_ATT, name);
+				Element groupTitle = doc.createElement(GSXML.TITLE_ELEM);
+				groupTitle.setTextContent(name);
+				defaultDescription.appendChild(groupTitle);
+				groupDesc.appendChild(defaultDescription);
+			}
+		} 
+		
+	}
+
+	private Element getGroupDescription(String name) {
+		Element description = (Element) GSXML.getNamedElement(groupDesc, GSXML.GROUP_ELEM, GSXML.NAME_ATT, name);
+		if (description == null) {
+			logger.error("GroupDescription is not defined. Check your groupConfig.xml");
+		}
+		return description;
+	}
+
+	private Element getUngroupedCollections(Element mr_collection_list) {
+		
+		Document doc = XMLConverter.newDOM();
+		// Create Set
+		Set<String> hierarchy_unique_collections = new HashSet<String>();
+		// Element hierarchyCollections = doc.createElement("coll_list");
+		// Get collection nodes
+		
+		NodeList hierarchy_all_collection_list = hierarchy.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		// Save hierarchy collection names to Hashset
+		for (int i = 0; i < hierarchy_all_collection_list.getLength(); i++) {
+			Element collection_element = (Element) hierarchy_all_collection_list.item(i);
+			hierarchy_unique_collections.add(collection_element.getAttribute(GSXML.NAME_ATT));
+		}
+		// Save available by message router collection names to Hashset
+		Set<String> mr_collections = new HashSet<String>();
+		NodeList mr_coll_list = mr_collection_list.getElementsByTagName(GSXML.COLLECTION_ELEM);
+
+		for (int i = 0; i < mr_coll_list.getLength(); i++) {
+			Element collection_element = (Element) mr_coll_list.item(i);
+			mr_collections.add(collection_element.getAttribute(GSXML.NAME_ATT));
+		}
+		//Save collections available by message router and not existed in hierarchy to ungrouped collections set
+		Set<String> ungrouped_collections = new HashSet<String>();
+		for (String string : mr_collections) {
+			if (!hierarchy_unique_collections.contains(string)){
+				ungrouped_collections.add(string);
+			}
+		}
+		//Output
+		Element result = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		for (String name : ungrouped_collections) {
+			Element collection = doc.createElement(GSXML.COLLECTION_ELEM);
+			collection.setAttribute(GSXML.NAME_ATT, name);
+			result.appendChild(collection);
+		}
+		return result;
+
+	}
+	private Element getPathInfo(String path) {
+
+		Document doc = XMLConverter.newDOM();
+		
+		if (path == null) {
+			 path = "";
+		}
+		String[] pathSteps = path.split("/");
+
+		Element pathInfo = doc.createElement(GSXML.PATH_ELEM + GSXML.LIST_MODIFIER);
+		
+		String currentPath = "";
+		for (int i = 0; i < pathSteps.length; i++) {
+			if (!pathSteps[i].isEmpty()) {
+				currentPath += "/" + pathSteps[i];
+				Element pathStepDescription = getGroupDescription(pathSteps[i]);
+				if (pathStepDescription != null){
+					pathStepDescription.setAttribute(GSXML.POSITION_ATT, String.valueOf(i));
+					pathStepDescription.setAttribute(GSXML.PATH_ATT, currentPath);
+				}
+				pathInfo.appendChild(doc.importNode(pathStepDescription, true));
+			}
+		}
+		
+		return pathInfo;
+	}
+	private Element getAvailableCollectionList(UserContext userContext){
+		Document doc = XMLConverter.newDOM();
+		// Get the message router info
+		Element mr_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		mr_info_message.appendChild(mr_request);
+		Element mr_info_response_message = (Element) this.router.process(mr_info_message);
+		if (mr_info_response_message == null) {
+			logger.error(" couldn't query the message router!");
+			return null;
+		}
+		Element mr_info_response = (Element) GSXML.getChildByTagName(mr_info_response_message, GSXML.RESPONSE_ELEM);
+		if (mr_info_response == null) {
+			logger.error("Message router response is null!");
+			return null;
+		}
+		
+		Element mr_collection_list = (Element) GSXML.getChildByTagName(mr_info_response, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		
+		return mr_collection_list;
+	}
+	private Element getSearchableCollectionList(UserContext userContext){
+		Document doc = XMLConverter.newDOM();
+		Element collectionList = doc.createElement(GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		// Get the message router info
+		Element mr_info_message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element mr_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "TextQuery" , userContext);
+		mr_info_message.appendChild(mr_request);
+		Element mr_info_response_message = (Element) this.router.process(mr_info_message);
+		if (mr_info_response_message == null) {
+			logger.error(" couldn't query the message router!");
+			return null;
+		}
+		NodeList options = mr_info_response_message.getElementsByTagName(GSXML.PARAM_OPTION_ELEM);
+		for (int i = 0; i < options.getLength(); i++) {
+			Element option = (Element) options.item(i);
+			String name = option.getAttribute(GSXML.NAME_ATT);
+			if (name.equals("all")){
+				continue;
+			}
+			Element collection = doc.createElement(GSXML.COLLECTION_ELEM);
+			collection.setAttribute(GSXML.NAME_ATT, name);
+			collectionList.appendChild(collection);
+		}
+		return collectionList;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CoverageMetadataRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CoverageMetadataRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CoverageMetadataRetrieve.java	(revision 31634)
@@ -0,0 +1,276 @@
+/*
+ * MyNewServicesTemplate.java - a dummy class showing how to create new 
+ * services for Greenstone3
+ *
+ * This class has two dummy services: TextQuery and MyDifferentService
+ */
+
+// This file needs to be put in org/greenstone/gsdl3/service
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.apache.log4j.*;
+
+import java.util.Iterator;
+import java.util.Vector;
+import java.util.Set;
+
+// change the class name (and the filename) to something more appropriate
+public class CoverageMetadataRetrieve extends ServiceRack
+{
+
+	// add in a logger for error messages
+	static Logger logger = Logger.getLogger("CoverageMetadataRetrieve");
+
+	protected SimpleCollectionDatabase coll_db = null;
+	protected String index_stem = null;
+
+	// the new service names
+	protected static final String COVERAGE_SERVICE = "CoverageMetadataRetrieve";
+
+	// initialize any custom variables
+	public CoverageMetadataRetrieve()
+	{
+
+	}
+
+	// clean up anything that we need to 
+	public void cleanUp()
+	{
+		super.cleanUp();
+	}
+
+	// Configure the class based in info in buildConfig.xml and collectionConfig.xml
+	// info is the <serviceRack name="MyNewServicesTemplate"/> element from
+	// buildConfig.xml, and extra_info is the whole collectionConfig.xml file
+	// in case its needed
+	public boolean configure(Element info, Element extra_info)
+	{
+
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring CoverageMetadataRetrieve...");
+
+		// set up short_service_info - this currently is a list of services, 
+		// with their names and service types
+		// we have two services, a new textquery, and a new one of a new type
+		//Element tq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		//tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		//tq_service.setAttribute(GSXML.NAME_ATT, QUERY_SERVICE);
+		//this.short_service_info.appendChild(tq_service);
+
+		Element diff_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		diff_service.setAttribute(GSXML.TYPE_ATT, "retrieve");
+		diff_service.setAttribute(GSXML.NAME_ATT, COVERAGE_SERVICE);
+		this.short_service_info.appendChild(diff_service);
+
+		// the index stem is either specified in the config file or is  the collection name
+		Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+		if (index_stem_elem != null)
+		{
+			this.index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (this.index_stem == null || this.index_stem.equals(""))
+		{
+			logger.error("CoverageMetadataRetrieve.configure(): indexStem element not found, stem will default to collection name");
+			this.index_stem = this.cluster_name;
+		}
+
+		// find out what kind of database we have
+		Element database_type_elem = (Element) GSXML.getChildByTagName(info, GSXML.DATABASE_TYPE_ELEM);
+		String database_type = null;
+		if (database_type_elem != null)
+		{
+			database_type = database_type_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (database_type == null || database_type.equals(""))
+		{
+			database_type = "gdbm"; // the default
+		}
+		coll_db = new SimpleCollectionDatabase(database_type);
+		if (!coll_db.databaseOK())
+		{
+			logger.error("Couldn't create the collection database of type " + database_type);
+			return false;
+		}
+
+		// Open database for querying
+		String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, this.index_stem, database_type);
+		if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ))
+		{
+			logger.error("Could not open collection database!");
+			return false;
+		}
+
+		// Extract any relevant information from info and extra_info
+		// This can be used to set up variables.
+
+		// If there is any formatting information, add it in to format_info_map
+
+		// Do this for all services as appropriate
+		Element format = null; // find it from info/extra_info
+		if (format != null)
+		{
+			this.format_info_map.put(COVERAGE_SERVICE, this.desc_doc.importNode(format, true));
+		}
+
+		return true;
+
+	}
+
+	// get the desription of a service. Could include parameter lists, displayText
+  protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+
+		// check that we have been asked for the right service
+		if (!service.equals(COVERAGE_SERVICE))
+		{
+			return null;
+		}
+
+		/*
+		 * if (service.equals(QUERY_SERVICE)) { Element tq_service =
+		 * doc.createElement(GSXML.SERVICE_ELEM);
+		 * tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		 * tq_service.setAttribute(GSXML.NAME_ATT, QUERY_SERVICE); if
+		 * (subset==null ||
+		 * subset.equals(GSXML.DISPLAY_TEXT_ELEM+GSXML.LIST_MODIFIER)) { // add
+		 * in any <displayText> elements // name, for example - get from
+		 * properties file
+		 * tq_service.appendChild(GSXML.createDisplayTextElement(doc,
+		 * GSXML.DISPLAY_TEXT_NAME, getTextString(QUERY_SERVICE+".name", lang)
+		 * )); }
+		 * 
+		 * if (subset==null ||
+		 * subset.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) { // add in a
+		 * param list if this service has parameters Element param_list =
+		 * doc.createElement(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+		 * tq_service.appendChild(param_list); // create any params and append
+		 * to param_list } return tq_service; }
+		 */
+
+		if (service.equals(COVERAGE_SERVICE))
+		{
+			Element diff_service = doc.createElement(GSXML.SERVICE_ELEM);
+			diff_service.setAttribute(GSXML.TYPE_ATT, "retrieve");
+			diff_service.setAttribute(GSXML.NAME_ATT, COVERAGE_SERVICE);
+			if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+			{
+				// add in any <displayText> elements
+				// name, for example - get from properties file
+				diff_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(COVERAGE_SERVICE + ".name", lang)));
+			}
+
+			if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+			{
+				// add in a param list if this service has parameters
+				Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				diff_service.appendChild(param_list);
+				// create any params and append to param_list
+			}
+
+			return diff_service;
+		}
+
+		// not a valid service for this class
+		return null;
+
+	}
+
+	/** This is the method that actually handles the TextQuery Service */
+	//protected Element processTextQuery(Element request) {
+
+	//Element result = this.doc.createElement(GSXML.RESPONSE_ELEM);
+	//result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+	//result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// fill in the rest
+	//return result;
+	//}
+
+	/** This is the method that actually handles the MyDifferentService service */
+	protected Element processCoverageMetadataRetrieve(Element request)
+	{
+
+		if (!this.coll_db.databaseOK())
+		{
+			logger.error("No valid database found\n");
+			return null;
+		}
+
+		DBInfo collection_info = this.coll_db.getInfo("collection");
+
+		Set<String> keys = collection_info.getKeys();
+
+		Vector<String> valid_keys = new Vector<String>();
+
+		// Iterate over keys and add valid ones to the valid_keys vector
+		String current_key = null;
+		Iterator<String> iter = keys.iterator();
+
+		while (iter.hasNext())
+		{
+			current_key = iter.next();
+			if (current_key.matches("^metadatalist-([a-zA-Z][^-])*$"))
+			{
+				logger.error("********** ADDING " + current_key + " TO VALID KEYS LIST **********\n");
+				valid_keys.add(current_key);
+			}
+		}
+
+		// Create response
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, COVERAGE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		Element metadataSetList = result_doc.createElement("metadataSetList");
+		result.appendChild(metadataSetList);
+
+		// Iterate over valid keys and build up response
+		Element metadataSet = null;
+		Element metadata = null;
+		String value = null;
+		String name = null;
+		iter = valid_keys.iterator();
+
+		while (iter.hasNext())
+		{
+			current_key = iter.next();
+
+			// Create metadataSet using the current key and add to metadataSetList
+			metadataSet = result_doc.createElement("metadataSet");
+			if (current_key.indexOf("-") != -1)
+			{
+				name = current_key.split("-")[1];
+			}
+			metadataSet.setAttribute(GSXML.NAME_ATT, name);
+			metadataSetList.appendChild(metadataSet);
+
+			// Create a metadata element for each value and add to metadataSet
+			Vector<String> sub_info = collection_info.getMultiInfo(current_key);
+			Iterator<String> iter2 = sub_info.iterator();
+			while (iter2.hasNext())
+			{
+				value = iter2.next();
+				metadata = result_doc.createElement("metadata");
+				metadata.setAttribute(GSXML.NAME_ATT, value);
+				metadataSet.appendChild(metadata);
+			}
+
+		}
+
+		return result;
+
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CrossCollectionSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CrossCollectionSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/CrossCollectionSearch.java	(revision 31634)
@@ -0,0 +1,602 @@
+/*
+ *    CrossCollectionSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+/**
+ * This ServiceRack gets specified in siteConfig.xml. So it is loaded by the MessaegRouter, and two services get activated: TextQuery, DocumentMetadataRetrieve.
+These are located at MR level, not inside a collection. QueryAction will send messages to "TextQuery", rather than eg "mgppdemo/TextQuery".
+These two services will requery the MR for search results/document metadata based on collections or documents listed.
+ */
+
+public class CrossCollectionSearch extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.CrossCollectionSearch.class.getName());
+	protected static final String QUERY_PARAM = "query";
+	protected static final String COLLECTION_PARAM = "collection";
+	protected static final String GROUP_PARAM = "group";
+  protected static final String MAXDOCS_PARAM = "maxDocs"; // matches standard maxDocs, but in this case, means max docs per collection
+  protected static final String HITS_PER_PAGE_PARAM = "hitsPerPage";
+  protected static final String MAXDOCS_DEFAULT = "20";
+	// the services on offer - these proxy the actual collection ones
+	protected static final String TEXT_QUERY_SERVICE = "TextQuery";
+	protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+
+	protected String[] coll_ids_list = null;
+	protected String[] coll_ids_list_no_all = null;
+	// maps lang to coll names list
+	protected HashMap<String, String[]> coll_names_map = null;
+
+	//protected String[] coll_names_list = null;
+
+	/** constructor */
+	public CrossCollectionSearch()
+	{
+	}
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		// any parameters? colls to include??
+		logger.info("Configuring CrossCollectionSearch...");
+		// query service
+		Element ccs_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		ccs_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		ccs_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+		this.short_service_info.appendChild(ccs_service);
+
+		// metadata service
+		Element dmr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+		this.short_service_info.appendChild(dmr_service);
+
+		// get any format info
+		Element format_info = (Element) GSXML.getChildByTagName(info, GSXML.FORMAT_ELEM);
+		if (format_info != null)
+		{
+			this.format_info_map.put(TEXT_QUERY_SERVICE, this.desc_doc.importNode(format_info, true));
+		}
+		else
+		{
+			// add in a default format statement
+		  //"xmlns:gsf='" + GSXML.GSF_NAMESPACE + "' xmlns:xsl='" + GSXML.XSL_NAMESPACE + "
+		  String format_string = "<format "+GSXML.STD_NAMESPACES_ATTS + "><gsf:template match='documentNode'><td><a><xsl:attribute name='href'>?a=d&amp;c=<xsl:value-of select='@collection'/>&amp;d=<xsl:value-of select='@nodeID'/><xsl:if test=\"@nodeType='leaf'\">&amp;sib=1</xsl:if>&amp;dt=<xsl:value-of select='@docType'/>&amp;p.a=q&amp;p.s=" + TEXT_QUERY_SERVICE + "&amp;p.c=";
+			if (this.cluster_name != null)
+			{
+				format_string += this.cluster_name;
+			}
+			format_string += "</xsl:attribute><gsf:icon/></a></td><td><gsf:metadata name='Title'/> (<xsl:value-of select='@collection'/>) </td></gsf:template></format>";
+			this.format_info_map.put(TEXT_QUERY_SERVICE, this.desc_doc.importNode(this.converter.getDOM(format_string).getDocumentElement(), true));
+		}
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+		if (service.equals(TEXT_QUERY_SERVICE))
+		{
+
+			Element ccs_service = doc.createElement(GSXML.SERVICE_ELEM);
+			ccs_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			ccs_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+
+			// display info
+			if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+			{
+				ccs_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(TEXT_QUERY_SERVICE + ".name", lang)));
+				ccs_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(TEXT_QUERY_SERVICE + ".submit", lang)));
+				ccs_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(TEXT_QUERY_SERVICE + ".description", lang)));
+			}
+			// param info
+			if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+			{
+				Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				// collection list
+				if (coll_ids_list == null)
+				{
+					initCollectionList(lang);
+				}
+				if (!coll_names_map.containsKey(lang))
+				{
+					addCollectionNames(lang);
+				}
+				Element param = GSXML.createParameterDescription(doc, COLLECTION_PARAM, getTextString("param." + COLLECTION_PARAM, lang), GSXML.PARAM_TYPE_ENUM_MULTI, "all", coll_ids_list, coll_names_map.get(lang));
+				param_list.appendChild(param);
+				// max docs param
+				param = GSXML.createParameterDescription(doc, MAXDOCS_PARAM, getTextString("param." + MAXDOCS_PARAM, lang), GSXML.PARAM_TYPE_INTEGER, MAXDOCS_DEFAULT, null, null);
+				param_list.appendChild(param);
+				// query param
+				param = GSXML.createParameterDescription(doc, QUERY_PARAM, getTextString("param." + QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+				param_list.appendChild(param);
+				ccs_service.appendChild(param_list);
+			}
+
+			logger.debug("service description=" + this.converter.getPrettyString(ccs_service));
+			return ccs_service;
+		}
+		// these ones are probably never called, but put them here just in case
+		Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+		service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		service_elem.setAttribute(GSXML.NAME_ATT, service);
+		return service_elem;
+
+	}
+
+	protected Element processTextQuery(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("TextQuery request had no paramList.");
+			return result; // Return the empty result
+		}
+		// get the collection list
+		String[] colls_list = coll_ids_list_no_all;
+		Element coll_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, COLLECTION_PARAM);
+		if (coll_param != null)
+		{
+			String coll_list = GSXML.getValue(coll_param);
+			if (!coll_list.equals("all") && !coll_list.equals(""))
+			{
+				colls_list = coll_list.split(",");
+			}
+		}
+			
+		colls_list = mergeGroups(userContext, param_list, colls_list);
+		
+		String maxdocs = MAXDOCS_DEFAULT;
+		Element maxdocs_param = GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, MAXDOCS_PARAM);
+		if (maxdocs_param != null) {
+		  maxdocs = GSXML.getValue(maxdocs_param);
+		}
+		
+		Document msg_doc = XMLConverter.newDOM();
+		Element query_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		// we are sending the same request to each collection - build up the to
+		// attribute for the request
+		StringBuffer to_att = new StringBuffer();
+		for (int i = 0; i < colls_list.length; i++)
+		{
+			if (i > 0)
+			{
+				to_att.append(",");
+			}
+			to_att.append(GSPath.appendLink(colls_list[i], "TextQuery"));
+
+		}
+		// send the query to all colls
+		Element query_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, to_att.toString(), userContext);
+		query_message.appendChild(query_request);
+		// should we add params individually?
+		Element new_param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		query_request.appendChild(new_param_list);
+		new_param_list.appendChild(msg_doc.importNode(GSXML.getNamedElement(param_list, GSXML.PARAM_ELEM, GSXML.NAME_ATT, QUERY_PARAM), true));
+
+		// for cross coll search, we only want maxdocs from each collection
+		// some colls use maxdocs, some use hits per page so lets send both
+		new_param_list.appendChild(GSXML.createParameter(msg_doc, MAXDOCS_PARAM, maxdocs));
+		new_param_list.appendChild(GSXML.createParameter(msg_doc, HITS_PER_PAGE_PARAM, maxdocs));
+		Element query_result = (Element) this.router.process(query_message);
+		// create the doc list for the response
+		Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(doc_node_list);
+		Element result_snippet_list = result_doc.createElement(GSXML.HL_SNIPPET_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(result_snippet_list);
+		NodeList hl_snippet_list = query_result.getElementsByTagName(GSXML.HL_SNIPPET_ELEM);
+		if (hl_snippet_list != null){
+			for (int hls = 0; hls < hl_snippet_list.getLength(); hls++){
+				result_snippet_list.appendChild(result_doc.importNode(hl_snippet_list.item(hls), true));
+			}
+		}
+
+		NodeList responses = query_result.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		int num_docs = 0;
+		for (int k = 0; k < responses.getLength(); k++)
+		{
+			String coll_name = GSPath.removeLastLink(((Element) responses.item(k)).getAttribute(GSXML.FROM_ATT));
+			NodeList nodes = ((Element) responses.item(k)).getElementsByTagName(GSXML.DOC_NODE_ELEM);
+			if (nodes == null || nodes.getLength() == 0)
+				continue;
+			num_docs += nodes.getLength();
+			Element last_node = null;
+			Element this_node = null;
+			for (int n = 0; n < nodes.getLength(); n++)
+			{
+				this_node = (Element) nodes.item(n);
+				this_node.setAttribute("collection", coll_name);
+	
+				if (k == 0)
+				{
+
+					doc_node_list.appendChild(result_doc.importNode(this_node, true));
+				}
+				else
+				{
+					if (last_node == null)
+					{
+						last_node = (Element) GSXML.getChildByTagName(doc_node_list, GSXML.DOC_NODE_ELEM);
+					}
+					last_node = GSXML.insertIntoOrderedList(doc_node_list, GSXML.DOC_NODE_ELEM, last_node, this_node, "rank", true);
+				}
+
+			}
+		}
+		// just send back num docs returned. Too hard to work out number of matches etc as each index type
+		// does it differently
+		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER); 
+		result.appendChild(metadata_list);
+		GSXML.addMetadata(metadata_list, "numDocsReturned", "" + num_docs);
+		return result;
+	}
+
+	//     protected Element processAdvTextQuery(Element request) 
+	//     {
+
+	//     }
+	
+	
+	
+	protected boolean initCollectionList(String lang)
+	{
+		UserContext userContext = new UserContext();
+		userContext.setLanguage(lang);
+		userContext.setUserID("");
+
+		// first, get the message router info
+		Document msg_doc = XMLConverter.newDOM();
+		Element coll_list_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element coll_list_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext); // uid
+		coll_list_message.appendChild(coll_list_request);
+		logger.debug("coll list request = " + this.converter.getPrettyString(coll_list_request));
+		Element coll_list_response = (Element) this.router.process(coll_list_message);
+		if (coll_list_response == null)
+		{
+			logger.error("couldn't query the message router!");
+			return false;
+		}
+		logger.debug("coll list response = " + this.converter.getPrettyString(coll_list_response));
+		// second, get some info from each collection. we want the coll name 
+		// and whether its got a text query service 
+
+		NodeList colls = coll_list_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+		// we can send the same request to multiple collections at once by using a comma separated list
+		Element metadata_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		StringBuffer colls_sb = new StringBuffer();
+		for (int i = 0; i < colls.getLength(); i++)
+		{
+			Element c = (Element) colls.item(i);
+			String name = c.getAttribute(GSXML.NAME_ATT);
+			if (i != 0)
+			{
+				colls_sb.append(",");
+			}
+			colls_sb.append(name);
+			//Element metadata_request = GSXML.createBasicRequest(this.doc, GSXML.REQUEST_TYPE_DESCRIBE, name, userContext);
+			//metadata_message.appendChild(metadata_request);
+		}
+
+		Element metadata_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, colls_sb.toString(), userContext);
+		metadata_message.appendChild(metadata_request);
+		logger.debug("metadata request = " + this.converter.getPrettyString(metadata_message));
+		Element metadata_response = (Element) this.router.process(metadata_message);
+		logger.debug("metadata response = " + this.converter.getPrettyString(metadata_response));
+		NodeList coll_responses = metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		ArrayList<String> valid_colls = new ArrayList<String>();
+		ArrayList<String> valid_coll_names = new ArrayList<String>();
+		for (int i = 0; i < coll_responses.getLength(); i++)
+		{
+			Element response = (Element) coll_responses.item(i);
+			Element coll = (Element) GSXML.getChildByTagName(response, GSXML.COLLECTION_ELEM);
+			Element service_list = (Element) GSXML.getChildByTagName(coll, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+			if (service_list == null)
+				continue;
+			Element query_service = GSXML.getNamedElement(service_list, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, TEXT_QUERY_SERVICE); // should be AbstractTextSearch.TEXT_QUERY_SERVICE
+			if (query_service == null)
+				continue;
+			// use the name of the response in case we are talking to a remote collection, not the name of the collection.
+			String coll_id = response.getAttribute(GSXML.FROM_ATT);
+			String coll_name = getDisplayText(coll, GSXML.DISPLAY_TEXT_NAME, lang, "en");
+			valid_colls.add(coll_id);
+			valid_coll_names.add(coll_name);
+		}
+
+		this.coll_names_map = new HashMap<String, String[]>();
+
+		// ids no all has the list without 'all' option.
+		this.coll_ids_list_no_all = new String[1];
+		this.coll_ids_list_no_all = valid_colls.toArray(coll_ids_list_no_all);
+
+		valid_colls.add(0, "all");
+		valid_coll_names.add(0, getTextString("param." + COLLECTION_PARAM + ".all", lang));
+
+		this.coll_ids_list = new String[1];
+		this.coll_ids_list = valid_colls.toArray(coll_ids_list);
+
+		String[] coll_names_list = new String[1];
+		coll_names_list = valid_coll_names.toArray(coll_names_list);
+		this.coll_names_map.put(lang, coll_names_list);
+		return true;
+	}
+
+	protected void addCollectionNames(String lang)
+	{
+
+		UserContext userContext = new UserContext();
+		userContext.setLanguage(lang);
+		userContext.setUserID("");
+
+		ArrayList<String> coll_names = new ArrayList<String>();
+		coll_names.add(getTextString("param." + COLLECTION_PARAM + ".all", lang));
+
+		// need to request MR for collection descriptions
+		Document msg_doc = XMLConverter.newDOM();
+		Element metadata_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+
+		// get a comma separated list of coll ids to send to MR
+		// the first item is the place holder for 'all'
+		StringBuffer colls_sb = new StringBuffer();
+		for (int i = 1; i < coll_ids_list.length; i++)
+		{
+			if (i != 1)
+			{
+				colls_sb.append(",");
+			}
+			colls_sb.append(coll_ids_list[i]);
+		}
+		Element metadata_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, colls_sb.toString(), userContext);
+		// param_list to request just displayTextList
+		Element param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element param = GSXML.createParameter(msg_doc, GSXML.SUBSET_PARAM, GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER);
+		param_list.appendChild(param);
+		metadata_request.appendChild(param_list);
+		metadata_message.appendChild(metadata_request);
+		logger.debug("coll names metadata request = " + this.converter.getPrettyString(metadata_message));
+		Element metadata_response = (Element) this.router.process(metadata_message);
+		logger.debug("coll names metadata response = " + this.converter.getPrettyString(metadata_response));
+		NodeList coll_responses = metadata_response.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		for (int i = 0; i < coll_responses.getLength(); i++)
+		{
+			Element response = (Element) coll_responses.item(i);
+			Element coll = (Element) GSXML.getChildByTagName(response, GSXML.COLLECTION_ELEM);
+			String coll_name = getDisplayText(coll, GSXML.DISPLAY_TEXT_NAME, lang, "en");
+			coll_names.add(coll_name);
+		}
+
+		String[] coll_names_list = new String[1];
+		coll_names_list = coll_names.toArray(coll_names_list);
+		this.coll_names_map.put(lang, coll_names_list);
+
+	}
+
+	protected Element processDocumentMetadataRetrieve(Element request)
+	{
+		// Create a new (empty) result message
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		UserContext userContext = new UserContext(request);
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("DocumentMetadataRetrieve request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		NodeList query_doc_list = request.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+		if (query_doc_list.getLength() == 0)
+		{
+			logger.error("DocumentMetadataRetrieve request had no documentNodes.");
+			return result; // Return the empty result
+		}
+
+		// the resulting doc node list
+		Element result_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(result_node_list);
+
+
+		// organise the nodes into collection lists
+		HashMap<String, Node> coll_map = new HashMap<String, Node>();
+
+		for (int i = 0; i < query_doc_list.getLength(); i++)
+		{
+			Element doc_node = (Element) query_doc_list.item(i);
+			String coll_name = doc_node.getAttribute("collection");
+			Element coll_items = (Element) coll_map.get(coll_name);
+			if (coll_items == null)
+			{
+				coll_items = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				coll_map.put(coll_name, coll_items);
+			}
+			coll_items.appendChild(result_doc.importNode(doc_node, true));
+		}
+
+		// create teh individual requests
+		Document msg_doc = XMLConverter.newDOM();
+		Element meta_request_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+
+		Set mapping_set = coll_map.entrySet();
+		Iterator iter = mapping_set.iterator();
+
+		while (iter.hasNext())
+		{
+			Map.Entry e = (Map.Entry) iter.next();
+			String cname = (String) e.getKey();
+			Element doc_nodes = (Element) e.getValue();
+			Element meta_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, GSPath.appendLink(cname, DOCUMENT_METADATA_RETRIEVE_SERVICE), userContext);
+			meta_request.appendChild(msg_doc.importNode(doc_nodes, true));
+			meta_request.appendChild(msg_doc.importNode(param_list, true));
+			meta_request_message.appendChild(meta_request);
+
+		}
+
+		Node meta_result_node = this.router.process(meta_request_message);
+		Element meta_result = GSXML.nodeToElement(meta_result_node);
+
+		// now need to put the doc nodes back in the right order
+		// go through the original list again. keep an element pointer to
+		// the next element in each collections list
+		NodeList meta_responses = meta_result.getElementsByTagName(GSXML.RESPONSE_ELEM);
+		for (int i = 0; i < meta_responses.getLength(); i++)
+		{
+			String collname = GSPath.removeLastLink(((Element) meta_responses.item(i)).getAttribute(GSXML.FROM_ATT));
+			Element first_elem = (Element) GSXML.getNodeByPath(meta_responses.item(i), "documentNodeList/documentNode");
+			coll_map.put(collname, first_elem);
+		}
+
+		for (int i = 0; i < query_doc_list.getLength(); i++)
+		{
+			Element doc_node = (Element) query_doc_list.item(i);
+			Element new_node = (Element) result_doc.importNode(doc_node, false);
+			result_node_list.appendChild(new_node);
+			String coll_name = doc_node.getAttribute("collection");
+
+			Element meta_elem = (Element) coll_map.get(coll_name);
+			GSXML.mergeMetadataLists(new_node, meta_elem);
+			coll_map.put(coll_name, meta_elem.getNextSibling());
+		}
+		return result;
+	}
+	
+	private String[] mergeGroups(UserContext userContext, Element paramList, String[] collArray){
+		Document doc = XMLConverter.newDOM();
+		boolean allSelected = false;
+		Element groupParamList = extractGroupParams(paramList, collArray, doc);		
+		Element collParam = GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, COLLECTION_PARAM);
+		if (collParam != null) {
+			String collValue = GSXML.getValue(collParam);
+			if (collValue != null) {
+				allSelected = collValue.matches(".*\\ball\\b.*");
+			}
+		}
+		//Group param not empty and coll param null or not 'all'
+		if ( allSelected || !groupParamList.hasChildNodes())
+		{	
+			return collArray;
+		}
+		//GroupInfo service to get uniq collections
+		String uniqCollServiceName = "UniqueCollections";
+		Element infoResponse = getMRInfo(userContext);
+		Element serviceList = (Element) GSXML.getChildByTagName(infoResponse, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		if (serviceList == null) { 
+			logger.error("Service list is null!");
+			return collArray;
+		}
+		Element groupInfoService = GSXML.getNamedElement(serviceList, GSXML.SERVICE_ELEM, GSXML.NAME_ATT,	uniqCollServiceName);
+		if (groupInfoService == null){
+			logger.error("UniqueCollections service unavailable; Check your groupConfig.xml");
+			return collArray;
+		}
+		Element groupQueryMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element groupQueryRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, uniqCollServiceName, userContext);
+		groupQueryMessage.appendChild(groupQueryRequest);
+		groupQueryRequest.appendChild(groupParamList);
+		if (collParam != null){
+			groupParamList.appendChild(doc.importNode(GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GSXML.COLLECTION_ELEM), true));	
+		}
+		Element groupQueryResult = (Element) this.router.process(groupQueryMessage);
+		Element groupQueryResponse = (Element) GSXML.getChildByTagName(groupQueryResult, GSXML.RESPONSE_ELEM);
+		Element collList = (Element) GSXML.getChildByTagName(groupQueryResponse, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+		NodeList collections = GSXML.getChildrenByTagName(collList, GSXML.COLLECTION_ELEM);
+		collArray = new String[collections.getLength()];
+		for (int i = 0; i < collections.getLength(); i++){
+			String collName = ((Element) collections.item(i)).getAttribute(GSXML.NAME_ATT);
+			collArray[i] = collName;
+		}
+		return collArray;
+		
+	}
+
+	private Element extractGroupParams(Element paramList, String[] collArray, Document doc) {
+		Element groupParamList = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element groupParam = null;
+		Element groupNode = GSXML.getNamedElement(paramList, GSXML.PARAM_ELEM, GSXML.NAME_ATT, GROUP_PARAM);
+		if (groupNode != null){
+			groupParam = (Element) doc.importNode(groupNode, true);
+			groupParamList.appendChild(groupParam);	
+		} else {
+			groupParam = doc.createElement(GSXML.PARAM_ELEM);
+			groupParam.setAttribute(GSXML.NAME_ATT, GSXML.GROUP_ELEM);
+			groupParamList.appendChild(groupParam);
+		}
+		String prefix = GSXML.GROUP_ELEM + "/";
+		for (int i = 0; i < collArray.length; i++) {
+			String collectionParam = collArray[i];
+			if (collectionParam.startsWith(prefix)){
+				String value = groupParam.getAttribute(GSXML.VALUE_ATT);
+				value += "," + collectionParam.substring(prefix.length() - 1 );
+				groupParam.setAttribute(GSXML.VALUE_ATT, value);
+			}
+		}
+		return groupParamList;
+	}
+	
+	private Element getMRInfo(UserContext userContext){
+		Document doc = XMLConverter.newDOM();
+		Element infoMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element infoRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, "", userContext);
+		Element paramList = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		infoRequest.appendChild(paramList);
+		GSXML.addParameterToList(paramList, GSXML.SUBSET_PARAM, GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		infoMessage.appendChild(infoRequest);
+		Element responseMessage = (Element) this.router.process(infoMessage);
+		if (responseMessage == null)
+		{
+			logger.error("couldn't query the message router!");
+			return null;
+		}
+		Element infoResponse = (Element) GSXML.getChildByTagName(responseMessage, GSXML.RESPONSE_ELEM);
+		if (infoResponse == null)
+		{
+			logger.error("response from message router is null!");
+			return null;
+		}
+		
+		return infoResponse;
+		
+	}
+	
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DebugService.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DebugService.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DebugService.java	(revision 31634)
@@ -0,0 +1,676 @@
+package org.greenstone.gsdl3.service;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSXSLT;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class DebugService extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.DebugService.class.getName());
+
+	/**********************************************************
+	 * The list of services the utility service rack supports *
+	 *********************************************************/
+	protected static final String GET_TEMPLATE_FROM_XML_FILE = "GetXMLTemplateFromFile";
+	protected static final String SAVE_TEMPLATE_TO_XML_FILE = "SaveXMLTemplateToFile";
+	protected static final String GET_TEMPLATE_LIST_FROM_FILE = "GetTemplateListFromFile";
+	protected static final String GET_XSLT_FILES_FOR_COLLECTION = "GetXSLTFilesForCollection";
+	protected static final String RESOLVE_CALL_TEMPLATE = "ResolveCallTemplate";
+	/*********************************************************/
+
+	String[] services = { GET_TEMPLATE_FROM_XML_FILE, SAVE_TEMPLATE_TO_XML_FILE, GET_TEMPLATE_LIST_FROM_FILE, GET_XSLT_FILES_FOR_COLLECTION, RESOLVE_CALL_TEMPLATE };
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring DebugServices...");
+		this.config_info = info;
+
+		for (int i = 0; i < services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, services[i]);
+			this.short_service_info.appendChild(service);
+		}
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < services.length; i++)
+		{
+			if (service_id.equals(services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+
+	protected Element processResolveCallTemplate(Element request)
+	{
+	  	Document result_doc = XMLConverter.newDOM();
+
+		Element result = GSXML.createBasicResponse(result_doc, RESOLVE_CALL_TEMPLATE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, RESOLVE_CALL_TEMPLATE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			GSXML.addError(result, RESOLVE_CALL_TEMPLATE + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collectionName = (String) params.get("collectionName");
+
+		// check permissions
+		if (!userHasEditPermissions(collectionName, request)) {
+		  GSXML.addError(result, "This user does not have the required permissions to perform this action.");
+		  return result;
+		}
+		String interfaceName = (String) params.get("interfaceName");
+		String siteName = (String) params.get("siteName");
+		String fileName = (String) params.get("fileName");
+		String nameToGet = (String) params.get("templateName");
+
+		fileName = fileName.replace("\\", "/");
+
+		Document xslDoc = GSXSLT.mergedXSLTDocumentCascade(fileName, siteName, collectionName, interfaceName, new ArrayList<String>(), true);
+
+		int sepIndex = fileName.lastIndexOf("/");
+		String pathExtra = null;
+		if (sepIndex != -1)
+		{
+			pathExtra = fileName.substring(0, sepIndex + 1);
+			fileName = fileName.substring(sepIndex + 1);
+		}
+
+		GSXSLT.inlineImportAndIncludeFilesDebug(xslDoc, pathExtra, true, fileName, siteName, collectionName, interfaceName, new ArrayList<String>());
+
+		NodeList templateList = xslDoc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "template");
+		for (int i = 0; i < templateList.getLength(); i++)
+		{
+			Element current = (Element) templateList.item(i);
+			if (current.hasAttribute("name") && current.getAttribute("name").equals(nameToGet))
+			{
+				Element debugElement = (Element) current.getElementsByTagName("debug").item(0);
+
+				Element requestedTemplate = result_doc.createElement("requestedTemplate");
+				requestedTemplate.setTextContent(debugElement.getAttribute("filename"));
+				result.appendChild(requestedTemplate);
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processGetXMLTemplateFromFile(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, GET_TEMPLATE_FROM_XML_FILE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, GET_TEMPLATE_FROM_XML_FILE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			GSXML.addError(result, GET_TEMPLATE_FROM_XML_FILE + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+
+		String collectionName = (String) params.get("collectionName");
+		// check permissions
+		if (!userHasEditPermissions(collectionName, request)) {
+		  GSXML.addError(result, "This user does not have the required permissions to perform this action.");
+		  return result;
+		}
+
+		String locationName = (String) params.get("locationName");
+		String interfaceName = (String) params.get("interfaceName");
+		String siteName = (String) params.get("siteName");
+		String fileName = (String) params.get("fileName");
+		String namespace = (String) params.get("namespace");
+		String nodeName = (String) params.get("nodename");
+		String nameToGet = (String) params.get("name");
+		String matchToGet = (String) params.get("match");
+		String xPath = (String) params.get("xpath");
+
+
+
+		String fullNamespace;
+		if (namespace.toLowerCase().equals("gsf"))
+		{
+			fullNamespace = GSXML.GSF_NAMESPACE;
+		}
+		else if (namespace.toLowerCase().equals("xsl"))
+		{
+			fullNamespace = GSXML.XSL_NAMESPACE;
+		}
+		else
+		{
+			GSXML.addError(result, "A valid namespace was not specified.");
+			return result;
+		}
+
+		File xslFile = createFileFromLocation(locationName, fileName, interfaceName, siteName, collectionName);
+		if (xslFile.exists())
+		{
+			XMLConverter converter = new XMLConverter();
+			Document xslDoc = converter.getDOM(xslFile, "UTF-8");
+			Element rootElem = xslDoc.getDocumentElement();
+
+			if (xPath != null && xPath.length() > 0)
+			{
+				String[] pathSegments = xPath.split("/");
+				for (int i = 1; i < pathSegments.length; i++)
+				{
+					String currentSegment = pathSegments[i];
+					int count = 1;
+					if (currentSegment.contains("["))
+					{
+						count = Integer.parseInt(currentSegment.substring(currentSegment.indexOf("[") + 1, currentSegment.indexOf("]")));
+						currentSegment = currentSegment.substring(0, currentSegment.indexOf("["));
+					}
+					Node child = rootElem.getFirstChild();
+					while (count > 0)
+					{
+						if (child.getNodeType() == Node.ELEMENT_NODE && ((Element) child).getNodeName().equals(currentSegment))
+						{
+							rootElem = (Element) child;
+							count--;
+						}
+						child = child.getNextSibling();
+					}
+				}
+			}
+
+			NodeList templateElems = rootElem.getElementsByTagNameNS(fullNamespace, nodeName);
+
+			if (nameToGet != null && nameToGet.length() != 0)
+			{
+				for (int i = 0; i < templateElems.getLength(); i++)
+				{
+					Element template = (Element) templateElems.item(i);
+					if (template.getAttribute("name").equals(nameToGet))
+					{
+						fixAttributes(template);
+
+						Element requestedTemplate = result_doc.createElement("requestedNameTemplate");
+						requestedTemplate.appendChild(result_doc.importNode(template, true));
+						result.appendChild(requestedTemplate);
+					}
+				}
+			}
+
+			//Maybe should look for highest priority
+			if (matchToGet != null && matchToGet.length() != 0)
+			{
+				for (int i = 0; i < templateElems.getLength(); i++)
+				{
+					Element template = (Element) templateElems.item(i);
+					if (template.getAttribute("match").equals(matchToGet))
+					{
+						fixAttributes(template);
+
+						Element requestedTemplate = result_doc.createElement("requestedMatchTemplate");
+						requestedTemplate.appendChild(result_doc.importNode(template, true));
+						result.appendChild(requestedTemplate);
+					}
+				}
+			}
+		}
+
+		return result;
+	}
+
+	protected void fixAttributes(Element template)
+	{
+		NodeList nodes = template.getElementsByTagName("*");
+		for (int j = 0; j < nodes.getLength(); j++)
+		{
+			Node current = nodes.item(j);
+			NamedNodeMap attributes = current.getAttributes();
+			for (int k = 0; k < attributes.getLength(); k++)
+			{
+				Node currentAttr = attributes.item(k);
+				String value = currentAttr.getNodeValue();
+				if (value.contains("&") || value.contains("<") || value.contains(">") || value.contains("\""))
+				{
+					currentAttr.setNodeValue(value.replace("&", "&amp;amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;"));
+				}
+			}
+		}
+	}
+
+	protected Element processSaveXMLTemplateToFile(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, SAVE_TEMPLATE_TO_XML_FILE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collectionName = (String) params.get("collectionName");
+		// check permissions
+		if (!userHasEditPermissions(collectionName, request)) {
+		  GSXML.addError(result, "This user does not have the required permissions to perform this action.");
+		  return result;
+		}
+
+		String locationName = (String) params.get("locationName");
+		String fileName = (String) params.get("fileName");
+		String interfaceName = (String) params.get("interfaceName");
+		String siteName = (String) params.get("siteName");
+		String namespace = (String) params.get("namespace");
+		String nodeName = (String) params.get("nodename");
+		String nameToSave = (String) params.get("name");
+		String matchToSave = (String) params.get("match");
+		String xPath = (String) params.get("xpath");
+		String xml = (String) params.get("xml");
+
+		String fullNamespace;
+		if (namespace.toLowerCase().equals("gsf"))
+		{
+			fullNamespace = GSXML.GSF_NAMESPACE;
+		}
+		else if (namespace.toLowerCase().equals("xsl"))
+		{
+			fullNamespace = GSXML.XSL_NAMESPACE;
+		}
+		else
+		{
+			GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + ": The specified namespace was not valid", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		File xslFile = createFileFromLocation(locationName, fileName, interfaceName, siteName, collectionName);
+		if (xslFile.exists())
+		{
+			XMLConverter converter = new XMLConverter();
+			Document xslDoc = converter.getDOM(xslFile, "UTF-8");
+
+			Element rootElem = xslDoc.getDocumentElement();
+
+			if (xPath != null && xPath.length() > 0)
+			{
+				String[] pathSegments = xPath.split("/");
+				for (int i = 1; i < pathSegments.length; i++)
+				{
+					String currentSegment = pathSegments[i];
+					int count = 1;
+					if (currentSegment.contains("["))
+					{
+						count = Integer.parseInt(currentSegment.substring(currentSegment.indexOf("[") + 1, currentSegment.indexOf("]")));
+						currentSegment = currentSegment.substring(0, currentSegment.indexOf("["));
+					}
+					Node child = rootElem.getFirstChild();
+					while (count > 0)
+					{
+						if (child.getNodeType() == Node.ELEMENT_NODE && ((Element) child).getNodeName().equals(currentSegment))
+						{
+							rootElem = (Element) child;
+							count--;
+						}
+						child = child.getNextSibling();
+					}
+				}
+			}
+
+			NodeList templateElems = rootElem.getElementsByTagNameNS(fullNamespace, nodeName);
+
+			boolean found = false;
+			if (nameToSave != null && nameToSave.length() != 0)
+			{
+				for (int i = 0; i < templateElems.getLength(); i++)
+				{
+					Element template = (Element) templateElems.item(i);
+					if (template.getAttribute("name").equals(nameToSave))
+					{
+						try
+						{
+							Element newTemplate = (Element) converter.getDOM("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<xsl:stylesheet version=\"1.0\" "+ GSXML.ALL_NAMESPACES_ATTS + "\">" + xml + "</xsl:stylesheet>", "UTF-8").getDocumentElement().getElementsByTagNameNS(fullNamespace, nodeName).item(0);
+							template.getParentNode().replaceChild(xslDoc.importNode(newTemplate, true), template);
+							found = true;
+						}
+						catch (Exception ex)
+						{
+							ex.printStackTrace();
+						}
+					}
+				}
+			}
+			//Maybe should look for highest priority match
+			if (matchToSave != null && matchToSave.length() != 0)
+			{
+				for (int i = 0; i < templateElems.getLength(); i++)
+				{
+					Element template = (Element) templateElems.item(i);
+					if (template.getAttribute("match").equals(matchToSave))
+					{
+						Element newTemplate = (Element) converter.getDOM("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<xsl:stylesheet version=\"1.0\"" + GSXML.ALL_NAMESPACES_ATTS + "\">" + xml + "</xsl:stylesheet>", "UTF-8").getDocumentElement().getElementsByTagNameNS(fullNamespace, nodeName).item(0);
+						template.getParentNode().replaceChild(xslDoc.importNode(newTemplate, true), template);
+						found = true;
+					}
+				}
+			}
+
+			if (!found)
+			{
+				GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + ": Could not save as the specified template could not be found", GSXML.ERROR_TYPE_SYNTAX);
+			}
+			else
+			{
+
+				try
+				{
+					Transformer transformer = TransformerFactory.newInstance().newTransformer();
+
+					//initialize StreamResult with File object to save to file
+					StreamResult sresult = new StreamResult(new FileWriter(xslFile));
+					DOMSource source = new DOMSource(xslDoc);
+					transformer.transform(source, sresult);
+				}
+				catch (Exception ex)
+				{
+					GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + ": There was an error writing out the XML file", GSXML.ERROR_TYPE_SYNTAX);
+				}
+
+			}
+		}
+		else
+		{
+			GSXML.addError(result, SAVE_TEMPLATE_TO_XML_FILE + "File: " + xslFile.getAbsolutePath() + " does not exist", GSXML.ERROR_TYPE_SYNTAX);
+		}
+
+		return result;
+	}
+
+	protected Element processGetTemplateListFromFile(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, GET_TEMPLATE_LIST_FROM_FILE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, GET_TEMPLATE_LIST_FROM_FILE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			GSXML.addError(result, GET_TEMPLATE_FROM_XML_FILE + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collectionName = (String) params.get("collectionName");
+		// check permissions
+		if (!userHasEditPermissions(collectionName, request)) {
+		  GSXML.addError(result, "This user does not have the required permissions to perform this action.");
+		  return result;
+		}
+		String locationName = (String) params.get("locationName");
+		String siteName = (String) params.get("siteName");
+		String interfaceName = (String) params.get("interfaceName");
+		String fileName = (String) params.get("fileName");
+
+		fileName.replace("/", File.separator);
+
+		if (locationName == null || locationName.length() == 0)
+		{
+			GSXML.addError(result, "Parameter \"locationName\" was null or empty.");
+			return result;
+		}
+
+		File xslFile = createFileFromLocation(locationName, fileName, interfaceName, siteName, collectionName);
+
+		if (xslFile.exists())
+		{
+			XMLConverter converter = new XMLConverter();
+			Document xslDoc = converter.getDOM(xslFile, "UTF-8");
+			Element xslDocElem = xslDoc.getDocumentElement();
+
+			if (locationName.equals("collectionConfig"))
+			{
+				GSXSLT.modifyCollectionConfigForDebug(xslDocElem);
+			}
+
+			Element templateList = result_doc.createElement("templateList");
+			StringBuilder templateListString = new StringBuilder("[");
+
+			NodeList xslTemplateElems = xslDocElem.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "template");
+			NodeList gsfTemplateElems = xslDocElem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "template");
+			for (int i = 0; i < xslTemplateElems.getLength() + gsfTemplateElems.getLength(); i++)
+			{
+				Element currentElem = (i < xslTemplateElems.getLength()) ? (Element) xslTemplateElems.item(i) : (Element) gsfTemplateElems.item(i - xslTemplateElems.getLength());
+				if (!currentElem.hasAttribute(GSXML.NAME_ATT) && !currentElem.hasAttribute(GSXML.MATCH_ATT))
+				{
+					continue;
+				}
+
+				templateListString.append("{");
+				templateListString.append("\"namespace\":\"" + ((i < xslTemplateElems.getLength()) ? "xsl" : "gsf") + "\",");
+				if (currentElem.hasAttribute(GSXML.NAME_ATT))
+				{
+					templateListString.append("\"name\":\"" + currentElem.getAttribute(GSXML.NAME_ATT) + "\",");
+				}
+				else if (currentElem.hasAttribute(GSXML.MATCH_ATT))
+				{
+					templateListString.append("\"match\":\"" + currentElem.getAttribute(GSXML.MATCH_ATT) + "\",");
+				}
+
+				if (currentElem.getUserData("xpath") != null)
+				{
+					templateListString.append("\"xpath\":\"" + currentElem.getUserData("xpath") + "\",");
+				}
+				templateListString.deleteCharAt(templateListString.length() - 1);
+				templateListString.append("},");
+			}
+
+			templateListString.deleteCharAt(templateListString.length() - 1);
+			templateListString.append("]");
+
+			templateList.setTextContent(templateListString.toString());
+			result.appendChild(templateList);
+		}
+		else
+		{
+			GSXML.addError(result, "File: " + xslFile.getAbsolutePath() + " does not exist");
+			return result;
+		}
+
+		return result;
+	}
+
+	protected Element processGetXSLTFilesForCollection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, GET_XSLT_FILES_FOR_COLLECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, GET_XSLT_FILES_FOR_COLLECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		if (param_list == null)
+		{
+			GSXML.addError(result, GET_TEMPLATE_FROM_XML_FILE + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collectionName = (String) params.get("collectionName");
+		// check permissions
+		if (!userHasEditPermissions(collectionName, request)) {
+		  GSXML.addError(result, "This user does not have the required permissions to perform this action.");
+		  return result;
+		}
+		String interfaceName = (String) params.get("interfaceName");
+		String siteName = (String) params.get("siteName");
+
+		Element fileList = result_doc.createElement("fileListJSON");
+		StringBuilder fileListString = new StringBuilder("[");
+
+		String[] placesToCheck = new String[] { GlobalProperties.getGSDL3Home() + File.separator + "interfaces" + File.separator + interfaceName + File.separator + "transform", //INTERFACE FILE PATH
+		GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "transform", //SITE FILE PATH
+		GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "collect" + File.separator + collectionName + File.separator + "transform", //COLLECTION FILE PATH
+		GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "collect" + File.separator + collectionName + File.separator + "etc" //COLLECTION CONFIG FILE PATH
+		};
+
+		String[] shortPlaceNames = new String[] { "interface", "site", "collection", "collectionConfig" };
+
+		for (int i = 0; i < placesToCheck.length; i++)
+		{
+			ArrayList<File> xslFiles = getXSLTFilesFromDirectoryRecursive(new File(placesToCheck[i]));
+			for (File f : xslFiles)
+			{
+				fileListString.append("{\"location\":\"" + shortPlaceNames[i] + "\",\"path\":\"" + f.getAbsolutePath().replace(placesToCheck[i] + File.separator, "") + "\"},");
+			}
+		}
+		fileListString.deleteCharAt(fileListString.length() - 1); //Remove the last , character
+		fileListString.append("]");
+
+		fileList.setTextContent(fileListString.toString());
+		result.appendChild(fileList);
+
+		return result;
+	}
+
+	protected File createFileFromLocation(String location, String filename, String interfaceName, String siteName, String collectionName)
+	{
+		String filePath = "";
+		if (location.equals("interface"))
+		{
+			filePath = GlobalProperties.getGSDL3Home() + File.separator + "interfaces" + File.separator + interfaceName + File.separator + "transform" + File.separator + filename;
+		}
+		else if (location.equals("site"))
+		{
+			filePath = GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "transform" + File.separator + filename;
+		}
+		else if (location.equals("collection"))
+		{
+			filePath = GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "collect" + File.separator + collectionName + File.separator + "transform" + File.separator + filename;
+		}
+		else if (location.equals("collectionConfig"))
+		{
+			filePath = GlobalProperties.getGSDL3Home() + File.separator + "sites" + File.separator + siteName + File.separator + "collect" + File.separator + collectionName + File.separator + "etc" + File.separator + filename;
+		}
+
+		return new File(filePath);
+	}
+
+	protected ArrayList<File> getXSLTFilesFromDirectoryRecursive(File dir)
+	{
+		ArrayList<File> result = new ArrayList<File>();
+
+		if (dir != null && dir.exists() && dir.isDirectory())
+		{
+			for (File f : dir.listFiles())
+			{
+				if (f.isDirectory())
+				{
+					result.addAll(getXSLTFilesFromDirectoryRecursive(f));
+				}
+				else if (f.getName().endsWith(".xsl") || f.getName().endsWith(".xml"))
+				{
+					result.add(f);
+				}
+			}
+		}
+
+		return result;
+	}
+
+  protected boolean userHasEditPermissions(String collection, Element request) {
+    UserContext context = new UserContext(request);
+    for (String group : context.getGroups()) {
+      // administrator always has permission
+      if (group.equals("administrator")) {
+	return true;
+      }
+      // all-collections-editor can edit any collection
+      if (!collection.equals("")) {
+	if (group.equals("all-collections-editor")) {
+	  return true;
+	}
+	if (group.equals(collection+"-collection-editor")) {
+	  return true;
+	}
+      }
+    }
+    // haven't found a group with edit permissions
+    return false;
+    
+  }
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocXMLUtil.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocXMLUtil.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocXMLUtil.java	(revision 31634)
@@ -0,0 +1,475 @@
+/*
+ *    DocXMLUtil.java
+ *    Used to manipulate archive doc.xml files
+ *
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.GSDocumentModel;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import org.apache.log4j.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.transform.Result;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+public class DocXMLUtil extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.DocXMLUtil.class.getName());
+	GSDocumentModel _GSDM = null;
+
+	/******************************************************************
+	 * The list of services the doc.xml utility service rack supports *
+	 *****************************************************************/
+	protected static final String DOC_XML_CREATE_EMPTY_FILE = "DocXMLCreateEmptyFile";
+	protected static final String DOC_XML_GET_METADATA = "DocXMLGetMetadata";
+	protected static final String DOC_XML_SET_METADATA = "DocXMLSetMetadata";
+	protected static final String DOC_XML_CREATE_SECTION = "DocXMLCreateSection";
+	protected static final String DOC_XML_DELETE_SECTION = "DocXMLDeleteSection";
+	protected static final String DOC_XML_GET_SECTION = "DocXMLGetSection";
+	protected static final String DOC_XML_SET_SECTION = "DocXMLSetSection";
+	protected static final String DOC_XML_GET_TEXT = "DocXMLGetText";
+	protected static final String DOC_XML_SET_TEXT = "DocXMLSetText";
+	protected static final String DOC_XML_DELETE_TEXT = "DocXMLDeleteText";
+	/*****************************************************************/
+
+	String[] services = { DOC_XML_CREATE_EMPTY_FILE, DOC_XML_GET_METADATA, DOC_XML_SET_METADATA, DOC_XML_GET_SECTION, DOC_XML_SET_SECTION, DOC_XML_DELETE_SECTION, DOC_XML_GET_TEXT, DOC_XML_SET_TEXT };
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring DocXMLUtil...");
+		this.config_info = info;
+
+		for (int i = 0; i < services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, services[i]);
+			this.short_service_info.appendChild(service);
+		}
+		
+		_GSDM = new GSDocumentModel(this.site_home, this.router);
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < services.length; i++)
+		{
+			if (service_id.equals(services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+
+	/************
+	 * Services *
+	 ***********/
+
+	protected Element processDocXMLCreateEmptyFile(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_CREATE_EMPTY_FILE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_CREATE_EMPTY_FILE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the document items that are requested
+		NodeList docList = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < docList.getLength(); i++)
+		{
+			Element currentDoc = (Element) docList.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentXMLCreateDocXML(oid, collection, userContext);
+			if(_GSDM.checkError(result, DOC_XML_CREATE_EMPTY_FILE))
+			{
+				return result;
+			}
+		}
+		return result;
+	}
+
+	protected Element processDocXMLGetMetadata(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_GET_METADATA);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_GET_METADATA + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the metadata items that are requested
+		NodeList metadataList = request.getElementsByTagName(GSXML.METADATA_ELEM);
+		for (int i = 0; i < metadataList.getLength(); i++)
+		{
+			Element currentMetadata = (Element) metadataList.item(i);
+			String oid = currentMetadata.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentMetadata.getAttribute(GSXML.COLLECTION_ATT);
+			String metadataName = currentMetadata.getAttribute(GSXML.NAME_ATT);
+
+			ArrayList<Element> metadataValues = _GSDM.documentXMLGetMetadata(oid, collection, metadataName, userContext);
+			if(_GSDM.checkError(result, DOC_XML_GET_METADATA))
+			{
+				return result;
+			}
+			
+			for(Element metadataValue : metadataValues)
+			{
+				Element metadataElem = result_doc.createElement(GSXML.METADATA_ELEM);
+				metadataElem.setAttribute(GSXML.NAME_ATT, metadataName);
+				metadataElem.setAttribute(GSXML.VALUE_ATT, metadataValue.getFirstChild().getNodeValue());
+				result.appendChild(metadataElem);
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocXMLSetMetadata(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_SET_METADATA);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_SET_METADATA + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the metadata items that are requested
+		NodeList metadataList = request.getElementsByTagName(GSXML.METADATA_ELEM);
+		for (int i = 0; i < metadataList.getLength(); i++)
+		{
+			Element currentMetadata = (Element) metadataList.item(i);
+			String oid = currentMetadata.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentMetadata.getAttribute(GSXML.COLLECTION_ATT);
+			String metadataName = currentMetadata.getAttribute(GSXML.NAME_ATT);
+			String newMetadataValue = currentMetadata.getAttribute(GSXML.VALUE_ATT);
+			
+			//Optional values
+			String oldMetadataValue = currentMetadata.getAttribute("old" + GSXML.VALUE_ATT);
+			String position = currentMetadata.getAttribute("position"); //TODO: Replace "position" with a constant
+			String operation = currentMetadata.getAttribute("operation");
+			
+			int op = GSDocumentModel.OPERATION_APPEND;
+			if(operation.toLowerCase().equals("insertbefore"))
+			{
+				op = GSDocumentModel.OPERATION_INSERT_BEFORE;
+			}
+			else if (operation.toLowerCase().equals("insertafter"))
+			{
+				op = GSDocumentModel.OPERATION_INSERT_AFTER;
+			}
+			else if (operation.toLowerCase().equals("replace"))
+			{
+				op = GSDocumentModel.OPERATION_REPLACE;
+			}
+			
+			//If we are given a position then set the value at that position
+			if(position != null && !position.equals(""))
+			{
+				int pos = -1;
+				try
+				{
+					pos = Integer.parseInt(position);
+				}
+				catch(Exception ex)
+				{
+					GSXML.addError(result, DOC_XML_SET_METADATA + ": Error converting the position attribute to an integer", GSXML.ERROR_TYPE_SYNTAX);
+					return result;
+				}
+				_GSDM.documentXMLSetMetadata(oid, collection, metadataName, newMetadataValue, pos, op, userContext);
+				if(_GSDM.checkError(result, DOC_XML_SET_METADATA))
+				{
+					return result;
+				}
+			}
+			//If we are given a value to replace with then call the replacement method
+			else if (oldMetadataValue != null && !oldMetadataValue.equals(""))
+			{
+				_GSDM.documentXMLReplaceMetadata(oid, collection, metadataName, oldMetadataValue, newMetadataValue, userContext);
+				if(_GSDM.checkError(result, DOC_XML_SET_METADATA))
+				{
+					return result;
+				}
+			}
+			else
+			{
+				GSXML.addError(result, DOC_XML_SET_METADATA + ": A position or previous value was not given", GSXML.ERROR_TYPE_SYNTAX);
+				return result;
+			}
+		}
+
+		return result;
+	}
+	
+	protected Element processDocXMLCreateSection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_CREATE_SECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_CREATE_SECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList sectionList = request.getElementsByTagName(GSXML.DOCXML_SECTION_ELEM); //TODO: Replace "Section" with a constant
+		for (int i = 0; i < sectionList.getLength(); i++)
+		{
+			Element currentSection = (Element) sectionList.item(i);
+			String oid = currentSection.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentSection.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentXMLCreateSection(oid, collection, userContext);
+			if(_GSDM.checkError(result, DOC_XML_CREATE_SECTION))
+			{
+				return result;
+			}
+		}
+		return result;
+	}
+	
+	protected Element processDocXMLDeleteSection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_DELETE_SECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_DELETE_SECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList sectionList = request.getElementsByTagName(GSXML.DOCXML_SECTION_ELEM); 
+		for (int i = 0; i < sectionList.getLength(); i++)
+		{
+			Element currentSection = (Element) sectionList.item(i);
+			String oid = currentSection.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentSection.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentXMLDeleteSection(oid, collection, userContext);
+			if(_GSDM.checkError(result, DOC_XML_DELETE_SECTION))
+			{
+				return result;
+			}
+		}
+		return result;
+	}
+	
+	protected Element processDocXMLGetSection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_GET_SECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_GET_SECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList sectionList = request.getElementsByTagName(GSXML.DOCXML_SECTION_ELEM);
+		for (int i = 0; i < sectionList.getLength(); i++)
+		{
+			Element currentSection = (Element) sectionList.item(i);
+			String oid = currentSection.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentSection.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentXMLGetSection(oid, collection, userContext);
+			if(_GSDM.checkError(result, DOC_XML_GET_SECTION))
+			{
+				return result;
+			}
+		}
+		return result;
+	}
+
+	protected Element processDocXMLSetSection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_SET_SECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_SET_SECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList sectionList = request.getElementsByTagName(GSXML.DOCXML_SECTION_ELEM);
+		for (int i = 0; i < sectionList.getLength(); i++)
+		{
+			Element currentSection = (Element) sectionList.item(i);
+			String oid = currentSection.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentSection.getAttribute(GSXML.COLLECTION_ATT);
+			String operation = currentSection.getAttribute("operation");
+			
+			int op = GSDocumentModel.OPERATION_REPLACE;
+			if(operation.equals("insertbefore"))
+			{
+				op = GSDocumentModel.OPERATION_INSERT_BEFORE;
+			}
+			else if (operation.equals("insertafter"))
+			{
+				op = GSDocumentModel.OPERATION_INSERT_AFTER;
+			}
+			else if (operation.equals("append"))
+			{
+				op = GSDocumentModel.OPERATION_APPEND;
+			}
+			
+			_GSDM.documentXMLSetSection(oid, collection, currentSection, op, userContext);
+			if(_GSDM.checkError(result, DOC_XML_SET_SECTION))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocXMLGetText(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_GET_TEXT);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_GET_TEXT + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList contentList = request.getElementsByTagName(GSXML.DOCXML_CONTENT_ELEM);
+		for (int i = 0; i < contentList.getLength(); i++)
+		{
+			Element currentContent = (Element) contentList.item(i);
+			String oid = currentContent.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentContent.getAttribute(GSXML.COLLECTION_ATT);
+
+			String content = _GSDM.documentXMLGetText(oid, collection, userContext);
+			if(_GSDM.checkError(result, DOC_XML_GET_TEXT))
+			{
+				return result;
+			}
+			
+			if (content == null)
+			{
+				result.appendChild(result_doc.createElement(GSXML.DOCXML_CONTENT_ELEM));
+			}
+			else
+			{
+				Element contentElem = result_doc.createElement(GSXML.DOCXML_CONTENT_ELEM);
+				Node textNode = result_doc.createTextNode(content);
+				contentElem.appendChild(textNode);
+				result.appendChild(contentElem); 
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocXMLSetText(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOC_XML_SET_TEXT);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOC_XML_SET_TEXT + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Go through each of the requests 
+		NodeList contentList = request.getElementsByTagName(GSXML.DOCXML_CONTENT_ELEM);
+		for (int i = 0; i < contentList.getLength(); i++)
+		{
+			Element currentContent = (Element) contentList.item(i);
+			String oid = currentContent.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentContent.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentXMLSetText(oid, collection, currentContent, userContext);
+			if(_GSDM.checkError(result, DOC_XML_SET_TEXT))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentBasket.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentBasket.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentBasket.java	(revision 31634)
@@ -0,0 +1,738 @@
+/*
+ *    DocumentBasket.java
+ *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+import java.sql.Statement;
+import java.util.Hashtable;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import java.net.InetAddress;
+import java.util.Properties;
+import java.util.Date;
+
+import javax.mail.*;
+import javax.mail.internet.*;
+
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import javax.swing.Timer;
+
+import java.io.*;
+
+import org.apache.log4j.*;
+
+public class DocumentBasket extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.DocumentBasket.class.getName());
+
+	// the services on offer
+	// these strings must match what is found in the properties file
+	protected static final String ADD_ITEM_SERVICE = "AddDocument";
+	protected static final String DISPLAY_ITEMS_SERVICE = "DisplayDocumentList";
+	protected static final String ITEM_NUM_SERVICE = "GetDocuments";
+	protected static final String DELETE_ITEMS_SERVICE = "DeleteDocuments";
+	protected static final String DELETE_ITEM_SERVICE = "DeleteDocument";
+	protected static final String CLEAR_ITEMS_SERVICE = "ClearDocuments";
+	//added
+	protected static final String MERGE_ITEM_SERVICE = "MergeDocument";
+	protected static final String ITEM_PARAM = "item";
+	protected static final String delimiter = "|";
+	protected static final int delay = 1800000;
+
+	protected static final String BASKET_BOOK = "documentBasketBook";
+
+	protected Hashtable<String, Hashtable<String, Hashtable<String, Item>>> userMap = null;
+	protected Hashtable<String, UserTimer> timerMap = null;
+	protected String username = "";
+	protected String password = "";
+
+	/** constructor */
+	public DocumentBasket()
+	{
+		userMap = new Hashtable<String, Hashtable<String, Hashtable<String, Item>>>();
+		timerMap = new Hashtable<String, UserTimer>();
+	}
+
+	private Hashtable<String, Hashtable<String, Item>> updateDocMap(Element request)
+	{
+		String id = request.getAttribute("uid");
+
+		if (userMap.containsKey(id))
+		{
+			if (timerMap.containsKey(id))
+			{
+				UserTimer timer = timerMap.get(id);
+				timer.restart();
+			}
+			return userMap.get(id);
+		}
+		else
+		{
+			UserTimer timer = new UserTimer(delay, id);
+			timerMap.put(id, timer);
+			timer.start();
+			Hashtable<String, Hashtable<String, Item>> newDocs = new Hashtable<String, Hashtable<String, Item>>();
+			userMap.put(id, newDocs);
+			return newDocs;
+		}
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		logger.info("Configuring DocumentBasket...");
+		this.config_info = info;
+
+		// set up short_service_info_ - for now just has name and type
+		Element add_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		add_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		add_service.setAttribute(GSXML.NAME_ATT, ADD_ITEM_SERVICE);
+		this.short_service_info.appendChild(add_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element disp_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		disp_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		disp_service.setAttribute(GSXML.NAME_ATT, DISPLAY_ITEMS_SERVICE);
+		this.short_service_info.appendChild(disp_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element num_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		num_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		num_service.setAttribute(GSXML.NAME_ATT, ITEM_NUM_SERVICE);
+		this.short_service_info.appendChild(num_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element delete_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		delete_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		delete_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEMS_SERVICE);
+		this.short_service_info.appendChild(delete_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element deleteone_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		deleteone_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		deleteone_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEM_SERVICE);
+		this.short_service_info.appendChild(deleteone_service);
+		
+		// set up short_service_info_ - for now just has name and type
+		Element deleteall_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		deleteall_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		deleteall_service.setAttribute(GSXML.NAME_ATT, CLEAR_ITEMS_SERVICE);
+		this.short_service_info.appendChild(deleteall_service);
+
+		// set up short_service_info_ - for now just has name and type
+		Element merge_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		merge_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+		merge_service.setAttribute(GSXML.NAME_ATT, MERGE_ITEM_SERVICE);
+		this.short_service_info.appendChild(merge_service);
+
+		return true;
+	}
+
+	/** returns a specific service description */
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		if (service_id.equals(ADD_ITEM_SERVICE))
+		{
+			Element add_service = doc.createElement(GSXML.SERVICE_ELEM);
+			add_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			add_service.setAttribute(GSXML.NAME_ATT, ADD_ITEM_SERVICE);
+			return add_service;
+		}
+		if (service_id.equals(DISPLAY_ITEMS_SERVICE))
+		{
+
+			Element disp_service = doc.createElement(GSXML.SERVICE_ELEM);
+			disp_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			disp_service.setAttribute(GSXML.NAME_ATT, DISPLAY_ITEMS_SERVICE);
+			return disp_service;
+		}
+
+		if (service_id.equals(ITEM_NUM_SERVICE))
+		{
+
+			Element num_service = doc.createElement(GSXML.SERVICE_ELEM);
+			num_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			num_service.setAttribute(GSXML.NAME_ATT, ITEM_NUM_SERVICE);
+			return num_service;
+		}
+
+		if (service_id.equals(DELETE_ITEMS_SERVICE))
+		{
+
+			Element del_service = doc.createElement(GSXML.SERVICE_ELEM);
+			del_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			del_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEMS_SERVICE);
+			return del_service;
+		}
+		
+		if (service_id.equals(CLEAR_ITEMS_SERVICE))
+		{
+
+			Element clear_service = doc.createElement(GSXML.SERVICE_ELEM);
+			clear_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			clear_service.setAttribute(GSXML.NAME_ATT, CLEAR_ITEMS_SERVICE);
+			return clear_service;
+		}
+
+		if (service_id.equals(DELETE_ITEM_SERVICE))
+		{
+
+			Element delone_service = doc.createElement(GSXML.SERVICE_ELEM);
+			delone_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			delone_service.setAttribute(GSXML.NAME_ATT, DELETE_ITEM_SERVICE);
+			return delone_service;
+		}
+		if (service_id.equals(MERGE_ITEM_SERVICE))
+		{
+			Element merge_service = doc.createElement(GSXML.SERVICE_ELEM);
+			merge_service.setAttribute(GSXML.TYPE_ATT, "gather"); // what??
+			merge_service.setAttribute(GSXML.NAME_ATT, MERGE_ITEM_SERVICE);
+			return merge_service;
+		}
+		return null;
+	}
+
+	protected Element processAddDocument(Element request)
+	{
+		//System.err.println("REQUEST = " + GSXML.xmlNodeToString(request));
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+		//System.err.println("DOCSMAP = " + docsMap);
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("DocumentBasket Error: AddDocument request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		//System.err.println("PARAMS = " + params);
+		String item = (String) params.get("item");
+
+		int startIndex = item.startsWith(BASKET_BOOK) ? BASKET_BOOK.length() : 0;
+
+		String collection = "";
+		int pos = item.indexOf(":");
+		if (pos != -1)
+		{
+			collection = item.substring(startIndex, pos);
+			item = item.substring(pos + 1);
+		}
+		
+		//logger.error("COLLECTION = " + collection + " *** ITEM = " + item);
+		if (docsMap.containsKey(collection))
+		{
+			Hashtable<String, Item> items = docsMap.get(collection);
+			if (!items.containsKey(item))
+			{
+				Item newItem = generateItem(collection, item);
+				items.put(item, newItem);
+				result.appendChild(newItem.wrapIntoElement(result_doc));
+			}
+		}
+		else
+		{
+			Hashtable<String, Item> items = new Hashtable<String, Item>();
+			Item newItem = generateItem(collection, item);
+			items.put(item, newItem);
+			docsMap.put(collection, items);
+			result.appendChild(newItem.wrapIntoElement(result_doc));
+		}
+
+		return result;
+	}
+
+	protected Element processMergeDocument(Element request)
+	{
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("DocumentBasket Error: MergeDocument request had no paramList.");
+			return null; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String docString = (String) params.get("docs");
+		String[] docs = docString.split("-");
+
+		for (String d : docs)
+		{
+			logger.error("DOC = " + d);
+		}
+
+		/*
+		 * try{ System.out.println("Concatenate Two PDF"); PdfReader reader1 =
+		 * new PdfReader("3A01-01_Part_1-001.pdf"); PdfReader reader2 = new
+		 * PdfReader("3A01-01_Part_1-017.pdf"); PdfCopyFields copy = new
+		 * PdfCopyFields(new FileOutputStream("concatenatedPDF.pdf"));
+		 * copy.addDocument(reader1); copy.addDocument(reader2); copy.close(); }
+		 * catch(Exception ex) { ex.printStackTrace(); }
+		 */
+		// added -->
+
+		try
+		{
+			{
+				PrintWriter pw = new PrintWriter(new FileOutputStream("G:/output1.xml"));
+				File file = new File("G:/greenstone3-svn/web/sites/localsite/collect/peij21/archives/HASH0189.dir/");
+				File[] files = file.listFiles();
+
+				for (int i = 0; i < files.length; i++)
+				{
+
+					//System.out.println(files[i].getName());
+					String fileName = files[i].getName();
+
+					if (fileName.equals("doc.xml"))
+					{
+
+						System.out.println("Processing " + files[i].getPath() + "... ");
+						BufferedReader br = new BufferedReader(new FileReader(files[i].getPath()));
+						String line = br.readLine();
+						while (line != null)
+						{
+							pw.println(line);
+							line = br.readLine();
+						}
+						br.close();
+
+					}
+
+				}
+
+				File file1 = new File("G:/greenstone3-svn/web/sites/localsite/collect/peij21/archives/HASHfdc0.dir/");
+				File[] files1 = file1.listFiles();
+
+				for (int i = 0; i < files1.length; i++)
+				{
+
+					//System.out.println(files[i].getName());
+					String fileName = files1[i].getName();
+
+					if (fileName.equals("doc.xml"))
+					{
+
+						System.out.println("Processing " + files1[i].getPath() + "... ");
+						BufferedReader br = new BufferedReader(new FileReader(files1[i].getPath()));
+						String line = br.readLine();
+						while (line != null)
+						{
+							pw.println(line);
+							line = br.readLine();
+						}
+						br.close();
+
+					}
+
+				}
+
+				pw.close();
+
+				System.out.println("All doc.xml files have been concatenated into output1.xml");
+			}
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+		return null;
+	}
+
+	//end
+	private Item generateItem(String collection, String id)
+	{
+		Item item = new Item(collection, id);
+		String to = GSPath.appendLink(collection, "DocumentMetadataRetrieve");
+		ArrayList<String> tmp = new ArrayList<String>();
+		tmp.add(id);
+		
+		UserContext userContext = new UserContext();
+		userContext.setLanguage("en");
+		userContext.setUserID("dumy");
+		
+		Element response = getDocumentMetadata(to, userContext, tmp.iterator());
+		Element doc_node = (Element) response.getElementsByTagName(GSXML.DOC_NODE_ELEM).item(0);
+
+		String node_id = doc_node.getAttribute(GSXML.NODE_ID_ATT);
+		Element metadata_list = (Element) doc_node.getElementsByTagName(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER).item(0);
+
+		//assign title metadata if any
+		Element metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "Title");
+		if (metadata != null)
+		{
+			item.title = GSXML.getNodeText(metadata).trim();
+		}
+		//assign date metadata if any
+		metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "Date");
+		if (metadata != null)
+		{
+			item.date = GSXML.getNodeText(metadata).trim();
+		}
+
+		//assign root title metadata if any
+		metadata = GSXML.getNamedElement(metadata_list, "metadata", "name", "root_Title");
+		if (metadata != null)
+		{
+			String rootTitle = GSXML.getNodeText(metadata).trim();
+			if (!rootTitle.equals(item.title))
+			{
+				item.rootTitle = rootTitle;
+			}
+
+		}
+
+		return item;
+	}
+	
+	protected Element processClearDocuments(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		
+		if (param_list == null)
+		{
+			logger.error("DocumentBasket Error: DeleteDocument request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		String collection = (String) params.get(GSParams.COLLECTION);
+
+		if (collection == null)
+			return result;
+		
+		docsMap.put(collection, new Hashtable<String, Item>());
+		
+		return result;
+	}
+
+	protected Element processDeleteDocuments(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		//GSXML.printXMLNode(param_list);
+
+		if (param_list == null)
+		{
+			logger.error("DocumentBasket Error: DeleteDocument request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String param = (String) params.get("items");
+
+		if (param == null)
+			return result;
+
+		String[] items = param.split("\\|");
+
+		for (int i = 0; i < items.length; i++)
+		{
+			String item = items[i];
+			if (item.trim().length() == 0)
+				continue;
+
+			String collection = "";
+			int pos = item.indexOf(":");
+			if (pos != -1)
+			{
+				collection = item.substring(0, pos);
+				item = item.substring(pos + 1);
+			}
+
+			if (docsMap.containsKey(collection))
+			{
+				Hashtable itemMap = docsMap.get(collection);
+				if (itemMap.containsKey(item))
+				{
+					itemMap.remove(item);
+				}
+				if (itemMap.size() == 0)
+				{
+					docsMap.remove(collection);
+				}
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDeleteDocument(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		//GSXML.printXMLNode(param_list);
+
+		if (param_list == null)
+		{
+			logger.error("DocumentBasket Error: DeleteDocument request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String item = (String) params.get("item");
+
+		if (item == null)
+			return result;
+
+		String collection = "";
+		int pos = item.indexOf(":");
+
+		if (pos != -1)
+		{
+			collection = item.substring(0, pos);
+			item = item.substring(pos + 1);
+		}
+
+		if (docsMap.containsKey(collection))
+		{
+			Hashtable itemMap = docsMap.get(collection);
+			if (itemMap.containsKey(item))
+			{
+				itemMap.remove(item);
+			}
+			if (itemMap.size() == 0)
+			{
+				docsMap.remove(collection);
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processGetDocuments(Element request)
+	{
+		// GSXML.printXMLNode(request);
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		int size = 0;
+		String ids = "";
+		Iterator<String> keys = docsMap.keySet().iterator();
+
+		while (keys.hasNext())
+		{
+			Hashtable items = docsMap.get(keys.next());
+			size += items.size();
+			Iterator values = items.values().iterator();
+			while (values.hasNext())
+			{
+				Item item = (Item) values.next();
+				result.appendChild(item.wrapIntoElement(result_doc));
+			}
+		}
+
+		Element selement = result_doc.createElement("size");
+		selement.setAttribute("value", size + "");
+		result.appendChild(selement);
+
+		return result;
+	}
+
+  private Element getDocumentMetadata(String to, UserContext userContext, Iterator<String> ids)
+	{
+
+		// Build a request to obtain some document metadata
+	  Document msg_doc = XMLConverter.newDOM();
+		Element dm_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		Element dm_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+		dm_message.appendChild(dm_request);
+
+		// Create a parameter list to specify the required metadata information
+		HashSet<String> meta_names = new HashSet<String>();
+		meta_names.add("Title"); // the default
+		meta_names.add("root_Title");
+		meta_names.add("Date");
+
+		Element param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+
+		Element param = null;
+		Iterator<String> i = meta_names.iterator();
+		while (i.hasNext())
+		{
+			String name = i.next();
+			param = msg_doc.createElement(GSXML.PARAM_ELEM);
+			param_list.appendChild(param);
+			param.setAttribute(GSXML.NAME_ATT, "metadata");
+			param.setAttribute(GSXML.VALUE_ATT, name);
+		}
+
+		dm_request.appendChild(param_list);
+
+		// create the doc node list for the metadata request
+		Element dm_doc_list = msg_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		dm_request.appendChild(dm_doc_list);
+
+		while (ids.hasNext())
+		{
+			// Add the documentNode to the list
+			Element dm_doc_node = msg_doc.createElement(GSXML.DOC_NODE_ELEM);
+			dm_doc_list.appendChild(dm_doc_node);
+			dm_doc_node.setAttribute(GSXML.NODE_ID_ATT, ids.next());
+		}
+
+		return (Element) this.router.process(dm_message);
+	}
+
+	protected Element processDisplayDocumentList(Element request)
+	{
+		Hashtable<String, Hashtable<String, Item>> docsMap = updateDocMap(request);
+
+		// Create a new (empty) result message
+		Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+		Iterator<String> keys = docsMap.keySet().iterator();
+
+		while (keys.hasNext())
+		{
+			String collection = keys.next();
+			Hashtable items = docsMap.get(collection);
+			Iterator itemItr = items.values().iterator();
+
+			Element collectionNode = result_doc.createElement("documentList");
+			collectionNode.setAttribute("name", collection);
+			result.appendChild(collectionNode);
+
+			while (itemItr.hasNext())
+			{
+				Item item = (Item) itemItr.next();
+				Element itemElement = result_doc.createElement("item");
+
+				collectionNode.appendChild(itemElement);
+				itemElement.setAttribute("name", item.docid);
+				itemElement.setAttribute("collection", item.collection);
+				itemElement.setAttribute("title", item.title);
+				itemElement.setAttribute("date", item.date);
+				itemElement.setAttribute("root_title", item.rootTitle);
+			}
+		}
+
+		return result;
+	}
+
+	protected class Item
+	{
+		public String collection;
+		public String docid;
+		public String title = "";
+		public String query = "";
+		public String date = "";
+		public String rootTitle = "";
+
+		public Item(String coll, String id)
+		{
+			this.collection = coll;
+			this.docid = id;
+		}
+
+		public boolean equals(Object o)
+		{
+			if (!(o instanceof Item))
+			{
+				return false;
+			}
+
+			Item item = (Item) o;
+			String id = collection + ":" + docid;
+			String idin = item.collection + ":" + item.docid;
+			return id.equals(idin);
+		}
+
+		public String toString()
+		{
+			return collection + ":" + docid + ":" + "[" + ((!rootTitle.equals("")) ? (rootTitle + ":") : "") + title + "]";
+		}
+
+		public Element wrapIntoElement(Document doc)
+		{
+			Element itemElement = doc.createElement("item");
+			itemElement.setAttribute("name", docid);
+			itemElement.setAttribute("collection", collection);
+			itemElement.setAttribute("title", title);
+			itemElement.setAttribute("date", date);
+			itemElement.setAttribute("root_title", rootTitle);
+			return itemElement;
+		}
+	}
+
+	private class UserTimer extends Timer implements ActionListener
+	{
+		String id = "";
+
+		public UserTimer(int delay, String id)
+		{
+			super(delay, (ActionListener) null);
+			addActionListener(this);
+			this.id = id;
+		}
+
+		public void actionPerformed(ActionEvent e)
+		{
+			userMap.remove(id);
+			timerMap.remove(id);
+			stop();
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentMaker.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentMaker.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/DocumentMaker.java	(revision 31634)
@@ -0,0 +1,508 @@
+/*
+ *    DocumentMaker.java
+ *    The Document Maker service that can be used to create/modify custom
+ *    documents in Greenstone collection
+ *
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.*;
+
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+
+
+import org.greenstone.gsdl3.util.GSDocumentModel;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+public class DocumentMaker extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.ArchiveIO.class.getName());
+
+	GSDocumentModel _GSDM = null;
+
+	/****************************************************
+	 * The list of services the Document Maker supports *
+	 ***************************************************/
+	//Document/section services
+	protected static final String DOCUMENT_CREATE = "DocumentCreate";
+	protected static final String DOCUMENT_DELETE = "DocumentDelete";
+	protected static final String DOCUMENT_DUPLICATE = "DocumentDuplicate";
+	protected static final String DOCUMENT_MOVE = "DocumentMove";
+	protected static final String DOCUMENT_MERGE = "DocumentMerge";
+	protected static final String DOCUMENT_SPLIT = "DocumentSplit";
+	protected static final String DOCUMENT_GET_INFORMATION = "DocumentGetInformation";
+
+	//Other services
+	protected static final String DOCUMENT_EXECUTE_TRANSACTION = "DocumentExecuteTransaction";
+	/***************************************************/
+
+	String[] services = { DOCUMENT_CREATE, DOCUMENT_DELETE, DOCUMENT_DUPLICATE, DOCUMENT_GET_INFORMATION, DOCUMENT_MOVE, DOCUMENT_MERGE, DOCUMENT_SPLIT, DOCUMENT_EXECUTE_TRANSACTION };
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring DocumentMaker...");
+		this.config_info = info;
+
+		for (int i = 0; i < services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, services[i]);
+			this.short_service_info.appendChild(service);
+		}
+		
+		_GSDM = new GSDocumentModel(this.site_home, this.router);
+
+		return true;
+	}
+
+  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < services.length; i++)
+		{
+			if (service_id.equals(services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+
+	/************
+	 * Services *
+	 ***********/
+
+	protected Element processDocumentCreate(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_CREATE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_CREATE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to create
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			//Get information about the current new document
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentCreate(oid, collection, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_CREATE))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentDelete(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_DELETE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_DELETE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to delete
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+
+			_GSDM.documentDelete(oid, collection, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_DELETE))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentDuplicate(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_DUPLICATE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_DUPLICATE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to duplicate
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+			String newOID = currentDoc.getAttribute("new" + GSXML.NODE_ID_ATT);
+			String newCollection = currentDoc.getAttribute("new" + GSXML.COLLECTION_ATT);
+			String operation = currentDoc.getAttribute("operation");
+
+			_GSDM.documentMoveOrDuplicate(oid, collection, newOID, newCollection, _GSDM.operationStringToInt(operation), false, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_DUPLICATE))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentGetInformation(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_GET_INFORMATION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_GET_INFORMATION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to duplicate
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+
+			NodeList requestedInfoList = currentDoc.getElementsByTagName("info"); //TODO: Replace info with a constant
+			String[] requestedInfo = new String[requestedInfoList.getLength()];
+
+			for (int j = 0; j < requestedInfoList.getLength(); j++)
+			{
+				requestedInfo[j] = ((Element) requestedInfoList.item(j)).getAttribute(GSXML.NAME_ATT);
+			}
+
+			_GSDM.documentGetInformation(oid, collection, requestedInfo, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_GET_INFORMATION))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentMove(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_MOVE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_MOVE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to duplicate
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+			String newOID = currentDoc.getAttribute("new" + GSXML.NODE_ID_ATT);
+			String newCollection = currentDoc.getAttribute("new" + GSXML.COLLECTION_ATT);
+			String operation = currentDoc.getAttribute("operation");
+
+			_GSDM.documentMoveOrDuplicate(oid, collection, newOID, newCollection, _GSDM.operationStringToInt(operation), true, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_MOVE))
+			{
+				return result;
+			}
+		}
+		return result;
+	}
+
+	protected Element processDocumentMerge(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_MERGE);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_MERGE + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to duplicate
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+			String mergeOID = currentDoc.getAttribute("merge" + GSXML.NODE_ID_ATT);
+
+			_GSDM.documentMerge(oid, collection, mergeOID, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_MERGE))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentSplit(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_SPLIT);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_SPLIT + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		//Get the list of documents to duplicate
+		NodeList documents = request.getElementsByTagName(GSXML.DOCUMENT_ELEM);
+		for (int i = 0; i < documents.getLength(); i++)
+		{
+			Element currentDoc = (Element) documents.item(i);
+			String oid = currentDoc.getAttribute(GSXML.NODE_ID_ATT);
+			String collection = currentDoc.getAttribute(GSXML.COLLECTION_ATT);
+			String splitPoint = currentDoc.getAttribute("splitpoint");
+
+			int split;
+			try
+			{
+				split = Integer.parseInt(splitPoint);
+			}
+			catch (Exception ex)
+			{
+				GSXML.addError(result, DOCUMENT_SPLIT + ": The split point was not an integer", GSXML.ERROR_TYPE_SYNTAX);
+				return result;
+			}
+
+			_GSDM.documentSplit(oid, collection, split, userContext);
+			if (_GSDM.checkError(result, DOCUMENT_SPLIT))
+			{
+				return result;
+			}
+		}
+
+		return result;
+	}
+
+	protected Element processDocumentExecuteTransaction(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, DOCUMENT_EXECUTE_TRANSACTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, DOCUMENT_EXECUTE_TRANSACTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			GSXML.addError(result, DOCUMENT_EXECUTE_TRANSACTION + ": Request has no parameter list", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		String transactionString = (String) params.get("transactions");
+		transactionString = transactionString.replace("%26", "&");
+
+		List<Map<String, String>> transactions = null;
+		try
+		{
+			Gson gson = new Gson();
+			Type type = new TypeToken<List<Map<String, String>>>()
+			{
+			}.getType();
+			transactions = gson.fromJson(transactionString, type);
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+
+		if (transactions != null && transactions.size() > 0)
+		{
+			for (int j = 0; j < transactions.size(); j++)
+			{
+				Map keyValueMap = transactions.get(j);
+				String operation = (String) keyValueMap.get("operation");
+				if (operation.equals("move") || operation.equals("duplicate"))
+				{
+					String origCollection = (String) keyValueMap.get("collection");
+					String origOID = (String) keyValueMap.get("oid");
+					String newCollection = (String) keyValueMap.get("newCollection");
+					String newOID = (String) keyValueMap.get("newOID");
+					String subOperation = (String) keyValueMap.get("subOperation");
+
+					_GSDM.documentMoveOrDuplicate(origOID, origCollection, newOID, newCollection, _GSDM.operationStringToInt(subOperation), operation.equals("move"), userContext);
+				}
+				else if (operation.equals("createDocument"))
+				{
+					String oid = (String) keyValueMap.get("oid");
+					String collection = (String) keyValueMap.get("collection");
+
+					_GSDM.documentCreate(oid, collection, userContext);
+				}
+				else if (operation.equals("create"))
+				{
+					String oid = (String) keyValueMap.get("oid");
+					String collection = (String) keyValueMap.get("collection");
+					String subOperation = (String) keyValueMap.get("subOperation");
+
+					//_GSDM.documentCreate(oid, collection, userContext); <--- Maybe go back to this
+					_GSDM.documentXMLSetSection(oid, collection, result_doc.createElement(GSXML.DOCXML_SECTION_ELEM), _GSDM.operationStringToInt(subOperation), userContext);
+				}
+				else if (operation.equals("delete"))
+				{
+					String oid = (String) keyValueMap.get("oid");
+					String collection = (String) keyValueMap.get("collection");
+
+					_GSDM.documentDelete(oid, collection, userContext);
+				}
+				else if (operation.equals("setText"))
+				{
+					String oid = (String) keyValueMap.get("oid");
+					String collection = (String) keyValueMap.get("collection");
+					String newContent = (String) keyValueMap.get("text");
+					
+					_GSDM.documentXMLSetText(oid, collection, newContent, userContext);
+					// make sure NoText metadata is not set (in case we have added text where there was none before)
+					_GSDM.documentXMLDeleteMetadata(oid, collection, "NoText", userContext);
+					// only top level doc ids are in database
+					markDocumentInFlatDatabase("R", collection, OID.getTop(oid));
+
+				}
+
+				if (_GSDM.checkError(result, DOCUMENT_EXECUTE_TRANSACTION))
+				{
+					return result;
+				}
+			}
+		}
+		return result;
+		
+	}
+	protected void markDocumentInFlatDatabase(String mark, String collection, String oid) {
+	  
+		Document msg_doc = XMLConverter.newDOM();
+		Element message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		UserContext userContext = new UserContext();
+		Element query_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE , collection, userContext);		
+		message.appendChild(query_request);
+		Element result = (Element) this.router.process(message);
+		Element resp_elem = (Element) GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
+		Element coll_elem = (Element) GSXML.getChildByTagName(resp_elem, GSXML.COLLECTION_ELEM);
+		String dbtype = coll_elem.getAttribute(GSXML.DB_TYPE_ATT);
+		
+		SimpleCollectionDatabase coll_db = new SimpleCollectionDatabase(dbtype);
+		if (!coll_db.databaseOK())
+		{
+			logger.error("Couldn't create the collection database of type " + dbtype);
+			return;
+		}
+		
+		// Open database for reading
+		String coll_db_file = GSFile.archivesDatabaseFile(this.site_home, collection, dbtype);
+		if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ))
+		{
+			logger.error("Could not open collection archives database. Database doesn't exist or else somebody's already using it?");
+			return;
+		}
+		String old_value = coll_db.getValue(oid);
+		String new_value = old_value.replace("<index-status>B", "<index-status>" + mark);
+		// Close database for reading
+		coll_db.closeDatabase();
+		if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.WRITE))
+		{
+			logger.error("Could not open collection archives database. Somebody already using this database!");
+			return;
+		}
+		coll_db.setValue(oid, new_value);
+		coll_db.closeDatabase();
+		
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraSearch.java.tmp
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraSearch.java.tmp	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraSearch.java.tmp	(revision 31634)
@@ -0,0 +1,303 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+//Fedora Web Services classes
+import fedora.client.utility.AutoFinder;
+import fedora.client.APIAStubFactory;
+import fedora.client.Downloader;
+import fedora.server.access.FedoraAPIA;
+import fedora.server.types.gen.FieldSearchQuery;
+import fedora.server.types.gen.FieldSearchResult;
+import fedora.server.types.gen.ListSession;
+import fedora.server.types.gen.ObjectFields;
+import fedora.server.types.gen.Condition;
+import fedora.server.types.gen.ComparisonOperator;
+
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.transform.Templates;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.MalformedURLException;
+import java.lang.Object;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ *
+ */
+
+public class FedoraSearch
+    extends AbstractSearch {
+    
+     static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.FedoraSearch.class.getName());
+    // the services on offer
+    // these strings must match what is found in the properties file
+    protected static final String FIELD_PARAM = "fields";
+    
+    //Parameters connect to Proxy Server
+    private String proxyUser = null;
+    private String proxyPassword = null;
+    
+    private int maxResult, matchNum;
+    
+    protected String fedora_server_url = null;
+    
+    public FedoraSearch()
+    {
+      QUERY_SERVICE = "TextQuery";
+    }
+    
+    //Configure FedoraSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "fedoraServer");
+	if (server_elem == null) {
+	    logger.error("no fedoraServer element found");
+	    return false;
+	}
+	fedora_server_url = server_elem.getAttribute("url");
+	logger.info("Fedora Server:" + fedora_server_url);
+
+	if (fedora_server_url.equals("")) {
+	    logger.error("no url for the fedoraServer element");
+	    return false;
+	}
+	Element tq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+	tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+	tq_service.setAttribute(GSXML.NAME_ATT, QUERY_SERVICE);
+	this.short_service_info.appendChild(tq_service);
+	
+	// add some format info to service map if there is any
+	String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+	Element format = (Element) GSXML.getNodeByPath(extra_info, path);
+	if (format != null) {
+	    this.format_info_map.put(QUERY_SERVICE, this.desc_doc.importNode(format, true));
+	}
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+	// Create a new (empty) result message
+	Document result_doc = XMLConverter.newDOM();
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+	
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+	
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+	//Check Max number of Documents hit
+	String maxDocs = (String) params.get(MAXDOCS_PARAM);
+       	String hits_per_page = (String) params.get(HITS_PER_PAGE_PARAM);
+	
+	//Check the start_page number
+	String start_page = (String) params.get(START_PAGE_PARAM);	
+
+	// tidy whitespace
+	//query = query.replaceAll("\\s+", "+");
+	matchNum = 0;
+	maxResult = 20;
+
+	try {
+	    //Search Fedora Server
+	    String host = "toetoe";
+	    int port = 8200;
+	    String user = "fedoraAdmin";
+	    String pass = "fedoraAdmin";
+	    
+	    AutoFinder finder=new AutoFinder(host, port, user, pass);
+
+	    String[] fieldsArray = new String[] {"pid", "label", "fType", "bDef",
+						 "bMech", "cModel", "state", "ownerId", "cDate", "mDate",
+						 "dcmDate", "title", "creator", "subject", "description",
+						 "publisher", "contributor", "date", "type", "format",
+						 "identifier", "source", "language", "relation", "coverage",
+						 "rights"};
+	    
+	    String match_fields = join(fieldsArray, "|");
+	    
+	    FieldSearchQuery query_fedora=fedoraSearch(query, match_fields);
+	    
+            FieldSearchResult result_fedora = finder.findObjects(fieldsArray, maxResult, query_fedora);
+	    
+	    while (result_fedora != null) {
+		logger.info("Search Result:" + result_fedora.getResultList().length);
+		for (int i=0; i<result_fedora.getResultList().length; i++) {
+		    ObjectFields o = result_fedora.getResultList()[i];
+		    matchNum++;
+		    String pid = o.getPid();
+		    String fedora_url = fedora_server_url +"/fedora/get/" + pid;
+		    String label = o.getLabel();
+		    String[] creator = o.getCreator();
+		    String[] subject = o.getSubject();
+		    String[] publisher = o.getPublisher();
+		    String[] contributor = o.getContributor();
+		    String[] title = o.getTitle();
+		    String[] description = o.getDescription();
+		    String[] format = o.getFormat();
+		    String[] identifier = o.getIdentifier();
+		    String[] source = o.getSource();
+		    String[] language = o.getLanguage();
+
+		    Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+		    doc_node_list.appendChild(doc_node);
+		    
+		    Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		    doc_node.appendChild(metadata_list);
+		    
+		    if (fedora_url != null) {
+			GSXML.addMetadata( metadata_list, "URL", fedora_url);
+		    }
+		    if (o.getDescription() != null) {
+			GSXML.addMetadata(metadata_list, "Description", description[0]);
+		    }
+		    if (title[0] != null) {
+			GSXML.addMetadata(metadata_list, "Title", title[0]);
+		    }
+		}
+		ListSession sess=result_fedora.getListSession();
+		if (sess!=null) {
+		    result_fedora = finder.resumeFindObjects(sess.getToken());
+		} else {
+		    result_fedora = null;
+		}
+	    }
+	} catch (Exception e) {
+	    logger.error("ERROR: " + e.getClass().getName()
+			       + ((e.getMessage()==null) ? "" : ": " + e.getMessage()));
+	}
+    	
+	Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	//logger.info("FedoraSearch Number of Documents:" + matchNum);
+	GSXML.addMetadata(metadata_list, "numDocsMatched", ""+matchNum);
+	
+	return result;
+    }
+
+    protected FieldSearchQuery fedoraSearch(String query_terms, String matchFields) {
+
+	FieldSearchQuery fedora_query=new FieldSearchQuery();
+	try {
+	    // check if query contains any fields name (pid, label, fType....)
+	    if (query_terms.matches("^("+ matchFields+").+")){
+		logger.info("Implementing Specific Query....");
+
+		List conditions=fedora.server.search.Condition.getConditions(query_terms);
+		
+		Condition[] cond=new Condition[conditions.size()];
+		for (int i=0; i<conditions.size(); i++) {
+		    fedora.server.search.Condition c = (fedora.server.search.Condition) conditions.get(i);
+		    try {
+			String operator = transOperator(c.getOperator().getAbbreviation());
+			cond[i] = new Condition();
+			cond[i].setProperty(c.getProperty());
+			cond[i].setOperator(ComparisonOperator.fromValue(operator));
+			cond[i].setValue(c.getValue());
+		    } catch (Exception e){
+			e.printStackTrace();
+		    }
+		}
+		fedora_query.setConditions(cond);
+	    } else {
+		logger.info("Implementing Simple Query...");
+		fedora_query.setTerms(query_terms);
+	    }
+	} catch (Exception e){
+	    e.printStackTrace();
+	}
+	return fedora_query;
+    }
+    
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
+       	index_ids.add("kw");
+	index_ids.add("au");	
+	index_ids.add("su");	
+	index_ids.add("ti");
+	index_ids.add("de");
+	index_ids.add("fu");
+	index_names.add(getTextString("param."+FIELD_PARAM+".kw", lang)); 
+	index_names.add(getTextString("param."+FIELD_PARAM+".au", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".su", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".ti", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".de", lang));
+	index_names.add(getTextString("param."+FIELD_PARAM+".fu", lang));
+    }
+    
+    protected String getDocType (String node_id){
+	return node_id;
+    }
+    protected boolean hasChildren(String node_id){
+	return false;
+    }
+
+    protected boolean hasParent(String node_id){
+	return false;
+    }
+    private String transOperator(String operator){
+	String oper_char ="";
+	if (operator == "~") {
+	    oper_char = "has";
+	} else if (operator == "=") {
+	    oper_char = "eq";
+	} else if (operator == ">"){
+	    oper_char = "gt";
+	} else if (operator == ">="){
+	    oper_char = "ge";
+	} else if (operator == "<"){
+	    oper_char = "lt";
+   	} else if (operator == "<="){
+	    oper_char = "le";
+	}
+	return oper_char;
+    }
+    protected String join( String[] array, String delim ) {
+        String j = "";
+        for ( int i=0; i<array.length; i++ ) {
+            if (i!=0) j += delim;
+            j += array[i];
+        }
+        return j;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraServiceProxy.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraServiceProxy.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/FedoraServiceProxy.java	(revision 31634)
@@ -0,0 +1,885 @@
+/*
+ *    ServiceRack.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gs3client.dlservices.DigitalLibraryServicesAPIA;
+import org.greenstone.gs3client.dlservices.FedoraServicesAPIA;
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import org.apache.commons.lang3.StringUtils;
+
+/*
+// greenstone classes
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.core.*;
+
+// for fedora
+import org.greenstone.gs3client.dlservices.*;
+import org.greenstone.fedora.services.FedoraGS3Exception.CancelledException;
+
+// xml classes
+import org.w3c.dom.Node; 
+import org.w3c.dom.NodeList; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document; 
+import org.xml.sax.InputSource;
+import javax.xml.parsers.*;
+import org.apache.xpath.XPathAPI;
+
+// general java classes
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Properties;
+import java.util.ResourceBundle;
+import java.util.regex.*;
+import java.lang.reflect.Method;
+*/
+
+import org.apache.log4j.*;
+
+/**
+ * FedoraServiceProxy - communicates with the FedoraGS3 interface.
+ *
+ * @author Anupama Krishnan
+ */
+public class FedoraServiceProxy
+    extends ServiceRack implements OID.OIDTranslatable
+{
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.FedoraServiceProxy.class.getName());
+    protected MacroResolver macro_resolver = null;
+
+
+    /** The handle to the fedora connection */
+    private DigitalLibraryServicesAPIA fedoraServicesAPIA;
+
+    private String prevLanguage = "";
+
+    public void cleanUp() { 
+	super.cleanUp();
+    }
+    
+    /** sets the message router */
+    public void setMessageRouter(MessageRouter m) {
+       this.router = m;
+       setLibraryName(m.getLibraryName());
+    }
+
+    /** the no-args constructor */
+    public FedoraServiceProxy() {
+	super();
+
+	this.macro_resolver = new BasicTextMacroResolver();
+    }
+    
+
+    /* configure the service module
+     *
+     * @param info the XML node <serviceRack name="XXX"/> with name equal
+     * to the class name (of the subclass)
+     *
+     * must configure short_service_info_ and service_info_map_
+     * @return true if configured ok
+     * must be implemented in subclasses
+     */
+    /*public boolean configure(Element info) {
+	return configure(info, null);
+	}*/
+    
+    public boolean configure(Element info, Element extra_info) {
+	// set up the class loader
+		
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	// Try to instantiate a Fedora dl handle
+	try {
+	    // Fedora connection settings defaults. 
+	    // Read host and port from global.properties, since by default, we expect the Greenstone server to be used
+	    Properties globalProperties = new Properties();
+	    globalProperties.load(Class.forName("org.greenstone.util.GlobalProperties").getClassLoader().getResourceAsStream("global.properties"));
+	    String host = globalProperties.getProperty("tomcat.server", "localhost");
+	    String port = globalProperties.getProperty("tomcat.port", "8383");
+	    String protocol = "http";
+	    String username = "fedoraIntCallUser"; //"fedoraAdmin"
+	    String password = "changeme"; //"<user password>"
+
+	    // See if buildConfig.xml overrides any of the defaults
+	    // info is the <serviceRack> Element from buildConfig.xml (extra_info are the Elements of collectionConfig.xml)
+
+	    NodeList nodes = info.getElementsByTagName("fedoraConnection");
+	    if(nodes != null && nodes.getLength() > 0) {
+
+		Element fedoraElement = (Element)nodes.item(0);
+		if(fedoraElement.hasAttribute("protocol")) {
+		    protocol = fedoraElement.getAttribute("protocol");
+		}		
+		if(fedoraElement.hasAttribute("host")) {
+		    host = fedoraElement.getAttribute("host");
+		}
+		if(fedoraElement.hasAttribute("port")) {
+		    port = fedoraElement.getAttribute("port");
+		}
+		if(fedoraElement.hasAttribute("username")) {
+		    username = fedoraElement.getAttribute("username");
+		}
+		if(fedoraElement.hasAttribute("password")) {
+		    password = fedoraElement.getAttribute("password");
+		}		
+	    }	
+
+	    fedoraServicesAPIA = new FedoraServicesAPIA(protocol, host, Integer.parseInt(port), username, password);
+
+	} catch(org.greenstone.fedora.services.FedoraGS3Exception.CancelledException e) {
+	    // The user pressed cancel in the fedora services instantiation dialog
+	    return false;
+	} catch(Exception e) {
+	    logger.error("Error instantiating the interface to the Fedora Repository:\n", e); // second parameter prints e's stacktrace
+	    return false;
+	} 
+
+	
+	// Need to put the available services into short_service_info
+	// This is used by DefaultReceptionist.process() has an exception. But DefaultReceptionist.addExtraInfo() 
+	// isn't helpful, and the problem actually already occurs in 
+	// Receptionist.process() -> PageAction.process() -> MessageRouter.process() 
+	// -> Collection/ServiceCluster.process() -> ServiceCluster.configureServiceRackList() 
+	// -> ServiceRack.process() -> ServiceRack.processDescribe() -> ServiceRack.getServiceList().
+	// ServiceRack.getServiceList() requires this ServiceRack's services to be filled into the 
+	// short_service_info Element which needs to be done in this FedoraServiceProxy.configure().
+	
+	// get the display and format elements from the coll config file for
+	// the classifiers
+	AbstractBrowse.extractExtraClassifierInfo(info, extra_info);
+
+	// Copied from IViaProxy.java:
+	String collection = fedoraServicesAPIA.describeCollection(this.cluster_name);
+
+	Element collNode = getResponseAsDOM(collection);
+	Element serviceList = (Element)collNode.getElementsByTagName(GSXML.SERVICE_ELEM+GSXML.LIST_MODIFIER).item(0);
+
+//this.short_service_info.appendChild(short_service_info.getOwnerDocument().importNode(serviceList, true));
+	// we want the individual service Elements, not the serviceList Element which will wrap it later
+	NodeList services = collNode.getElementsByTagName(GSXML.SERVICE_ELEM);
+	for(int i = 0; i < services.getLength(); i++) {
+	    Node service = services.item(i);
+	    this.short_service_info.appendChild(short_service_info.getOwnerDocument().importNode(service, true));
+	}
+
+	// add some format info to service map if there is any
+	String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+	Element search_format = (Element) GSXML.getNodeByPath(extra_info, path);
+	if (search_format != null) {
+	    this.format_info_map.put("TextQuery", this.desc_doc.importNode(search_format, true));
+	    this.format_info_map.put("FieldQuery", this.desc_doc.importNode(search_format, true));
+	}
+	
+	// look for document display format
+	path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+	Element display_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (display_format != null) {
+	    this.format_info_map.put("DocumentContentRetrieve", this.desc_doc.importNode(display_format, true));
+	    // should we make a copy?
+	}
+
+	// the format info
+	Element cb_format_info = this.desc_doc.createElement(GSXML.FORMAT_ELEM);
+	boolean format_found = false;
+
+	// look for classifier <browse><format>
+	path = GSPath.appendLink(GSXML.BROWSE_ELEM, GSXML.FORMAT_ELEM);
+	Element browse_format = (Element)GSXML.getNodeByPath(extra_info, path);
+	if (browse_format != null) {
+	    cb_format_info.appendChild(GSXML.duplicateWithNewName(this.desc_doc, browse_format, GSXML.DEFAULT_ELEM, true));
+	    format_found = true;
+	} 
+	
+	// add in to the description a simplified list of classifiers
+	Element browse = (Element)GSXML.getChildByTagName(extra_info, "browse"); // the <browse>
+	NodeList classifiers = browse.getElementsByTagName(GSXML.CLASSIFIER_ELEM);
+	for(int i=0; i<classifiers.getLength(); i++) {
+	    Element cl = (Element)classifiers.item(i);
+	    Element new_cl = (Element)this.desc_doc.importNode(cl, false); // just import this node, not the children
+	    
+	    // get the format info out, and put inside a classifier element
+	    Element format_cl = (Element)new_cl.cloneNode(false);
+	    Element format = (Element)GSXML.getChildByTagName(cl, GSXML.FORMAT_ELEM);
+	    if (format != null) {
+		
+		//copy all the children
+		NodeList elems = format.getChildNodes();
+		for (int j=0; j<elems.getLength();j++) {
+		    format_cl.appendChild(this.desc_doc.importNode(elems.item(j), true));
+		}
+		cb_format_info.appendChild(format_cl);
+		format_found = true;
+	    }
+	    	    
+	}
+	    
+	if (format_found) {
+	    this.format_info_map.put("ClassifierBrowse", cb_format_info);
+	}
+	
+
+	// set up the macro resolver
+	macro_resolver.setSiteDetails(this.site_http_address, this.cluster_name, this.getLibraryName());
+	Element replacement_elem = (Element)GSXML.getChildByTagName(extra_info, "replaceList");
+	if (replacement_elem != null) {
+	    macro_resolver.addMacros(replacement_elem);
+	}
+	// look for any refs to global replace lists
+	NodeList replace_refs_elems = extra_info.getElementsByTagName("replaceListRef");
+	for (int i=0; i<replace_refs_elems.getLength(); i++) {
+	    String id = ((Element)replace_refs_elems.item(i)).getAttribute("id");
+	    if (!id.equals("")) {
+		Element replace_list = GSXML.getNamedElement(this.router.config_info, "replaceList", "id", id);
+		if (replace_list != null) {
+		    macro_resolver.addMacros(replace_list);
+		}
+	    }
+	}
+
+	// configured ok
+	return true;
+    }
+
+  
+    /* "DocumentContentRetrieve", "DocumentMetadataRetrieve", "DocumentStructureRetrieve", 
+      "TextQuery", "FieldQuery", "ClassifierBrowse", "ClassifierBrowseMetadataRetrieve" */
+
+    protected Element processDocumentContentRetrieve(Element request) {
+	String[] docIDs = parse(request, GSXML.DOC_NODE_ELEM, GSXML.NODE_ID_ATT); 
+	String[] relLinks = parse(request, GSXML.DOC_NODE_ELEM, "externalURL");
+	
+	//logger.error("### request:"); 
+	//logger.error(GSXML.elementToString(request, true));
+
+	if(docIDs == null) {
+	    logger.error("DocumentContentRetrieve request specified no doc nodes.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	} else {
+	    for(int i = 0; i < docIDs.length; i++) {
+		//logger.error("BEFORE: docIDs[" + i + "]: " + docIDs[i]);
+		if(relLinks[i] != null && docIDs[i].startsWith("http://")) { // need to do a look up
+		    docIDs[i] = translateExternalId(docIDs[i]);
+		} else {
+		    docIDs[i] = translateId(docIDs[i]);		    
+		}
+		//logger.error("AFTER: docIDs[" + i + "]: " + docIDs[i]);
+	    }
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+	
+	// first param (the collection) is not used by Fedora	
+	Element response = getResponseAsDOM(fedoraServicesAPIA.retrieveDocumentContent(this.cluster_name, docIDs));
+
+	
+	// resolve any collection specific macros
+	NodeList nodeContents = response.getElementsByTagName(GSXML.NODE_CONTENT_ELEM);
+	for(int i = 0; i < nodeContents.getLength(); i++) {
+	    Element nodeContent = (Element)nodeContents.item(i);
+	    /*if(nodeContent != null) {
+		nodeContent = (Element)nodeContent.getFirstChild(); // textNode
+		}*/
+	    //logger.error("GIRAFFE 1. content retrieve response - nodeContent: " + GSXML.nodeToFormattedString(nodeContent));
+	    String docContent = nodeContent.getFirstChild().getNodeValue(); // getTextNode and get its contents.
+	    //logger.error("GIRAFFE 2. content retrieve response - docContent: " + docContent);
+	    
+	    if(docContent != null) {
+		// get document text and resolve and macros. Rel and external links have _httpextlink_ set by HTMLPlugin
+		docContent = macro_resolver.resolve(docContent, lang, MacroResolver.SCOPE_TEXT, ""); // doc_id
+		nodeContent.getFirstChild().setNodeValue(docContent);
+		//logger.error("GIRAFFE 3. content retrieve response. Updated docContent: " + docContent);
+	    }
+	}
+
+	return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0); 
+    }
+
+    protected Element processDocumentStructureRetrieve(Element request) {
+	String[] docIDs = parse(request, GSXML.DOC_NODE_ELEM, GSXML.NODE_ID_ATT);
+	String[] relLinks = parse(request, GSXML.DOC_NODE_ELEM, "externalURL");	
+
+	if(docIDs == null) {
+	    logger.error("DocumentStructureRetrieve request specified no doc nodes.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	} else {
+	    for(int i = 0; i < docIDs.length; i++) {
+		//logger.error("BEFORE: docIDs[" + i + "]: " + docIDs[i]);
+		if(relLinks[i] != null && docIDs[i].startsWith("http://")) { // need to do a look up
+		    docIDs[i] = translateExternalId(docIDs[i]);
+		} else {
+		    docIDs[i] = translateId(docIDs[i]);
+		}
+	    }
+	}
+
+	NodeList params = request.getElementsByTagName(GSXML.PARAM_ELEM);
+	String structure="";
+	String info="";
+	for(int i = 0; i < params.getLength(); i++) {
+	    Element param = (Element)params.item(i);
+	    if(param.getAttribute("name").equals("structure")) {
+		structure = structure + param.getAttribute("value") + "|";
+	    } else if(param.getAttribute("name").equals("info")) {
+		info = info + param.getAttribute("value") + "|";
+	    }
+	}	
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+	Element response = getResponseAsDOM(fedoraServicesAPIA.retrieveDocumentStructure(
+		this.cluster_name, docIDs, new String[]{structure}, new String[]{info}));
+	return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0); 
+    }
+
+    protected Element processDocumentMetadataRetrieve(Element request) {
+	String[] docIDs = parse(request, GSXML.DOC_NODE_ELEM, GSXML.NODE_ID_ATT);
+	String[] relLinks = parse(request, GSXML.DOC_NODE_ELEM, "externalURL");
+
+	if(docIDs == null) {
+	    logger.error("DocumentMetadataRetrieve request specified no doc nodes.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	} else {
+	    for(int i = 0; i < docIDs.length; i++) {
+		//logger.error("**** relLinks[i]: " + relLinks[i]);
+		//logger.error("**** docIDs[i]: " + docIDs[i]);
+		if(relLinks[i] != null && docIDs[i].startsWith("http://")) { // need to do a look up
+		    docIDs[i] = translateExternalId(docIDs[i]);
+		} else {
+		    docIDs[i] = translateId(docIDs[i]);
+		}
+		//logger.error("AFTER: docIDs[" + i + "]: " + docIDs[i]);
+	    }
+	}
+	
+	NodeList params = request.getElementsByTagName(GSXML.PARAM_ELEM);
+	String[] metafields = {};
+	if(params.getLength() > 0) {
+	    metafields = new String[params.getLength()];
+	    for(int i = 0; i < metafields.length; i++) {
+		Element param = (Element)params.item(i);
+		//if(param.hasAttribute(GSXML.NAME_ATT) && param.getAttribute(GSXML.NAME_ATT).equals("metadata") && param.hasAttribute(GSXML.VALUE_ATT)) {
+		if(param.hasAttribute(GSXML.VALUE_ATT)){ 
+		    metafields[i] = param.getAttribute(GSXML.VALUE_ATT);
+		} else {
+		    metafields[i] = "";
+		}
+	    }
+	}
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+	Element response = getResponseAsDOM(fedoraServicesAPIA.retrieveDocumentMetadata(
+					    this.cluster_name, docIDs, metafields));
+	return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0); 
+    }
+
+    protected Element processClassifierBrowseMetadataRetrieve(Element request) {	
+	String[] classIDs = parse(request, GSXML.CLASS_NODE_ELEM, GSXML.NODE_ID_ATT);
+	//String[] relLinks = parse(request, GSXML.CLASS_NODE_ELEM, "externalURL");
+
+	if(classIDs == null) {
+	    logger.error("ClassifierBrowseMetadataRetrieve request specified no classifier nodes.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	} else {
+	    for(int i = 0; i < classIDs.length; i++) {
+		classIDs[i] = translateId(classIDs[i]);
+	    }
+	}
+	
+	NodeList params = request.getElementsByTagName(GSXML.PARAM_ELEM);
+	String[] metafields = {};
+	if(params.getLength() > 0) {
+	    metafields = new String[params.getLength()];
+	    for(int i = 0; i < metafields.length; i++) {
+		Element param = (Element)params.item(i);
+		if(param.hasAttribute(GSXML.VALUE_ATT)){ 
+		    metafields[i] = param.getAttribute(GSXML.VALUE_ATT);
+		} else {
+		    metafields[i] = "";
+		}
+	    }
+	}
+	
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+	Element response = getResponseAsDOM(fedoraServicesAPIA.retrieveBrowseMetadata(
+	       this.cluster_name, "ClassifierBrowseMetadataRetrieve", classIDs, metafields));
+	//logger.error("**** Response from retrieveBrowseMeta: " + GSXML.elementToString(response, true));
+	return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0);
+    }
+
+    protected Element processClassifierBrowse(Element request) {
+	String collection = this.cluster_name;
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+
+	NodeList classNodes = request.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+	if(classNodes == null || classNodes.getLength() <= 0) {
+	    logger.error("ClassifierBrowse request specified no classifier IDs.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	}
+	String classifierIDs[] = new String[classNodes.getLength()];
+	for(int i = 0; i < classifierIDs.length; i++) {
+	    Element e = (Element)classNodes.item(i);
+	    classifierIDs[i] = e.getAttribute(GSXML.NODE_ID_ATT);
+	    classifierIDs[i] = translateId(classifierIDs[i]);	
+	}
+	
+	NodeList params = request.getElementsByTagName(GSXML.PARAM_ELEM);
+	String structure="";
+	String info="";
+	for(int i = 0; i < params.getLength(); i++) {
+	    Element param = (Element)params.item(i);
+	    if(param.getAttribute("name").equals("structure")) {
+		structure = structure + param.getAttribute("value") + "|";
+	    } else if(param.getAttribute("name").equals("info")) {
+		info = info + param.getAttribute("value") + "|";
+	    }
+	}
+	///structure = structure + "siblings"; //test for getting with classifier browse structure: siblings
+	
+	Element response 
+	    = getResponseAsDOM(fedoraServicesAPIA.retrieveBrowseStructure(collection, "ClassifierBrowse", classifierIDs,
+									  new String[] {structure}, new String[] {info}));
+	//logger.error("**** FedoraServiceProxy - Response from retrieveBrowseStructure: " + GSXML.elementToString(response, true));	
+	
+	return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0);
+    }
+
+    protected Element processTextQuery(Element request) {
+	return processQuery(request, "TextQuery");
+    }
+
+    protected Element processFieldQuery(Element request) {
+	return processQuery(request, "FieldQuery");
+    }
+
+    protected Element processQuery(Element request, String querytype) {
+	String collection = this.cluster_name;
+
+	String lang = request.getAttribute(GSXML.LANG_ATT);
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+
+	NodeList paramNodes = request.getElementsByTagName(GSXML.PARAM_ELEM);
+	if(paramNodes.getLength() > 0) {
+	    HashMap<String, String> params = new HashMap<String, String>(paramNodes.getLength()); 
+	    for(int i = 0; i < paramNodes.getLength(); i++) {
+		Element param = (Element)paramNodes.item(i);
+		params.put(param.getAttribute(GSXML.NAME_ATT), param.getAttribute(GSXML.VALUE_ATT));
+	    }
+
+	    Element response = getResponseAsDOM(fedoraServicesAPIA.query(collection, querytype, params));
+	    return (Element)response.getElementsByTagName(GSXML.RESPONSE_ELEM).item(0);
+	} else {
+	    logger.error("TextQuery request specified no parameters.\n");
+	    return XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM); // empty response
+	}
+    }
+
+    // get the requested nodeIDs out of a request message
+    protected String[] parse(Element request, String nodeType, String attribute) {	
+	String[] nodevalues = null;
+	int count = 0;
+
+	Element docList = (Element) GSXML.getChildByTagName(request, nodeType+GSXML.LIST_MODIFIER);
+	if (docList != null) {	    
+	    NodeList docNodes = docList.getElementsByTagName(nodeType);
+	    if(docNodes.getLength() > 0) {
+		nodevalues = new String[docNodes.getLength()];
+		for(int i = 0; i < nodevalues.length; i++) {
+		    Element e = (Element)docNodes.item(i);
+		    String id = e.getAttribute(attribute);
+		    // Not sure why there are at times requests for hashXXX.dir, which is not a fedora PID
+		    // To skip these: if not requesting an externalURL and if requesting a docNode, 
+		    // then the ID has to contain the : character special to fedora PIDs
+		    if(attribute == "externalURL" || (nodeType != GSXML.DOC_NODE_ELEM || id.contains(":"))) {
+			nodevalues[count++] = id;
+		    }
+		}
+	    }
+	}
+
+	if(count == 0) {
+	    return null;
+	}
+
+	String[] tmp = new String[count];
+	for(int i = 0; i < count; i++) {
+	    tmp[i] = nodevalues[i];
+	}
+	nodevalues = null;
+	nodevalues = tmp;
+
+	return nodevalues;
+    }
+
+
+    /** if id ends in .fc, .pc etc, then translate it to the correct id 
+     * For now (for testing things work) the default implementation is to just remove the suffix */
+    protected String translateId(String id) {
+	if (OID.needsTranslating(id)) {
+	    return OID.translateOID(this, id); //return translateOID(id);
+	}
+	return id;
+    }
+    
+    /** if an id is not a greenstone id (an external id) then translate 
+     * it to a greenstone one 
+     * default implementation: return the id. Custom implementation: 
+     * the id is a url that maps to a fedorapid whose dc.title contains the required HASHID */
+    protected String translateExternalId(String id) {
+	//logger.error("*** to translate an external ID: " + id); /////return id;
+	return this.externalId2OID(id);
+    }
+
+    /** converts an external id to greenstone OID. External ID is a URL link 
+     * that, if relative, maps to a fedorapid that has an entry in fedora.
+     * The dc:title meta for that fedorapid will contain the required OID. */
+    public String externalId2OID(String extid) {
+	if(extid.endsWith(".rt") && (extid.indexOf('.') != extid.lastIndexOf('.'))) { 
+	       // .rt is not file extension, but Greenstone request for root of document
+	       // not relevant for external ID
+	       extid = extid.substring(0, extid.length()-3);
+	}
+
+	   // the following method is unique to FedoraServicesAPIA
+	String response = ((FedoraServicesAPIA)fedoraServicesAPIA).getDocIDforURL(extid, this.cluster_name);
+	if(response.indexOf(GSXML.ERROR_ELEM) != -1) {
+	    logger.error("**** The following error occurred when trying to find externalID for ID " + extid);
+	    logger.error(response);
+	    return extid;
+	}
+	if(response.equals("")) {
+	    return extid;
+	} else {
+	    return response;
+	}
+    }
+
+
+  /** translates relative oids into proper oids:
+   * .pr (parent), .rt (root) .fc (first child), .lc (last child),
+   * .ns (next sibling), .ps (previous sibling) 
+   * .np (next page), .pp (previous page) : links sections in the order that you'd read the document
+   * a suffix is expected to be present so test before using 
+   */
+    public String processOID(String doc_id, String top, String suff, int sibling_num) {
+
+    // send off request to get sibling etc. information from Fedora
+    Element response = null;
+    String[] children = null;
+    if(doc_id.startsWith("CL")) { // classifiernode
+	response = getResponseAsDOM(fedoraServicesAPIA.retrieveBrowseStructure(this.cluster_name, "ClassifierBrowse", new String[]{doc_id},
+									       new String[]{"children"}, new String[]{"siblingPosition"}));
+	NodeList nl = response.getElementsByTagName(GSXML.NODE_STRUCTURE_ELEM);
+	if(nl.getLength() > 0) {
+	    Element nodeStructure = (Element)nl.item(0);
+
+	    if(nodeStructure != null) {
+		Element root = (Element) GSXML.getChildByTagName(nodeStructure, GSXML.CLASS_NODE_ELEM);
+		if(root != null) { // get children
+		    NodeList classNodes = root.getElementsByTagName(GSXML.CLASS_NODE_ELEM);
+		    if(classNodes != null) {
+			children = new String[classNodes.getLength()];
+			for(int i = 0; i < children.length; i++) {
+			    Element child = (Element)classNodes.item(i);
+			    children[i] = child.getAttribute(GSXML.NODE_ID_ATT);
+			}
+		    }
+		}
+	    }
+	}
+    } else { // documentnode
+	response = getResponseAsDOM(fedoraServicesAPIA.retrieveDocumentStructure(this.cluster_name, new String[]{doc_id},
+										 new String[]{"children"}, new String[]{"siblingPosition"}));
+	String path = GSPath.createPath(new String[]{GSXML.RESPONSE_ELEM, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER, 
+					      GSXML.DOC_NODE_ELEM, GSXML.NODE_STRUCTURE_ELEM, GSXML.DOC_NODE_ELEM});	
+	Element parentDocNode = (Element) GSXML.getNodeByPath(response, path);
+
+	if (parentDocNode == null) {
+	    return top;
+	} // else 
+	NodeList docNodes = parentDocNode.getElementsByTagName(GSXML.DOC_NODE_ELEM); // only children should remain, since that's what we requested
+	if(docNodes.getLength() > 0) {
+	    children = new String[docNodes.getLength()];
+	    
+	    for(int i = 0; i < children.length; i++) {
+		Element e = (Element)docNodes.item(i);
+		children[i] = e.getAttribute(GSXML.NODE_ID_ATT);
+	    }
+	} else { // return root node
+	    children = new String[]{doc_id};
+	}
+    }
+    
+    if (suff.equals("fc")) {
+      return children[0];
+    } else if (suff.equals("lc")) {
+      return children[children.length-1];
+    } else {
+      if (suff.equals("ss")) {
+	return children[sibling_num-1];
+      }
+      // find the position that we are at.
+      int i=0;
+      while(i<children.length) {
+	if (children[i].equals(top)) {
+	  break;
+	}
+	i++;
+      }
+	    
+      if (suff.equals("ns")) {
+	if (i==children.length-1) {
+	  return children[i];
+	}
+	return children[i+1];
+      } else if (suff.equals("ps")) {
+	if (i==0) {
+	  return children[i];
+	}
+	return children[i-1];
+      }
+    }
+
+    return top;
+  }
+
+
+    protected Element getResponseAsDOM(String response) {
+	if(response == null) { // will not be the case, because an empty  
+	    return null;    // response message will be sent instead
+	}
+
+	Element message = null;		
+	try{
+	    // turn the String xml response into a DOM tree:	
+	    DocumentBuilder builder 
+		= DocumentBuilderFactory.newInstance().newDocumentBuilder();
+	    Document doc 
+		= builder.parse(new InputSource(new StringReader(response)));
+	    message = doc.getDocumentElement();
+	} catch(Exception e){
+	    if(response == null) {
+		response = "";
+	    }
+	    logger.error("An error occurred while trying to parse the response: ");
+	    logger.error(response);
+	    logger.error(e.getMessage());
+	}
+	
+	// Error elements in message will be processed outside of here, just return the message
+	return message;
+    }
+
+    /* //process method for stylesheet requests    
+    protected Element processFormat(Element request) {}	*/
+    
+    /* returns the service list for the subclass */
+    /* protected Element getServiceList(String lang) {
+	// for now, it is static and has no language stuff
+	return (Element) this.short_service_info.cloneNode(true);
+	}*/
+
+    /** returns a specific service description */
+  protected Element getServiceDescription(Document doc, String service, String lang, String subset) {
+	if(!lang.equals(prevLanguage)) {
+	    prevLanguage = lang;
+	    fedoraServicesAPIA.setLanguage(lang);
+	}
+	String serviceResponse = fedoraServicesAPIA.describeService(service);
+	Element response = getResponseAsDOM(serviceResponse);
+
+	// should be no chance of an npe, since FedoraGS3 lists the services, so will have descriptions for each
+	Element e = (Element)response.getElementsByTagName(GSXML.SERVICE_ELEM).item(0);
+	e = (Element)doc.importNode(e, true);
+	return e; 
+    }
+
+    /** overloaded version for no args case */
+    protected String getTextString(String key, String lang) {
+	return getTextString(key, lang, null, null);
+    }
+
+    protected String getTextString(String key, String lang, String dictionary) {
+	return getTextString(key, lang, dictionary, null);
+    }
+    protected String getTextString(String key, String lang, String [] args) {
+	return getTextString(key, lang, null, args);
+    }
+	
+    /** getTextString - retrieves a language specific text string for the given
+key and locale, from the specified resource_bundle (dictionary)
+    */
+    protected String getTextString(String key, String lang, String dictionary, String[] args) {
+
+	// we want to use the collection class loader in case there are coll specific files
+	if (dictionary != null) {
+	    // just try the one specified dictionary
+	    Dictionary dict = new Dictionary(dictionary, lang, this.class_loader);
+	    String result = dict.get(key, args);
+	    if (result == null) { // not found
+		return "_"+key+"_";
+	    }
+	    return result;
+	}
+
+	// now we try class names for dictionary names
+	String class_name = this.getClass().getName();
+	class_name = class_name.substring(class_name.lastIndexOf('.')+1);
+	Dictionary dict = new Dictionary(class_name, lang, this.class_loader);
+	String result = dict.get(key, args);
+	if (result != null) {
+	    return result;
+	}
+
+	// we have to try super classes
+	Class c = this.getClass().getSuperclass();
+	while (result == null && c != null) {
+	    class_name = c.getName();
+	    class_name = class_name.substring(class_name.lastIndexOf('.')+1);
+	    if (class_name.equals("ServiceRack")) {
+		// this is as far as we go
+		break;
+	    }
+	    dict = new Dictionary(class_name, lang, this.class_loader);
+	    result = dict.get(key, args);
+	    c = c.getSuperclass();
+	}
+	if (result == null) {
+	    return "_"+key+"_";
+	}
+	return result;
+	
+    }
+
+    protected String getMetadataNameText(String key, String lang) {
+
+	String properties_name = "metadata_names";
+	Dictionary dict = new Dictionary(properties_name, lang);
+	
+	String result = dict.get(key);
+	if (result == null) { // not found
+	    return null;
+	} 
+	return result;
+    }
+
+    public static class BasicTextMacroResolver extends MacroResolver {	
+	private static final Pattern p_back_slash = Pattern.compile("\\\"");// create a pattern "\\\"", but it matches both " and \"
+
+	public String resolve(String text, String lang, String scope, String doc_oid) 
+	{
+
+	    if (text == null || text.equals("")) {
+		return text;
+	    }	    
+	    if (!scope.equals(SCOPE_TEXT) || text_macros.size()==0) {
+		return text;
+	    }
+
+	    java.util.ArrayList macros = text_macros;
+	    for (int i=0; i<macros.size(); i++) {
+		String new_text = null;
+		Macro m = (Macro)macros.get(i);
+		
+		if(m.type == TYPE_TEXT) {
+		    // make sure we resolve any macros in the text
+
+		    if(text.contains(m.macro)) {
+			if (m.resolve) {
+			    new_text = this.resolve(m.text, lang, scope, doc_oid);
+			} else {
+			    new_text = m.text;
+			}
+			text = StringUtils.replace(text, m.macro, new_text);//text = text.replaceAll(m.macro, new_text);
+			if (m.macro.endsWith("\\\\")) { // to get rid of "\" from the string like: "src="http://www.greenstone.org:80/.../mw.gif\">"
+			    Matcher m_slash = p_back_slash.matcher(text);
+			    String clean_str = "";
+			    int s=0;
+			    while (m_slash.find()) {
+				if (!text.substring(m_slash.end()-2, m_slash.end()-1).equals("\\")) {
+				    clean_str = clean_str + text.substring(s,m_slash.end()-1); // it matches ", so get a substring before "
+				}else{
+				    clean_str = clean_str + text.substring(s,m_slash.end()-2);// it matches \", so get a substring before \
+				}
+				s = m_slash.end();// get the index of the last match
+				clean_str = clean_str + "\"";
+			    }
+			    text = clean_str + text.substring(s,text.length());
+			}
+		    }
+		}
+	    }
+	    return text;
+	}
+    }
+
+
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Browse.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Browse.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Browse.java	(revision 31634)
@@ -0,0 +1,229 @@
+/*
+ *    GS2Browse.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.BasicDocumentDatabase;
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.util.MacroResolver;
+import org.greenstone.gsdl3.util.GS2MacroResolver;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Greenstone 2 collection classifier service
+ * 
+ */
+
+public class GS2Browse extends AbstractBrowse
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2Browse.class.getName());
+
+	protected SimpleCollectionDatabase coll_db = null;
+  BasicDocumentDatabase gs_doc_db = null;
+	public GS2Browse()
+	{
+	  this.macro_resolver = new GS2MacroResolver(this.coll_db, this.class_loader);
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+		this.coll_db.closeDatabase();
+		this.gs_doc_db.cleanUp();
+	}
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring GS2Browse...");
+		// the index stem is either specified in the config file or is  the collection name
+		Element index_stem_elem = (Element) GSXML.getChildByTagName(info, GSXML.INDEX_STEM_ELEM);
+		String index_stem = null;
+		if (index_stem_elem != null)
+		{
+			index_stem = index_stem_elem.getAttribute(GSXML.NAME_ATT);
+		}
+		if (index_stem == null || index_stem.equals(""))
+		{
+			index_stem = this.cluster_name;
+		}
+
+		// find out what kind of database we have
+		Element database_type_elem = (Element) GSXML.getChildByTagName(info, GSXML.DATABASE_TYPE_ELEM);
+		String database_type = null;
+		if (database_type_elem != null)
+		{
+			database_type = database_type_elem.getAttribute(GSXML.NAME_ATT);
+		}
+
+		if (database_type == null || database_type.equals(""))
+		{
+			database_type = "gdbm"; // the default
+		}
+
+		// do we still need this????
+		coll_db = new SimpleCollectionDatabase(database_type);
+		if (!coll_db.databaseOK())
+		{
+			logger.error("Couldn't create the collection database of type " + database_type);
+			return false;
+		}
+
+		// Open database for querying
+		String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, index_stem, database_type);
+		if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ))
+		{
+			logger.error("Could not open collection database!");
+			return false;
+		}
+		
+		
+		gs_doc_db = new BasicDocumentDatabase(database_type, this.site_home, this.cluster_name, index_stem);
+		if (!gs_doc_db.isValid())
+		{
+			logger.error("Failed to open Document Database.");
+			return false;
+		}
+		this.gs_doc = gs_doc_db;
+
+	
+		return true;
+	}
+
+	/** if id ends in .fc, .pc etc, then translate it to the correct id */
+	protected String translateId(String node_id)
+	{
+		return OID.translateOID(this.coll_db, node_id); //return this.coll_db.translateOID(node_id);
+	}
+
+	protected String getChildType(String node_id)
+	{
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return null;
+		}
+		return info.getInfo("childtype");
+	}
+
+
+
+
+	protected String getMetadata(String node_id, String key)
+	{
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return "";
+		}
+
+		Set<String> keys = info.getKeys();
+		Iterator<String> it = keys.iterator();
+		while (it.hasNext())
+		{
+			String key_in = it.next();
+			String value = info.getInfo(key);
+			if (key_in.equals(key))
+			{
+				return value;
+			}
+		}
+
+		return "";
+
+	}
+
+	/**
+	 * get the metadata for the classifier node node_id returns a metadataList
+	 * element: <metadataList><metadata
+	 * name="xxx">value</metadata></metadataList> if all_metadata is true,
+	 * returns all available metadata, otherwise just returns requested metadata
+	 */
+	// assumes only one value per metadata
+  // does no macro resolving. assumes classifier metadata will not have macros.
+  protected Element getMetadataList(Document doc, String node_id, boolean all_metadata, ArrayList<String> metadata_names)
+	{
+		String lang = "en";
+		Element metadata_list = doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return null;
+		}
+		if (all_metadata)
+		{
+			// return everything out of the database
+			Set<String> keys = info.getKeys();
+			Iterator<String> it = keys.iterator();
+			while (it.hasNext())
+			{
+				String key = it.next();
+				String value = info.getInfo(key);
+				GSXML.addMetadata(metadata_list, key, this.macro_resolver.resolve(value, lang, MacroResolver.SCOPE_META, node_id));
+			}
+
+		}
+		else
+		{
+			for (int i = 0; i < metadata_names.size(); i++)
+			{
+				String meta_name = metadata_names.get(i);
+				String value = (String) info.getInfo(meta_name);
+				GSXML.addMetadata(metadata_list, meta_name, this.macro_resolver.resolve(value, lang, MacroResolver.SCOPE_META, node_id));
+			}
+		}
+		return metadata_list;
+	}
+
+
+	protected int getNumChildren(String node_id)
+	{
+	  return this.gs_doc.getNumChildren(node_id);
+	}
+
+	/**
+	 * returns true if the id refers to a document (rather than a classifier
+	 * node)
+	 */
+	protected boolean isDocumentId(String node_id)
+	{
+		if (node_id.startsWith("CL"))
+		{
+			return false;
+		}
+		return true;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Construct.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Construct.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2Construct.java	(revision 31634)
@@ -0,0 +1,1212 @@
+/*
+ *    GS2Construct.java 
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.build.GS2PerlConstructor;
+import org.greenstone.gsdl3.build.GS2PerlListener;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSParams;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSStatus;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.UserContext;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+// https://developer.android.com/reference/org/json/JSONObject.html
+// https://developer.android.com/reference/org/json/JSONArray.html
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+
+/**
+ * A Services class for building collections provides a wrapper around the old
+ * perl scripts
+ * 
+ * @author Katherine Don
+ */
+public class GS2Construct extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2Construct.class.getName());
+
+    // default error message
+    private static final String NO_PERMISSIONS_ERROR = "This user does not have the required permissions to perform this action.";
+
+	// services offered
+	private static final String NEW_SERVICE = "NewCollection";
+	private static final String ADD_DOC_SERVICE = "AddDocument";
+	private static final String IMPORT_SERVICE = "ImportCollection";
+	private static final String BUILD_SERVICE = "BuildCollection";
+	private static final String ACTIVATE_SERVICE = "ActivateCollection";
+	private static final String BUILD_AND_ACTIVATE_SERVICE = "BuildAndActivateCollection";
+	private static final String DELETE_SERVICE = "DeleteCollection";
+	private static final String RELOAD_SERVICE = "ReloadCollection";
+	private static final String MODIFY_METADATA_SERVICE = "ModifyMetadata"; // set or remove metadata
+
+
+	// params used
+	private static final String COL_PARAM = "collection";
+	private static final String NEW_COL_TITLE_PARAM = "collTitle";
+	private static final String NEW_COL_ABOUT_PARAM = "collAbout";
+	private static final String CREATOR_PARAM = "creator";
+	private static final String NEW_FILE_PARAM = "newfile";
+	private static final String PROCESS_ID_PARAM = GSParams.PROCESS_ID;
+	private static final String BUILDTYPE_PARAM = "buildType";
+	private static final String BUILDTYPE_MG = "mg";
+	private static final String BUILDTYPE_MGPP = "mgpp";
+
+	protected static String DATABASE_TYPE = null;
+	protected SimpleCollectionDatabase coll_db = null;
+	
+	// the list of the collections - store between some method calls
+	private String[] collection_list = null;
+
+	// set of listeners for any construction commands
+	protected Map<String, GS2PerlListener> listeners = null;
+	protected HashMap<String, Boolean> collectionOperationMap = new HashMap<String, Boolean>();
+
+	public GS2Construct()
+	{
+		this.listeners = Collections.synchronizedMap(new HashMap<String, GS2PerlListener>());
+	}
+
+	/** returns a specific service description */
+	protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+
+		Element description = doc.createElement(GSXML.SERVICE_ELEM);
+		description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		description.setAttribute(GSXML.NAME_ATT, service);
+		if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+		{
+			description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service + ".name", lang)));
+			description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service + ".description", lang)));
+			description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service + ".submit", lang)));
+		}
+		if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+		{
+			Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			description.appendChild(param_list);
+
+			if (service.equals(NEW_SERVICE))
+			{
+
+				Element param = GSXML.createParameterDescription(doc, NEW_COL_TITLE_PARAM, getTextString("param." + NEW_COL_TITLE_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+				param_list.appendChild(param);
+				param = GSXML.createParameterDescription(doc, CREATOR_PARAM, getTextString("param." + CREATOR_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+				param_list.appendChild(param);
+				param = GSXML.createParameterDescription(doc, NEW_COL_ABOUT_PARAM, getTextString("param." + NEW_COL_ABOUT_PARAM, lang), GSXML.PARAM_TYPE_TEXT, null, null, null);
+				param_list.appendChild(param);
+				String[] types = { BUILDTYPE_MGPP, BUILDTYPE_MG };
+				String[] type_texts = { getTextString("param." + BUILDTYPE_PARAM + "." + BUILDTYPE_MGPP, lang), getTextString("param." + BUILDTYPE_PARAM + "." + BUILDTYPE_MG, lang) };
+
+				param = GSXML.createParameterDescription(doc, BUILDTYPE_PARAM, getTextString("param." + BUILDTYPE_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, BUILDTYPE_MGPP, types, type_texts);
+				param_list.appendChild(param);
+			}
+			else if (service.equals(ACTIVATE_SERVICE) || service.equals(IMPORT_SERVICE) || service.equals(BUILD_SERVICE) || service.equals(RELOAD_SERVICE) || service.equals(DELETE_SERVICE) || service.equals(MODIFY_METADATA_SERVICE))
+			{
+
+				this.collection_list = getCollectionList();
+				Element param = GSXML.createParameterDescription(doc, COL_PARAM, getTextString("param." + COL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, null, this.collection_list, this.collection_list);
+				param_list.appendChild(param);
+			}
+			else
+			{
+				// invalid service name
+				return null;
+			}
+		}
+		return description;
+	}
+
+	// each service must have a method "process<New service name>"
+
+	protected Element processNewCollection(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processNewCollection", NO_PERMISSIONS_ERROR);
+	    }
+	    return runCommand(request, GS2PerlConstructor.NEW);
+	}
+
+	/** TODO:implement this */
+	protected Element processAddDocument(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processAddDocument", NO_PERMISSIONS_ERROR);
+	    }
+
+	  Document result_doc = XMLConverter.newDOM();
+		// decode the file name, add it to the import directory
+		String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, name);
+		Element status = result_doc.createElement(GSXML.STATUS_ELEM);
+		response.appendChild(status);
+		//String lang = request.getAttribute(GSXML.LANG_ATT);
+		//String request_type = request.getAttribute(GSXML.TYPE_ATT);
+		Text t = result_doc.createTextNode("AddDocument: not implemented yet");
+		status.appendChild(t);
+		status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+		return response;
+	}
+
+	protected Element processBuildAndActivateCollection(Element request)
+	{
+	    // check permissions
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processBuildAndActivateCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+		    
+		waitUntilReady(request);
+		Element buildResponse = processBuildCollection(request);
+		if (buildResponse.getElementsByTagName(GSXML.ERROR_ELEM).getLength() > 0)
+		{
+			signalReady(request);
+			return buildResponse;
+		}
+
+		Element statusElem = (Element) buildResponse.getElementsByTagName(GSXML.STATUS_ELEM).item(0);
+		String id = statusElem.getAttribute("pid");
+
+		GS2PerlListener currentListener = this.listeners.get(id);
+		int statusCode = currentListener.getStatus();
+		while (!GSStatus.isCompleted(statusCode))
+		{
+			// wait for the process, and keep checking the status code
+			// there is probably a better way to do this.
+			try
+			{
+				Thread.currentThread().sleep(100);
+			}
+			catch (Exception e)
+			{ // ignore
+			}
+			statusCode = currentListener.getStatus();
+		}
+
+		Element activateResponse = processActivateCollection(request);
+		signalReady(request);
+		return activateResponse;
+	}
+
+	protected Element processImportCollection(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processImportCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		if (params == null)
+		{
+			return null;
+		}
+
+		//If we have been requested to only build certain documents then we need to create a manifest file
+		String documentsParam = (String) params.get("documents");
+		if (documentsParam != null && !documentsParam.equals(""))
+		{
+			String s = File.separator;
+			String manifestFolderPath = this.site_home + s + "collect" + s + params.get(COL_PARAM) + s + "manifests";
+			String manifestFilePath = manifestFolderPath + File.separator + "tempManifest.xml";
+
+			File manifestFolderFile = new File(manifestFolderPath);
+			if (!manifestFolderFile.exists())
+			{
+				manifestFolderFile.mkdirs();
+			}
+
+			File manifestFile = new File(manifestFilePath);
+			if (!manifestFile.exists())
+			{
+				try
+				{
+					manifestFile.createNewFile();
+				}
+				catch (Exception ex)
+				{
+					ex.printStackTrace();
+					return null; //Probably should return an actual error
+				}
+			}
+			String[] docList = documentsParam.split(",");
+
+			try
+			{
+				BufferedWriter bw = new BufferedWriter(new FileWriter(manifestFile));
+				bw.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+				bw.write("<Manifest>\n");
+				bw.write("  <Index>\n");
+				for (int j = 0; j < docList.length; j++)
+				{
+					bw.write("    <Filename>" + docList[j] + "</Filename>\n");
+				}
+				bw.write("  </Index>\n");
+				bw.write("</Manifest>\n");
+				bw.close();
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+				return null; //Probably should return an actual error
+			}
+		}
+
+		return runCommand(request, GS2PerlConstructor.IMPORT);
+	}
+
+	protected Element processBuildCollection(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processBuildCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+		return runCommand(request, GS2PerlConstructor.BUILD);
+	}
+
+	protected Element processModifyMetadata(Element request)
+	{
+
+	    // There are two types of operations whereby metadata gets modified:
+	    // - document including document-meta editing: user needs document editing powers
+	    // - adding user comments: user just needs an account and needs to be logged in
+	    // We handle both cases in this service.
+
+	    Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+	    HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+	    
+	    String metaserver_command = (String) params.get("a"); // e.g. set-archives-metadata or set-metadata-array
+	    boolean supportsSettingMultipleMeta = metaserver_command.equals("set-metadata-array") ? true : false;
+	    String json_str = (String) params.get("json"); // String or null if no param named "json"
+
+	    String[] docids = null;
+	    
+
+	    if (userHasCollectionEditPermissions(request, params)) { // means user can modify ANY metadata
+
+		// if dealing with an array of meta, then parse out the docids from the json
+		if(supportsSettingMultipleMeta) {
+		    docids = getDocIdsWithOptFilter(json_str, null);
+		} // else set-meta operation on single metadata field of single doc, 
+		  // and docid will be obtained in runCommand() where it's needed
+
+	    } else {
+		// check if user logged in
+		// shouldn't be able to do any meta modification if not logged in
+
+		UserContext context = new UserContext(request);
+		if (context.getUsername().equals("")) {
+		    
+		    return errorResponse("processModifyMetadata", "Cannot modify any metadata when not logged in.");
+		} else { // User is logged in at least, see whether they can do any restricted set-meta ops
+		    // that are open to regular users (those without permissions to edit this collection).
+		    // For now, there's only one restricted set-meta operation open to any logged in users
+		    // who don't otherwise have editing permissions for the collection: adding user comments.
+		    
+		    boolean isAddingUserComments = false;
+		    Pattern allowedMetaFieldsPattern = Pattern.compile("^(username|usertimestamp|usercomment)$");
+		    if(supportsSettingMultipleMeta) {
+			
+			docids = getDocIdsWithOptFilter(json_str, allowedMetaFieldsPattern);
+			if(docids != null) {
+			    isAddingUserComments = true;
+			}
+		    } else {
+			String metaname = (String) params.get("metaname");
+			if(isAllowedToSetMeta(metaname, allowedMetaFieldsPattern)) {
+			    isAddingUserComments = true;
+			}
+		    }
+		
+		    if(!isAddingUserComments) { // logged in user is attempting to set meta outside restricted set,
+			     // In this case, they're attempting to set meta not related to user comments
+			return errorResponse("processModifyMetadata", NO_PERMISSIONS_ERROR);
+		    }
+		}
+	    }
+		
+		// wait until we can reserve the collection for processing
+		waitUntilReady(request);
+	
+		
+		// process
+		Element response = runCommand(request, GS2PerlConstructor.MODIFY_METADATA_SERVER, docids);
+		
+		if (response.getElementsByTagName(GSXML.ERROR_ELEM).getLength() <= 0) // if no errors, wait for process to finish
+		{
+			Element statusElem = (Element) response.getElementsByTagName(GSXML.STATUS_ELEM).item(0);
+			String id = statusElem.getAttribute("pid");
+			
+			GS2PerlListener currentListener = this.listeners.get(id);
+			int statusCode = currentListener.getStatus();
+			while (!GSStatus.isCompleted(statusCode))
+			{
+				// wait for the process, and keep checking the status code
+				// there is probably a better way to do this.
+				try
+				{		
+					Thread.currentThread().sleep(100);
+				}
+				catch (Exception e)
+				{ // ignore
+				}
+				statusCode = currentListener.getStatus();
+			}	    
+		}		
+		
+		Element statusElem = (Element) response.getElementsByTagName(GSXML.STATUS_ELEM).item(0);
+		String statusString = GSXML.getNodeText(statusElem);
+		statusString += " and monitored until done."; 
+		GSXML.setNodeText(statusElem, statusString);
+		
+		// release hold on collection
+		signalReady(request);
+		return response;
+		
+	}
+
+	protected Element processActivateCollection(Element request)
+	{
+
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processActivateCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+		// this activates the collection on disk. but now we need to tell
+		// the MR about it. but we have to wait until the process is finished.
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		String coll_name = (String) params.get(COL_PARAM);
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+
+		UserContext userContext = new UserContext(request);
+		String request_type = request.getAttribute(GSXML.TYPE_ATT);
+		
+		// now we de-activate the collection before running activate.pl, and then re-activate at end 
+		// So activate.pl only does the moving, no activation. This way will prevent java from launching 
+		// perl, exiting and then leaving dangling file handles (on index/text/col.gdb) in perl.
+		if (!request_type.equals(GSXML.REQUEST_TYPE_STATUS)) {			
+			systemRequest("delete", coll_name, null, userContext); // deactivate collection
+		}	
+
+		Element response = runCommand(request, GS2PerlConstructor.ACTIVATE); // if request is for STATUS, then this won't run activate.pl
+
+		Element status = (Element) GSXML.getChildByTagName(response, GSXML.STATUS_ELEM);
+
+		// check for finished
+		int status_code = Integer.parseInt(status.getAttribute(GSXML.STATUS_ERROR_CODE_ATT));
+		if (GSStatus.isCompleted(status_code) && GSStatus.isError(status_code))
+		{
+			// we shouldn't carry out the next bit, just return the response
+			return response;
+		}
+		String id = status.getAttribute(GSXML.STATUS_PROCESS_ID_ATT);
+		GS2PerlListener listener = this.listeners.get(id);
+		if (listener == null)
+		{
+			logger.error("somethings gone wrong, couldn't find the listener");
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		}
+
+		while (!GSStatus.isCompleted(status_code))
+		{
+			// wait for the process, and keep checking the status code
+			// there is probably a better way to do this.
+			try
+			{
+				Thread.currentThread().sleep(100);
+			}
+			catch (Exception e)
+			{ // ignore
+			}
+			status_code = listener.getStatus();
+		}
+
+		// add the rest of the messages to the status node
+		Text t = status.getOwnerDocument().createTextNode("\n" + listener.getUpdate());
+		status.appendChild(t);
+		status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus()));
+		if (GSStatus.isError(status_code))
+		{
+			return response; // without doing the next bit
+		}
+
+		t = status.getOwnerDocument().createTextNode("\n");
+		status.appendChild(t);
+		// once have got here, we assume
+		// the first bit proceeded successfully, now reload the collection (sends a collection reactivation request)
+		systemRequest("reload", coll_name, status, userContext); // this will append more messages to the status, and overwrite the error code att
+		return response;
+
+	}
+
+	protected Element processDeleteCollection(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processDeleteCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+	  Document result_doc = XMLConverter.newDOM();
+		// the response to send back
+		String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, name);
+		Element status = result_doc.createElement(GSXML.STATUS_ELEM);
+		response.appendChild(status);
+		Text t = null; // the text node for the error/success message
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String request_type = request.getAttribute(GSXML.TYPE_ATT);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		boolean get_status_only = false;
+		if (request_type.equals(GSXML.REQUEST_TYPE_STATUS))
+		{
+			get_status_only = true;
+		}
+		if (get_status_only)
+		{
+			// at the moment, delete is synchronous. but it may take ages so should do the command in another thread maybe? in which case we will want to ask for status
+			logger.error("had a status request for delete - this shouldn't happen!!");
+			//t = result_doc.createTextNode("");
+			//status.appendChild(t);
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		}
+		String coll_name = (String) params.get(COL_PARAM);
+		String[] args = { coll_name };
+		File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, coll_name));
+		// check that the coll is there in the first place
+		if (!coll_dir.exists())
+		{
+			t = result_doc.createTextNode(getTextString("delete.exists_error", lang, args));
+			status.appendChild(t);
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		}
+
+		// try to delete the directory
+		if (!GSFile.deleteFile(coll_dir))
+		{
+			t = result_doc.createTextNode(getTextString("delete.delete_error", lang, args));
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			status.appendChild(t);
+			return response;
+		}
+
+		UserContext userContext = new UserContext(request);
+
+		systemRequest("delete", coll_name, status, userContext);
+		return response;
+	}
+
+	protected Element processReloadCollection(Element request)
+	{
+	    if (!userHasCollectionEditPermissions(request)) {
+		return errorResponse("processReloadCollection", NO_PERMISSIONS_ERROR);
+	    }
+
+	  Document result_doc = XMLConverter.newDOM();
+		// the response to send back
+		String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, name);
+		Element status = result_doc.createElement(GSXML.STATUS_ELEM);
+		response.appendChild(status);
+		Text t = null; // the text node for the error/success message
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String request_type = request.getAttribute(GSXML.TYPE_ATT);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		boolean get_status_only = false;
+		if (request_type.equals(GSXML.REQUEST_TYPE_STATUS))
+		{
+			get_status_only = true;
+		}
+		if (get_status_only)
+		{
+			// reload is synchronous - this makes no sense
+			logger.error("had a status request for reload - this shouldn't happen!!");
+			//t = result_doc.createTextNode("");
+			//status.appendChild(t);
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		}
+
+		String coll_name = (String) params.get(COL_PARAM);
+
+		UserContext userContext = new UserContext(request);
+
+		systemRequest("reload", coll_name, status, userContext);
+		return response;
+
+	}
+
+	/**
+	 * send a configure request to the message router action name should be
+	 * "delete" or "reload" response will be put into the status element
+	 */
+	protected void systemRequest(String operation, String coll_name, Element status, UserContext userContext)
+	{
+		// send the request to the MR
+	  Document doc = XMLConverter.newDOM();
+		Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_SYSTEM, "", userContext);
+		message.appendChild(request);
+		Element command = doc.createElement(GSXML.SYSTEM_ELEM);
+		request.appendChild(command);
+		command.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, GSXML.COLLECTION_ELEM);
+		command.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, coll_name);
+
+		if (operation.equals("delete"))
+		{
+			command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_DEACTIVATE);
+		}
+		else if (operation.equals("reload"))
+		{
+			command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ACTIVATE);
+		}
+		else
+		{
+			logger.error("invalid action name passed to systemRequest:" + operation);
+			return;
+		}
+		request.appendChild(command);
+		Node response = this.router.process(message); // at the moment, get no info in response so ignore it
+		Text t;
+		String[] args = { coll_name };
+
+		if (status != null)
+		{
+			if (response == null)
+			{
+			  t = status.getOwnerDocument().createTextNode(getTextString(operation + ".configure_error", userContext.getLanguage(), args));
+				status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+				status.appendChild(t);
+				return;
+			}
+
+			// if we got here, we have succeeded!
+			t = status.getOwnerDocument().createTextNode(getTextString(operation + ".success", userContext.getLanguage(), args));
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.SUCCESS));
+			status.appendChild(t);
+		}
+	}
+
+	/**
+	 * configure the service module for now, all services have type=build - need
+	 * to think about this
+	 */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("configuring GS2Construct");
+
+		Element e = null;
+		// hard code in the services for now
+
+		// set up short_service_info_ - for now just has name and type
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, NEW_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, IMPORT_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, BUILD_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, ACTIVATE_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, BUILD_AND_ACTIVATE_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, DELETE_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, RELOAD_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		//e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		//e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		//e.setAttribute(GSXML.NAME_ATT, ADD_DOC_SERVICE);
+		//this.short_service_info.appendChild(e);
+
+		e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS);
+		e.setAttribute(GSXML.NAME_ATT, MODIFY_METADATA_SERVICE);
+		this.short_service_info.appendChild(e);
+
+		return true;
+	}
+
+	protected Element runCommand(Element request, int type) {
+		return runCommand(request, type, null);
+	}
+
+	/** returns a response element */
+	protected Element runCommand(Element request, int type, String[] docids)
+	{
+		Document result_doc = XMLConverter.newDOM();
+		// the response to send back
+		String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, name);
+		Element status = result_doc.createElement(GSXML.STATUS_ELEM);
+		response.appendChild(status);
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String request_type = request.getAttribute(GSXML.TYPE_ATT);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		boolean get_status_only = false;
+		if (request_type.equals(GSXML.REQUEST_TYPE_STATUS))
+		{
+			get_status_only = true;
+		}
+
+		// just check for status messages if that's all that's required
+		if (get_status_only)
+		{
+			String id = (String) params.get(PROCESS_ID_PARAM);
+			status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id);
+			GS2PerlListener listener = this.listeners.get(id);
+			if (listener == null)
+			{
+				Text t = result_doc.createTextNode(getTextString("general.process_id_error", lang));
+				status.appendChild(t);
+				status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			}
+			else
+			{
+				Text t = result_doc.createTextNode(listener.getUpdate());
+				status.appendChild(t);
+				status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus()));
+				// check that we actually should be removing the listener here
+				if (listener.isFinished())
+				{ // remove this listener - its job is done
+					this.listeners.remove(id); // not working
+				}
+			}
+			return response;
+
+		}
+
+		// do the actual command
+		String coll_name = null;
+		if (type == GS2PerlConstructor.NEW)
+		{
+			String coll_title = (String) params.get(NEW_COL_TITLE_PARAM);
+			coll_name = createNewCollName(coll_title);
+		}
+		else
+		{
+			coll_name = (String) params.get(COL_PARAM);
+		}
+
+		// makes a paramList of the relevant params
+		Element other_params = extractOtherParams(params, type);
+
+		//create the constructor to do the work
+		GS2PerlConstructor constructor = new GS2PerlConstructor("perl_build");
+		if (!constructor.configure())
+		{
+			Text t = result_doc.createTextNode(getTextString("general.configure_constructor_error", lang));
+			status.appendChild(t);
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		}
+
+		constructor.setSiteHome(this.site_home);
+		constructor.setCollectionName(coll_name);
+		constructor.setActionType(type);
+		constructor.setProcessParams(other_params);
+		if (type == GS2PerlConstructor.IMPORT)
+		{
+			constructor.setManifestFile(this.site_home + File.separator + "collect" + File.separator + params.get(COL_PARAM) + File.separator + "manifests" + File.separator + "tempManifest.xml");
+		}
+		else if (type == GS2PerlConstructor.MODIFY_METADATA_SERVER) {
+		    StringBuffer querystring = new StringBuffer();
+		    
+		    // convert params into a single string again?
+		    Set<Map.Entry<String, Serializable>> entries = params.entrySet();
+		    Iterator<Map.Entry<String, Serializable>> i = entries.iterator();
+			
+		    String oid = null;
+
+		    while (i.hasNext()) {
+
+				Map.Entry<String, Serializable> entry = i.next();
+				String paramname = entry.getKey();
+				paramname = paramname.replace("s1.", ""); // replaces all occurrences
+				if (paramname.equals("collection")) {
+					paramname = "c";
+				}
+				if (paramname.equals("d")){
+					oid = (String) entry.getValue();
+				}
+				
+				String paramvalue = (String) entry.getValue();
+
+				querystring.append(paramname + "=" + paramvalue);
+				if (i.hasNext()) {
+					querystring.append("&");
+				}
+		    }
+		    
+		    if(oid != null) { // if we have only one oid
+			markDocumentInFlatDatabase("R", coll_name, OID.getTop(oid));
+		    } else if (docids != null) { // check if we are dealing with many doc ids, as cold in theory happen when set-metadata-array is called
+			
+			for(int index = 0; index < docids.length; index++) {
+			    String docid = docids[index];
+			    markDocumentInFlatDatabase("R", coll_name, OID.getTop(docid));
+			}
+		    } else {
+			String msg = getTextString("general.no_valid_docid_error", lang);
+			logger.error("*** " + msg);
+			Text t = result_doc.createTextNode(msg);
+			status.appendChild(t);
+			status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR));
+			return response;
+		    }
+		    
+		    constructor.setQueryString(querystring.toString());
+		}
+
+		GS2PerlListener listener = new GS2PerlListener();
+		constructor.addListener(listener);
+		constructor.start();
+
+		String id = newID();
+		this.listeners.put(id, listener);
+
+		status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id);
+		status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ACCEPTED));
+		Text t = result_doc.createTextNode(getTextString("general.process_start", lang));
+		status.appendChild(t);
+		return response;
+	}
+
+	//************************
+	// some helper functions
+	//************************
+
+	/** parse the collect directory and return a list of collection names */
+	protected String[] getCollectionList()
+	{
+
+		File collectDir = new File(GSFile.collectDir(this.site_home));
+		if (!collectDir.exists())
+		{
+			logger.error("couldn't find collect dir: " + collectDir.toString());
+			return null;
+		}
+		logger.info("GS2Construct: reading thru directory " + collectDir.getPath() + " to find collections.");
+		File[] contents = collectDir.listFiles();
+		int num_colls = 0;
+		for (int i = 0; i < contents.length; i++)
+		{
+			if (contents[i].isDirectory() && !contents[i].getName().startsWith("CVS"))
+			{
+				num_colls++;
+			}
+		}
+
+		String[] names = new String[num_colls];
+
+		for (int i = 0, j = 0; i < contents.length; i++)
+		{
+			if (contents[i].isDirectory())
+			{
+				String colName = contents[i].getName();
+				if (!colName.startsWith("CVS"))
+				{
+					names[j] = colName;
+					j++;
+				}
+
+			}
+		}
+
+		return names;
+
+	}
+
+	/** ids used for process id */
+	private int current_id = 0;
+
+	private String newID()
+	{
+		current_id++;
+		return Integer.toString(current_id);
+	}
+
+	/** creates a new short name from the collection title */
+	protected String createNewCollName(String coll_title)
+	{
+
+		String base_name = null;
+		// take the first 6 letters
+		if (coll_title.length() < 6)
+		{
+			base_name = coll_title;
+		}
+		else
+		{
+			base_name = coll_title.substring(0, 6);
+		}
+		File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name));
+		if (!coll_dir.exists())
+		{ // this name is ok - not used yet
+			return base_name;
+		}
+
+		// now we have to make a new name until we get a good one
+		// try name1, name2 name3 etc
+		int i = 0;
+		while (coll_dir.exists())
+		{
+			i++;
+			coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name + Integer.toString(i)));
+		}
+		return base_name + Integer.toString(i);
+
+	}
+
+	/**
+	 * takes the params from the request (in the HashMap) and extracts any that
+	 * need to be passed to the constructor and puts them into a paramList
+	 * element
+	 */
+	protected Element extractOtherParams(HashMap<String, Serializable> params, int type)
+	{
+	  Document doc = XMLConverter.newDOM();
+		Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (type == GS2PerlConstructor.NEW)
+		{
+			Element param = doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, "creator");
+			param.setAttribute(GSXML.VALUE_ATT, (String) params.get(CREATOR_PARAM));
+
+			param_list.appendChild(param);
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, "about");
+			param.setAttribute(GSXML.VALUE_ATT, (String) params.get(NEW_COL_ABOUT_PARAM));
+			param_list.appendChild(param);
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, "title");
+			param.setAttribute(GSXML.VALUE_ATT, (String) params.get(NEW_COL_TITLE_PARAM));
+			param_list.appendChild(param);
+			param = doc.createElement(GSXML.PARAM_ELEM);
+			param.setAttribute(GSXML.NAME_ATT, "buildtype");
+			param.setAttribute(GSXML.VALUE_ATT, (String) params.get(BUILDTYPE_PARAM));
+			param_list.appendChild(param);
+			return param_list;
+		}
+
+		// other ones dont have params yet
+		return null;
+	}
+
+	protected void waitUntilReady(Element request)
+	{
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collection = (String) params.get(COL_PARAM);
+
+		if (checkCollectionIsNotBusy(collection))
+		{
+			return;
+		}
+
+		while (!checkCollectionIsNotBusy(collection)) // When the collection ceases to be busy, we place a hold on it
+		{
+			try
+			{
+				Thread.currentThread().sleep(1000);
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+		}
+	}
+
+	protected void signalReady(Element request)
+	{
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		String collection = (String) params.get(COL_PARAM);
+
+		collectionOperationMap.remove(collection);
+	}
+
+	// If collection is NOT busy, then reserve it
+	protected synchronized boolean checkCollectionIsNotBusy(String collection)
+	{
+		if (collectionOperationMap.get(collection) == null)
+		{
+			collectionOperationMap.put(collection, true);
+			return true;
+		}
+		return false;
+	}
+
+    protected boolean isAllowedToSetMeta(String metaname, Pattern allowedMetaFieldsPattern) 
+    {
+	if(allowedMetaFieldsPattern == null) { // null when user has edit permissions, so they can set any meta
+	    ///logger.info("### User has permissions to set any meta.");
+	    return true;
+	}
+	if(metaname == null) {
+	    ///logger.info("### Can't check null metaname against pattern");
+	    return false;
+	}
+	
+	Matcher m = allowedMetaFieldsPattern.matcher(metaname);
+	if(!m.matches()) {
+	    ///logger.info("### metaname: " + metaname + " doesn't match allowed allowed fields: " + allowedMetaFieldsPattern.toString());
+	    return false;
+	} else {
+	    return true;
+	}
+    }
+    
+    protected String[] getDocIdsWithOptFilter(String json_str, Pattern filterFields) // boolean strictOrPermissible
+    {
+	if(json_str == null) {
+	    logger.error("### Shouldn't be happening: null json string");
+	    return null;
+	}
+
+	String[] docids = null;
+
+	// check that the name of each metadata being set matches the pattern filterFields.
+	// The presence of any other meta means something other than adding user comments is being attempted,
+	// which is invalid
+	try {
+	    
+	    JSONArray docInfoList = new JSONArray(json_str);
+	    docids = new String[docInfoList.length()];
+	    for(int index = 0; index < docInfoList.length(); index++) {
+		JSONObject docInfo = docInfoList.getJSONObject(index);
+		if(docInfo.has("metatable")) { // should exist for metadata arrays
+		    
+		    docids[index] = docInfo.getString("docid"); // should exist if metatable existed
+
+		    ///logger.info("@@@ Found docid: " + docids[index]);
+
+		    JSONArray metatable = docInfo.getJSONArray("metatable");
+		    for(int i = 0; i < metatable.length(); i++) {
+			JSONObject meta = metatable.getJSONObject(i);
+
+			String metaname = meta.getString("metaname");
+			///logger.info("### metaname: " + metaname);
+			
+			if(!isAllowedToSetMeta(metaname, filterFields)) {
+			    return null;
+			}
+		    }
+		}
+	    }
+	} catch(JSONException jsonex) {
+	    logger.error("Exception when parsing json string: " + json_str);
+	    logger.error(jsonex);
+	    
+	}
+	
+	// if we're here, then it means that the JSON only asked for username|usercomment|usertimestamp meta
+	// meaning that the setmeta operation was a valid user comment operation.
+	// In that case, we have a docid for which we need to add a user comment 
+	// set-metadata-array can take more docids, but doesn't happen for a user comment. And one comment
+	// is added at a time, but 3 meta fields are set for each comment: username, usercomment and timestamp
+	// hence the use of set-meta-array.
+	return docids;
+
+    }
+
+    protected Element errorResponse(String serviceName, String errorMsg) {
+	Document result_doc = XMLConverter.newDOM();
+	Element result = GSXML.createBasicResponse(result_doc, serviceName);
+	GSXML.addError(result, errorMsg);
+	return result;
+    }
+
+    /** Copy from DebugService.userHasEditPermissions
+     This function checks that the user is logged in and that the user 
+     is in the right group to edit the collection */
+    protected boolean userHasCollectionEditPermissions(Element request) {
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+	HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+	return userHasCollectionEditPermissions(request, params);
+
+    }
+
+    protected boolean userHasCollectionEditPermissions(Element request, HashMap<String, Serializable> params) {
+	String collection = (String) params.get(COL_PARAM); // could be null on newcoll operation
+
+
+    UserContext context = new UserContext(request);
+    if(collection == null) {
+	return !context.getUsername().equals("");
+    }
+
+    /* 
+    // FOR DEBUGGING
+    Set<Map.Entry<String, Serializable>> entries = params.entrySet();
+    Iterator<Map.Entry<String, Serializable>> i = entries.iterator();
+    
+    StringBuffer parametersLine = new StringBuffer();
+    while (i.hasNext()) {
+	
+	Map.Entry<String, Serializable> entry = i.next();
+	String paramname = entry.getKey();
+	String paramvalue = (String) entry.getValue();
+	
+	parametersLine.append("\t" + paramname + ": " + paramvalue + "\n");
+    }
+    
+    logger.info("XXXXXXXXXXXXX PARAMETERS:\n" + parametersLine);
+    */
+
+    for (String group : context.getGroups()) {
+      // administrator always has permission
+      if (group.equals("administrator")) {
+	return true;
+      }
+      // all-collections-editor can edit any collection
+      if (!collection.equals("")) {
+	if (group.equals("all-collections-editor")) {
+	  return true;
+	}
+	if (group.equals(collection+"-collection-editor")) {
+	  return true;
+	}
+      }
+    }
+    // haven't found a group with edit permissions
+    return false;
+    
+  }
+    protected void markDocumentInFlatDatabase(String mark, String collection, String oid) {
+		
+		Document msg_doc = XMLConverter.newDOM();
+		Element message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+		UserContext userContext = new UserContext();
+		Element query_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE , collection, userContext);		
+		message.appendChild(query_request);
+		Element result = (Element) this.router.process(message);
+		Element resp_elem = (Element) GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
+		Element coll_elem = (Element) GSXML.getChildByTagName(resp_elem, GSXML.COLLECTION_ELEM);
+		String dbtype = coll_elem.getAttribute(GSXML.DB_TYPE_ATT);
+		
+		SimpleCollectionDatabase coll_db = new SimpleCollectionDatabase(dbtype);
+		if (!coll_db.databaseOK())
+		{
+			logger.error("Couldn't create the collection database of type " + dbtype);
+			return;
+		}
+		
+		// Open database for reading. It may not exist if collection is pre-built without archives (such as demo collections)
+		String coll_db_file = GSFile.archivesDatabaseFile(this.site_home, collection, dbtype);
+		if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ))
+		{
+			logger.error("Could not open collection archives database. Database doesn't exist or else somebody's already using it?");
+			return;
+		}
+		// now we know we have an archives folder
+		String old_value = coll_db.getValue(oid);
+		String new_value = "<index-status>" + mark;
+		if(old_value == null) {
+		    logger.error("### null old_value in flat DB for oid " + oid);
+		} else {
+		    new_value = old_value.replace("<index-status>B", "<index-status>" + mark);
+		    logger.info("### Replacing db entry for oid " + oid + " which has old_value " + old_value);
+		    logger.info("### with new value " + new_value);
+				    
+		}
+		// Close database for reading
+		coll_db.closeDatabase();
+
+		if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.WRITE))
+		    {
+			logger.error("Could not open collection archives database. Somebody already using this database!");
+			return;
+		    }
+		
+		coll_db.setValue(oid, new_value);
+		
+		coll_db.closeDatabase();		
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneRetrieve.java	(revision 31634)
@@ -0,0 +1,176 @@
+/*
+ *    GS2LuceneRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.util.GSHTML;
+import org.greenstone.gsdl3.util.OID;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Retrieve documents from a gs2 lucene collection. Note that this doesn't
+ * actually use lucene, as the documents are stored in XML files
+ */
+public class GS2LuceneRetrieve extends AbstractGS2DocumentRetrieve
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2LuceneRetrieve.class.getName());
+
+	protected static final String DOC_LEVEL = "Doc";
+	protected static final String SEC_LEVEL = "Sec";
+	protected static final String ID_ATT = "gs2:docOID";
+
+	// Parameters used
+	private static final String LEVEL_PARAM = "level";
+
+	// Elements used in the config file that are specific to this class
+	private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+
+	private String default_level = null;
+	private String text_dir = null;
+
+	private boolean text_available = true;
+
+	public GS2LuceneRetrieve()
+	{
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		// Do specific configuration
+		logger.info("Configuring GS2LuceneRetrieve...");
+
+		text_dir = GSFile.collectionIndexDir(this.site_home, this.cluster_name) + File.separatorChar + "text" + File.separatorChar;
+		if (!(new File(text_dir).isDirectory()))
+		{
+			logger.error("Text directory " + text_dir + " does not exist, will be unable to retrieve text for " + cluster_name);
+			text_available = false;
+			return true; // return true so that we still get the other services for the collection
+		}
+		// Get the default level out of <defaultLevel> (buildConfig.xml)
+		Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+		if (def != null)
+		{
+			this.default_level = def.getAttribute(GSXML.SHORTNAME_ATT);
+		}
+		if (this.default_level == null || this.default_level.equals(""))
+		{
+			logger.error("Default level not specified for " + this.cluster_name + ", assuming " + DOC_LEVEL);
+			this.default_level = DOC_LEVEL;
+		}
+
+		return true;
+
+	}
+
+	/**
+	 * returns the content of a node should return a nodeContent element:
+	 * <nodeContent>text content or other elements</nodeContent>
+	 */
+  protected Element getNodeContent(Document doc, String doc_id, String lang) throws GSException
+	{
+		String[] args = new String[1];
+		args[0] = doc_id;
+		String doc_content = getTextString("TextRetrievalError", lang, args);
+		try
+		{
+			if (!text_available)
+			{
+				throw new Exception("No text directory available");
+			}
+
+			DBInfo info = this.coll_db.getInfo(OID.getTop(doc_id));
+			if (info == null)
+			{
+				throw new Exception("Couldn't get database entry for " + OID.getTop(doc_id));
+			}
+
+			String archivedir = info.getInfo("archivedir");
+			File doc_xml_file = new File(text_dir + archivedir + File.separatorChar + "doc.xml");
+			if (!doc_xml_file.isFile())
+			{
+				throw new Exception("Doc XML file " + doc_xml_file.getPath() + " does not exist");
+			}
+			Document doc_xml_doc = this.converter.getDOM(doc_xml_file, "utf-8");
+			if (doc_xml_doc == null)
+			{
+				throw new Exception("Couldn't parse file " + doc_xml_file.getPath());
+			}
+			Element full_document = doc_xml_doc.getDocumentElement();
+			if (full_document == null)
+			{
+				throw new Exception("Couldn't parse file " + doc_xml_file.getPath());
+			}
+			Element current_section = null;
+			if (default_level.equals(DOC_LEVEL))
+			{
+				current_section = full_document;
+			}
+			else
+			{
+				current_section = GSXML.getNamedElement(full_document, SEC_LEVEL, ID_ATT, doc_id);
+			}
+			if (current_section == null)
+			{
+				throw new Exception("Couldn't find section " + doc_id + " in file " + doc_xml_file.getPath());
+			}
+			doc_content = GSXML.getNodeText(current_section);
+			if (doc_content == null)
+			{
+				doc_content = "";
+			}
+			else
+			{
+				doc_content = resolveTextMacros(doc_content, doc_id, lang);
+			}
+		}
+		catch (Exception e)
+		{
+			logger.error("Error trying to get document text for " + doc_id + " in collection " + this.cluster_name + ": " + e);
+		}
+
+		Element content_node = doc.createElement(GSXML.NODE_CONTENT_ELEM);
+		Text t = doc.createTextNode(doc_content);
+		content_node.appendChild(t);
+		return content_node;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2LuceneSearch.java	(revision 31634)
@@ -0,0 +1,322 @@
+/*
+ *    GS2LuceneSearch.java
+ *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+import org.greenstone.LuceneWrapper4.GS2LuceneQuery;
+import org.greenstone.LuceneWrapper4.LuceneQueryResult;
+import org.greenstone.gsdl3.util.FacetWrapper;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public class GS2LuceneSearch extends SharedSoleneGS2FieldSearch
+{
+
+  protected static final String SORT_ORDER_PARAM = "reverseSort";
+  protected static final String SORT_ORDER_REVERSE = "1";
+  protected static final String SORT_ORDER_NORMAL = "0";
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2LuceneSearch.class.getName());
+
+	private GS2LuceneQuery lucene_src = null;
+
+	public GS2LuceneSearch()
+	{
+	  does_paging = true;
+		paramDefaults.put(SORT_ORDER_PARAM, SORT_ORDER_NORMAL);
+		this.lucene_src = new GS2LuceneQuery();
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+		this.lucene_src.cleanUp();
+	}
+
+	/** add in the Lucene specific params to TextQuery */
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		super.addCustomQueryParams(param_list, lang);
+		/** Add in the reverse sort on/off param */
+		createParameter(SORT_ORDER_PARAM, param_list, lang);
+	}
+  /** add in Lucene specific params for AdvancedFieldQuery */
+  protected void addCustomQueryParamsAdvField(Element param_list, String lang)
+	{
+		super.addCustomQueryParamsAdvField(param_list, lang);
+		createParameter(SORT_ORDER_PARAM, param_list, lang);
+		
+	}
+	/** create a param and add to the list */
+  protected void createParameter(String name, Element param_list, String lang)
+  {
+    Document doc = param_list.getOwnerDocument();
+    Element param = null;
+    String param_default = paramDefaults.get(name);
+    if (name.equals(SORT_ORDER_PARAM)) {
+	    String[] vals = { SORT_ORDER_REVERSE, SORT_ORDER_NORMAL };
+	    String[] vals_texts = { getTextString("param." + SORT_ORDER_PARAM + "." + SORT_ORDER_REVERSE, lang), getTextString("param." + SORT_ORDER_PARAM + "." + SORT_ORDER_NORMAL, lang) };
+
+	    param = GSXML.createParameterDescription(doc, SORT_ORDER_PARAM, getTextString("param." + SORT_ORDER_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals, vals_texts);
+    }
+
+    if (param != null)
+      {
+	param_list.appendChild(param);
+      }
+    else
+      {
+	super.createParameter(name, param_list, lang);
+      }
+    
+  }
+   
+	/** methods to handle actually doing the query */
+
+	/** do any initialisation of the query object */
+	protected boolean setUpQueryer(HashMap params)
+	{
+		String indexdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar + "index" + File.separatorChar;
+
+		String index = "didx";
+		if (this.default_level.toUpperCase().equals("SEC")) {
+		  index = "sidx";
+		}
+		String physical_index_language_name = null;
+		String physical_sub_index_name = null;
+		int hits_per_page = Integer.parseInt(paramDefaults.get(HITS_PER_PAGE_PARAM));
+		int start_page = Integer.parseInt(paramDefaults.get(START_PAGE_PARAM));
+		String sort_field = getLuceneSort(default_sort);
+		String sort_order = paramDefaults.get(SORT_ORDER_PARAM);
+	       
+		// set up the query params
+		Set entries = params.entrySet();
+		Iterator i = entries.iterator();
+		while (i.hasNext())
+		{
+			Map.Entry m = (Map.Entry) i.next();
+			String name = (String) m.getKey();
+			String value = (String) m.getValue();
+
+			if (name.equals(HITS_PER_PAGE_PARAM))
+			{
+			  if (value.equals("all")) {
+			    hits_per_page = -1;
+			  } else {
+			    hits_per_page = Integer.parseInt(value);
+			  }
+			}
+			else if (name.equals(START_PAGE_PARAM))
+			{
+				start_page = Integer.parseInt(value);
+
+			}
+			else if (name.equals(MATCH_PARAM))
+			{
+				if (value.equals(MATCH_PARAM_ALL))
+				{
+					this.lucene_src.setDefaultConjunctionOperator("AND");
+				}
+				else
+				{
+					this.lucene_src.setDefaultConjunctionOperator("OR");
+				}
+			}
+			else if (name.equals(RANK_PARAM))
+			{
+			  sort_field = getLuceneSort(value);
+			  this.lucene_src.setSortField(sort_field);
+
+			}
+			else if (name.equals(SORT_ORDER_PARAM)) {
+			  sort_order = value;
+			}
+			else if (name.equals(LEVEL_PARAM))
+			{
+				if (value.toUpperCase().equals("SEC"))
+				{
+					index = "sidx";
+				}
+				else
+				{
+					index = "didx";
+				}
+			}
+			else if (name.equals(INDEX_SUBCOLLECTION_PARAM))
+			{
+				physical_sub_index_name = value;
+			}
+			else if (name.equals(INDEX_LANGUAGE_PARAM))
+			{
+				physical_index_language_name = value;
+			} // ignore any others
+		}
+		// set up start and end results if necessary
+		// start results always start at 0
+		int start_results = 0; 
+		if (start_page > 1 && hits_per_page > 0)
+		{
+		  start_results = ((start_page - 1) * hits_per_page) ; 
+		}
+		int end_results = Integer.MAX_VALUE;
+		if (hits_per_page > 0) {
+		  end_results = hits_per_page * start_page;
+		}
+		this.lucene_src.setStartResults(start_results);
+		this.lucene_src.setEndResults(end_results);
+
+		if (index.equals("sidx") || index.equals("didx"))
+		{
+			if (physical_sub_index_name != null)
+			{
+				index += physical_sub_index_name;
+			}
+			if (physical_index_language_name != null)
+			{
+				index += physical_index_language_name;
+			}
+		}
+
+		if (sort_order.equals(SORT_ORDER_REVERSE)) {
+		  this.lucene_src.setReverseSort(true);
+		} else {
+		  this.lucene_src.setReverseSort(false);
+		}
+		this.lucene_src.setIndexDir(indexdir + index);
+		this.lucene_src.initialise();
+		return true;
+	}
+
+	/** do the query */
+	protected Object runQuery(String query)
+	{
+		try
+		{
+			LuceneQueryResult lqr = this.lucene_src.runQuery(query);
+			return lqr;
+		}
+		catch (Exception e)
+		{
+			logger.error("Exception happened in runQuery(): ", e);
+		}
+
+		return null;
+	}
+
+	/** get the total number of docs that match */
+	protected long numDocsMatched(Object query_result)
+	{
+		return ((LuceneQueryResult) query_result).getTotalDocs();
+	}
+
+	/** get the list of doc ids */
+	protected String[] getDocIDs(Object query_result)
+	{
+		Vector docs = ((LuceneQueryResult) query_result).getDocs();
+		String[] doc_nums = new String[docs.size()];
+		for (int d = 0; d < docs.size(); d++)
+		{
+			String doc_num = ((LuceneQueryResult.DocInfo) docs.elementAt(d)).id_;
+			doc_nums[d] = doc_num;
+		}
+		return doc_nums;
+	}
+
+	/** get the list of doc ranks */
+	protected String[] getDocRanks(Object query_result)
+	{
+		Vector docs = ((LuceneQueryResult) query_result).getDocs();
+		String[] doc_ranks = new String[docs.size()];
+		for (int d = 0; d < docs.size(); d++)
+		{
+			doc_ranks[d] = Float.toString(((LuceneQueryResult.DocInfo) docs.elementAt(d)).rank_);
+		}
+		return doc_ranks;
+	}
+
+	/** add in term info if available */
+	protected boolean addTermInfo(Element term_list, HashMap params, Object query_result)
+	{
+	  Document doc = term_list.getOwnerDocument();
+		String query_level = (String) params.get(LEVEL_PARAM); // the current query level
+
+		Vector terms = ((LuceneQueryResult) query_result).getTerms();
+		for (int t = 0; t < terms.size(); t++)
+		{
+			LuceneQueryResult.TermInfo term_info = (LuceneQueryResult.TermInfo) terms.get(t);
+
+			Element term_elem = doc.createElement(GSXML.TERM_ELEM);
+			term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
+			term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
+			term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
+			term_elem.setAttribute(FIELD_ATT, term_info.field_);
+			term_list.appendChild(term_elem);
+		}
+
+		Vector stopwords = ((LuceneQueryResult) query_result).getStopWords();
+		for (int t = 0; t < stopwords.size(); t++)
+		{
+			String stopword = (String) stopwords.get(t);
+
+			Element stopword_elem = doc.createElement(GSXML.STOPWORD_ELEM);
+			stopword_elem.setAttribute(GSXML.NAME_ATT, stopword);
+			term_list.appendChild(stopword_elem);
+		}
+
+		return true;
+	}
+	
+	protected ArrayList<FacetWrapper> getFacets(Object query_result)
+	{
+		return null;
+	}
+
+  protected String getLuceneSort(String gs3_sort) {
+
+    if (gs3_sort.equals(RANK_PARAM_RANK)) {
+      return GS2LuceneQuery.SORT_RANK;
+    }
+    if (gs3_sort.equals(RANK_PARAM_NONE)) {
+      return GS2LuceneQuery.SORT_NATURAL;
+    }
+    return gs3_sort;
+  }
+
+@Override
+protected Map<String, Map<String, List<String>>> getHighlightSnippets(
+		Object query_result) {
+	// TODO Auto-generated method stub
+	return null;
+}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPRetrieve.java	(revision 31634)
@@ -0,0 +1,128 @@
+/*
+ *    GS2MGPPRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+import org.apache.log4j.*;
+
+public class GS2MGPPRetrieve
+  extends AbstractGS2DocumentRetrieve {
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2MGPPRetrieve.class.getName());
+  
+  // Parameters used
+  private static final String LEVEL_PARAM = "level";
+  
+  // Elements used in the config file that are specific to this class
+  private static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
+  
+  private static MGPPRetrieveWrapper mgpp_src = null;
+  
+  private String default_level = null;
+  private String mgpp_textdir = null;
+  
+  public GS2MGPPRetrieve() {
+    if(mgpp_src == null) {
+      mgpp_src = new MGPPRetrieveWrapper();
+    }
+  }
+  
+  public void cleanUp() {
+    super.cleanUp();
+  }
+  
+  /** configure this service */
+  public boolean configure(Element info, Element extra_info) {
+    if (!super.configure(info, extra_info)){
+      return false;
+    }
+    
+    // Do specific configuration
+    logger.info("Configuring GS2MGPPRetrieve...");
+    
+    // Get the default level out of <defaultLevel> (buildConfig.xml)
+    Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
+    if (def != null) {
+      this.default_level = def.getAttribute(GSXML.SHORTNAME_ATT);
+    }
+    if (this.default_level == null || this.default_level.equals("")) {
+      logger.error("default level not specified!");
+      return false;
+    }
+    
+    // The location of the MGPP text files
+    mgpp_textdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) +
+      File.separatorChar + GSFile.collectionTextPath(this.index_stem);
+    
+    // Do generic configuration
+    return true;
+    
+  }
+  
+  /** returns the content of a node
+   * should return a nodeContent element:
+   * <nodeContent>text content or other elements</nodeContent>
+   */
+  protected Element getNodeContent(Document doc, String doc_id, String lang) throws GSException {
+    long doc_num = this.coll_db.OID2DocnumLong(doc_id);
+    if (doc_num == -1) {
+      logger.error("OID "+doc_id +" couldn't be converted to mgpp num");
+      return null;
+    }
+    Element content_node = doc.createElement(GSXML.NODE_CONTENT_ELEM);
+    synchronized (mgpp_src) {
+    	String doc_content = "";
+    	try {
+    		
+    		doc_content = mgpp_src.getDocument(this.mgpp_textdir,
+    				this.default_level,
+    				doc_num);
+    		
+    		if (doc_content != null) {
+    			doc_content = resolveTextMacros(doc_content, doc_id, lang);
+    		}
+    		
+		// remove the <Doc></Doc> and <Sec></Sec> tags
+		//doc_content = doc_content.replace("<Doc>", "").replace("</Doc>", "").replace("<Sec>", "").replace("</Sec>", "");
+		doc_content = doc_content.replaceAll("</?(Doc|Sec)>", "");
+    	} catch (Exception e) {
+    		logger.info("exception happended with mgpp_src.getDocument()" + e);
+    		doc_content = "this is the content for section hash id "+ doc_id+", mgpp doc num "+doc_num+"\n";
+    		
+    	}
+    	Text t = doc.createTextNode(doc_content);
+    	content_node.appendChild(t);
+    	return content_node;
+    }//end of synchronized
+  }
+  
+  
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGPPSearch.java	(revision 31634)
@@ -0,0 +1,366 @@
+/*
+ *    GS2MGPPSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.FacetWrapper;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.mgpp.MGPPDocInfo;
+import org.greenstone.mgpp.MGPPQueryResult;
+import org.greenstone.mgpp.MGPPSearchWrapper;
+import org.greenstone.mgpp.MGPPTermInfo;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public class GS2MGPPSearch extends AbstractGS2FieldSearch
+{
+	private static MGPPSearchWrapper mgpp_src = null;
+
+	private String physical_index_name = "idx";
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2MGPPSearch.class.getName());
+
+	/** constructor */
+	public GS2MGPPSearch()
+	{
+	  does_chunking = true;
+		if (mgpp_src == null)
+		{
+			mgpp_src = new MGPPSearchWrapper();
+		}
+	}
+
+	public void cleanUp()
+	{
+		super.cleanUp();
+		mgpp_src.unloadIndexData();
+		mgpp_src.reset(); // reset stored settings to defaults
+	}
+
+	/** process a query */
+	protected Element processAnyQuery(Element request, int query_type)
+	{
+		synchronized (mgpp_src)
+		{
+			return super.processAnyQuery(request, query_type);
+		}
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		// set up the defaults which are not dependent on query parameters
+		// the default level is also the level which the database is expecting
+		// this must not be overwritten
+		mgpp_src.setReturnLevel(this.default_db_level);
+		// return term info
+		mgpp_src.setReturnTerms(true);
+		mgpp_src.setMaxNumeric(this.maxnumeric);
+		// mgpp internal default is 50, so set it here so the interface params agree
+		paramDefaults.put(MAXDOCS_PARAM, "50");
+		return true;
+	}
+
+	/** add in the mgpp specific params to TextQuery */
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		super.addCustomQueryParams(param_list, lang);
+		createParameter(RANK_PARAM, param_list, lang);
+	}
+
+	protected boolean setUpQueryer(HashMap<String, Serializable> params)
+	{
+
+		// set up the defaults that may be changed by query params
+		mgpp_src.setQueryLevel(this.default_level);
+		// we have case folding on by default
+		if (this.does_case) {
+		  mgpp_src.setCase(paramDefaults.get(CASE_PARAM).equals(BOOLEAN_PARAM_ON) ? true : false);
+		}
+		if (this.does_stem) {
+		  mgpp_src.setStem(paramDefaults.get(STEM_PARAM).equals(BOOLEAN_PARAM_ON) ? true : false);
+		}
+		if (this.does_accent) {
+		  mgpp_src.setAccentFold(paramDefaults.get(ACCENT_PARAM).equals(BOOLEAN_PARAM_ON) ? true : false);
+		}
+		// set up the query params
+		Set entries = params.entrySet();
+		Iterator i = entries.iterator();
+		String current_physical_index_name = this.physical_index_name;
+		String physical_sub_index_name = this.default_index_subcollection;
+		String physical_index_language_name = this.default_index_language;
+		while (i.hasNext())
+		{
+			Map.Entry m = (Map.Entry) i.next();
+			String name = (String) m.getKey();
+			String value = (String) m.getValue();
+
+			if (name.equals(CASE_PARAM) && this.does_case)
+			{
+				boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+				mgpp_src.setCase(val);
+			}
+			else if (name.equals(STEM_PARAM) && this.does_stem)
+			{
+				boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+				mgpp_src.setStem(val);
+			}
+			else if (name.equals(ACCENT_PARAM) && this.does_accent)
+			{
+				boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+				mgpp_src.setAccentFold(val);
+			}
+			else if (name.equals(MAXDOCS_PARAM) && !value.equals(""))
+			{
+				int docs = Integer.parseInt(value);
+				mgpp_src.setMaxDocs(docs);
+			}
+			else if (name.equals(LEVEL_PARAM))
+			{
+				mgpp_src.setQueryLevel(value);
+			}
+			else if (name.equals(MATCH_PARAM))
+			{
+				int mode;
+				if (value.equals(MATCH_PARAM_ALL))
+					mode = 1;
+				else
+					mode = 0;
+				mgpp_src.setMatchMode(mode);
+			}
+			else if (name.equals(RANK_PARAM))
+			{
+				if (value.equals(RANK_PARAM_RANK))
+				{
+					mgpp_src.setSortByRank(true);
+				}
+				else if (value.equals(RANK_PARAM_NONE))
+				{
+					mgpp_src.setSortByRank(false);
+				}
+			}
+			else if (name.equals(INDEX_SUBCOLLECTION_PARAM))
+			{
+				physical_sub_index_name = value;
+			}
+			else if (name.equals(INDEX_LANGUAGE_PARAM))
+			{
+				physical_index_language_name = value;
+			} // ignore any others
+		}
+
+		if (physical_index_name.equals("idx"))
+		{
+			if (physical_sub_index_name != null)
+			{
+				current_physical_index_name += physical_sub_index_name;
+			}
+			if (physical_index_language_name != null)
+			{
+				current_physical_index_name += physical_index_language_name;
+			}
+		}
+
+		// set up mgpp_src
+		String indexdir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separatorChar + GSFile.collectionIndexPath(this.index_stem, current_physical_index_name);
+		mgpp_src.loadIndexData(indexdir);
+
+		return true;
+	}
+
+	protected Object runQuery(String query)
+	{
+		mgpp_src.runQuery(query);
+		MGPPQueryResult mqr = mgpp_src.getQueryResult();
+		return mqr;
+
+	}
+
+	protected long numDocsMatched(Object query_result)
+	{
+		return ((MGPPQueryResult) query_result).getTotalDocs();
+	}
+
+	protected String[] getDocIDs(Object query_result)
+	{
+
+		Vector docs = ((MGPPQueryResult) query_result).getDocs();
+		String[] doc_nums = new String[docs.size()];
+		for (int d = 0; d < docs.size(); d++)
+		{
+			doc_nums[d] = Long.toString((((MGPPDocInfo) docs.elementAt(d)).num_));
+		}
+		return doc_nums;
+	}
+
+	protected String[] getDocRanks(Object query_result)
+	{
+
+		Vector docs = ((MGPPQueryResult) query_result).getDocs();
+		String[] doc_ranks = new String[docs.size()];
+		for (int d = 0; d < docs.size(); d++)
+		{
+			doc_ranks[d] = Float.toString(((MGPPDocInfo) docs.elementAt(d)).rank_);
+		}
+		return doc_ranks;
+	}
+
+	protected boolean addTermInfo(Element term_list, HashMap<String, Serializable> params, Object query_result)
+	{
+	  Document doc = term_list.getOwnerDocument();
+		String query_level = (String) params.get(LEVEL_PARAM); // the current query level
+
+		Vector terms = ((MGPPQueryResult) query_result).getTerms();
+		for (int t = 0; t < terms.size(); t++)
+		{
+			MGPPTermInfo term_info = (MGPPTermInfo) terms.get(t);
+
+			Element term_elem = doc.createElement(GSXML.TERM_ELEM);
+			term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
+			term_elem.setAttribute(STEM_ATT, "" + term_info.stem_method_);
+			term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
+			term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
+			String field = term_info.tag_;
+			if (field.equals(query_level))
+			{
+				// ignore
+				field = "";
+			}
+			term_elem.setAttribute(FIELD_ATT, field);
+
+			Vector equiv_terms = term_info.equiv_terms_;
+			Element equiv_term_list = doc.createElement(EQUIV_TERM_ELEM + GSXML.LIST_MODIFIER);
+			term_elem.appendChild(equiv_term_list);
+
+			for (int et = 0; et < equiv_terms.size(); et++)
+			{
+				String equiv_term = (String) equiv_terms.get(et);
+
+				Element equiv_term_elem = doc.createElement(GSXML.TERM_ELEM);
+				equiv_term_elem.setAttribute(GSXML.NAME_ATT, equiv_term);
+				equiv_term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "");
+				equiv_term_elem.setAttribute(FREQ_ATT, "");
+				equiv_term_list.appendChild(equiv_term_elem);
+			}
+
+			term_list.appendChild(term_elem);
+		}
+		return true;
+	}
+
+	protected String addFieldInfo(String query, String field)
+	{
+		if (field.equals("") || field.equals("ZZ"))
+		{
+			return query;
+		}
+		return "[" + query + "]:" + field;
+	}
+
+	protected void addQueryElem(StringBuffer final_query, String query, String field, String combine)
+	{
+
+		String comb = "";
+		if (final_query.length() > 0)
+		{
+			comb = " " + combine + " ";
+		}
+		final_query.append(comb + addFieldInfo(query, field));
+	}
+
+	protected String addStemOptions(String query, String stem, String casef, String accent)
+	{
+		String mods = "#";
+		if (casef != null)
+		{
+			if (casef.equals("1"))
+			{
+				mods += "i";
+			}
+			else
+			{
+				mods += "c";
+			}
+		}
+		if (stem != null)
+		{
+			if (stem.equals("1"))
+			{
+				mods += "s";
+			}
+			else
+			{
+				mods += "u";
+			}
+		}
+		if (accent != null)
+		{
+			if (accent.equals("1"))
+			{
+				mods += "f";
+			}
+			else
+			{
+				mods += "a";
+			}
+		}
+
+		StringBuffer temp = new StringBuffer();
+		String[] terms = query.split(" ");
+		for (int i = 0; i < terms.length; i++)
+		{
+			String t = terms[i].trim();
+			// what is the TX bit about???
+			if (!t.equals("") && !t.equals("TX"))
+			{
+				temp.append(" " + t + mods);
+			}
+		}
+		return temp.toString();
+	}
+
+	protected ArrayList<FacetWrapper> getFacets(Object query_result)
+	{
+		return null;
+	}
+
+	@Override
+	protected Map<String, Map<String, List<String>>> getHighlightSnippets(
+			Object query_result) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGRetrieve.java	(revision 31634)
@@ -0,0 +1,153 @@
+/*
+ *    GS2MGRetrieve.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.mg.MGRetrieveWrapper;
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+// General Java classes
+import java.io.File;
+
+import org.apache.log4j.*;
+
+// Apache Commons
+import org.apache.commons.lang3.*;
+
+public class GS2MGRetrieve
+extends AbstractGS2DocumentRetrieve {
+    static Logger logger = Logger.getLogger (org.greenstone.gsdl3.service.GS2MGRetrieve.class.getName ());
+    
+    // Elements used in the config file that are specific to this class
+    private static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    private static final String INDEX_LIST_ELEM = "indexList";
+    private static final String INDEX_ELEM = "index";
+    private static final String DEFAULT_INDEX_SUBCOLLECTION_ELEM = "defaultIndexSubcollection";
+    private static final String DEFAULT_INDEX_LANGUAGE_ELEM = "defaultIndexLanguage";
+    
+    private static MGRetrieveWrapper mg_src = null;
+    private String mg_basedir = null;
+    private String mg_textdir = null;
+    private String default_index = null;
+    private boolean has_default_index = false;
+    
+    public GS2MGRetrieve () {
+	if (this.mg_src == null){
+	    this.mg_src = new MGRetrieveWrapper ();
+	}
+    }
+    
+    public void cleanUp () {
+        super.cleanUp ();
+        this.mg_src.unloadIndexData ();
+    }
+    
+    /** configure this service */
+    public boolean configure (Element info, Element extra_info) {
+        if (!super.configure (info, extra_info)){
+            return false;
+        }
+        
+        // Do specific configuration
+        logger.info ("Configuring GS2MGRetrieve...");
+        
+        // Get the default index out of <defaultIndex> (buildConfig.xml)
+        Element def = (Element) GSXML.getChildByTagName (info, DEFAULT_INDEX_ELEM);
+        if (def != null) {
+            this.default_index = def.getAttribute (GSXML.SHORTNAME_ATT);
+        }
+        Element defSub = (Element) GSXML.getChildByTagName (info, DEFAULT_INDEX_SUBCOLLECTION_ELEM);
+        if (defSub != null) {
+            this.default_index += defSub.getAttribute (GSXML.SHORTNAME_ATT);
+            logger.info ("default indexSubcollection is "+defSub.getAttribute (GSXML.SHORTNAME_ATT));
+        } //concate defaultIndex + defaultIndexSubcollection
+        
+        //get the default indexLanguage out of <defaultIndexLanguage> (buildConfig.xml)
+        Element defLang = (Element) GSXML.getChildByTagName (info, DEFAULT_INDEX_LANGUAGE_ELEM);
+        if (defLang != null) {
+            this.default_index += defLang.getAttribute (GSXML.SHORTNAME_ATT);
+            logger.info ("default indexLanguage is "+defLang.getAttribute (GSXML.SHORTNAME_ATT));
+        } //concate defaultIndex + defaultIndexSubcollection + defaultIndexLanguage
+        
+        
+        if (this.default_index == null || this.default_index.equals ("")) {
+            logger.error ("default index is not specified, the content of a document will not be retrieved");
+            has_default_index = false;
+            return true;
+            //}
+        }
+        logger.debug ("Default index: " + this.default_index);
+        
+        // The location of the MG index and text files
+        mg_basedir = GSFile.collectionBaseDir (this.site_home, this.cluster_name) + File.separatorChar;  // Needed by MG
+        mg_textdir = GSFile.collectionTextPath (this.index_stem);
+        // index is only needed to start up MG, not used so just use the default index
+        String indexpath = GSFile.collectionIndexPath (this.index_stem, this.default_index);
+        this.mg_src.setIndex (indexpath);
+        has_default_index = true;
+        return true;
+    }
+    
+    /** returns the content of a node
+     * should return a nodeContent element:
+     * <nodeContent>text content or other elements</nodeContent>
+     */
+  protected Element getNodeContent (Document doc, String doc_id, String lang) throws GSException {
+        long doc_num = this.coll_db.OID2DocnumLong (doc_id);
+        if (doc_num == -1) {
+            logger.error ("OID "+doc_id +" couldn't be converted to mg num");
+            return null;
+        }
+        Element content_node = doc.createElement (GSXML.NODE_CONTENT_ELEM);
+                
+        String doc_content = null;
+        
+        //means that this.mg_src is up and running
+        if (has_default_index ){
+	    synchronized(this.mg_src){
+                String indexpath = GSFile.collectionIndexPath(this.index_stem, this.default_index);        
+		this.mg_src.setIndex(indexpath);
+		doc_content =  this.mg_src.getDocument (this.mg_basedir,
+							this.mg_textdir, doc_num);
+	    }
+        }
+        
+        if (doc_content!=null) {
+            // remove any ctrl-c or ctrl-b
+            doc_content = StringUtils.replace(doc_content, "\u0002|\u0003", "");
+            // replace _httpimg_ with the correct address
+            doc_content = resolveTextMacros (doc_content, doc_id, lang);
+            //GSXML.addDocText(doc, doc_content);
+        } else {
+            logger.error ("the doc content was null, not getting that section\n");
+            doc_content = "couldn't retrieve content for this section, please check the log file for more detail\n";
+        }
+        Text t = doc.createTextNode (doc_content);
+        content_node.appendChild (t);
+        return content_node;
+        
+    }       
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GS2MGSearch.java	(revision 31634)
@@ -0,0 +1,256 @@
+/*
+ *    GS2MGSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.mg.*;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+// java
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.Iterator;
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ * @author Katherine Don
+ * @author <a href="mailto:mdewsnip@cs.waikato.ac.nz">Michael Dewsnip</a>
+ */
+
+public class GS2MGSearch
+extends AbstractGS2TextSearch {
+    
+    protected static MGSearchWrapper mg_src = null;
+    
+    static Logger logger = Logger.getLogger (org.greenstone.gsdl3.service.GS2MGSearch.class.getName ());
+    
+    
+    /** constructor */
+    public GS2MGSearch () {
+        does_chunking = true;
+	if(this.mg_src == null){
+	    this.mg_src = new MGSearchWrapper ();
+        }
+    }
+    public void cleanUp () {
+        super.cleanUp ();
+        this.mg_src.unloadIndexData ();
+    }
+    
+    /** configure this service */
+    public boolean configure (Element info, Element extra_info) {
+        if (!super.configure (info, extra_info)){
+            return false;
+        }
+	// internally mg uses 50, so set this here
+	paramDefaults.put(MAXDOCS_PARAM, "50");
+        this.mg_src.setMaxNumeric (this.maxnumeric);
+        return true;
+    }
+    
+    
+    
+    /** do the actual query */
+    protected Element processTextQuery (Element request) {
+        synchronized(this.mg_src){
+        // Create a new (empty) result message ('doc' is in ServiceRack.java)
+	  Document result_doc = XMLConverter.newDOM();
+        Element result = result_doc.createElement (GSXML.RESPONSE_ELEM);
+        result.setAttribute (GSXML.FROM_ATT, QUERY_SERVICE);
+        result.setAttribute (GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+        
+        // Get the parameters of the request
+        Element param_list = (Element) GSXML.getChildByTagName (request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+        if (param_list == null) {
+            logger.error ("TextQuery request had no paramList.");
+            return result;  // Return the empty result
+        }
+        
+        // Process the request parameters
+        HashMap<String, Serializable> params = GSXML.extractParams (param_list, false);
+        
+        // Make sure a query has been specified
+        String query = (String) params.get (QUERY_PARAM);
+        if (query == null || query.equals ("")) {
+            return result;  // Return the empty result
+        }
+        
+        // If an index hasn't been specified, use the default
+        String index = (String) params.get (INDEX_PARAM);
+        if (index == null) {
+            index = this.default_index;
+        }
+        
+        // If a subcollection index has been specified, use it
+        String indexSub = (String) params.get (INDEX_SUBCOLLECTION_PARAM);
+        if (indexSub != null) {
+            index += indexSub;
+        }
+        else{
+            if (!this.default_index_subcollection.equals ("")){
+                index += this.default_index_subcollection;
+            }
+        }
+        
+        // If a subcollection index has been specified, use it
+        String indexLang = (String) params.get (INDEX_LANGUAGE_PARAM);
+        if (indexLang != null) {
+            index += indexLang;
+        }
+        else{
+            if (!this.default_index_language.equals ("")){
+                index += this.default_index_language;
+            }
+        }
+        
+        // The location of the MG index and text files
+        String basedir = GSFile.collectionBaseDir (this.site_home, this.cluster_name) +  File.separatorChar;  // Needed for MG
+        String textdir = GSFile.collectionTextPath (this.index_stem);
+        String indexpath = GSFile.collectionIndexPath (this.index_stem, index);
+        this.mg_src.setIndex (indexpath);
+        
+        // set the mg query parameters to the values the user has specified
+        setStandardQueryParams (params);
+        this.mg_src.runQuery (basedir, textdir, query);
+        MGQueryResult mqr = this.mg_src.getQueryResult ();
+        if (mqr.isClear ()) {
+            // something has gone wrong
+            GSXML.addError (result, "Couldn't query the mg database", GSXML.ERROR_TYPE_SYSTEM);
+            return result;
+        }
+        long totalDocs = mqr.getTotalDocs ();
+        
+        // Get the docnums out, and convert to HASH ids
+        Vector docs = mqr.getDocs ();
+        if (docs.size () == 0) {
+            logger.error ("No results found...\n");
+        }
+        
+        // Create a metadata list to store information about the query results
+        Element metadata_list = result_doc.createElement (GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+        result.appendChild (metadata_list);
+        
+        // Add a metadata element specifying the number of matching documents
+        // because teh total number is just the number returned, use numDocsReturned, not numDocsMatched
+        GSXML.addMetadata (metadata_list, "numDocsReturned", ""+totalDocs);
+        // add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
+        GSXML.addMetadata (metadata_list, "query", query);
+        
+        if (docs.size () > 0) {
+            // Create a document list to store the matching documents, and add them
+            Element document_list = result_doc.createElement (GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+            result.appendChild (document_list);
+            for (int d = 0; d < docs.size (); d++) {
+                long docnum = ((MGDocInfo) docs.elementAt (d)).num_;
+                float rank = ((MGDocInfo) docs.elementAt (d)).rank_;
+                String doc_id = internalNum2OID (docnum);
+                Element doc_node = createDocNode (result_doc, doc_id, Float.toString (rank));
+                document_list.appendChild (doc_node);
+            }
+        }
+        
+        // Create a term list to store the term information, and add it
+        Element term_list = result_doc.createElement (GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+        result.appendChild (term_list);
+        Vector terms = mqr.getTerms ();
+        for (int t = 0; t < terms.size (); t++) {
+            MGTermInfo term_info = (MGTermInfo) terms.get (t);
+            
+            String term = term_info.term_;
+            int stem_method = term_info.stem_method_;
+            Vector equiv_terms = term_info.equiv_terms_;
+            
+            Element term_elem = result_doc.createElement (GSXML.TERM_ELEM);
+            term_elem.setAttribute (GSXML.NAME_ATT, term);
+            term_elem.setAttribute (STEM_ATT, "" + stem_method);
+            
+            Element equiv_term_list = result_doc.createElement (EQUIV_TERM_ELEM+GSXML.LIST_MODIFIER);
+            term_elem.appendChild (equiv_term_list);
+            
+            long total_term_freq = 0;
+            for (int et = 0; et < equiv_terms.size (); et++) {
+                MGEquivTermInfo equiv_term_info = (MGEquivTermInfo) equiv_terms.get (et);
+                
+                Element equiv_term_elem = result_doc.createElement (GSXML.TERM_ELEM);
+                equiv_term_elem.setAttribute (GSXML.NAME_ATT, equiv_term_info.term_);
+                equiv_term_elem.setAttribute (NUM_DOCS_MATCH_ATT, "" + equiv_term_info.match_docs_);
+                equiv_term_elem.setAttribute (FREQ_ATT, "" + equiv_term_info.term_freq_);
+                equiv_term_list.appendChild (equiv_term_elem);
+                
+                total_term_freq += equiv_term_info.term_freq_;
+            }
+            
+            term_elem.setAttribute (FREQ_ATT, "" + total_term_freq);
+            term_list.appendChild (term_elem);
+        }
+        return result;
+	}//end of synchronized
+    }
+    
+    // should probably use a list rather than map
+    protected boolean setStandardQueryParams(HashMap<String, Serializable> params)
+    {
+	// set the default settings that gs uses
+      this.mg_src.setMaxDocs(Integer.parseInt(paramDefaults.get(MAXDOCS_PARAM)));
+	this.mg_src.setReturnTerms(true);
+	this.mg_src.setCase(paramDefaults.get(CASE_PARAM).equals(BOOLEAN_PARAM_ON) ? true : false);
+	this.mg_src.setStem(paramDefaults.get(STEM_PARAM).equals(BOOLEAN_PARAM_ON) ? true : false);	
+	Set entries = params.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+
+	    if (name.equals(CASE_PARAM) && this.does_case) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setCase(val);
+	    }
+	    else if (name.equals(STEM_PARAM) && this.does_stem) {
+		boolean val = (value.equals(BOOLEAN_PARAM_ON) ? true : false);
+		this.mg_src.setStem(val);
+	    }
+	    else if (name.equals(MATCH_PARAM)) {
+		int mode = (value.equals(MATCH_PARAM_ALL) ? 1 : 0);
+		this.mg_src.setMatchMode(mode);
+	    }
+	    else if (name.equals(MAXDOCS_PARAM)) {
+		int docs = Integer.parseInt(value);
+		this.mg_src.setMaxDocs(docs);
+	    } // ignore any others
+	}
+	return true;
+    }
+    
+    
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleNgramMGPPSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleNgramMGPPSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleNgramMGPPSearch.java	(revision 31634)
@@ -0,0 +1,188 @@
+/*
+ *    GS2MGPPSearch.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.mgpp.*;
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Element; 
+
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.Collections;
+import org.apache.log4j.*;
+
+/**
+ *
+ * @author Shaoqun Wu
+ */
+
+public class GoogleNgramMGPPSearch
+  extends GS2MGPPSearch {
+   static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GoogleNgramMGPPSearch.class.getName());
+						
+  protected String default_max_docs = "-1";
+  protected String default_hits_per_page = "30";
+    /** constructor */
+    public GoogleNgramMGPPSearch(){
+	this.does_paging = true;
+    }
+    
+  /** configure this service */
+    public boolean configure(Element info, Element extra_info) {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	this.default_max_docs = "-1";
+	this.default_hits_per_page = "30";
+        this.does_stem = false;
+	return true;
+    }
+  
+    // sort the doc_nums by their frequency
+    protected String [] getDocIDs(Object query_result) {
+	try{
+	  Vector docs = ((MGPPQueryResult)query_result).getDocs();
+	  //ArrayList docList_past = new ArrayList(); 
+          //ArrayList docList_future = new ArrayList();	 
+	  //ArrayList docList_present = new ArrayList();
+	  
+	 ArrayList<DocWrapper> docList = new ArrayList<DocWrapper>();
+
+	  for (int d = 0; d < docs.size(); d++) {
+	      String num = Long.toString((((MGPPDocInfo) docs.elementAt(d)).num_));
+	      String doc_id = internalNum2OID(num);
+	      DBInfo dbInfo = this.gs_doc_db.getInfo(doc_id);
+	      String fre = (String)dbInfo.getInfo("Frequency");
+	      String tense = (String)dbInfo.getInfo("Tense");
+	     
+	      if(!fre.equals("")){
+		  //  if (tense.equals("past")){
+		  //    docList_past.add(new DocWrapper(num,Integer.parseInt(fre),tense));
+		  // }
+		  // else{
+		  //    if (tense.equals("future")){
+		  //	  docList_future.add(new DocWrapper(num,Integer.parseInt(fre),tense));
+		  //     }
+		  //    else{
+		  //	  if(tense.equals("present")){
+		  //	      docList_present.add(new DocWrapper(num,Integer.parseInt(fre),tense));   
+		  //	  }
+		  //    }
+		  //}
+		  docList.add(new DocWrapper(num,Integer.parseInt(fre),tense));
+	      }
+	      
+	  }
+	  
+	  
+	  //Collections.sort(docList_past);
+	  //Collections.sort(docList_future);
+	  //Collections.sort(docList_present);
+	  
+	  Collections.sort(docList);
+	  int i_pa =  0;
+	  int i_f =  0;
+	  int i_pre = 0;
+	  
+	  //String [] doc_nums = new String [docList_past.size()+docList_future.size()+docList_present.size()];
+	  String [] doc_nums = new String [docList.size()];
+	  int interval = 10;
+
+	  for(int d = 0; d < doc_nums.length; d++){
+
+	      // for(;i_pre < docList_present.size() && interval > 0;i_pre++){
+// 		  doc_nums[d] = ((DocWrapper)docList_present.get(i_pre)).num;
+// 		  d++;
+// 		  interval--;
+// 	      }
+	      
+// 	      interval = 10+interval;
+	 
+// 	      for(;i_pa < docList_past.size() && interval > 0;i_pa++){
+// 		  doc_nums[d] = ((DocWrapper)docList_past.get(i_pa)).num;
+// 		  d++;
+// 		  interval--;
+// 	      }
+
+	     
+// 	      interval = 10+interval;
+
+// 	      for(;i_f < docList_future.size() && interval > 0;i_f++){
+// 		  doc_nums[d] = ((DocWrapper)docList_future.get(i_f)).num;
+// 		  d++;
+// 		  interval--;
+// 	      }
+
+// 	    interval = 10;
+
+	      doc_nums[d] = docList.get(d).num;
+	    
+	  }
+
+	  return doc_nums;
+    }
+    catch(Exception e){
+	e.printStackTrace();
+    }
+
+    return null;
+  }
+
+  static class DocWrapper implements Comparable{
+	public int fre = 0;
+	public String num = "";
+        public String tense = "";
+
+
+      public DocWrapper(String num, int fre, String tense){
+	    this.fre = fre;
+	    this.num = num;
+	    this.tense = tense;
+	}
+
+	public int compareTo(Object o){
+	    
+	    if (!(o instanceof DocWrapper)) return -1;
+	    DocWrapper docIn = (DocWrapper)o;
+	    if (num.equals(docIn.num)){
+		return 0;
+	    }
+
+	    if (fre > docIn.fre) return -1; 
+	    return 1;	
+	}
+
+	public boolean equals(Object o){
+	    if (!(o instanceof DocWrapper)) return false;
+	    DocWrapper docIn = (DocWrapper)o;
+	    if (num.equals(docIn.num)){
+		return true;
+	    }
+	    return false;
+	}
+
+
+    }
+    
+      
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleSearch.java.tmp
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleSearch.java.tmp	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GoogleSearch.java.tmp	(revision 31634)
@@ -0,0 +1,231 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+//Google Web Services API classes
+//import com.google.soap.search.GoogleSearch;
+import com.google.soap.search.GoogleSearchFault;
+import com.google.soap.search.GoogleSearchResult;
+import com.google.soap.search.GoogleSearchResultElement;
+import com.google.soap.search.GoogleSearchDirectoryCategory;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.MalformedURLException;
+import java.lang.Object;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ *
+ */
+
+public class GoogleSearch
+    extends AbstractSearch {
+    
+     static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GoogleSearch.class.getName());
+     
+    //Parameters connect to Proxy Server
+    private boolean using_proxy = false;
+    private String proxy_host = null;
+    private int proxy_port;
+    private String proxy_user = null;
+    private char [] proxy_passwd = null;
+
+    // google key
+    private String client_key = null;
+    
+    public GoogleSearch()
+    {
+	does_paging = true;
+	does_chunking = false;
+      QUERY_SERVICE = "TextQuery";
+    }
+	    
+    //Configure GoogleSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	logger.info("Configuring GoogleSearch");
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "googleServer");
+	if (server_elem == null) {
+	    logger.error("no googleServer element found");
+	    return false;
+	}
+	
+	client_key = server_elem.getAttribute("key");
+	if (client_key.equals("")) {
+	    logger.error("no client_key for the googleServer element");
+	    return false;
+	}
+	
+
+	// are we behind a proxy??
+	// all the details should have been set up by the Message Router
+	proxy_host = System.getProperty("http.proxyHost");
+	if (proxy_host != null && !proxy_host.equals("")) {
+	    using_proxy = true;
+	    try {
+		proxy_port = Integer.parseInt(System.getProperty("http.proxyPort").trim());
+	    } catch (Exception e) {
+		logger.error("couldn't get proxy port, defaulting to 80");
+		proxy_port = 80;
+	    }
+	    PasswordAuthentication pa = Authenticator.requestPasswordAuthentication(proxy_host, null, proxy_port, "http", "", null);
+	    proxy_user = pa.getUserName();
+	    proxy_passwd = pa.getPassword();
+	}
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+	
+	//Connect to Google Web API service
+	com.google.soap.search.GoogleSearch search = new com.google.soap.search.GoogleSearch();
+
+	// Set mandatory attributes
+	search.setKey(client_key);
+
+	// proxy??
+	if (using_proxy) {
+	    search.setProxyHost(proxy_host);
+	    search.setProxyPort(proxy_port);
+	    search.setProxyUserName(proxy_user);
+	    search.setProxyPassword(new String(proxy_passwd));
+	}
+
+	//set optional attributes
+	search.setSafeSearch(true);
+	
+	// Create a new (empty) result message
+	Document result_doc = XMLConverter.newDOM();
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	Element query_metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(query_metadata_list);
+
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    logger.error("TextQuery request had no query string.");
+	    return result;  // Return the empty result
+	}
+       	// tidy whitespace
+	query = query.replaceAll("\\s+", "+");
+	
+	search.setQueryString(query);
+
+	//Check hits_per_page
+	int hits_per_page;
+	try {
+	    hits_per_page = Integer.parseInt(((String)params.get(HITS_PER_PAGE_PARAM)).trim());
+	} catch (Exception e) {
+	    logger.error("couldn't get hits per page param, defaulting to 10");
+	    hits_per_page = 10;
+	}
+	//Check the start_page number
+	int start_page;
+	try {
+	    start_page = Integer.parseInt(((String) params.get(START_PAGE_PARAM)).trim());
+	} catch (Exception e) {
+	    logger.error("couldn't get start page param, defaulting to 1");
+	    start_page = 1;
+	}
+	
+	//Invoke Actual Search
+	
+	// Google only allows 10 hits per request
+	int loop = hits_per_page/10;
+	int remainder = hits_per_page%10;
+	int google_start_page = (start_page-1)*hits_per_page;
+	int pages_per_loop;
+	for (int j=0; j < loop; j++){
+	    if (j < (loop-1) || remainder == 0) {
+		pages_per_loop = 10;
+	    } else {
+		pages_per_loop = remainder;
+	    }
+	    search.setMaxResults(pages_per_loop); 
+	    search.setStartResult(google_start_page);
+	    google_start_page = google_start_page + pages_per_loop;
+	    GoogleSearchResult google_result;
+	    try{
+		google_result = search.doSearch();
+	    } catch (GoogleSearchFault ex) {
+		logger.error("the call to the Google Web APIs failed:" + ex.toString());
+		// add the error to the result
+		return result;
+	    }
+	    if (j==0) {
+		
+		//Total amount of documents Google Search returned
+		// only need to do this on the first loop
+		
+		long numdocs_matched = google_result.getEstimatedTotalResultsCount();	    
+		GSXML.addMetadata(query_metadata_list, "numDocsMatched", ""+numdocs_matched);
+		
+	    }
+	    GoogleSearchResultElement[] details = google_result.getResultElements();
+	    for (int i=0; i<details.length; i++){
+		Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+		doc_node_list.appendChild(doc_node);
+		
+		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+		doc_node.appendChild(metadata_list);
+		
+		String google_url = details[i].getURL();  
+		String google_title = details[i].getTitle();
+		String google_snippet = details[i].getSnippet();
+		if (google_url !=null) {
+		    GSXML.addMetadata(metadata_list, "URL", google_url);
+		}
+		if (google_title != null) {
+		    GSXML.addMetadata(metadata_list, "Title", google_title);
+		}
+		if (google_snippet != null) {
+		    GSXML.addMetadata(metadata_list, "Snippet", google_snippet);
+		}
+	    }
+	} // for each loop
+	return result;
+    }
+    
+    protected void getIndexData(ArrayList index_ids, ArrayList index_names,String lang){
+       	index_ids.add("idx");
+	index_names.add("Google main index");
+    }
+    
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GutBookRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GutBookRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/GutBookRetrieve.java	(revision 31634)
@@ -0,0 +1,41 @@
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+
+public class GutBookRetrieve 
+    extends XMLRetrieve {
+
+  protected Element extractTitleMeta(Document result_doc, Element section) {
+	Element meta_elem = result_doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+
+	Element title_elem = (Element)GSXML.getChildByTagName(section, "title");
+	if (title_elem == null) {
+	    title_elem = (Element)GSXML.getNodeByPath(section, "chapheader/title");
+	}
+	if (title_elem == null) {
+	    // find teh first title elem
+	    NodeList titles = section.getElementsByTagName("title");
+	    if (titles != null) {
+		title_elem = (Element)titles.item(0);
+	    }
+	}
+
+	if (title_elem == null) {
+	    return null; // no title found
+	}
+	
+	
+	Text t = (Text) GSXML.getNodeTextNode(title_elem);
+	
+	meta_elem.appendChild(result_doc.importNode(t, false));
+	return meta_elem;
+
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaProxy.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaProxy.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaProxy.java	(revision 31634)
@@ -0,0 +1,572 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+/**
+ * Service class to proxy IVia
+ * 
+ */
+
+public class IViaProxy extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.IViaProxy.class.getName());
+
+	// the services on offer
+	// these strings must match what is found in the properties file
+	protected static final String TEXT_QUERY_SERVICE = "TextQuery";
+	protected static final String DOC_CONTENT_SERVICE = "DocumentContentRetrieve";
+	protected static final String DOC_META_SERVICE = "DocumentMetadataRetrieve";
+	protected static final String QUERY_PARAM = "query";
+	protected static final String FIELD_PARAM = "fields";
+	// have standard gs param names for hits per page, and start page
+	// these need to be mapped to iVia params
+	protected static final String GS_HITS_PARAM = "hitsPerPage";
+	protected static final String IM_HITS_PARAM = "no_of_records_per_page";
+	protected static final String GS_START_PAGE_PARAM = "startPage";
+	protected static final String IM_START_PAGE_PARAM = "start_page_no";
+
+	protected String ivia_server_url = null;
+
+	public boolean configure(Element info, Element extra_info)
+	{
+
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		Element server_elem = (Element) GSXML.getChildByTagName(info, "iViaServer");
+		if (server_elem == null)
+		{
+			logger.error("no iViaServer element found");
+			return false;
+		}
+		ivia_server_url = server_elem.getAttribute("url");
+		if (ivia_server_url.equals(""))
+		{
+			logger.error("no url for the iViaServer element");
+			return false;
+		}
+		Element tq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+		tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+		this.short_service_info.appendChild(tq_service);
+
+		Element dc_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dc_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dc_service.setAttribute(GSXML.NAME_ATT, DOC_CONTENT_SERVICE);
+		this.short_service_info.appendChild(dc_service);
+
+		Element dm_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dm_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dm_service.setAttribute(GSXML.NAME_ATT, DOC_META_SERVICE);
+		this.short_service_info.appendChild(dm_service);
+
+		// 
+		// add some format info to service map if there is any
+		String path = GSPath.appendLink(GSXML.SEARCH_ELEM, GSXML.FORMAT_ELEM);
+		Element format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (format != null)
+		{
+			this.format_info_map.put(TEXT_QUERY_SERVICE, this.desc_doc.importNode(format, true));
+		}
+
+		// look for document display format
+		path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+		Element display_format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (display_format != null)
+		{
+			this.format_info_map.put(DOC_CONTENT_SERVICE, this.desc_doc.importNode(display_format, true));
+			// shoudl we make a copy?
+		}
+
+		return true;
+
+	}
+
+	protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+
+		if (service.equals(TEXT_QUERY_SERVICE))
+		{
+			Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+			tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			tq_service.setAttribute(GSXML.NAME_ATT, TEXT_QUERY_SERVICE);
+			if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
+			{
+				tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(TEXT_QUERY_SERVICE + ".name", lang)));
+				tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(TEXT_QUERY_SERVICE + ".submit", lang)));
+				tq_service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(TEXT_QUERY_SERVICE + ".description", lang)));
+			}
+			if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
+			{
+				Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+				tq_service.appendChild(param_list);
+				Element param = GSXML.createParameterDescription(doc, QUERY_PARAM, getTextString("param." + QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);
+				param_list.appendChild(param);
+				String[] field_ids = { "kw", "au", "su", "ti", "de", "fu" };
+				String[] field_names = { getTextString("param." + FIELD_PARAM + ".kw", lang), getTextString("param." + FIELD_PARAM + ".au", lang), getTextString("param." + FIELD_PARAM + ".su", lang), getTextString("param." + FIELD_PARAM + ".ti", lang), getTextString("param." + FIELD_PARAM + ".de", lang), getTextString("param." + FIELD_PARAM + ".fu", lang) };
+
+				param = GSXML.createParameterDescription(doc, FIELD_PARAM, getTextString("param." + FIELD_PARAM, lang), GSXML.PARAM_TYPE_ENUM_MULTI, "kw,au,su,ti,de,fu", field_ids, field_names);
+				param_list.appendChild(param);
+
+				String[] hits_options = { "10", "30", "50" };
+				param = GSXML.createParameterDescription(doc, GS_HITS_PARAM, getTextString("param." + GS_HITS_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, "10", hits_options, hits_options);
+				param_list.appendChild(param);
+
+				param = GSXML.createParameterDescription(doc, GS_START_PAGE_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, "1", null, null);
+				param_list.appendChild(param);
+			}
+			return tq_service;
+		}
+		if (service.equals(DOC_META_SERVICE))
+		{
+			Element dm_service = doc.createElement(GSXML.SERVICE_ELEM);
+			dm_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			dm_service.setAttribute(GSXML.NAME_ATT, DOC_META_SERVICE);
+			return dm_service;
+
+		}
+		if (service.equals(DOC_CONTENT_SERVICE))
+		{
+			Element dc_service = doc.createElement(GSXML.SERVICE_ELEM);
+			dc_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			dc_service.setAttribute(GSXML.NAME_ATT, DOC_CONTENT_SERVICE);
+			return dc_service;
+
+		}
+		return null;
+	}
+
+	/** Process a text query - implemented by concrete subclasses */
+	protected Element processTextQuery(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		// Create a new (empty) result message
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, TEXT_QUERY_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(doc_node_list);
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("TextQuery request had no paramList.");
+			return result; // Return the empty result
+		}
+
+		// Process the request parameters
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+		// Make sure a query has been specified
+		String query = (String) params.get(QUERY_PARAM);
+		if (query == null || query.equals(""))
+		{
+			return result; // Return the empty result
+		}
+		// tidy whitespace
+		query = query.replaceAll("\\s+", "+");
+		String url_string = ivia_server_url + "/cgi-bin/canned_search?theme=gsdl3&query=" + query;
+
+		// check for fields
+		String fields = (String) params.get(FIELD_PARAM);
+		if (fields != null && !fields.equals(""))
+		{
+			url_string += "&fields=" + fields;
+		}
+
+		//check for hits per page
+		String hits_per_page = (String) params.get(GS_HITS_PARAM);
+		if (hits_per_page != null && !hits_per_page.equals(""))
+		{
+			url_string += "&" + IM_HITS_PARAM + "=" + hits_per_page;
+		}
+
+		// check for start page
+		String start_page = (String) params.get(GS_START_PAGE_PARAM);
+		if (start_page != null && !start_page.equals(""))
+		{
+			url_string += "&" + IM_START_PAGE_PARAM + "=" + start_page;
+		}
+		String results_num = null;
+		String doc_ids = null;
+		try
+		{
+			logger.debug("IViaProxy, sending " + url_string);
+			BufferedReader reader = makeConnection(url_string);
+			results_num = reader.readLine();
+			doc_ids = reader.readLine();
+
+		}
+		catch (Exception e)
+		{
+			logger.error("exception happened during query");
+			e.printStackTrace();
+			return result;
+		}
+
+		if (results_num.startsWith("Resources: "))
+		{
+			results_num = results_num.substring(11);
+		}
+		else
+		{
+			logger.error("badly formatted results line: " + results_num);
+			return result;
+		}
+		if (doc_ids.startsWith("Ids: "))
+		{
+			doc_ids = doc_ids.substring(5).trim();
+		}
+		else
+		{
+			logger.error("badly formatted docs line: " + doc_ids);
+			return result;
+		}
+
+		// get the num docs and add to a metadata list
+		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(metadata_list);
+
+		// Add a metadata element specifying the number of matching documents
+		long numdocs = Long.parseLong(results_num);
+		GSXML.addMetadata(metadata_list, "numDocsMatched", "" + numdocs);
+		String[] ids = doc_ids.split(" ");
+
+		for (int d = 0; d < ids.length; d++)
+		{
+			Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+			doc_node.setAttribute(GSXML.NODE_ID_ATT, ids[d]);
+			doc_node_list.appendChild(doc_node);
+		}
+		logger.debug("IViaProxy result:");
+		logger.debug(this.converter.getString(result));
+		return result;
+
+	}
+
+	protected Element processDocumentMetadataRetrieve(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOC_META_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("missing paramList.\n");
+			return result; // Return the empty result
+		}
+
+		// The metadata information required
+		StringBuffer field_list = new StringBuffer();
+		Element param = GSXML.getFirstElementChild(param_list);//(Element) param_list.getFirstChild();
+		while (param != null)
+		{
+			// Identify the metadata information desired
+			if (param.getAttribute(GSXML.NAME_ATT).equals("metadata"))
+			{
+				String metadata = GSXML.getValue(param);
+				if (isAcceptableMetadata(metadata))
+				{
+					field_list.append(metadata);
+					field_list.append(",");
+				}
+			}
+			param = (Element) param.getNextSibling();
+		}
+
+		if (field_list.length() == 0)
+		{
+			logger.error("no metadata specified.\n");
+			return result;
+		}
+
+		// Get the documents
+		Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (request_node_list == null)
+		{
+			logger.error("DocumentMetadataRetrieve request had no " + GSXML.DOC_NODE_ELEM + "List.\n");
+			return result;
+		}
+
+		StringBuffer record_id_list = new StringBuffer();
+
+		NodeList request_nodes = request_node_list.getChildNodes();
+		for (int i = 0; i < request_nodes.getLength(); i++)
+		{
+			Element request_node = (Element) request_nodes.item(i);
+			String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+			record_id_list.append(node_id);
+			record_id_list.append(",");
+		}
+
+		// do the query to the iVia server
+		String url_string = ivia_server_url + "/cgi-bin/view_record_set?theme=gsdl3&record_id_list=" + record_id_list.toString() + "&field_list=" + field_list.toString();
+
+		Element node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(node_list);
+		try
+		{
+			BufferedReader reader = makeConnection(url_string);
+			String line;
+			while ((line = reader.readLine()) != null)
+			{
+				if (!line.startsWith("Record:"))
+				{
+					continue;
+				}
+				// the first line is the record
+				line = line.substring(8);
+				Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+				doc_node.setAttribute(GSXML.NODE_ID_ATT, line);
+				Element meta_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+				doc_node.appendChild(meta_list);
+				while ((line = reader.readLine()) != null)
+				{
+					//metadata entry
+					int col_pos = line.indexOf(':');
+					if (col_pos == -1)
+					{
+						// end of the metadata for this doc
+						break;
+					}
+					String name = line.substring(0, col_pos);
+					String value = line.substring(col_pos + 2); // includes a space
+					GSXML.addMetadata(meta_list, name, value);
+				}
+				node_list.appendChild(doc_node);
+
+			}
+		}
+		catch (Exception e)
+		{
+			logger.error("exception happened");
+			e.printStackTrace();
+		}
+		logger.debug("IViaProxy: returning result: ");
+		logger.debug(this.converter.getPrettyString(result));
+		return result;
+
+	}
+
+	protected Element processDocumentContentRetrieve(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOC_CONTENT_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		// Get the request doc_list
+		Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (query_doc_list == null)
+		{
+			logger.error("DocumentContentRetrieve request specified no doc nodes.\n");
+			return result;
+		}
+
+		Element doc_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(doc_list);
+
+		// Get the documents
+		String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, GSXML.NODE_ID_ATT);
+		for (int i = 0; i < doc_ids.length; i++)
+		{
+			String doc_id = doc_ids[i];
+			Element doc_node = getDocument(result_doc, doc_id);
+			doc_list.appendChild(doc_node);
+		}
+		return result;
+
+	}
+
+	/**
+	 * gets a document by sending a request to iVia, then processes it and
+	 * creates a documentNode around the text
+	 */
+  protected Element getDocument(Document result_doc, String doc_id)
+	{
+
+		String url_string = ivia_server_url + "/cgi-bin/view_record?theme=gsdl3&record_id=" + doc_id;
+		StringBuffer buffer = new StringBuffer();
+		try
+		{
+			BufferedReader reader = makeConnection(url_string);
+
+			String line;
+			while ((line = reader.readLine()) != null)
+			{
+				buffer.append(line);
+			}
+
+		}
+		catch (Exception e)
+		{
+			logger.error("exception happened");
+			e.printStackTrace();
+		}
+
+		String node_content = buffer.toString();
+		String escaped_content = GSXML.xmlSafe(node_content);
+
+		StringBuffer processed_content = new StringBuffer(escaped_content.length());
+		processed_content.append("<nodeContent>");
+		int pos = 0;
+		int lastpos = 0;
+		while ((pos = escaped_content.indexOf("&lt;a ", lastpos)) != -1)
+		{
+			processed_content.append(escaped_content.substring(lastpos, pos));
+			int endpos = escaped_content.indexOf("&lt;/a&gt;", pos);
+			if (endpos == -1)
+			{
+				break;
+			}
+			String link = escaped_content.substring(pos, endpos + 10);
+			link = convertLink(link);
+			processed_content.append(link);
+			lastpos = endpos + 10;
+		}
+		processed_content.append(escaped_content.substring(lastpos)); // get the last bit
+		processed_content.append("</nodeContent>");
+
+		Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+		doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+
+		Document content_doc = this.converter.getDOM(processed_content.toString());
+		if (content_doc != null)
+		{
+			Element content_element = content_doc.getDocumentElement();
+			doc_node.appendChild(result_doc.importNode(content_element, true));
+		}
+		else
+		{
+			logger.error("Couldn't parse the node content");
+		}
+		return doc_node;
+
+	}
+
+	/** converts a url from an <a> element into a greenstone suitable one */
+	protected String convertLink(String aref)
+	{
+
+		if (aref.indexOf("href=&quot;http") != -1)
+		{
+			return aref; // an external link
+		}
+		String type = "other";
+		if (aref.indexOf("/cgi-bin/canned_search") != -1)
+		{
+			type = "query";
+		}
+		else if (aref.indexOf("/cgi-bin/click_through") != -1)
+		{
+			type = "external";
+		}
+		else if (aref.indexOf("/cgi-bin/view_record") != -1)
+		{
+			type = "document";
+		}
+
+		int href_start = aref.indexOf("href=&quot;") + 11;
+		int href_end = aref.indexOf("&gt;", href_start);
+		String href = aref.substring(href_start, href_end);
+		String link_content = aref.substring(href_end + 4, aref.length() - 10);
+
+		if (type.equals("external"))
+		{
+			// the external link is everything after the http at the end.
+			String address = href.substring(href.lastIndexOf("http"));
+			address = address.replaceAll("%3[aA]", ":");
+			address = address.replaceAll("%2[fF]", "/");
+
+			return "&lt;a href=\"" + address + "\"&gt;" + link_content + "&lt;/a&gt;";
+		}
+		if (type.equals("other"))
+		{
+			return "other type of link (" + link_content + ")";
+		}
+		StringBuffer result = new StringBuffer();
+		result.append("<link type='");
+		result.append(type);
+		result.append("'");
+		if (type.equals("query"))
+		{
+			result.append(" service='TextQuery'");
+		}
+		result.append(">");
+		// add in the parameters
+		href = href.substring(href.indexOf("?") + 1);
+		String[] params = href.split("&amp;");
+		for (int i = 0; i < params.length; i++)
+		{
+			String param = params[i];
+			int eq_pos = param.indexOf("=");
+			if (eq_pos != -1)
+			{
+
+				result.append("<param name='" + param.substring(0, eq_pos) + "' value='" + param.substring(eq_pos + 1) + "'/>");
+			}
+		}
+		result.append(link_content);
+		result.append("</link>");
+
+		return result.toString();
+
+	}
+
+	// iVia craps out if we ask for a metadata which is not valid. So need
+	// to make sure we only ask for acceptable fields.
+	protected boolean isAcceptableMetadata(String meta)
+	{
+		String valid_metadata = ",title,url,ivia_description,keywords,subjects,";
+		if (valid_metadata.indexOf("," + meta + ",") != -1)
+		{
+			return true;
+		}
+		return false;
+	}
+
+	protected BufferedReader makeConnection(String url_string)
+	{
+		BufferedReader reader = null;
+		try
+		{
+			URL url = new URL(url_string);
+			HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+			InputStream input = connection.getInputStream();
+			reader = new BufferedReader(new InputStreamReader(input));
+		}
+		catch (java.net.MalformedURLException e)
+		{
+
+			logger.error("Malformed URL: " + url_string);
+		}
+		catch (java.io.IOException e)
+		{
+			logger.error("An error occurred during IO to url " + url_string);
+		}
+		return reader;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaRetrieve.java	(revision 31634)
@@ -0,0 +1,264 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.util.Misc;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.io.File;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URLConnection;
+import java.net.URL;
+import java.net.Authenticator;
+import java.net.MalformedURLException;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ * @author Katherine Don
+ * @version $Revision$
+ * Modified by  <a href="mailto:chi@cs.waikato.ac.nz">Chi-Yu Huang</a>
+ */
+
+public class IViaRetrieve 
+    extends AbstractDocumentRetrieve {
+
+      static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.IViaRetrieve.class.getName());
+    
+
+    protected String ivia_server_url = null;
+    
+    public IViaRetrieve() {
+	does_structure = false;
+    }
+    
+    //Configure IViaRetrieve Service
+    public boolean configure(Element info, Element extra_info) 
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "iViaServer");
+	if (server_elem == null) {
+	    logger.error("no iViaServer element found");
+	    return false;
+	}
+	ivia_server_url = server_elem.getAttribute("url");
+	if (ivia_server_url.equals("")) {
+	    logger.error("no url for the iViaServer element");
+	    return false;
+	}
+	return true;
+	
+    }
+        
+    /** gets a document by sending a request to iVia, then processes it and creates a documentNode around the text */
+  protected Element getNodeContent(Document doc, String doc_id, String lang) 
+	throws GSException {
+
+	String url_string = ivia_server_url+"/cgi-bin/view_record?theme=gsdl3&record_id="+doc_id;
+
+	StringBuffer buffer = new StringBuffer();
+	try {
+	    BufferedReader reader = Misc.makeHttpConnection(url_string);
+    	    String line;
+	    while((line = reader.readLine())!= null) {
+		buffer.append(line);
+	    }
+   	} catch (java.net.MalformedURLException e) {
+	    throw new GSException("Malformed URL: "+url_string, GSXML.ERROR_TYPE_SYSTEM);
+	} catch (java.io.IOException e) {
+	    throw new GSException("IOException during connection to "+url_string+": "+e.toString(),GSXML.ERROR_TYPE_SYSTEM);
+	}
+
+	String node_content = buffer.toString(); 
+
+	String escaped_content = GSXML.xmlSafe(node_content);
+	
+	StringBuffer processed_content = new StringBuffer(escaped_content.length());
+	processed_content.append("<nodeContent>");
+	int pos = 0;
+	int lastpos = 0;
+	while ((pos = escaped_content.indexOf("&lt;a ", lastpos))!= -1) {
+	    processed_content.append(escaped_content.substring(lastpos, pos));
+	    int endpos = escaped_content.indexOf("&lt;/a&gt;", pos);
+	    if (endpos == -1) {
+		break;
+	    }
+	    String link = escaped_content.substring(pos, endpos+10);
+	    link = convertLink(link);
+	    processed_content.append(link);
+	    lastpos = endpos+10;
+	}
+	processed_content.append(escaped_content.substring(lastpos)); // get the last bit
+	processed_content.append("</nodeContent>");
+	    
+	Document content_doc = this.converter.getDOM(processed_content.toString());
+	if (content_doc == null) {
+	    logger.error("Couldn't parse node content");
+	    logger.error(processed_content.toString());
+	    return null;
+	}
+	
+	Element content_element = content_doc.getDocumentElement();
+
+	return (Element)doc.importNode(content_element,true);
+    }
+	
+    /** converts a url from an <a> element into a greenstone suitable one */
+    protected String convertLink(String aref) {
+	if (aref.indexOf("href=&quot;http") != -1) {
+	    return aref; // an external link
+	}
+	String type = "other";
+	if (aref.indexOf("/cgi-bin/canned_search")!=-1) {
+	    type="query";
+	} else if (aref.indexOf("/cgi-bin/click_through") != -1) {
+	    type = "external";
+	} else if (aref.indexOf("/cgi-bin/view_record") != -1) {
+	    type="document";
+	} 
+	
+	int href_start = aref.indexOf("href=&quot;")+11;
+	int href_end = aref.indexOf("&gt;", href_start);
+	String href = aref.substring(href_start, href_end);
+	String link_content = aref.substring(href_end+4, aref.length()-10);
+	
+	if (type.equals("external")) {
+	    // the external link is everything after the http at the end.
+	    String address = href.substring(href.lastIndexOf("http"));
+	    address = address.replaceAll("%3[aA]", ":");
+	    address = address.replaceAll("%2[fF]", "/");
+
+	    return "&lt;a href=\""+address+"\"&gt;"+link_content+"&lt;/a&gt;";
+	}
+	if (type.equals("other")) {
+	    return "other type of link ("+link_content+")";
+	}
+	StringBuffer result = new StringBuffer();
+	result.append("<link type='");
+	result.append(type);
+	result.append("'");
+	if (type.equals("query")) {
+	    result.append(" service='TextQuery'");
+	}
+	result.append(">");
+	// add in the parameters
+	href = href.substring(href.indexOf("?")+1);
+	String [] params = href.split("&amp;");
+	for (int i=0; i<params.length; i++) {
+	    String param = params[i];
+	    int eq_pos = param.indexOf("=");
+	    if (eq_pos != -1) {
+		
+		result.append("<param name='"+param.substring(0, eq_pos)+"' value='"+param.substring(eq_pos+1)+"'/>");
+	    }
+	}
+	result.append(link_content);
+	result.append("</link>");
+
+	return result.toString();
+    }
+
+    // iVia craps out if we ask for a metadata which is not valid. So need
+    // to make sure we only ask for acceptable fields.
+    protected boolean isAcceptableMetadata(String meta) {
+	String valid_metadata = ",title,url,ivia_description,keywords,subjects,";
+	if (valid_metadata.indexOf(","+meta+",")!=-1) {
+	    return true;
+	}
+	return false;
+    }
+
+    protected String translateId(String oid){
+	int p = oid.lastIndexOf('.');
+	if (p != oid.length()-3) {
+	    logger.info("translateoid error: '.' is not the third to last char!!");
+	    return oid;
+	}
+	String top = oid.substring(0, p);
+	return top;
+    }
+
+    protected String translateExternalId(String id){
+	return id;
+    }
+
+    protected String getDocType(String node_id){
+    	return GSXML.DOC_TYPE_SIMPLE;
+    }
+    protected String getRootId(String node_id){
+   	return node_id; 
+    }
+
+    protected ArrayList<String> getChildrenIds(String node_id){
+	return null; 
+    }
+    
+    protected String getParentId(String node_id){
+	return null;
+    }
+    
+  protected Element getMetadataList (Document doc, String doc_id,
+				       boolean all_metadata,
+				       ArrayList<String> metadata_names,
+				       String lang) 
+	throws GSException {
+	
+	Element meta_list = doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+
+	// do the query to the iVia server
+	StringBuffer field_list= new StringBuffer();
+	boolean metadata_found = false;
+
+	for (int i=0; i<metadata_names.size();i++){
+	    if (isAcceptableMetadata(metadata_names.get(i))){
+		metadata_found = true;
+		field_list.append(metadata_names.get(i));
+		field_list.append(",");
+	    }
+	}
+	if (!metadata_found){
+	    return meta_list;
+	}
+	
+	String url_string = ivia_server_url+"/cgi-bin/view_record_set?theme=gsdl3&record_id_list="+doc_id+"&field_list="+field_list.toString(); 
+	try {
+	    BufferedReader reader = Misc.makeHttpConnection(url_string);
+	    String line;
+	    while  ((line = reader.readLine()) != null) {
+		//metadata entry
+		int col_pos = line.indexOf(':');
+		if (col_pos == -1) {
+		    // end of the metadata for this doc
+		    break; 
+		}
+		String name = line.substring(0,col_pos);
+		String value = line.substring(col_pos+2); // includes a space
+		GSXML.addMetadata(meta_list, name, value);
+	    }
+	} catch (java.net.MalformedURLException e) {
+	    throw new GSException("Malformed URL: "+url_string, GSXML.ERROR_TYPE_SYSTEM);
+	} catch (java.io.IOException e) {
+	    throw new GSException("IOException: "+e.toString(), GSXML.ERROR_TYPE_SYSTEM);
+	}
+	return meta_list;
+    }
+    
+    protected String getStructureInfo(String doc_id, String info_type){
+	return "";	
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/IViaSearch.java	(revision 31634)
@@ -0,0 +1,209 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.util.Misc;
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+//Java classes
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.Serializable;
+import java.net.Authenticator;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ * @author Katherine Don
+ * @author Chi-Yu Huang
+ */
+
+public class IViaSearch
+    extends AbstractTextSearch {
+    
+
+     static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.IViaSearch.class.getName());
+ 
+    // have standard gs param names for hits per page, and start page
+    // these need to be mapped to iVia params
+    protected static final String IM_HITS_PARAM = "no_of_records_per_page";
+    protected static final String IM_START_PAGE_PARAM = "start_page_no";
+
+    protected String ivia_server_url = null;
+    protected ArrayList<String> index_ids = null;
+    public IViaSearch()
+    {
+	does_paging = true;
+	does_multi_index_search = true;
+    }
+    
+    //Configure IViaSearch Service
+    public boolean configure(Element info, Element extra_info) 
+    { 
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+	Element server_elem = (Element)GSXML.getChildByTagName(info, "iViaServer");
+	if (server_elem == null) {
+	    logger.error("no iViaServer element found");
+	    return false;
+	}
+	ivia_server_url = server_elem.getAttribute("url");
+	if (ivia_server_url.equals("")) {
+	    logger.error("no url for the iViaServer element");
+	    return false;
+	}
+	this.default_index = ",kw,au,su,ti,de,fu,"; // all of them
+	index_ids = new ArrayList<String>();
+	index_ids.add("kw");
+	index_ids.add("au");	
+	index_ids.add("su");	
+	index_ids.add("ti");
+	index_ids.add("de");
+	index_ids.add("fu");
+
+	return true;
+    }
+    
+    /** Process a text query - implemented by concrete subclasses */
+    protected Element processTextQuery(Element request) {
+      Document result_doc = XMLConverter.newDOM();
+	// Create a new (empty) result message
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(doc_node_list);
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String query = (String) params.get(QUERY_PARAM);
+	if (query == null || query.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// tidy whitespace
+	query = query.replaceAll("\\s+", "+");
+	String url_string = ivia_server_url+"/cgi-bin/canned_search?theme=gsdl3&query="+query; 
+
+	// check for fields
+	String fields = (String) params.get(INDEX_PARAM);
+	fields = checkFieldParam(fields); // removes invalid fields
+	if (!fields.equals("")) {
+	    url_string += "&fields="+fields;
+	}
+	//check for hits per page
+	String hits_per_page = (String) params.get(HITS_PER_PAGE_PARAM);
+	if (hits_per_page != null && !hits_per_page.equals("")) {
+	    url_string += "&"+IM_HITS_PARAM+"="+hits_per_page;
+	}
+
+	// check for start page
+	String start_page = (String) params.get(START_PAGE_PARAM);
+	if (start_page != null && !start_page.equals("")) {
+	    url_string += "&"+IM_START_PAGE_PARAM+"="+start_page;
+	}
+	String results_num = null;
+	String doc_ids = null;
+	BufferedReader reader = null;
+	try {
+	    logger.debug("sending "+url_string);
+	    reader = Misc.makeHttpConnection(url_string);
+	    results_num = reader.readLine();
+	    doc_ids = reader.readLine();
+	} catch (java.net.MalformedURLException e) {
+	    GSXML.addError(result, "Malformed URL: "+url_string);
+	    return result;
+	} catch (java.io.IOException e) {
+	    GSXML.addError(result, "IOException during connection to "+url_string+": "+e.toString());
+	    return result;
+	}
+	    
+	if (results_num.startsWith("Resources: ") && doc_ids.startsWith("Ids: ")) {
+	    results_num = results_num.substring(11);
+	    doc_ids = doc_ids.substring(5).trim();
+
+	} else {
+	    logger.error("badly formatted results:");
+	    StringBuffer result_string = new StringBuffer();
+	    result_string.append("Error: badly formatted result from IVia server:\n ");
+	    result_string.append(results_num);
+	    result_string.append(doc_ids);
+	    String line;
+	    try {
+		while((line = reader.readLine()) != null) {
+		    result_string.append(line);
+		}
+	    } catch (Exception e) {
+		result_string.append("Exception: "+e);
+	    }
+	    GSXML.addError(result, result_string.toString());
+	    
+	    return result;
+	}
+	
+	// get the num docs and add to a metadata list
+	Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+	
+	// Add a metadata element specifying the number of matching documents
+	long numdocs = Long.parseLong(results_num);
+	GSXML.addMetadata(metadata_list, "numDocsMatched", ""+numdocs);	
+	String [] ids = doc_ids.split(" ");
+	
+	for (int d=0; d<ids.length; d++) {
+	    Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, ids[d]);
+	    doc_node_list.appendChild(doc_node);
+	}
+	return result;
+    }
+     
+    protected String checkFieldParam(String fields) {
+	
+	if (fields == null) {
+	    // return the default
+	    return "";
+	}
+	StringBuffer new_fields = new StringBuffer();
+	String [] ids = fields.split(",");
+	for (int i=0; i<ids.length; i++) {
+	    if(index_ids.contains(ids[i])) {
+		new_fields.append(ids[i]);
+		new_fields.append(",");
+	    }
+	}
+	if (new_fields.length() == 0) {
+	    return "";
+	}
+	return new_fields.toString();
+    }
+    /**
+       An IVia server has a fixed list of fields to search (I think) so we can hard code them here rather than reading them in from a config file
+    */
+    protected void getIndexData(ArrayList<String> index_ids, ArrayList<String> index_names,String lang){
+	index_ids.addAll(this.index_ids);
+	index_names.add(getTextString("param."+INDEX_PARAM+".kw", lang)); 
+	index_names.add(getTextString("param."+INDEX_PARAM+".au", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".su", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".ti", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".de", lang));
+	index_names.add(getTextString("param."+INDEX_PARAM+".fu", lang));
+    }
+    
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/LuceneSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/LuceneSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/LuceneSearch.java	(revision 31634)
@@ -0,0 +1,208 @@
+/** This is a simple Lucene XML Search class. This is NOT the one 
+    used by Greenstone Lucene collections. This is used by the custom 
+    gberg collection */
+
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class LuceneSearch extends AbstractTextSearch
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.LuceneSearch.class.getName());
+
+	protected static final String INDEX_ELEM = "index";
+
+	protected ArrayList<String> index_ids;
+
+	public LuceneSearch()
+	{
+		index_ids = new ArrayList<String>();
+	}
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		default_index = "idx";
+
+		// cache index info read from config file
+		Element index_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_ELEM + GSXML.LIST_MODIFIER);
+		if (index_list != null)
+		{
+			NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+			int len = indexes.getLength();
+			// now add even if there is only one
+			for (int i = 0; i < len; i++)
+			{
+				Element index = (Element) indexes.item(i);
+				index_ids.add(index.getAttribute(GSXML.NAME_ATT));
+			}
+		}
+		else
+		{
+			// there is only one index, so we assume the default
+			index_ids.add(this.default_index);
+		}
+
+		return true;
+	}
+
+	protected void getIndexData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang)
+	{
+		// copying exercise for index_ids, 
+		for (int i = 0; i < this.index_ids.size(); i++)
+		{
+			index_ids.add(this.index_ids.get(i));
+		}
+
+		// But need to work out display name from scratch as this uses
+		// the 'lang' parameter
+
+		Element index_list = (Element) GSXML.getChildByTagName(this.config_info, INDEX_ELEM + GSXML.LIST_MODIFIER);
+		if (index_list != null)
+		{
+			NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+			int len = indexes.getLength();
+			// now add even if there is only one
+			for (int i = 0; i < len; i++)
+			{
+				Element index = (Element) indexes.item(i);
+				index_names.add(getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+
+			}
+		}
+		else
+		{
+			// there is only one index, so we assume the default
+			index_names.add("default index");
+		}
+	}
+
+	protected void initResultElement(Element result, Element doc_node_list, Element metadata_list)
+	{
+		// Create a new (empty) result message
+		result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		result.appendChild(doc_node_list);
+		result.appendChild(metadata_list);
+	}
+
+	protected boolean hasParamList(Element request, Element metadata_list)
+	{
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (param_list == null)
+		{
+			logger.error("TextQuery request had no paramList.");
+			GSXML.addMetadata(metadata_list, "numDocsMatched", "0");
+			return false; // signal that an empty result should be return
+		}
+
+		return true;
+	}
+
+	protected boolean hasQueryString(Element param_list, Element metadata_list)
+	{
+		// Process the request parameters to make sure a query has been specified
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		String query_string = (String) params.get(QUERY_PARAM);
+
+		if (query_string == null || query_string.equals(""))
+		{
+			logger.error("TextQuery request had no query string.");
+			GSXML.addMetadata(metadata_list, "numDocsMatched", "0");
+			return false; // signal that an empty result should be return
+		}
+
+		return true;
+	}
+
+	/** Process a text query - implemented by concrete subclasses */
+	protected Element processTextQuery(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		Element doc_node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+		initResultElement(result, doc_node_list, metadata_list);
+
+		if (!hasParamList(request, metadata_list))
+		{
+			return result;
+		}
+
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (!hasQueryString(param_list, metadata_list))
+		{
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		String query_string = (String) params.get(QUERY_PARAM);
+
+		// Get the index
+		String index = (String) params.get(INDEX_PARAM);
+		if (index == null || index.equals(""))
+		{
+			index = this.default_index; // assume the default
+		}
+
+		try
+		{
+			String index_dir = GSFile.collectionIndexDir(this.site_home, this.cluster_name);
+			index_dir += File.separator + index;
+			Directory index_dir_dir = FSDirectory.open(new File(index_dir));
+			IndexReader reader = DirectoryReader.open(index_dir_dir); //deprecated: IndexReader.open(index_dir_dir);
+			IndexSearcher searcher = new IndexSearcher(reader);
+
+			Term term = new Term("content", query_string);
+
+			Query query = new TermQuery(term);
+
+			TopDocs hits = searcher.search(query, Integer.MAX_VALUE);
+
+			GSXML.addMetadata(metadata_list, "numDocsMatched", "" + hits.scoreDocs.length);
+
+			for (int i = 0; i < hits.scoreDocs.length; i++)
+			{
+				int lucene_doc_num = hits.scoreDocs[i].doc;
+				org.apache.lucene.document.Document luc_doc = reader.document(lucene_doc_num);
+				String node_id = luc_doc.get("nodeID");
+				Element node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+				node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+				doc_node_list.appendChild(node);
+			}
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapRetrieve.java	(revision 31634)
@@ -0,0 +1,945 @@
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.Vector;
+
+import javax.swing.tree.DefaultMutableTreeNode;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class MapRetrieve extends ServiceRack
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.MapRetrieve.class.getName());
+
+	// the services on offer
+	protected static final String DOCUMENT_STRUCTURE_RETRIEVE_SERVICE = "DocumentStructureRetrieve";
+	protected static final String DOCUMENT_METADATA_RETRIEVE_SERVICE = "DocumentMetadataRetrieve";
+	protected static final String DOCUMENT_CONTENT_RETRIEVE_SERVICE = "DocumentContentRetrieve";
+
+	protected static final int DOCUMENT = 1;
+
+	protected ArrayList<String> index_name_list = null;
+	protected ArrayList<String> index_display_list = null;
+	protected String files_home_dir = null;
+	protected String temp_files_dir = null;
+	//protected String temp_image_file = null;
+	//protected String temp_image_file2 = null;
+	protected final String jpg_ext = ".jpg";
+	protected String http_image_dir = null;
+	protected String http_temp_image_dir = null;
+	private LinkedList<String> namesInList = new LinkedList<String>();
+
+	private DefaultMutableTreeNode tree;
+
+	/** constructor */
+	public MapRetrieve()
+	{
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring MapRetrieve...");
+		this.config_info = info;
+
+		// set up short_service_info_ - for now just has name and type
+		Element dmr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dmr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dmr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+		short_service_info.appendChild(dmr_service);
+
+		Element dcr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dcr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dcr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+		short_service_info.appendChild(dcr_service);
+
+		Element dsr_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+		dsr_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+		dsr_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+		short_service_info.appendChild(dsr_service);
+
+		// set the files_home variable for this collection
+		this.files_home_dir = GSFile.collectionIndexDir(this.site_home, this.cluster_name) + File.separator + "assoc" + File.separator;
+		this.temp_files_dir = GSFile.collectionBaseDir(this.site_home, this.cluster_name) + File.separator + "temp" + File.separator;
+
+		//this.files_home_dir + "maps" + File.separator + "temp" + File.separator;
+		//this.temp_image_file = this.temp_files_dir+"temp_";
+		//this.temp_image_file2 = this.temp_files_dir+"temp2_";
+
+		this.http_image_dir = this.site_http_address + "/collect/" + this.cluster_name + "/index/assoc/maps/";
+		this.http_temp_image_dir = this.site_http_address + "/collect/" + this.cluster_name + "/temp/";
+
+		this.index_name_list = new ArrayList<String>();
+		this.index_display_list = new ArrayList<String>();
+		Element index_list = (Element) GSXML.getChildByTagName(this.config_info, GSXML.INDEX_ELEM + GSXML.LIST_MODIFIER);
+		Element display_index_list = (Element) GSXML.getChildByTagName(extra_info, "search");
+		if (index_list != null && display_index_list != null)
+		{
+			NodeList indexes = index_list.getElementsByTagName(GSXML.INDEX_ELEM);
+			for (int i = 0; i < indexes.getLength(); i++)
+			{
+				String name = ((Element) indexes.item(i)).getAttribute(GSXML.NAME_ATT);
+				// add the index name
+				this.index_name_list.add(name);
+				// now look for the display name
+				Element this_index = GSXML.getNamedElement(display_index_list, "index", "name", name);
+				if (this_index != null)
+				{
+					Element disp = GSXML.getNamedElement(this_index, "displayItem", "name", "name");
+					if (disp != null)
+					{
+						String display = GSXML.getNodeText(disp);
+						if (!display.equals(""))
+						{
+							this.index_display_list.add(display);
+							continue;
+						}
+
+					}
+				}
+				// add the id as default text
+				this.index_display_list.add(name);
+
+			}
+		}
+
+		// look for document display format
+		String path = GSPath.appendLink(GSXML.DISPLAY_ELEM, GSXML.FORMAT_ELEM);
+		Element display_format = (Element) GSXML.getNodeByPath(extra_info, path);
+		if (display_format != null)
+		{
+			this.format_info_map.put(DOCUMENT_CONTENT_RETRIEVE_SERVICE, this.desc_doc.importNode(display_format, true));
+			// shoudl we make a copy?
+		}
+
+		// load the 2-dimensional tree for searching
+		try
+		{
+			ObjectInputStream objectin = new ObjectInputStream(new FileInputStream(this.files_home_dir + "nametree.dat"));
+			tree = (DefaultMutableTreeNode) objectin.readObject();
+			objectin.close();
+		}
+		catch (IOException ioe)
+		{
+			ioe.printStackTrace();
+		}
+		catch (ClassNotFoundException cnfe)
+		{
+			cnfe.printStackTrace();
+		}
+
+		return true;
+	}
+
+	/** */
+	protected Element getServiceDescription(Document doc, String service, String lang, String subset)
+	{
+		if (service.equals(DOCUMENT_STRUCTURE_RETRIEVE_SERVICE))
+		{
+
+			Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+			tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_STRUCTURE_RETRIEVE_SERVICE);
+			return tq_service;
+		}
+		else if (service.equals(DOCUMENT_METADATA_RETRIEVE_SERVICE))
+		{
+
+			Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+			tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+			return tq_service;
+		}
+		else if (service.equals(DOCUMENT_CONTENT_RETRIEVE_SERVICE))
+		{
+
+			Element tq_service = doc.createElement(GSXML.SERVICE_ELEM);
+			tq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
+			tq_service.setAttribute(GSXML.NAME_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+			return tq_service;
+		}
+
+		return null;
+
+	}
+
+	/** Retrieve the structure of a document */
+	protected Element processDocumentStructureRetrieve(Element request)
+	{
+		// Create a new (empty) result message
+	  Element result = XMLConverter.newDOM().createElement(GSXML.RESPONSE_ELEM);
+		return result;
+	}
+
+	protected Element processDocumentMetadataRetrieve(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		// Create a new (empty) result message
+		try
+		{
+			Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+
+			String uid = request.getAttribute(GSXML.USER_ID_ATT);
+			if (uid.equals(""))
+			{
+				logger.info("in metadata retrieve, uid = ''\n" + converter.getPrettyString(request));
+			}
+			result.setAttribute(GSXML.FROM_ATT, DOCUMENT_METADATA_RETRIEVE_SERVICE);
+			result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+			// Get the parameters of the request
+			Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (param_list == null)
+			{
+				logger.error("missing paramList.\n");
+				return result; // Return the empty result
+			}
+
+			// The metadata information required
+			Vector<String> metadata_list = new Vector<String>();
+			boolean all_metadata = false;
+			// Process the request parameters
+			Element param = GSXML.getFirstElementChild(param_list);//(Element) param_list.getFirstChild();
+
+			while (param != null)
+			{
+				// Identify the metadata information desired
+				if (param.getAttribute(GSXML.NAME_ATT).equals("metadata"))
+				{
+					String metadata = GSXML.getValue(param);
+					if (metadata.equals("all"))
+					{
+						all_metadata = true;
+						break;
+					}
+					metadata_list.add(metadata);
+				}
+				param = (Element) param.getNextSibling();
+			}
+
+			Element node_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			result.appendChild(node_list);
+
+			// Get the documents
+			Element request_node_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+			if (request_node_list == null)
+			{
+				logger.error(" DocumentMetadataRetrieve request had no " + GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+				return result;
+			}
+
+			NodeList request_nodes = request_node_list.getChildNodes();
+
+			try
+			{
+				//used just to ensure outfile is initialised
+				BufferedWriter outfile = new BufferedWriter(new FileWriter(this.temp_files_dir + "emptynothingness"));
+
+				for (int i = 0; i < request_nodes.getLength(); i++)
+				{
+					Element request_node = (Element) request_nodes.item(i);
+					String node_id = request_node.getAttribute(GSXML.NODE_ID_ATT);
+					String place_data = "";
+					String year = "";
+					String coOrdinates = "";
+					String thumb = "";
+					String link = "";
+					int mapFreq = 0;
+					LinkedList place_data_list = new LinkedList();
+					LinkedList coOrdinates_list = new LinkedList();
+
+					//if this is a first displaying of the map
+					if (node_id.indexOf("```") != -1 && node_id.indexOf("imageChooser") == -1)
+					{
+						//get the number of query terms on this map
+						mapFreq = Integer.parseInt(node_id.substring(node_id.lastIndexOf('`', node_id.indexOf("```") - 1) + 1, node_id.indexOf("```")));
+						//get the place names on this map
+						place_data = node_id.substring(node_id.indexOf("```") + 3, node_id.length());
+						//get the map metadata
+						node_id = node_id.substring(0, node_id.indexOf('`', node_id.indexOf("```")));
+
+						try
+						{
+							// loop for each query term on the map
+							for (int r = 0; r < mapFreq; r++)
+							{
+								// get title, type, location, and the string for creating the hyperlink for this query term
+								String title = place_data.substring(0, place_data.indexOf('`'));
+								String type = place_data.substring(place_data.indexOf('`') + 1, place_data.indexOf('`', place_data.indexOf('`') + 1));
+								String location = place_data.substring(place_data.indexOf('`', place_data.indexOf('`') + 1) + 1, place_data.indexOf('`', place_data.indexOf('`', place_data.indexOf('`') + 1) + 1));
+								if (place_data.indexOf("```") != -1)
+									link = place_data.substring(0, place_data.indexOf("```"));
+								else
+									link = place_data;
+								// resolve the type and location
+								type = getType(type);
+								location = getLocation(location);
+								// remove this query term from the string
+								if (place_data.indexOf("```") != -1)
+									place_data = place_data.substring(place_data.indexOf("```") + 3, place_data.length());
+
+								//add the co-ordinates of this query term to the co-ordinates list
+								coOrdinates_list.add("`" + link.substring(link.lastIndexOf('`') + 1, link.length()) + "`" + link.substring(link.lastIndexOf('`', link.lastIndexOf('`') - 1) + 2, link.lastIndexOf('`')));
+
+								// add the title, type and location to the places list
+								place_data_list.add(title + ", " + type + ", " + location + ";");
+							}
+						}
+						catch (StringIndexOutOfBoundsException sioobe)
+						{
+							sioobe.printStackTrace();
+						}
+					}
+
+					// Add the document to the list
+					Element new_node = (Element) result_doc.importNode(request_node, false);
+					node_list.appendChild(new_node);
+
+					// Add the requested metadata information
+					Element node_meta_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+					new_node.appendChild(node_meta_list);
+
+					// create the navigation thumbnails.  This doesn't seem to work most of the time ???????
+					for (int m = 0; m < metadata_list.size(); m++)
+					{
+						String metadata = metadata_list.get(m);
+						thumb = "";
+						String value = "";
+						if (node_id.indexOf('.') != -1)
+							thumb = node_id.substring(0, node_id.indexOf('.'));
+						if (node_id.indexOf('`') != -1)
+							value = node_id.substring(node_id.lastIndexOf('`', node_id.lastIndexOf('`') - 1) + 1, node_id.lastIndexOf('`', node_id.lastIndexOf('`') - 1) + 5);
+						year = value;
+
+						place_data = "";
+						if (place_data_list.size() != 0)
+							for (int q = 0; q < mapFreq; q++)
+							{
+								link = (String) place_data_list.get(q);
+								if (q != 0)
+								{
+									place_data = place_data + "<br>" + link;
+								}
+								else
+								{
+									place_data = link;
+									coOrdinates = "``";
+								}
+								coOrdinates = coOrdinates + (String) coOrdinates_list.get(q);
+							}
+
+						link = "<a href=\"?a=d&c=" + this.cluster_name + "&d=" + node_id + coOrdinates + "&dt=map\">";
+						thumb = "<img src=\"" + this.http_image_dir + thumb + "thumb.jpg\" border=0>";
+						value = "<table><tr><td>" + link + "<p>" + place_data + "<br>" + value + "</a></td><td>" + link + thumb + "</a></td></tr></table>";
+						if (metadata.equals("Title"))
+							if (!(place_data.equals("")) && place_data.indexOf(", , ;") == -1 && node_id.indexOf("```") == -1)
+								GSXML.addMetadata(node_meta_list, "Title", value);//metadata, value);
+							else
+								GSXML.addMetadata(node_meta_list, metadata, "");
+
+						if (place_data.indexOf(", , ;") == -1)
+						{
+							if (i == 0)
+								if (!(mapFreq == 0))
+								{
+									outfile = new BufferedWriter(new FileWriter(this.temp_files_dir + "links" + uid));
+									outfile.write("<table align=\"center\"><tr>");
+								}
+
+							if (!(mapFreq == 0))
+							{
+								if (i % 7 == 0)
+									outfile.write("</tr><tr>");
+								outfile.write("<td align=\"center\">" + link + thumb + "</a><br>" + link + year + "</a></td>");
+							}
+
+							if (i == request_nodes.getLength() - 1)
+							{
+								outfile.write("</tr></table><p>");
+								outfile.flush();
+								outfile.close();
+							}
+						}
+					}
+
+				}
+			}
+			catch (IOException ioe)
+			{
+				ioe.printStackTrace();
+			}
+
+			return result;
+		}
+		catch (Exception excep)
+		{
+			excep.printStackTrace();
+		}
+		return null;
+
+	}
+
+	protected Element processDocumentContentRetrieve(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		// Create a new (empty) result message
+		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		result.setAttribute(GSXML.FROM_ATT, DOCUMENT_CONTENT_RETRIEVE_SERVICE);
+		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+		String uid = request.getAttribute(GSXML.USER_ID_ATT);
+		String temp_image_file = this.temp_files_dir + "temp_" + uid + ".jpg";
+		String temp_image_file2 = this.temp_files_dir + "temp_" + uid + "_2.jpg";
+
+		Element query_doc_list = (Element) GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		if (query_doc_list == null)
+		{
+			logger.error("DocumentContentRetrieve request specified no doc nodes.\n");
+			return result;
+		}
+
+		String legend_file = this.temp_files_dir + "legend_" + uid + ".jpg";
+		String blank_file = this.files_home_dir + "blank.jpg";
+		Element doc_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+		result.appendChild(doc_list);
+
+		// Get the documents
+		String[] doc_ids = GSXML.getAttributeValuesFromList(query_doc_list, GSXML.NODE_ID_ATT);
+
+		for (int i = 0; i < doc_ids.length; i++)
+		{
+			String doc_id = doc_ids[i];
+			// img_num is the name of the map file eg. 072.jpg
+			String img_num = doc_id.substring(0, doc_id.indexOf('.'));
+			// strings for inserting image in html
+			String img_left = "<img border=0 src=\"" + this.http_temp_image_dir;
+			String doc_content = "";
+			String co_ordinates = "";
+			String img_size = "";
+			int height = 0;
+			int width = 0;
+			// the number for the legend image if adding locations to the map
+			int leg_num = 0;
+			double ratio = 0;
+
+			// if user has clicked on the map.  This does not resolve maps that are not North oriented.
+			if (doc_id.indexOf("imageChooser") == 0)
+			{
+				try
+				{
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.length());
+					img_num = doc_id.substring(0, doc_id.indexOf('.'));
+
+					// get the map size
+					String get_size[] = { "identify", "-size", "10000", temp_image_file };
+					Process proc;
+					proc = Runtime.getRuntime().exec(get_size);
+					BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+					img_size = br.readLine();
+					proc.waitFor();
+					img_size = img_size.substring(img_size.indexOf("JPEG") + 5, img_size.indexOf(" ", img_size.indexOf("JPEG") + 5));
+					width = Integer.parseInt(img_size.substring(0, img_size.indexOf("x")));
+					height = Integer.parseInt(img_size.substring(img_size.indexOf("x") + 1, img_size.length()));
+
+					// scale image size according to the ratio
+					ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`', doc_id.indexOf("```") - 1) + 1, doc_id.indexOf("```")));
+					width = (int) (width * ratio);
+					height = (int) (height * ratio);
+
+					// get the position of the mouse click on the image
+					int xclick = Integer.parseInt(doc_id.substring(doc_id.indexOf("```") + 3, doc_id.lastIndexOf('`')));
+					int yclick = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`') + 1, doc_id.length()));
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.indexOf("```"));
+
+					// convert click position to percentage distance accross and down the image.
+					double xpercent = (xclick * 1.0) / width;
+					double ypercent = (yclick * 1.0) / height;
+
+					// get the top left and bottom right co-ordinates of the map
+					double ytop = Double.parseDouble(doc_id.substring(0, doc_id.indexOf('`'))) * -1;
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.length());
+					double xleft = Double.parseDouble(doc_id.substring(0, doc_id.indexOf('`')));
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.length());
+					double ybot = Double.parseDouble(doc_id.substring(0, doc_id.indexOf('`'))) * -1;
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.length());
+					double xright = Double.parseDouble(doc_id.substring(0, doc_id.indexOf('`')));
+					doc_id = doc_id.substring(doc_id.indexOf('`') + 1, doc_id.length());
+
+					// calculate the map co-ordinates of the mouse click
+					xpercent = ((xright - xleft) * xpercent) + xleft;
+					ypercent = ((ybot - ytop) * ypercent) + ytop;
+
+					// search the tree for nearby place names
+					namesInList.clear();
+					findName(xpercent, ypercent, 0.1, 0.1, tree, true);
+
+					int namesInArraySize = namesInList.size();
+					String returnNames[] = new String[namesInArraySize];
+
+					//put the names in an array
+					for (int ri = 0; namesInList.size() > 0; ri++)
+					{
+						returnNames[ri] = namesInList.getFirst();
+						returnNames[ri] = returnNames[ri].substring(0, returnNames[ri].indexOf('`'));
+						namesInList.removeFirst();
+					}
+
+					//sort the names
+					Arrays.sort(returnNames);
+					doc_content = "\n<script>\nfunction openIt(loc){\n\topener.location=loc;\n\tself.close();\n}\n</script>";
+					for (int nameIndex = 0; nameIndex < namesInArraySize; nameIndex++)
+					{
+						String tempName = returnNames[nameIndex];
+						//convert names with spaces for hyperlinks
+						if (returnNames[nameIndex].indexOf(' ') != -1)
+						{
+							returnNames[nameIndex] = returnNames[nameIndex].replaceAll(" ", "+");
+							returnNames[nameIndex] = "%22" + returnNames[nameIndex] + "%22";
+						}
+						// add the place name to the html
+						doc_content = doc_content + "<a href=\"\" onClick=openIt('?a=q&sa=&rt=r&s=MapQuery&c=" + this.cluster_name + "&startPage=1&s1.index=none&s1.maxDocs=10&s1.query=" + returnNames[nameIndex] + "')>" + tempName + "</a><br>";
+					}
+
+				}
+				catch (Exception ioexception)
+				{
+					ioexception.printStackTrace();
+				}
+			}
+			else
+			{
+
+				try
+				{
+					//file for converting image    
+					BufferedWriter bw = new BufferedWriter(new FileWriter(this.temp_files_dir + "add_x_" + uid));
+					;
+					Process proc;
+
+					// if a new search
+					if (doc_id.indexOf("```") != -1)
+					{
+						// copy requested map to temp.jpg
+						proc = Runtime.getRuntime().exec("cp " + this.files_home_dir + "maps" + File.separator + img_num + ".jpg " + temp_image_file);
+						proc.waitFor();
+					}
+
+					//get the image size
+					String get_size[] = { "identify", "-size", "10000", temp_image_file };
+					proc = Runtime.getRuntime().exec(get_size);
+					BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+					img_size = br.readLine();
+					proc.waitFor();
+					img_size = img_size.substring(img_size.indexOf("JPEG") + 5, img_size.indexOf(" ", img_size.indexOf("JPEG") + 5));
+					if (img_size.indexOf("+") != -1)
+					{
+						img_size = img_size.substring(0, img_size.indexOf("+"));
+					}
+					width = Integer.parseInt(img_size.substring(0, img_size.indexOf("x")));
+					height = Integer.parseInt(img_size.substring(img_size.indexOf("x") + 1, img_size.length()));
+
+					// if a new search
+					if (doc_id.indexOf("```") != -1)
+					{
+						co_ordinates = doc_id.substring(doc_id.indexOf("```") + 2, doc_id.length());
+						// get the number of places to mark on the map
+						int x_number = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`', doc_id.indexOf("```") - 1) + 1, doc_id.indexOf("```")));
+
+						//write the convert command
+						bw.write("convert -font helvetica -fill red -pointsize 48 ");
+
+						for (int xi = 0; xi < x_number; xi++)
+						{
+
+							// get the co-ordinates of the place name
+							double xco = Double.parseDouble(co_ordinates.substring(co_ordinates.lastIndexOf('`', co_ordinates.lastIndexOf('`') - 1) + 1, co_ordinates.lastIndexOf('`', co_ordinates.lastIndexOf('`') - 1) + 7));
+							double yco = Double.parseDouble(co_ordinates.substring(co_ordinates.lastIndexOf('`') + 1, co_ordinates.length()));
+							if (xi != x_number - 1)
+								co_ordinates = co_ordinates.substring(0, co_ordinates.lastIndexOf('`', co_ordinates.lastIndexOf('`') - 1));
+
+							int index = 0;
+							index = doc_id.indexOf('`') + 2;
+
+							// get the zoom ratio and the top left and bottom right co-ordinates of the map
+							ratio = 0.4;
+							double tly = Double.parseDouble(doc_id.substring(index, index + 5));
+							index = doc_id.indexOf('`', index) + 1;
+							double tlx = Double.parseDouble(doc_id.substring(index, index + 6));
+							index = doc_id.indexOf('`', index) + 2;
+							double bry = Double.parseDouble(doc_id.substring(index, index + 5));
+							index = doc_id.indexOf('`', index) + 1;
+							double brx = Double.parseDouble(doc_id.substring(index, index + 6));
+							index = doc_id.indexOf('`', index) + 1;
+							double orient = Double.parseDouble(doc_id.substring(index, doc_id.indexOf('`', index)));
+
+							// find the centre of the map
+							double xcent = ((brx - tlx) / 2) + tlx;
+							double ycent = ((bry - tly) / 2) + tly;
+
+							// get the orientation of the map
+							orient = Math.toRadians(orient);
+
+							// rotate the co-ordinates around the centre of the map 
+							xco = xco - xcent;
+							yco = yco - ycent;
+							double oldx = xco;
+							xco = xco * Math.cos(orient) - yco * Math.sin(orient);
+							yco = oldx * Math.sin(orient) + yco * Math.cos(orient);
+							xco = xco + xcent;
+							yco = yco + ycent;
+							xco = (xco - tlx) / (brx - tlx);
+							yco = (yco - tly) / (bry - tly);
+
+							// calculate the pixels for placing the mark
+							width = (int) (xco * width) - 5;
+							height = (int) (yco * height) - 5;
+
+							// write the options for convert
+							bw.write("-draw 'text " + width + "," + height + " \"x\"' ");
+							bw.flush();
+
+							// reset the width and height variables for the image
+							width = Integer.parseInt(img_size.substring(0, img_size.indexOf("x")));
+							height = Integer.parseInt(img_size.substring(img_size.indexOf("x") + 1, img_size.length()));
+						}
+						doc_id = doc_id.substring(0, doc_id.lastIndexOf('`', doc_id.indexOf("```") - 1));
+
+						// write the end of the convert command, then command for zooming image to the right level, then command to copy the blank image over legend.jpg
+						bw.write(temp_image_file + " " + temp_image_file + ";convert -scale " + ((int) (ratio * width)) + "x" + ((int) (ratio * height)) + " " + temp_image_file + " " + temp_image_file2 + ";cp " + blank_file + " " + legend_file);
+
+						// reset legend number
+						leg_num = 0;
+					}
+
+					else
+					// if changing zoom level
+					if (doc_id.substring(doc_id.lastIndexOf('`'), doc_id.length()).indexOf('.') != -1)
+					{
+						// get the ratio
+						ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`') + 1, doc_id.length()));
+						// write the command for scaling image
+						bw.write("convert -scale " + ((int) (ratio * width)) + "x" + ((int) (ratio * height)) + " " + temp_image_file + " " + temp_image_file2);
+						doc_id = doc_id.substring(0, doc_id.lastIndexOf('`'));
+					}
+
+					// if adding locations to map
+					else
+					{
+
+						// get the location type to add
+						String restricter = doc_id.substring(doc_id.lastIndexOf('`') + 1, doc_id.length());
+						doc_id = doc_id.substring(0, doc_id.lastIndexOf('`'));
+
+						// get the number for the legend image
+						leg_num = Integer.parseInt(doc_id.substring(doc_id.lastIndexOf('`') + 1, doc_id.length()));
+						doc_id = doc_id.substring(0, doc_id.lastIndexOf('`'));
+
+						//open file for location type
+						BufferedReader inType = new BufferedReader(new FileReader(this.files_home_dir + "place_types" + File.separator + restricter + ".txt"));
+
+						//check through the file and add any places that are in the bounds of this map.
+
+						// check value. set to true if a place to add is found
+						boolean add_place_type = false;
+
+						int index = 0;
+						index = doc_id.indexOf('`') + 2;
+						// get zoom ratio
+						ratio = Double.parseDouble(doc_id.substring(doc_id.lastIndexOf('`') + 1, doc_id.length()));
+						// get top left and bottom right co-ordinates of the map
+						double tly = Double.parseDouble(doc_id.substring(index, index + 5));
+						index = doc_id.indexOf('`', index) + 1;
+						double tlx = Double.parseDouble(doc_id.substring(index, index + 6));
+						index = doc_id.indexOf('`', index) + 2;
+						double bry = Double.parseDouble(doc_id.substring(index, index + 5));
+						index = doc_id.indexOf('`', index) + 1;
+						double brx = Double.parseDouble(doc_id.substring(index, index + 6));
+						index = doc_id.indexOf('`', index) + 1;
+						// get orientation of the map
+						double orient = Double.parseDouble(doc_id.substring(index, doc_id.indexOf('`', index)));
+						// calculate centre of map
+						double xcent = ((brx - tlx) / 2) + tlx;
+						double ycent = ((bry - tly) / 2) + tly;
+						orient = Math.toRadians(orient);
+
+						String type_point = "";
+						double xco = 0.0;
+						double yco = 0.0;
+
+						// read the file
+						while (inType.ready())
+						{
+							//read a line
+							type_point = inType.readLine();
+
+							// get the co-ordinates of the point
+							xco = Double.parseDouble(type_point.substring(type_point.lastIndexOf('`') + 1, type_point.length()));
+							yco = Double.parseDouble(type_point.substring(type_point.lastIndexOf('`', type_point.lastIndexOf('`') - 1) + 2, type_point.lastIndexOf('`')));
+
+							// if it is within the co-ordinates of the map
+							if (xco >= tlx && xco < brx && yco >= tly && yco < bry)
+							{
+								// rotate the point around the centre according to map orientation
+								xco = xco - xcent;
+								yco = yco - ycent;
+								double oldx = xco;
+								xco = xco * Math.cos(orient) - yco * Math.sin(orient);
+								yco = oldx * Math.sin(orient) + yco * Math.cos(orient);
+								xco = xco + xcent;
+								yco = yco + ycent;
+								// get the pixels for where to place the mark
+								xco = (xco - tlx) / (brx - tlx);
+								yco = (yco - tly) / (bry - tly);
+								width = (int) (xco * width) - 5;
+								height = (int) (yco * height) - 5;
+
+								//if this is the first point to be added
+								if (!add_place_type)
+								{
+									// write the initial convert command
+									bw.write("convert -font helvetica -fill red -pointsize 36 ");
+									// toggle check value
+									add_place_type = true;
+								}
+
+								// write the options for the convert command
+								bw.write("-draw 'text " + width + "," + height + " \"" + leg_num + "\"' ");
+								bw.flush();
+
+								// reset width and height variables for the image
+								width = Integer.parseInt(img_size.substring(0, img_size.indexOf("x")));
+								height = Integer.parseInt(img_size.substring(img_size.indexOf("x") + 1, img_size.length()));
+							}
+						}
+
+						//if there are places to mark on the map
+						if (add_place_type)
+						{
+							// finish the convert command and write command for scaling image
+							bw.write(temp_image_file + " " + temp_image_file + ";convert -scale " + ((int) (ratio * width)) + "x" + ((int) (ratio * height)) + " " + temp_image_file + " " + temp_image_file2);
+
+							// open file for converting the legend command
+							BufferedWriter buf = new BufferedWriter(new FileWriter(this.temp_files_dir + "add_l_" + uid));
+							if (leg_num == 1)
+								buf.write("cp " + blank_file + " " + legend_file + ";");
+							// write the command for adding to the legend
+							buf.write("convert -font helvetica -fill red -pointsize 12 -draw 'text 15," + (leg_num * 15 + 20) + " \"" + leg_num + " " + getType(restricter) + "\"' " + legend_file + " " + legend_file);
+							buf.flush();
+							buf.close();
+							// execute the command for the legend image
+							proc = Runtime.getRuntime().exec("sh " + this.temp_files_dir + "add_l_" + uid);
+							proc.waitFor();
+						}
+						inType.close();
+					}
+					bw.flush();
+					bw.close();
+
+					// execute the convert commands etc.
+					proc = Runtime.getRuntime().exec("sh " + this.temp_files_dir + "add_x_" + uid);
+					proc.waitFor();
+
+				}
+				catch (Exception ioe)
+				{
+					ioe.printStackTrace();
+				}
+
+				//write the html for the document
+
+				String doc_content_head = "?a=d&c=" + this.cluster_name + "&dt=map&d=";
+				doc_content = "<td valign=\"top\"><script>document.write('" + img_left + "legend_" + uid + ".jpg" + "?'+new Date().getTime())</script>\"></td>";
+				doc_content = doc_content + "<td><script>document.write('" + img_left + "temp_" + uid + "_2.jpg?'+new Date().getTime())</script>\" onclick=\"imgClickHandler(event, this, '" + img_num + "')\"></td>";
+
+				// replace _httpimg_ with the correct address
+				doc_content = "<table><tr>" + doc_content + "</tr></table>";
+
+				String javascript = getJavascript(ratio, doc_id, leg_num);
+
+				doc_content = "<center>You are currently viewing the map at " + (int) (ratio * 100) + "%<br>Select a radio button for a new zoom level.</center>" + javascript + doc_content;
+				doc_content = doc_content.replaceAll("zoomhead", doc_content_head);
+				doc_content = doc_content.replaceAll("placeChooserImage", doc_content_head + "imageChooser`" + doc_id + "`" + ratio);
+
+				// read the file for the navigation thumbnails
+				try
+				{
+					BufferedReader infile = new BufferedReader(new FileReader(this.temp_files_dir + "links" + uid));
+					doc_content = doc_content + infile.readLine();
+					infile.close();
+				}
+				catch (Exception ioexc)
+				{
+					ioexc.printStackTrace();
+				}
+			}
+
+			// put the html in a text node
+			Element text_doc = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+			text_doc.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+			GSXML.addDocText(text_doc, doc_content);
+			doc_list.appendChild(text_doc);
+
+		}
+		return result;
+	}
+
+	// resolves location codes for places from LINZ database
+	private String getLocation(String location)
+	{
+		String read;
+		try
+		{
+			BufferedReader in = new BufferedReader(new FileReader(this.files_home_dir + "files" + File.separator + "landdist.txt"));
+			in.readLine();
+			while (in.ready())
+			{
+				read = in.readLine();
+				if (read.substring(0, 2).equals(location))
+				{
+					in.close();
+					return read.substring(3, read.length());
+				}
+			}
+		}
+		catch (Exception e)
+		{
+		}
+		return "";
+	}
+
+	// resolves type codes for the LINZ database
+	private String getType(String type)
+	{
+		String read;
+		try
+		{
+			BufferedReader in = new BufferedReader(new FileReader(this.files_home_dir + "files" + File.separator + "pointdes.txt"));
+			in.readLine();
+			while (in.ready())
+			{
+				read = in.readLine();
+				if (read.substring(0, read.indexOf('`')).toLowerCase().equals(type.toLowerCase()))
+				{
+					in.close();
+					return read.substring(read.indexOf('`') + 1, read.indexOf(':'));
+				}
+			}
+		}
+		catch (Exception e)
+		{
+		}
+		return "";
+	}
+
+	//recursive pre-order traversal of the 2-dimensional tree
+	// x & y are co-ordinates of mouse click, xoff & yoff are the distance in degrees away from x & y to search,
+	// d is the current node of the tree, xcomp controls whether searching is by x co-ordinate or y co-ordinate.
+	private void findName(double x, double y, double xoff, double yoff, DefaultMutableTreeNode d, boolean xcomp)
+	{
+		//if a leaf node return.  All leaf nodes are empty
+		if (d.isLeaf())
+			return;
+		//get coordinates of the current node
+		double xco = Double.parseDouble(((String) d.getUserObject()).substring(((String) d.getUserObject()).length() - 6, ((String) d.getUserObject()).length()));
+		double yco = Double.parseDouble(((String) d.getUserObject()).substring(((String) d.getUserObject()).length() - 12, ((String) d.getUserObject()).length() - 7));
+		//if in range
+		if ((x - xoff) < xco && (x + xoff) > xco && (y - yoff) < yco && (y + yoff) > yco)
+		{
+			//add to list
+			namesInList.addFirst((String) d.getUserObject());
+		}
+		//if comparing the x axis
+		if (xcomp)
+		{
+			//if need to search left subtree
+			if ((x - xoff) < xco)
+				findName(x, y, xoff, yoff, (DefaultMutableTreeNode) d.getChildAt(0), !xcomp);
+			//if need to search right subtree
+			if ((x + xoff) > xco)
+				findName(x, y, xoff, yoff, (DefaultMutableTreeNode) d.getChildAt(1), !xcomp);
+		}
+		else
+		{
+			//if need to search left subtree
+			if ((y - yoff) < yco)
+				findName(x, y, xoff, yoff, (DefaultMutableTreeNode) d.getChildAt(0), !xcomp);
+			//if need to search right subtree
+			if ((y + yoff) > yco)
+				findName(x, y, xoff, yoff, (DefaultMutableTreeNode) d.getChildAt(1), !xcomp);
+		}
+	}
+
+	// a whole lot of stuff for the html that is hidden at the bottom here because it clutters more than I already have further up!
+	private String getJavascript(double ratio, String doc_id, int leg_num)
+	{
+		String javascript = "\n<script>\nfunction imgClickHandler (evt, img, num) {\n\tif (window.event){\n\t\tvar chooserString = 'placeChooserImage' + '```' + window.event.offsetX + '`' + window.event.offsetY\n\t\tmywindow = window.open(chooserString,'Choose','width=600, height=500, scrollbars=yes, resizable=yes, toolbar=no, location=no, status=yes, menubar=no');\n\t\t//alert(window.event.offsetX + ':' + window.event.offsetY + ':' + num);\n\t}\n\telse if (evt.target) {\n\t\tvar coords = {x: 0, y: 0 };\n\t\tvar el = evt.target;\n\t\tdo {\n\t\t\tcoords.x += el.offsetLeft;\n\t\t\tcoords.y += el.offsetTop;\n\t\t}\n\t\twhile ((el = el.offsetParent));\n\t\tvar offsetX = evt.clientX - coords.x;\n\t\tvar offsetY = evt.clientY - coords.y;\n\t\talert(offsetX + ':' + offsetY + ':' + num);\n\t}\n}\nfunction leapTo (link)\n{\n\tvar new_url=link;\n\tif (  (new_url != \"\")  &&  (new_url != null)  ) \n\t\twindow.location=new_url;\n\telse \n\t\talert(\"You must make a selection.\");\n}\nfunction jumpTo (front,link,back)\n{\n\tvar new_url=front+link+back;\n\tif (  (new_url != \"\")  &&  (new_url != null)  ) \n\t\twindow.location=new_url;\n\telse \n\t\talert(\"You must make a selection.\");\n}// Deactivate Cloaking -->\n</script>";
+
+		String radioButtons = "\n<center><FORM>";
+		if (ratio != 0.4)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.4')\">40%";
+		if (ratio != 0.5)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.5')\">50%";
+		if (ratio != 0.6)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.6')\">60%";
+		if (ratio != 0.7)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.7')\">70%";
+		if (ratio != 0.8)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.8')\">80%";
+		if (ratio != 0.9)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`0.9')\">90%";
+		if (ratio != 1.0)
+			radioButtons = radioButtons + "\n\t<INPUT TYPE=\"radio\" NAME=\"buttons1\" onClick=\"leapTo('zoomhead" + doc_id + "`1.0')\">100%";
+		radioButtons = radioButtons + "\n</FORM></center>";
+
+		leg_num = leg_num + 1;
+
+		String doc_link = "?a=d&c=" + this.cluster_name + "&dt=map&d=" + doc_id + "`" + ratio + "`" + leg_num + "`";
+
+		StringBuffer dropDownBox = new StringBuffer();
+
+		dropDownBox.append("Add Locations Of Type: <form name=\"myForm\"><select name=\"s1index\" onChange=\"jumpTo('");
+		dropDownBox.append(doc_link);
+		dropDownBox.append("',document.myForm.s1index.options[document.myForm.s1index.selectedIndex].value,'')\">\n");
+
+		for (int i = 0; i < this.index_name_list.size(); i++)
+		{
+			String name = this.index_name_list.get(i);
+			dropDownBox.append("<option value=\"");
+			dropDownBox.append(name);
+			dropDownBox.append("\">");
+			String display = this.index_display_list.get(i);
+			dropDownBox.append(display);
+			dropDownBox.append("\n");
+		}
+		// for each option, Malcolm had onClick=\"leapTo('"+doc_link+name+"')\" - dont seem to need this
+		dropDownBox.append("</select></form>\n");
+
+		return javascript + radioButtons + dropDownBox;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/MapSearch.java	(revision 31634)
@@ -0,0 +1,396 @@
+/*
+ *    MapSearch.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Element; 
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList; 
+
+// General java classes
+import java.util.Hashtable;
+import java.util.HashMap;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.lang.reflect.Array;
+import java.util.Comparator;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.FileInputStream;
+import java.io.Serializable;
+import java.lang.ClassNotFoundException;
+
+import org.apache.log4j.*;
+
+/**
+ *
+ * 
+ */
+public class MapSearch 
+    extends AbstractTextSearch {
+
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.MapSearch.class.getName());
+
+    protected String files_home_dir = null;
+
+    protected static final String FREQ_ATT = "freq";
+
+    // use the following if we add back in index searching
+    //protected String default_index = null;
+    //protected static final String DEFAULT_INDEX_ELEM = "defaultIndex";
+    //protected static final String INDEX_ELEM = "index";  
+    
+    /** Hashtable containing the place names database, indexed on the place name in lower case. */
+    public static Hashtable hashNames;
+
+    /** Constructor */
+    public MapSearch() {}
+
+    /** */
+    public boolean configure(Element info, Element extra_info)
+    {
+	if (!super.configure(info, extra_info)) {
+	    return false;
+	}
+	logger.info("Configuring MapSearch...");
+
+	// set the files_home_dir variable for this collection
+	this.files_home_dir = GSFile.collectionIndexDir(site_home, cluster_name)+File.separator+"assoc"+File.separator;
+
+	//load the hashtable
+	hashNames = new Hashtable();
+	try{
+	    ObjectInputStream in = new ObjectInputStream(new FileInputStream(this.files_home_dir+"hashnames.dat"));
+	    hashNames = (Hashtable)in.readObject();
+	    in.close();
+	}catch(IOException ioe){
+	    ioe.printStackTrace();
+	    return false;
+	}catch(ClassNotFoundException cnf){
+	    cnf.printStackTrace();
+	    return false;
+	}
+	 
+	// just in case we get the index stuff going, here is the code
+	
+	// get the display stuff for the indexes
+	
+// 	NodeList indexes = info.getElementsByTagName(GSXML.INDEX_ELEM);
+// 	if (extra_info != null) {
+// 	    Document owner = info.getOwnerDocument();
+// 	    Element config_search = (Element)GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+// 	    if (config_search != null) {
+// 		for (int i=0; i<indexes.getLength();i++) {
+// 		    Element ind = (Element)indexes.item(i);
+// 		    String name = ind.getAttribute(GSXML.NAME_ATT);
+// 		    Element node_extra = GSXML.getNamedElement(config_search,
+// 							       GSXML.INDEX_ELEM,
+// 							       GSXML.NAME_ATT,
+// 							       name);
+// 		    if (node_extra == null) {
+// 			// no index specification for this index
+// 			continue;
+// 		    }
+		    
+// 		    // get the display elements if any - displayName
+// 		    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+// 		    if (display_names !=null) {
+// 			for (int j=0; j<display_names.getLength(); j++) {
+// 			    Element e = (Element)display_names.item(j);
+// 			    ind.appendChild(owner.importNode(e, true));
+// 			}
+// 		    }
+// 		} // for each index
+// 	    }
+// 	}
+// 	// Get the default index out of <defaultIndex> (buildConfig.xml)
+// 	Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_INDEX_ELEM);
+// 	if (def != null) {
+// 	    default_index = def.getAttribute(GSXML.NAME_ATT);
+// 	}
+// 	if (default_index == null || default_index.equals("")) {
+// 	    // use the first one from the index list
+// 	    default_index = ((Element)indexes.item(0)).getAttribute(GSXML.NAME_ATT);
+// 	}
+
+	return true;
+    }
+
+    protected void getIndexData(ArrayList<String> index_ids, ArrayList<String> index_names, String lang) 
+    {
+	// for now, we just have one dummy index
+	index_ids.add("idx");
+	index_names.add("maps"); // get from properties
+
+	// use the following if we ever get the index search working
+// 	Element index_list = (Element)GSXML.getChildByTagName(this.config_info, INDEX_ELEM+GSXML.LIST_MODIFIER);
+// 	NodeList indexes = index_list.getElementsByTagName(INDEX_ELEM);
+// 	for (int i=0; i<indexes.getLength(); i++) {
+// 	    Element index = (Element)indexes.item(i);
+// 	    index_ids.add( index.getAttribute(GSXML.NAME_ATT));
+// 	    index_names.add(getDisplayText(index, GSXML.DISPLAY_TEXT_NAME, lang, "en"));
+		
+// 	}
+    }
+    
+
+    /** Process a map query */
+    protected Element processTextQuery(Element request) 
+    {
+      Document result_doc = XMLConverter.newDOM();
+	// Create a new (empty) result message
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, QUERY_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+	// Get the parameters of the request
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("TextQuery request had no paramList.");
+	    return result;  // Return the empty result
+	}
+
+	// Process the request parameters
+	HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+
+	// Make sure a query has been specified
+	String searchTerm = (String) params.get(QUERY_PARAM);
+	if (searchTerm == null || searchTerm.equals("")) {
+	    return result;  // Return the empty result
+	}
+
+	// If an index hasn't been specified, use the default
+// 	String index = (String) params.get(INDEX_PARAM); 
+// 	if (index == null) {
+// 	    index = default_index;
+// 	}
+
+	//convert the query string from the form '&quotTe Awamutu&quot Hamilton' to 'Te Awamutu+Hamilton'
+
+	//replace ' ' with '+'
+	searchTerm = searchTerm.replace(' ','+');
+
+	//replace '&quot' with '"'
+	int place = -1;
+	while(searchTerm.indexOf("&quot;",place+1) != -1){
+	    place = searchTerm.indexOf("&quot;",place+1);
+	    searchTerm = (searchTerm.substring(0,place) + "\"" + searchTerm.substring(place+6,searchTerm.length()));
+	}
+
+	//replace spaces in string enclosed in quotes
+	place = -1;
+	while(searchTerm.indexOf('"',place+1) != -1){
+	    place = searchTerm.indexOf('"',place+1);
+	    if(searchTerm.indexOf('"',place+1) != -1)
+		searchTerm = (searchTerm.substring(0,place) + searchTerm.substring(place, searchTerm.indexOf('"',place+1)).replace('+',' ') + searchTerm.substring(searchTerm.indexOf('"',place+1), searchTerm.length()));
+	    place = searchTerm.indexOf('"',place);
+	}
+
+	//remove speech marks
+	place = 0;
+	while(place != -1){
+	    place = searchTerm.indexOf('"', place);
+	    if(place != -1){
+		searchTerm = searchTerm.substring(0, place) + searchTerm.substring(place+1, searchTerm.length());
+		place=0;
+	    }
+	}
+
+	//find the number of search terms (number of '+' plus 1)
+	int words = 1;
+	place = 0;
+	while (place != -1){
+	    place = searchTerm.indexOf('+', place+1);
+	    if(place != -1) words++;
+	}
+	place = 0;
+
+	//store each search term in a string array
+	String terms[] = new String[words];
+	String terms_freq[] = new String[words];
+	for (int i=0; i<(words-1); i++){
+	    terms[i]=searchTerm.substring(place, searchTerm.indexOf('+', place));
+	    place = searchTerm.indexOf('+', place)+1;
+	}
+	terms[words-1] = searchTerm.substring(place, searchTerm.length());
+
+	Object nameArray[] = new Object[1];
+	int nameArraySize;
+	String nameData;
+	double xco, yco;
+	LinkedList<Object> mapList = new LinkedList<Object>();
+	LinkedList placeList = new LinkedList();
+	String readString = "";
+
+	//capitalise the search terms
+	for(int i=0;i<words;i++)
+	    terms[i] = terms[i].toUpperCase();
+
+	for(int k=0; k<words; k++){
+	    nameArraySize=0;
+
+
+	    if((nameArray = (Object[])hashNames.get(terms[k].toLowerCase())) != null){
+		logger.debug(hashNames.get(terms[k].toLowerCase()));
+		
+		nameArraySize = Arrays.asList(nameArray).size();
+	    }
+	    
+	    
+	    int h;
+	    for(h=0;h<nameArraySize;h++){
+
+		nameData = (String)nameArray[h];
+
+		// get the co-ordinates of the current place name
+		xco = Double.parseDouble(nameData.substring(nameData.lastIndexOf('`')+1,nameData.length()));
+		yco = Double.parseDouble(nameData.substring(nameData.lastIndexOf('`',nameData.lastIndexOf('`')-1)+2,nameData.lastIndexOf('`')));
+
+		//open the text file containing map metadata (first line is a description of the file)
+		try{
+		    BufferedReader inmap = new BufferedReader(new FileReader(this.files_home_dir + "files"+File.separator+"maps"+File.separator+"mapdata.txt"));
+		    inmap.readLine();
+
+		    // for each map
+		    while(inmap.ready()){
+			readString = inmap.readLine();
+			int indexer = 0;
+			indexer = readString.indexOf('`')+2;
+			// get the co-ordinates of the top left and bottom right corners of the map.
+			double ytop = Double.parseDouble(readString.substring(indexer,indexer+5));
+			indexer = readString.indexOf('`',indexer)+1;
+			double xtop = Double.parseDouble(readString.substring(indexer,indexer+6));
+			indexer = readString.indexOf('`',indexer)+2;
+			double ybot = Double.parseDouble(readString.substring(indexer,indexer+5));
+			indexer = readString.indexOf('`',indexer)+1;
+			double xbot = Double.parseDouble(readString.substring(indexer,indexer+6));
+
+			//if the place is within the map, put the map metadata and the placename metadata in linked list
+			if(xco >= xtop && xco < xbot && yco >= ytop && yco < ybot)
+			    mapList.add(readString+"```"+nameData);
+		    }
+		}catch(Exception e){e.printStackTrace();}
+	    }
+	}
+
+	// use an array object to sort the linked list by map number
+	Object[] mapListArray = mapList.toArray();
+	Arrays.sort(mapListArray);
+	mapList.clear();
+	for(int mla=0; mla<Array.getLength(mapListArray); mla++)
+	    mapList.addFirst(mapListArray[mla]);
+
+	//for each map, create a list of the query matches on that map
+	LinkedList<Object> tempList = new LinkedList<Object>();
+	String mapNumber = "";
+	String currentMap[] = {"",""};
+	int mapFreq = 0;
+	while(mapList.size()>0){
+	    readString = (String)mapList.removeFirst();
+	    if(mapNumber.equals(readString.substring(0,readString.indexOf('.')))){
+		currentMap[1] = currentMap[1]+readString.substring(readString.indexOf("```"),readString.length());
+		mapFreq++;
+	    }
+	    else{
+		if(!currentMap[0].equals(""))
+		    tempList.add(currentMap[0]+"`"+mapFreq+currentMap[1]);
+		currentMap[0] = readString.substring(0,readString.indexOf("```"));
+		currentMap[1] = readString.substring(readString.indexOf("```"),readString.length());
+		mapNumber = readString.substring(0,readString.indexOf('.'));
+		mapFreq=1;
+	    }
+	}
+	if(!currentMap[0].equals(""))
+	    tempList.add(currentMap[0]+"`"+mapFreq+currentMap[1]);
+
+	int totalDocs = tempList.size();
+
+	// use an array object to sort the linked list by number of matches on each map
+	Object[] tempListArray = tempList.toArray();
+	Arrays.sort(tempListArray, new MapResultSorter());
+	tempList.clear();
+	for(int tla=0; tla<Array.getLength(tempListArray); tla++)
+	    tempList.add(tempListArray[tla]);
+
+	// Create a metadata list to store information about the query results
+	Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER); 
+	result.appendChild(metadata_list);
+			
+	// Add a metadata element specifying the number of matching documents
+	GSXML.addMetadata(metadata_list, "numDocsMatched", "" + totalDocs);
+	GSXML.addMetadata(metadata_list, "numDocsReturned", ""+totalDocs);
+       	// Create a document list to store the matching documents, and add them
+	Element document_list = result_doc.createElement(GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(document_list);
+	for (int d = 0; d < totalDocs; d++) {
+	    String doc_id = (String)tempList.get(d);
+
+	    Element doc_node = result_doc.createElement(GSXML.DOC_NODE_ELEM);
+	    doc_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
+	    doc_node.setAttribute(GSXML.NODE_TYPE_ATT, "thumbnail");
+	    doc_node.setAttribute(GSXML.DOC_TYPE_ATT, "map");
+	    document_list.appendChild(doc_node);
+	}
+			
+	// Create a term list to store the term information, and add it
+	Element term_list = result_doc.createElement(GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
+	result.appendChild(term_list);	
+	
+
+	for (int t=0; t<words; t++){
+	    String term = terms[t];
+	
+	    Element term_elem = result_doc.createElement(GSXML.TERM_ELEM);
+  	    term_elem.setAttribute(GSXML.NAME_ATT, term);
+  	    term_elem.setAttribute(FREQ_ATT, "" + terms_freq[t]);
+
+	    term_list.appendChild(term_elem);
+	}
+	return result;
+    }
+ 
+
+    public class MapResultSorter implements Comparator {
+	
+	public int compare(Object o1, Object o2) {
+	    String first = (String)o1;
+	    String second = (String) o2;
+	    int firstInt;
+	    int secondInt;
+	    first = first.substring(first.lastIndexOf('`',first.indexOf("```")-1)+1,first.indexOf("```"));
+	    second = second.substring(second.lastIndexOf('`',second.indexOf("```")-1)+1,second.indexOf("```"));
+	    firstInt = Integer.parseInt(first);
+	    secondInt = Integer.parseInt(second);
+	    if(firstInt > secondInt)
+		return -1;
+	    else if(firstInt == secondInt)
+		return 0;
+	    else
+		return 1;
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ModuleWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ModuleWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ModuleWrapper.java	(revision 31634)
@@ -0,0 +1,240 @@
+/*
+ *    ModuleWrapper.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+import org.greenstone.gsdl3.core.ModuleInterface;
+
+// XML classes
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.SAXParser;
+import org.apache.xerces.dom.TextImpl;
+import org.apache.xerces.parsers.DOMParser;
+import org.w3c.dom.Document; 
+import org.w3c.dom.Element;
+import org.w3c.dom.Node; 
+import org.w3c.dom.Node;                                                
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+// java classes
+import java.io.StringReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.security.SecureRandom;
+
+import org.apache.log4j.*;
+
+/**
+ * A classes for logging and checking all traffic to and from a ModuleInterface
+ * 
+ * if logCounter hasn't been initialised, initialise it to a random 
+ * value. This has to use secure random rather than a normal random
+ * or every time the application is run the same numbers will be
+ * generated and the search for a free range will become a linear
+ * search rather than a hash.
+ *
+ *
+ * @author <a href="mailto:s.yeates@cs.waikato.ac.nz">Stuart Yeates</a>
+ * @version $Revision$
+ */
+public class ModuleWrapper 
+  implements ModuleInterface 
+{
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service. ModuleWrapper.class.getName());
+
+  /** the module we're wrapping */
+  protected ModuleInterface inner = null;
+  /** the module we're wrapping */
+  public ModuleInterface getInner(){return inner;};
+  /** the module we're wrapping */
+  public void setInner(ModuleInterface i){inner=i;};
+
+  /** The schema of the IN XML */
+  protected String fromSchema = null;
+  /** The schema of the IN XML */
+  public String getFromSchema(){return fromSchema;};
+  /** The schema of the IN XML */
+  public void setFromSchema(String s){fromSchema=s;};
+
+  /** The schema of the OUT XML */
+  protected String toSchema = null;
+  /** The schema of the OUT XML */
+  public String getToSchema(){return toSchema;};
+  /** The schema of the OUT XML */
+  public void setToSchema(String s){toSchema=s;};
+
+  /** Are we logging the XML ? */
+  protected boolean logging = true;
+  /** Are we logging the XML ? */
+  public boolean getLogging(){return logging;};
+  /** Are we logging the XML ? */
+  public void setLogging(boolean b){logging=b;};
+
+  /** Where we are logging the XML */
+  protected String logDirectory = "/tmp/";
+  /** Where we are logging the XML */
+  public String getLogDirectory(){return logDirectory;};
+  /** Where we are logging the XML */
+  public void setLogDirectory(String s){logDirectory=s;};
+
+  /** The number of the previous log file */
+  protected static long logCounter = 0;
+
+  /** The no-args constructor */
+  public ModuleWrapper(){
+
+  }
+  
+  public void cleanUp() {}
+  
+  /** The all-args constructor */
+  public ModuleWrapper(String in,String out, ModuleInterface inner){
+    this.setFromSchema(in);
+    this.setToSchema(out);
+    this.setInner(inner);
+  }
+
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xmlIn the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+  public String process(String xmlIn) {
+    
+    long logNumber = 0;
+    String xmlOut = "";
+    if (getLogging()) {
+      if (logCounter == 0)
+	logCounter = new SecureRandom().nextLong();
+      try {
+	logNumber = logCounter++;
+	String filename = getLogDirectory() + File.separator + logNumber + ".in";
+	File file = new File(filename);
+	
+	while (file.exists()){
+	  logCounter = new SecureRandom().nextLong();
+	  logNumber = logCounter++;
+	  filename = getLogDirectory() + File.separator + logNumber + ".in";
+	  file = new File(filename);
+	}
+	
+	FileWriter writer = new FileWriter(file);
+	writer.write(xmlIn);
+      } catch (IOException e) {
+	logger.error("caught exception: " +e );
+      }
+    }
+    xmlOut = processInner(xmlIn);
+
+    try {
+      String filename = getLogDirectory() + File.separator + logNumber + ".out";
+      File file = new File(filename);
+      FileWriter writer = new FileWriter(file);
+      writer.write(xmlOut);
+    } catch (IOException e) {
+      logger.error("caught exception: " +e );
+    }
+    return xmlOut;
+  }
+
+  /**
+   * Process an XML request - as a String
+   *
+   * @param xmlIn the request to process
+   * @return the response - contains any error messages
+   * @see java.lang.String
+   */
+  protected String processInner(String xmlIn) 
+  {
+    try {
+      DocumentBuilderFactory factory=DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder= factory.newDocumentBuilder();
+      DOMParser parser = new DOMParser();
+      try {
+	
+	parser.setFeature("http://xml.org/sax/features/validation",true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema",true);
+	parser.setFeature("http://apache.org/xml/features/validation/dynamic", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema-full-checking", true);
+	parser.setFeature("http://apache.org/xml/features/validation/schema/normalized-value", true);
+	parser.setFeature("http://apache.org/xml/features/validation/warn-on-duplicate-attdef", true);
+	parser.setFeature("http://apache.org/xml/features/validation/warn-on-undeclared-elemdef", true);
+	parser.setFeature("http://apache.org/xml/features/warn-on-duplicate-entitydef", true);
+      } catch (Exception a) {
+	logger.error("unable to set feature:" +a);
+	a.printStackTrace();
+      }
+
+      // do the pre-call checking
+      InputSource xml_source = new InputSource(new StringReader(xmlIn));  
+      try {
+	parser.parse(xml_source);
+      } catch (Exception e){  
+	logger.error("parsing error:" +e);
+	e.printStackTrace();
+	return "<response> <error class=\"ModuleWrapper\" code=1> Error: supplied string contained the parse error: " + e + " </error> </response>";
+      } 
+
+      String xmlOut = inner.process(xmlIn);
+
+      // do the post-call checking
+      InputSource xmlResult = new InputSource(new StringReader(xmlIn));  
+      try {
+	parser.parse(xmlResult);
+      } catch (Exception e){  
+	logger.error("parsing error:" +e);
+	e.printStackTrace();
+	return "<response> <error class=\"ModuleWrapper\" code=2> Error: returned string contained the parse error: " + e + "</error></response>";
+      } 
+
+      return xmlOut;
+
+    } catch (Exception e){  
+      logger.error("other error:" +e);
+      e.printStackTrace();
+      return "<response> <error class=\"ModuleWrapper\" code=3>Error: Unknown error or warning: " + e + " </error></response>";
+    } 
+  }
+
+  /**
+   * Process an XML request - as a DOM Element
+   *
+   * @param in the request to process
+   * @return the response - contains any error messages
+   * @see org.w3c.dom.Element
+   */
+  public Node process(Node xmlIn) {
+    throw new Error("Not implmented yet. Should be faked by stringizing the node.");
+  }
+}    
+                                                                           
+
+
+
+
+
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/OAIPMH.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/OAIPMH.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/OAIPMH.java	(revision 31634)
@@ -0,0 +1,866 @@
+/*
+ *    OAIPMH.java
+ *    Copyright (C) 2010 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import org.greenstone.gsdl3.core.GSException;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.OAIXML;
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.DBInfo;
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+// General Java classes
+import java.io.File;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+
+import org.apache.log4j.Logger;
+
+/** Implements the oai metadata retrieval service for GS3 collections.
+ *  Dig into each collection's database and retrieve the metadata
+ *
+ */
+
+public class OAIPMH extends ServiceRack {
+  
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.OAIPMH.class.getName());
+  
+  protected SimpleCollectionDatabase coll_db = null;
+  protected SimpleCollectionDatabase oaiinf_db = null;
+  
+  protected String site_name = "";
+  protected String coll_name = "";
+ 
+  // set this up during configure
+  protected Element list_sets_response = null;
+  
+  protected Element meta_formats_definition = null;
+  protected HashMap<String, HashSet<String>> format_elements_map = null;
+  protected HashMap<String, Element> format_response_map = null;
+  /** constructor */
+  public OAIPMH() {
+
+  }
+  
+  public void cleanUp() {
+    super.cleanUp();//??
+    this.coll_db.closeDatabase();
+    if (this.oaiinf_db != null){
+    	this.oaiinf_db.closeDatabase();
+    }
+    
+  }
+  /** configure this service 
+  info is the OAIPMH service rack from collectionConfig.xml, and 
+  extra_info is buildConfig.xml */
+  public boolean configure(Element info, Element extra_info) {
+    if (!super.configure(info, extra_info)){
+      logger.info("Configuring ServiceRack.java returns false.");
+      return false;
+    }
+    
+    //get the names from ServiceRack.java
+    this.site_name = this.router.getSiteName();
+    this.coll_name = this.cluster_name;
+    
+    logger.info("Configuring OAIPMH...");
+
+    this.config_info = info;
+    
+    // the index stem is either specified in the buildConfig.xml file (extra_info) or uses the collection name
+    Element metadata_list = (Element) GSXML.getChildByTagName(extra_info, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+    String index_stem = "";
+    String infodb_type = "";
+    if (metadata_list != null) {
+      
+      Element index_stem_elem = (Element) GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "indexStem");
+      
+      if (index_stem_elem != null) {
+	index_stem = GSXML.getNodeText(index_stem_elem);
+      }
+
+      Element infodb_type_elem = (Element) GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "infodbType");
+      if (infodb_type_elem != null) {
+	infodb_type = GSXML.getNodeText(infodb_type_elem);
+      }
+
+    }
+
+    if (index_stem == null || index_stem.equals("")) {
+	index_stem = this.cluster_name; // index_stem is the name of the db in indext/text, it is <colname>.<db>
+    }
+    if (infodb_type == null || infodb_type.equals("")) {
+      infodb_type = "gdbm"; // the default
+    }
+
+    coll_db = new SimpleCollectionDatabase(infodb_type);
+    if (!coll_db.databaseOK()) {
+      logger.error("Couldn't create the collection database of type "+infodb_type);
+      return false;
+    }
+
+    oaiinf_db = new SimpleCollectionDatabase(infodb_type);
+    if (!oaiinf_db.databaseOK()) {
+      logger.error("Couldn't create the oai-inf database of type "+infodb_type);
+      return false;
+    }
+
+    
+    // Open databases for querying
+    String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, index_stem, infodb_type);
+    if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ)) {
+      logger.error("Could not open collection database!");
+      return false;
+    }
+    // the oaiinf_db is called oai-inf.<infodb_type_extension>
+    String oaiinf_db_file = GSFile.OAIInfoDatabaseFile(this.site_home, this.cluster_name, "oai-inf", infodb_type);
+    File oaiinfFile = new File(oaiinf_db_file);
+    
+    if(!oaiinfFile.exists()) {
+	logger.warn("oai-inf database for collection + " + this.cluster_name + " does not exist.");
+	oaiinf_db = null;
+    } else if (!this.oaiinf_db.openDatabase(oaiinf_db_file, SimpleCollectionDatabase.READ)) {
+	logger.warn("Could not open oai-inf database for collection + " + this.cluster_name + "!");
+	oaiinf_db = null;
+    }
+    
+    // work out what sets this collection has. Will usually contain the collection itself, optional super collection, and maybe subcolls if appropriate classifiers are present.
+    configureSetInfo();
+    // the short_service_info is used by the message router to find the method names, 
+
+    Element list_records = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    list_records.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_RECORDS);
+    list_records.setAttribute(GSXML.TYPE_ATT, "oai");
+    this.short_service_info.appendChild(list_records);
+
+    Element list_identifiers = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    list_identifiers.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_IDENTIFIERS);
+    list_identifiers.setAttribute(GSXML.TYPE_ATT, "oai");
+    this.short_service_info.appendChild(list_identifiers);
+    
+    Element list_sets = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    list_sets.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_SETS);
+    list_sets.setAttribute(GSXML.TYPE_ATT, "oai");
+    this.short_service_info.appendChild(list_sets);
+    
+    Element list_metadata_formats = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    list_metadata_formats.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_METADATA_FORMATS);
+    list_metadata_formats.setAttribute(GSXML.TYPE_ATT, "oai");
+    this.short_service_info.appendChild(list_metadata_formats);
+
+    Element get_record = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    get_record.setAttribute(GSXML.NAME_ATT, OAIXML.GET_RECORD);
+    get_record.setAttribute(GSXML.TYPE_ATT, "oai");
+    this.short_service_info.appendChild(get_record);
+    
+    return true;
+  }
+
+  public boolean configureOAI(Element oai_config_elem) {
+    this.meta_formats_definition = this.desc_doc.createElement(OAIXML.LIST_METADATA_FORMATS);
+    this.format_response_map = new HashMap<String, Element>();
+    this.format_elements_map = new HashMap<String, HashSet<String>>();
+
+    // for now, all we want is the metadata prefix description and the mapping list
+    Element main_lmf_elem = (Element) GSXML.getChildByTagName(oai_config_elem, OAIXML.LIST_METADATA_FORMATS);
+    if (main_lmf_elem == null) {
+      logger.error("No listMetadataFormats element found in OAIConfig.xml");
+      return false;
+    }
+    NodeList meta_formats_list = this.config_info.getElementsByTagName(OAIXML.METADATA_FORMAT);
+    if (meta_formats_list.getLength() == 0) {
+      logger.error("no metadataFormat elements found in OAIPMH serviceRack element");
+      return false;
+    }
+    boolean found_meta_format = false;
+    for(int i=0; i<meta_formats_list.getLength(); i++) {
+      Element mf = (Element) meta_formats_list.item(i);
+      String prefix = mf.getAttribute(OAIXML.METADATA_PREFIX);
+      if (prefix.equals("")) {
+	logger.error("metadataFormat element had no metadataPrefix attribute");
+	continue;
+      }
+      // get the right format from OAICOnfig
+      Element meta_format = findNamedMetadataFormat(main_lmf_elem, prefix);
+      if (meta_format == null) {
+	logger.error("Couldn't find metadataFormat named "+prefix+" in OAIConfig.xml");
+	continue;
+      }
+      // copy the format definition into our stored Element
+      Element collection_version_format = (Element) this.desc_doc.importNode(meta_format, true);
+      collection_version_format.setAttribute(GSXML.NAME_ATT, prefix); // for convenience
+      this.meta_formats_definition.appendChild(collection_version_format);
+      // set up the response element for this format
+      format_response_map.put(prefix, OAIXML.getMetadataFormatShort(this.desc_doc, collection_version_format));
+      // add in collection specific mappings
+      addCollectionMappings(collection_version_format, mf);
+      // now set up a list of all collection elements for reverse lookup of the mapping
+      format_elements_map.put(prefix, getAllCollectionElements(collection_version_format));
+      
+    }
+    return true;
+  }
+
+  protected Element findNamedMetadataFormat(Element list_meta_formats, String prefix) {
+    NodeList formats = list_meta_formats.getElementsByTagName(OAIXML.METADATA_FORMAT);
+    for (int i=0; i<formats.getLength(); i++) {
+      Element format = (Element)formats.item(i);
+      String meta_name = GSXML.getNodeText((Element)GSXML.getChildByTagName(format, OAIXML.METADATA_PREFIX));
+      if (prefix.equals(meta_name)) {
+	return format;
+      }
+    }
+    return null;
+  }
+    
+    /** goes through the mappings from the collection one, and replaces existing ones in the main one */
+    protected void addCollectionMappings(Element main_meta_format, Element coll_meta_format) {
+
+      Element element_list = (Element)GSXML.getChildByTagName(main_meta_format, OAIXML.ELEMENT+GSXML.LIST_MODIFIER);
+      Document doc = element_list.getOwnerDocument();
+      NodeList coll_elements = coll_meta_format.getElementsByTagName(OAIXML.ELEMENT);
+      if (coll_elements.getLength()==0) {
+	// no mappings to include
+	return;
+      }
+      for (int i=0; i<coll_elements.getLength(); i++) {
+	Element e = (Element)coll_elements.item(i);
+	String elem_name = e.getAttribute(GSXML.NAME_ATT);
+	Element main_elem = GSXML.getNamedElement(element_list, OAIXML.ELEMENT, GSXML.NAME_ATT, elem_name);
+	if (main_elem == null) {
+	  logger.error(elem_name+" not found in meta format, not using it");
+	} else {
+	  element_list.replaceChild(doc.importNode(e, true),main_elem );
+      }
+      }
+    }
+
+    /** goes through all the mappings and makes a set of all collection
+	metadata names that could become an oai meta element - acts as
+	a reverse lookup for the mappings */
+    protected HashSet<String> getAllCollectionElements(Element meta_format) {
+      HashSet<String> meta_name_set = new HashSet<String>();
+      NodeList elements = meta_format.getElementsByTagName(OAIXML.ELEMENT);
+      for (int i=0; i<elements.getLength(); i++) {
+	Element e = (Element)elements.item(i);
+	Element map = (Element)GSXML.getChildByTagName(e, OAIXML.MAPPING);
+	if (map == null) {
+	  // there is no mapping, just use the element name
+	  meta_name_set.add(e.getAttribute(GSXML.NAME_ATT));
+	} else {
+	  String list_of_names = map.getAttribute(OAIXML.ELEMENTS);
+	  String[] name_array = list_of_names.split(",");
+	  for (int j=0; j<name_array.length; j++) {
+	    meta_name_set.add(name_array[j]);
+	  }
+	}
+      }
+      return meta_name_set;
+    }
+   
+  /** returns a specific service description */
+  public Element getServiceDescription(Document doc, String service_id, String lang, String subset) {
+    
+    if (service_id.equals(OAIXML.LIST_RECORDS)) {
+      Element list_records = doc.createElement(GSXML.SERVICE_ELEM);
+      list_records.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_RECORDS);
+      list_records.setAttribute(GSXML.TYPE_ATT, "oai");
+      return list_records;
+    }
+    
+    if (service_id.equals(OAIXML.LIST_IDENTIFIERS)) {
+      Element list_identifiers = doc.createElement(GSXML.SERVICE_ELEM);
+      list_identifiers.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_IDENTIFIERS);
+      list_identifiers.setAttribute(GSXML.TYPE_ATT, "oai");
+      return list_identifiers;
+    }
+    if (service_id.equals(OAIXML.LIST_SETS)) {
+      Element list_sets = doc.createElement(GSXML.SERVICE_ELEM);
+      list_sets.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_SETS);
+      list_sets.setAttribute(GSXML.TYPE_ATT, "oai");
+      return list_sets;
+    }
+    if (service_id.equals(OAIXML.LIST_METADATA_FORMATS)) {
+      Element list_metadata_formats = doc.createElement(GSXML.SERVICE_ELEM);
+      list_metadata_formats.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_METADATA_FORMATS);
+      list_metadata_formats.setAttribute(GSXML.TYPE_ATT, "oai");
+      return list_metadata_formats;
+    }
+    
+    if (service_id.equals(OAIXML.GET_RECORD)) {
+      Element get_record = doc.createElement(GSXML.SERVICE_ELEM);
+      get_record.setAttribute(GSXML.NAME_ATT, OAIXML.GET_RECORD);
+      get_record.setAttribute(GSXML.TYPE_ATT, "oai");
+      return get_record;
+    }
+    
+    return null;
+  }
+
+  /** The list sets service returns all the sets that this collection is/is part of/contains. This is gathered by Receptionist from all collections to answer the OAI ListSets request.  */
+  protected Element processListSets(Element req) {
+    return list_sets_response;
+  }
+  /** returns the actual record element used in the OAI GetRecord response */
+  protected Element processGetRecord(Element req) {
+    /** arguments:
+        identifier: required
+        metadataPrefix: required
+     *  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
+     */ 
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+    HashMap<String, String> param_map = GSXML.getParamMap(params);    
+    
+    String prefix = param_map.get(OAIXML.METADATA_PREFIX);
+    if (prefix == null || prefix.equals("")) {
+      //Just a double-check
+      logger.error("the value of metadataPrefix att is not present in the request.");
+      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+    
+    // check that we support this format
+    if (!format_response_map.containsKey(prefix)) {
+      logger.error("metadata prefix is not supported for collection "+this.coll_name);
+      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+
+    Document doc = XMLConverter.newDOM();
+    
+    String oid = param_map.get(OAIXML.OID); // TODO should this be identifier???
+    boolean OID_is_deleted = false;
+    long millis = -1;
+
+    DBInfo oai_info = null;
+    if(oaiinf_db != null) {
+	oai_info = this.oaiinf_db.getInfo(oid);
+	if (oai_info == null) {
+	    logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
+	} else  {
+	    String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
+	    if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
+		OID_is_deleted = true;
+
+		// get the right timestamp for deletion: from oaiinf db
+		String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // in seconds presumably, like oailastmodified in the collection index db		
+		
+		millis = Long.parseLong(timestamp)*1000; // in milliseconds
+	    }
+	}
+    }
+
+    //get a DBInfo object of the identifier; if this identifier is not present in the database,
+    // null is returned.
+    DBInfo info = this.coll_db.getInfo(oid);
+    if (info == null) {
+      logger.error("OID: " + oid + " is not present in the collection database.");
+      //return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, ""); // not an error: may exist as deleted (marked 'D') in oai-inf db
+    }
+    else if (millis == -1) { // so !OID_is_deleted, get oailastmodified from collection's index db
+	ArrayList<String> keys = new ArrayList<String>(info.getKeys());	
+	millis = getDateStampMillis(info);	
+    }
+    String oailastmodified = (millis == -1) ? "" : OAIXML.getTime(millis);
+    
+
+    Element get_record_response = doc.createElement(GSXML.RESPONSE_ELEM);
+    Element get_record = doc.createElement(OAIXML.GET_RECORD);
+    get_record_response.appendChild(get_record);
+    Element record = doc.createElement(OAIXML.RECORD);
+    //compose the header element
+    record.appendChild(createHeaderElement(doc, oid, oailastmodified, OID_is_deleted));      
+    if(!OID_is_deleted) {
+	//compose the metadata element
+	record.appendChild(createMetadataElement(doc, prefix, info));
+    }
+    get_record.appendChild(record);
+    return get_record_response;
+  }
+
+  /** return a list of records in specified set, containing metadata from specified prefix*/
+  protected Element processListRecords(Element req) {
+    return processListIdentifiersOrRecords(req, OAIXML.LIST_RECORDS, true);
+  }
+
+  /** return a list of identifiers in specified set that contain metadata belonging to specified prefix. */
+  protected Element processListIdentifiers(Element req) {
+    return processListIdentifiersOrRecords(req, OAIXML.LIST_IDENTIFIERS, false);
+  }
+
+  // Get a list of records/identifiers that match the parameters.
+  protected Element processListIdentifiersOrRecords(Element req, String response_name, boolean include_metadata) {
+    /** arguments:
+        metadataPrefix: required
+     *  from: optional
+     *  until: optional
+     *  set: optional
+     *  resumptionToken: exclusive and optional (ignored as it has been handled by OAIReceptionist)
+     *  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
+     */ 
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+    
+    if(params.getLength() == 0) {
+      logger.error("must at least have the metadataPrefix parameter, can't be none");
+      return OAIXML.createErrorResponse(OAIXML.BAD_ARGUMENT, "");
+    }
+    
+    HashMap<String, String> param_map = GSXML.getParamMap(params);  
+    
+    String prefix = "";
+    Date from_date = null;
+    Date until_date = null;
+    
+    if(param_map.containsKey(OAIXML.METADATA_PREFIX) == false) {    
+      //Just a double-check
+      logger.error("A param element containing the metadataPrefix is not present.");
+      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+    prefix = param_map.get(OAIXML.METADATA_PREFIX);
+    if (prefix == null || prefix.equals("")) {
+      //Just a double-check
+      logger.error("the value of metadataPrefix att is not present in the request.");
+      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+    
+    if(param_map.containsKey(OAIXML.FROM)) {
+      String from = param_map.get(OAIXML.FROM);
+      from_date = OAIXML.getDate(from);
+    }    
+    if(param_map.containsKey(OAIXML.UNTIL)) {
+      String until = param_map.get(OAIXML.UNTIL);
+      until_date = OAIXML.getDate(until);
+    }    
+
+    if (!format_response_map.containsKey(prefix)) {
+      logger.error(prefix + " metadata prefix is not supported for collection "+this.coll_name);
+      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
+    }
+
+    // get list of oids
+    ArrayList<String> oid_list = null;
+    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
+	oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
+    
+	if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
+	    logger.warn("@@@@@@@@@@@@@ NO OIDs in oai-inf db for " + this.cluster_name);
+	    oid_list = getChildrenIds(OAIXML.BROWSELIST);
+	}
+    }
+
+    if (oid_list == null) {
+      logger.error("No matched records found in collection: oai-inf and index db's browselist are empty");
+      return OAIXML.createErrorResponse(OAIXML.NO_RECORDS_MATCH, "");
+    }
+    // all validation is done
+
+    // get the list of elements that are in this metadata prefix
+    HashSet<String> set_of_elems = format_elements_map.get(prefix);
+
+    Document doc = XMLConverter.newDOM();
+    Element list_items_response = doc.createElement(GSXML.RESPONSE_ELEM);
+    Element list_items = doc.createElement(response_name);
+    list_items_response.appendChild(list_items);
+
+    for(int i=0; i<oid_list.size(); i++) {
+      String oid = oid_list.get(i);
+      boolean OID_is_deleted = false;
+      long millis = -1;
+
+      DBInfo oai_info = null;
+      if(oaiinf_db != null) {
+	  oai_info = this.oaiinf_db.getInfo(oid);
+	  if (oai_info == null) {
+	      logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
+	  } else  {
+	      String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
+	      if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
+		  OID_is_deleted = true;
+		  
+		  // get the right timestamp for deletion: from oaiinf db
+		  String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // in seconds presumably, like oailastmodified in the collection index db		
+		  
+		  millis = Long.parseLong(timestamp)*1000; // in milliseconds
+	      }
+	  }
+      }
+      DBInfo info = this.coll_db.getInfo(oid);
+      if (info == null) { // can happen if oid was deleted, in which case only oai_info keeps a record of the oid
+        logger.error("Collection database does not contain information about oid: " +oid);
+      }
+      else if (millis == -1) { // so !OID_is_deleted, get oailastmodified from collection's index db
+	  
+	  millis = getDateStampMillis(info);
+      }
+
+      Date this_date = null;
+      if (millis == -1) {
+	  if (from_date != null || until_date !=null) {
+	      continue; // if this doc doesn't have a date for some reason, and
+	      // we are doing a date range, then don't include it.
+	  }
+      } else {
+	  this_date = new Date(millis);
+	  if (from_date != null) {
+	      if(this_date.before(from_date)) {
+		  continue;
+	      }
+	  }
+	  if (until_date != null) {
+	      if (this_date.after(until_date)) {
+		  continue;
+	      }
+	  }    
+      }
+      
+      //compose the header element, which we'll be appending no matter what
+      Element header = createHeaderElement(doc, oid, OAIXML.getTime(millis), OID_is_deleted);
+
+      if (include_metadata) { // doing ListRecords
+	  // compose a record for adding header and metadata
+	  Element record = doc.createElement(OAIXML.RECORD);
+	  list_items.appendChild(record);
+	  //insert the header element
+	  record.appendChild(header);
+	  //Now check that this id has metadata for the required prefix.
+	  if (info != null && documentContainsMetadata(info, set_of_elems)) {
+	      // YES, it does have some metadata for this prefix	    
+	      //compose the metadata element
+	      record.appendChild(createMetadataElement(doc, prefix, info));
+	  } // otherwise the oid was 'deleted' and only in the oai-inf db and not in the info (collection index) db
+      } else { // doing ListIdentifiers
+	  //append the header element
+	  list_items.appendChild(header);      
+      }      
+      
+    }//end of for(int i=0; i<oid_list.size(); i++) of doing thru each record
+    
+    return list_items_response;        
+    
+  }
+
+  
+  // have implemented setDescription as an element, instead of a container containing metadata
+  private boolean configureSetInfo() {
+
+    Document doc = XMLConverter.newDOM();
+    this.list_sets_response = doc.createElement(GSXML.RESPONSE_ELEM);
+    Element list_sets_elem = doc.createElement(OAIXML.LIST_SETS);
+    this.list_sets_response.appendChild(list_sets_elem);
+    String set_name = this.coll_name;
+    String set_description = null;
+    Element name_elem = (Element)GSXML.getChildByTagName(this.config_info, OAIXML.SET_NAME);
+    if (name_elem!=null) {
+      set_name = GSXML.getNodeText(name_elem);
+      if (set_name.equals("")) {
+	set_name = this.coll_name; // default to coll name if can't find one
+      }
+    }
+    Element description_elem = (Element)GSXML.getChildByTagName(this.config_info, OAIXML.SET_DESCRIPTION);
+    if (description_elem!=null) {
+      set_description = GSXML.getNodeText(description_elem);
+      if (set_description.equals("")) {
+	set_description = null;
+      }
+    }
+    Element coll_set = OAIXML.createSet(doc, this.coll_name, set_name, set_description);
+    list_sets_elem.appendChild(coll_set);
+    
+    // are we part of any super sets?
+    NodeList super_set_list = GSXML.getChildrenByTagName(this.config_info, OAIXML.OAI_SUPER_SET);
+    for (int i=0; i<super_set_list.getLength(); i++) {
+      String super_name = ((Element)super_set_list.item(i)).getAttribute(GSXML.NAME_ATT);
+      if (super_name != null && !super_name.equals("")) {
+	list_sets_elem.appendChild(OAIXML.createSet(doc, super_name, super_name, null));
+      }
+    }
+    return true;
+  }
+
+ /** create the metadata element used when processing ListRecords/GetRecord requests
+   */
+  protected Element createMetadataElement(Document doc, String prefix, DBInfo info) {
+    // the <metadata> element
+    Element metadata = doc.createElement(OAIXML.METADATA);
+    // the <oai:dc namespace...> element
+    Element prfx_str_elem = OAIXML.getMetadataPrefixElement(doc, prefix, OAIXML.oai_version);
+    metadata.appendChild(prfx_str_elem);
+
+    Element meta_format_element = GSXML.getNamedElement(this.meta_formats_definition, OAIXML.METADATA_FORMAT, GSXML.NAME_ATT, prefix);
+    NodeList elements = meta_format_element.getElementsByTagName(OAIXML.ELEMENT);
+    // for each element in the definition
+    for (int i=0; i<elements.getLength(); i++) {
+      Element e = (Element)elements.item(i);
+      Element map = (Element)GSXML.getChildByTagName(e, OAIXML.MAPPING);
+      if (map == null) {
+	// look up the element name
+	addMetadata(prfx_str_elem, e.getAttribute(GSXML.NAME_ATT), info);
+      } else {
+	// we go though the list of names in the mapping
+	addMetadata(prfx_str_elem, e.getAttribute(GSXML.NAME_ATT), map.getAttribute(OAIXML.SELECT), map.getAttribute(OAIXML.ELEMENTS), info);
+      }
+    }
+    // output any metadata that is not just a simple mapping
+    addCustomMetadata(prfx_str_elem, prefix, info);
+    return metadata;
+  }
+
+  /** a simple addMetadata where we look for meta_name metadata, and add as that name*/
+  protected void addMetadata(Element meta_list_elem, String meta_name, DBInfo info) {
+    Vector<String> values = info.getMultiInfo(meta_name);
+    if (values != null && values.size()!=0) {
+      for (int i=0; i<values.size(); i++) {
+	addMetadataElement(meta_list_elem, meta_name, values.get(i));
+      }
+    }
+  }
+
+  /** more complicated addMetadata - can add multiple items. */
+  protected void addMetadata(Element meta_list_elem, String new_meta_name, String select_type, String name_list, DBInfo info) {
+    String[] names = name_list.split(",");
+    for (int i=0; i<names.length; i++) {
+      Vector<String> values = info.getMultiInfo(names[i]);
+      if (values == null || values.size()==0) {
+	continue;
+      }
+      for (int j=0; j<values.size(); j++) {
+	addMetadataElement(meta_list_elem, new_meta_name, values.get(j));
+	if (select_type.equals(OAIXML.SELECT_SINGLE_VALUE)) {
+	  return; // only want to add one value
+	}
+      }
+      if (select_type.equals(OAIXML.SELECT_FIRST_VALID_META)) {
+	return; // we have added all values of this meta elem
+      }
+      // otherwise, we will keep going through the list and add them all.
+    }
+  }
+  
+  // specific metadata formats might need to do some custom metadata that is not
+  //just a standard mapping. eg oai_dc outputting an identifier that is a link
+  protected void addCustomMetadata(Element meta_list_elem, String prefix, DBInfo info) {
+    
+    if (prefix.equals(OAIXML.META_FORMAT_DC)) {
+      // we want to add in another dc:identifier element with a link to the resource if possible
+      // try gs.OAIResourceURL first, then srclinkFile, then GS version of documnet
+      String gsURL = info.getInfo(OAIXML.GS_OAI_RESOURCE_URL);
+      if (gsURL.equals("")) {
+	String base_url = OAIXML.getBaseURL(); // e.g. e.g. http://host:port/greenstone3/oaiserver
+	// try srclinkFile
+	gsURL = info.getInfo("srclinkFile");
+	if (!gsURL.equals("")) {
+	  // make up the link to the file
+	  gsURL = base_url.replace("oaiserver", "") + "sites/" + this.site_name 
+	    + "/collect/" + this.coll_name + "/index/assoc/" 
+	    + info.getInfo("assocfilepath") + "/" + gsURL; 
+	} else {
+	  // no srclink file, lets provide a link to the greenstone doc
+	  gsURL = base_url.replace("oaiserver", "library") + "/collection/" + this.coll_name + "/document/" + info.getInfo("Identifier");
+	}
+      }
+      // now we have the url link, add as metadata
+      addMetadataElement(meta_list_elem, "dc:identifier", gsURL);
+    }
+  }
+
+  /** create the actual metadata element for the list */
+  protected void addMetadataElement(Element meta_list_elem, String name, String value) {
+    
+    Element meta = GSXML.createTextElement(meta_list_elem.getOwnerDocument(), name, value);
+    meta_list_elem.appendChild(meta);
+  }
+
+
+  /** create a header element used when processing requests like ListRecords/GetRecord/ListIdentifiers
+   */  
+  protected Element createHeaderElement(Document doc, String oid, String oailastmodified, boolean deleted) {
+
+        Element header = doc.createElement(OAIXML.HEADER);
+	
+	// if deleted, get the date and change oailastmodified to timestamp in oaiinfo
+	if(deleted) {
+	    header.setAttribute(OAIXML.OAI_INF_STATUS, OAIXML.HEADER_STATUS_ATTR_DELETED); // set the header status to deleted
+		// then the timestamp for deletion will be from oai-inf database 
+	}
+    
+        Element identifier = doc.createElement(OAIXML.IDENTIFIER);
+	GSXML.setNodeText(identifier, coll_name + ":" + oid);
+        header.appendChild(identifier);
+        Element set_spec = doc.createElement(OAIXML.SET_SPEC);
+	GSXML.setNodeText(set_spec, coll_name);
+        header.appendChild(set_spec);
+        Element datestamp = doc.createElement(OAIXML.DATESTAMP);
+        GSXML.setNodeText(datestamp, oailastmodified);
+        header.appendChild(datestamp);
+        return header;
+  }
+
+  /** return the metadata information  */
+  protected Element processListMetadataFormats(Element req) {
+    // the request sent here must contain an OID. see doListMetadataFormats() in OAIReceptionist
+    Element param = GSXML.getNamedElement(req, GSXML.PARAM_ELEM, GSXML.NAME_ATT, OAIXML.OID);
+    if (param == null) {
+      logger.error("An element containing the OID attribute not is present.");
+      return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
+    }
+    String oid = param.getAttribute(GSXML.VALUE_ATT);
+    if (oid == null || oid.equals("")) {
+      logger.error("No OID is present in the request.");
+      return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
+    }
+
+    /*
+    ArrayList<String> oid_list = null;
+    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
+	oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
+	
+	if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
+	    oid_list = getChildrenIds(OAIXML.BROWSELIST);
+	}
+    }
+    */
+    // assume meta formats are only for OIDs that have not been deleted
+    // so don't need to check oai-inf db, and can just check collection's index db for list of OIDs
+    ArrayList<String> oid_list = getChildrenIds(OAIXML.BROWSELIST);
+    if (oid_list == null || oid_list.contains(oid) == false) {
+      logger.error("OID: " + oid + " is not present in the database.");
+      Element e= OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
+//      logger.error((new XMLConverter()).getPrettyString (e));
+      return e;
+    }
+    
+    DBInfo info = null;    
+    info = this.coll_db.getInfo(oid);
+    if (info == null) { //just double check
+      return OAIXML.createErrorResponse(OAIXML.OAI_SERVICE_UNAVAILABLE, "");
+    }
+    
+    Document doc = XMLConverter.newDOM();
+    Element list_metadata_formats_response = doc.createElement(GSXML.RESPONSE_ELEM);
+    
+    Element list_metadata_formats = doc.createElement(OAIXML.LIST_METADATA_FORMATS);
+    list_metadata_formats_response.appendChild(list_metadata_formats);
+    boolean has_meta_format = false;
+    
+    // for each format in format_elements_map
+    Iterator<String> it = format_elements_map.keySet().iterator();
+    while (it.hasNext()) {
+      String format = it.next();
+      HashSet<String> set_of_elems = format_elements_map.get(format);
+      if (documentContainsMetadata(info, set_of_elems)) {
+	// add this format into the response
+	has_meta_format = true;
+	list_metadata_formats.appendChild(doc.importNode(format_response_map.get(format), true));
+      }
+    }
+
+    if (has_meta_format == false) {
+      logger.error("Specified metadata names are not contained in the database.");
+      return OAIXML.createErrorResponse(OAIXML.NO_METADATA_FORMATS, "");
+    } else {
+      return list_metadata_formats_response;
+    }
+  }
+
+  protected boolean documentContainsMetadata(DBInfo info, HashSet<String> set_of_elems) {
+    if (set_of_elems.size() == 0) {
+      return false;
+    }
+    Iterator<String> i = set_of_elems.iterator();
+    while (i.hasNext()) {
+      if (!info.getInfo(i.next()).equals("")) {
+	return true;
+      }
+    }
+    return false;
+  }
+
+  /** returns a list of the child ids in order, null if no children */
+  protected ArrayList<String> getChildrenIds(String node_id) {
+    DBInfo info = this.coll_db.getInfo(node_id);
+    if (info == null) {
+      return null;
+    }
+    
+    String contains = info.getInfo("contains");
+    if (contains.equals("")) {
+      return null;
+    }
+    ArrayList<String> children = new ArrayList<String>();
+    StringTokenizer st = new StringTokenizer(contains, ";");
+    while (st.hasMoreTokens()) {
+      String child_id = st.nextToken().replaceAll("\"", node_id);
+      children.add(child_id);
+    }
+    return children;    
+  }
+  /**method to check whether any of the 'metadata_names' is contained in the 'info'.
+   * The name may be in the form: <name>,<mapped name>, in which the mapped name is
+   * optional. The mapped name is looked up in the DBInfo; if not present, use the first
+   * name which is mandatory.
+   */
+  protected boolean containsMetadata(DBInfo info, String[] metadata_names) {
+    if (metadata_names == null) return false;
+    logger.info("checking metadata names in db.");
+    for(int i=0; i<metadata_names.length; i++) {
+      int index = metadata_names[i].indexOf(",");
+      String meta_name = (index == -1) ? metadata_names[i] :
+                              metadata_names[i].substring(index + 1);
+      
+      if(info.getInfo(meta_name).equals("") == false) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  protected long getDateStampMillis(DBInfo info) {
+    // gs.OAIDateStamp is in YYYY-MM-DD
+    String time_stamp = info.getInfo(OAIXML.GS_OAI_DATE_STAMP);
+    long millis = -1;
+    if (!time_stamp.equals("")) {
+      millis = OAIXML.getTime(time_stamp);
+    }
+    if (millis == -1) {
+      // oailastmodified is in seconds
+      time_stamp = info.getInfo(OAIXML.OAI_LASTMODIFIED);
+      if (!time_stamp.equals("")) {
+	millis = Long.parseLong(time_stamp)*1000;
+      }
+    }
+    return millis;
+    
+
+  }
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/PhindPhraseBrowse.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/PhindPhraseBrowse.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/PhindPhraseBrowse.java	(revision 31634)
@@ -0,0 +1,477 @@
+/*
+ *    PhindServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+
+import org.greenstone.mgpp.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.Text;
+
+import java.util.Vector;
+import java.util.HashMap;
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.log4j.*;
+
+/**
+ * PhindServices - the phind phrase browsing service
+ *
+ */
+public class PhindPhraseBrowse
+  extends ServiceRack {
+  
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.PhindPhraseBrowse.class.getName());
+  
+  // the services on offer
+  private static final String PHIND_SERVICE = "PhindApplet";
+  
+  private static MGPPRetrieveWrapper mgpp_retrieve_src=null;
+  private static MGPPSearchWrapper mgpp_search_src=null;
+  private String basepath = null;
+  
+  private Element applet_description = null;
+  
+  public PhindPhraseBrowse() {
+    if(this.mgpp_retrieve_src == null) {
+      this.mgpp_retrieve_src = new MGPPRetrieveWrapper();
+    }
+    if(this.mgpp_search_src == null) {
+      this.mgpp_search_src = new MGPPSearchWrapper();
+    }
+    // set up the default params
+    this.mgpp_search_src.setQueryLevel("Document");
+    this.mgpp_search_src.setReturnLevel("Document");
+    this.mgpp_search_src.setMaxDocs(5);
+    this.mgpp_search_src.setStem(false);
+    this.mgpp_search_src.setCase(true);
+  }
+  
+  public void cleanUp() {
+    super.cleanUp();
+    this.mgpp_search_src.unloadIndexData();
+  }
+  
+  /** configure the service module
+   *
+   * @param info a DOM Element containing any config info for the service
+   * @return true if configured
+   */
+  public boolean configure(Element info, Element extra_info) {
+    
+    if (!super.configure(info, extra_info)){
+      return false;
+    }
+    
+    logger.info("configuring PhindPhraseBrowse");
+    
+    // set up short_service_info_ - for now just has name and type
+    Element e = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+    e.setAttribute(GSXML.NAME_ATT, PHIND_SERVICE);
+    this.short_service_info.appendChild(e);
+    
+    // set up the static applet description
+    
+    applet_description = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+    applet_description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_APPLET);
+    applet_description.setAttribute(GSXML.NAME_ATT, PHIND_SERVICE);
+    
+    // add in the applet info for the phind applet
+    // need to make this dynamic - library names etc
+    // change the applet params - have a single param with the library name
+    // this is left blank at this end, and must be filled in by applet action - if the library name is not needed, this param is left out
+    // phindcgi param now is not complete - library must be prepended to it.
+    String app_info = "<"+GSXML.APPLET_ELEM+" CODEBASE='applet' CODE='org.greenstone.applet.phind.Phind.class' ARCHIVE='phind.jar, xercesImpl.jar, xml-apis.jar' WIDTH='500' HEIGHT='400'><PARAM NAME='library' VALUE=''/> <PARAM NAME='phindcgi' VALUE='?";
+    app_info += GSParams.ACTION +"=a&amp;"+GSParams.REQUEST_TYPE +"=r&amp;"+GSParams.SERVICE+"="+PHIND_SERVICE+"&amp;"+GSParams.OUTPUT+"=xml&amp;"+GSParams.RESPONSE_ONLY+"=1'/>";
+    app_info +="<PARAM NAME='collection'   VALUE='";
+    app_info += this.cluster_name;
+    app_info += "'/> <PARAM NAME='classifier' VALUE='1'/>  <PARAM NAME='orientation'  VALUE='vertical'/> <PARAM NAME='depth' VALUE='2'/> <PARAM NAME='resultorder' VALUE='L,l,E,e,D,d'/> <PARAM NAME='backdrop' VALUE='interfaces/default/images/phindbg1.jpg'/><PARAM NAME='fontsize' VALUE='10'/> <PARAM NAME='blocksize'    VALUE='10'/>The Phind java applet.</"+GSXML.APPLET_ELEM+">";
+    
+    Document dom = this.converter.getDOM(app_info);
+    if (dom==null) {
+      logger.error("Couldn't parse applet info");
+      return false;
+    }
+    Element app_elem = dom.getDocumentElement();
+    applet_description.appendChild(this.desc_doc.importNode(app_elem, true));
+    
+    return true;
+  }
+  
+  protected Element getServiceDescription(Document doc, String service, String lang, String subset) {
+    if (!service.equals(PHIND_SERVICE)) {
+      return null;
+    }
+    Element describe = (Element)doc.importNode(applet_description,true);
+	
+	Element el1 = GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME,  getTextString(PHIND_SERVICE+".name", lang));	
+    describe.appendChild(el1);
+	
+	Element el2 = GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION,  getTextString(PHIND_SERVICE+".description", lang));
+    describe.appendChild(el2);
+	
+    return describe;
+  }
+  
+  protected Element processPhindApplet(Element request) {
+    Document result_doc = XMLConverter.newDOM();
+    Element param_elem = (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+    HashMap<String, Serializable> params = GSXML.extractParams(param_elem, false);
+    
+    long first_e = Long.parseLong((String)params.get("pfe"));
+    long last_e = Long.parseLong((String)params.get("ple"));
+    long first_l = Long.parseLong((String)params.get("pfl"));
+    long last_l = Long.parseLong((String)params.get("pll"));
+    long first_d = Long.parseLong((String)params.get("pfd"));
+    long last_d = Long.parseLong((String)params.get("pld"));
+    
+    long phrase;
+    String phrase_str = (String)params.get("ppnum");
+    if (phrase_str == null || phrase_str.equals("")) {
+      phrase=0;
+    } else {
+      phrase = Long.parseLong(phrase_str);
+    }
+    String word = (String)params.get("pptext");
+    String phind_index = (String)params.get("pc");
+    // the location of the mgpp database files
+    this.basepath = GSFile.phindBaseDir(this.site_home, this.cluster_name, phind_index);
+    
+    // the result element
+    Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+    result.setAttribute(GSXML.FROM_ATT, PHIND_SERVICE);
+    result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+    
+    // applet result info must be in appletInfo element
+    Element applet_data = result_doc.createElement(GSXML.APPLET_DATA_ELEM);
+    result.appendChild(applet_data);
+    Element phind_data = result_doc.createElement("phindData");
+    applet_data.appendChild(phind_data);
+    
+    
+    // if we dont know the phrase number, look it up
+    if (phrase == 0) {
+      if (word==null || word.equals("")) {
+        Element error = phindError(result_doc, "no word or phrase");
+        phind_data.appendChild(error);
+        return result;
+      }
+      phrase = findPhraseNumberFromWord( word);
+    }
+    if (phrase==0) {
+      // the word is not in the collection
+      // return a phind error string
+      Element error = phindError(result_doc, "the term "+word+" is not in the collection");
+      phind_data.appendChild(error);
+      return result;
+    }
+    
+    // get the phrase data into the phind_data node
+    getPhraseData(phind_data, phrase, first_l, last_l,
+      first_e, last_e,  first_d, last_d);
+    return result;
+    
+    
+  }// processPhindApplet
+  
+  protected long findPhraseNumberFromWord(String word) {
+    synchronized (mgpp_search_src) {
+    	// set the mgpp index data - we are looking up pword
+    	mgpp_search_src.loadIndexData(this.basepath+File.separatorChar+"pword");
+    	
+    	mgpp_search_src.runQuery(word);
+    	
+    	MGPPQueryResult res = mgpp_search_src.getQueryResult();
+    	Vector docs = res.getDocs();
+    	if (docs.size()==0) {
+    		// phrase not found
+    		return 0;
+    	}
+    	MGPPDocInfo doc = (MGPPDocInfo)docs.firstElement();
+    	return doc.num_;
+    }
+  }
+  
+  protected boolean getPhraseData(Element phind_data,
+    long phrase, long first_l, long last_l,
+    long first_e, long last_e, long first_d,
+    long last_d) {
+    
+    synchronized (mgpp_retrieve_src) {
+    String record = this.mgpp_retrieve_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+      phrase);
+    if (record.equals("")) {
+      Element error = phindError(phind_data.getOwnerDocument(), "somethings gone wrong - we haven't got a record for phrase number "+phrase);
+      phind_data.appendChild(error);
+      return false;
+    }
+    
+    // parse the record - its in gordons cryptic form
+    // ":word:tf:ef:df:el:dl:lf:ll"
+    // el: e,e,e
+    // dl: d;f,d;f,
+    // lf and ll may be null
+    // l: type,dest, dest; type,dest,dest
+    
+    // ignore everything up to and including first colon (has
+    // <Document>3505: at the start)
+    record = record.substring(record.indexOf(':')+1);
+    
+    // split on ':'
+    String [] fields = record.split(":");
+    String word = fields[0];
+    String tf = fields[1];
+    String ef = fields[2];
+    String df = fields[3];
+    
+    
+    String expansions = fields[4];
+    String documents = fields[5];
+    String lf = "0";
+    String linklist = "";
+    if (fields.length > 7) {// have thesaurus stuff
+      lf =fields[6];
+      linklist = fields[7];
+    }
+    
+    // the phindData attributes and phrase
+    phind_data.setAttribute("id", Long.toString(phrase));
+    phind_data.setAttribute("df", df);
+    phind_data.setAttribute("ef", ef);
+    phind_data.setAttribute("lf", lf);
+    phind_data.setAttribute("tf", tf);
+    // GSXML.createTextElement(result_doc, "phrase", word); ??? - this needs to be appended somewhere????
+    
+    addExpansionList(phind_data, expansions, word, ef, first_e, last_e);
+    addDocumentList(phind_data, documents, word, df, first_d, last_d);
+    if (!lf.equals("0")) {
+      addThesaurusList(phind_data, linklist, word, lf, first_l, last_l);
+    }
+    return true;
+    } // end of synchronized (mgpp_retrieve_src) 
+  }
+  
+  protected boolean addExpansionList( Element phind_data, String record,
+    String word,
+    String freq,
+    long first, long last) {
+    Document phind_doc = phind_data.getOwnerDocument();
+    Element expansion_list = phind_doc.createElement("expansionList");
+    phind_data.appendChild(expansion_list);
+    expansion_list.setAttribute("length", freq);
+    expansion_list.setAttribute("start", Long.toString(first));
+    expansion_list.setAttribute("end", Long.toString(last));
+    
+    // get the list of strings
+    String [] expansions = record.split(",");
+    int length = expansions.length;
+    if (length < last) last = length;
+    for (long i = first; i < last; i++) {
+      long num  = Long.parseLong(expansions[(int)i]);
+      Element expansion = getExpansion(phind_doc, num, word);
+      expansion.setAttribute("num", Long.toString(i));
+      expansion_list.appendChild(expansion);
+    }
+    return true;
+  }
+  
+  protected Element getExpansion(Document phind_doc, long phrase_num,
+    String orig_phrase) {
+    
+    // look up the phrase in the pdata thingy
+    String record = this.mgpp_retrieve_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+      phrase_num);
+    
+    if (record ==null || record.equals("")) return null;
+    
+    // ignore everything up to and including first colon
+    record = record.substring(record.indexOf(':')+1);
+    
+    String [] fields = record.split(":");
+    String phrase = fields[0];
+    String tf = fields[1];
+    //String ef = fields[2]; dont use this
+    String df = fields[3];
+    
+    Element expansion = phind_doc.createElement("expansion");
+    expansion.setAttribute("tf", tf);
+    expansion.setAttribute("df", df);
+    expansion.setAttribute("id", Long.toString(phrase_num));
+    
+    // get teh suffix and prefix
+    String [] ends = splitPhraseOnWord(phrase, orig_phrase);
+    if (!ends[0].equals("")) {
+      expansion.appendChild(GSXML.createTextElement(phind_doc, "prefix", ends[0]));
+    }
+    if (!ends[1].equals("")) {
+      expansion.appendChild(GSXML.createTextElement(phind_doc, "suffix", ends[1]));
+    }
+    
+    return expansion;
+    
+  }
+  
+  protected boolean addDocumentList(Element phind_data, String record,
+    String word,
+    String freq,
+    long first, long last) {
+    Document phind_doc = phind_data.getOwnerDocument();
+    Element document_list = phind_doc.createElement("documentList");
+    phind_data.appendChild(document_list);
+    document_list.setAttribute("length", freq);
+    document_list.setAttribute("start", Long.toString(first));
+    document_list.setAttribute("end", Long.toString(last));
+    
+    // get the list of doc,freq
+    String [] doc_freqs = record.split(";");
+    int length = doc_freqs.length;
+    if (length<last) last=length;
+    
+    for (long i = first; i < last; i++) {
+      String doc_elem = doc_freqs[(int)i];
+      int p = doc_elem.indexOf(',');
+      long doc_num;
+      String doc_freq;
+      if (p == -1) { // there is no freq in the record
+        doc_num =Long.parseLong(doc_elem);
+        doc_freq = "1";
+      } else {
+        doc_num = Long.parseLong(doc_elem.substring(0,p));
+        doc_freq = doc_elem.substring(p+1);
+      }
+      Element document = getDocument(phind_doc, doc_num);
+      document.setAttribute("freq", doc_freq);
+      document.setAttribute("num", Long.toString(i));
+      document_list.appendChild(document);
+    }
+    
+    
+    return true;
+  }
+  
+  
+  protected Element getDocument(Document phind_doc, long doc_num) {
+    
+    // look up the phrase in the docs thingy
+    String record = this.mgpp_retrieve_src.getDocument(this.basepath+File.separatorChar+"docs", "Document",
+      doc_num);
+    
+    if (record ==null || record.equals("")) return null;
+    
+    // ignore everything up to and including first \t
+    record = record.substring(record.indexOf('\t')+1);
+    
+    String [] fields = record.split("\t");
+    String hash = fields[0];
+    String title = fields[1];
+    
+    Element d = phind_doc.createElement("document");
+    d.setAttribute("hash", hash);
+    d.appendChild(GSXML.createTextElement(phind_doc, "title", title));
+    
+    return d;
+    
+  }
+  protected boolean addThesaurusList(Element phind_data, String record,
+    String word,
+    String freq,
+    long first, long last) {
+    
+    Document phind_doc = phind_data.getOwnerDocument();
+    Element thesaurus_list = phind_doc.createElement("thesaurusList");
+    phind_data.appendChild(thesaurus_list);
+    thesaurus_list.setAttribute("length", freq);
+    thesaurus_list.setAttribute("start", Long.toString(first));
+    thesaurus_list.setAttribute("end", Long.toString(last));
+    
+    // get the list of type,dest,dest
+    String [] links = record.split(";");
+    int length = links.length;
+    long index = 0;
+    for (int i = 0; i < length; i++) { // go through the entries
+      String link_info = links[(int)i];
+      String [] items = link_info.split(",");
+      // the first entry is teh type
+      String type = items[0];
+      for (int j = 1; j<items.length; j++, index++) {
+        if (index >= first && index < last) { // only output the ones we want
+          long phrase = Long.parseLong(items[j]);
+          Element t = getThesaurus(phind_doc, phrase);
+          t.setAttribute("type", type);
+          thesaurus_list.appendChild(t);
+        }
+      }
+    }
+    
+    return true;
+  }
+  
+  protected Element getThesaurus(Document phind_doc, long phrase_num) {
+    
+    // look up the phrase in the pdata thingy
+    String record = this.mgpp_retrieve_src.getDocument(this.basepath+File.separatorChar+"pdata", "Document",
+      phrase_num);
+    
+    if (record ==null || record.equals("")) return null;
+    
+    // ignore everything up to and including first colon
+    record = record.substring(record.indexOf(':')+1);
+    
+    String [] fields = record.split(":");
+    String phrase = fields[0];
+    String tf = fields[1];
+    //String ef = fields[2]; dont use this
+    String df = fields[3];
+    
+    Element thesaurus = phind_doc.createElement("thesaurus");
+    thesaurus.setAttribute("tf", tf);
+    thesaurus.setAttribute("df", df);
+    thesaurus.setAttribute("id", Long.toString(phrase_num));
+    thesaurus.appendChild(GSXML.createTextElement(phind_doc, "phrase", phrase));
+    return thesaurus;
+    
+  }
+  
+  /** returns an array of two elements - the prefix and the suffix*/
+  protected String [] splitPhraseOnWord(String phrase, String word) {
+    
+    if (word.equals("")) {
+      
+      String [] res =  {phrase, ""};
+      return res;
+    }
+    // use 2 so that we only split on the first occurrance. trailing empty strings should be included
+    String [] result = phrase.split(word, 2);
+    return result;
+    
+  }
+  
+  protected Element phindError(Document phind_doc, String message) {
+    Element e = phind_doc.createElement("phindError");
+    Text t = phind_doc.createTextNode(message);
+    e.appendChild(t);
+    return e;
+  }
+  
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/RSSRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/RSSRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/RSSRetrieve.java	(revision 31634)
@@ -0,0 +1,294 @@
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.collection.Collection;
+import org.greenstone.util.GlobalProperties;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Attr;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.ProcessingInstruction;
+
+// General Java classes
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.Serializable;
+import java.util.Vector;
+import java.util.HashMap;
+import java.util.Date;
+import java.text.SimpleDateFormat;
+
+import org.apache.log4j.*;
+
+public class RSSRetrieve extends ServiceRack {
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.RSSRetrieve.class.getName());
+    protected static final String RSS_SERVICE = "RSSFeed";
+
+    public boolean configure(Element info, Element extra_info) {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+	logger.info("configuring RSSRetrieve...");	
+	
+	// set up short_service_info_ - for now just has name and type
+	Element rss_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+	rss_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	rss_service.setAttribute(GSXML.NAME_ATT, RSS_SERVICE);
+	this.short_service_info.appendChild(rss_service);
+
+	return true;
+    }
+
+    // this may get called but is not useful in the case of retrieve services
+    protected Element getServiceDescription(Document doc, String service_id, String lang, String subset) {
+
+	Element rss_service = doc.createElement(GSXML.SERVICE_ELEM);
+	rss_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	rss_service.setAttribute(GSXML.NAME_ATT, service_id);
+	return rss_service;
+    }
+    
+    // Sends off a collection 'describe' message and returns the <collection> element of the response.
+    // This contains the collection meta from collectionConfig.xml. Used to construct header of RSS feed
+    protected Element getCollMetadata(UserContext userContext) {
+      
+      Document msg_doc = XMLConverter.newDOM();
+	Element mr_request_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
+	String to = this.cluster_name;
+	Element meta_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE, to, userContext);
+	mr_request_message.appendChild(meta_request);
+	Element meta_response = (Element) this.router.process(mr_request_message);
+	meta_response = (Element) GSXML.getChildByTagName(meta_response, GSXML.RESPONSE_ELEM);
+
+	NodeList nl = meta_response.getElementsByTagName(GSXML.COLLECTION_ELEM);
+	Element collectionEl = (Element) nl.item(0);
+	return collectionEl;
+    }
+
+
+    /**
+       Generates the RSS feed XML by creating the header and footer with the contents
+       of the the collection's index/rss-items.rdf file embedded in the middle.       
+       @return the RSS feed XML.
+       @see http://cyber.law.harvard.edu/rss/rss.html
+     */
+    protected Element processRSSFeed(Element request) {
+
+	// Ask the MessageRouter for this collection's colConfig metadata 
+	// from which the RSS header values will be constructed
+	UserContext userContext = new UserContext(request);
+	Element collMeta = getCollMetadata(userContext);
+	
+	//logger.error("**** collection metadata:");
+	//GSXML.elementToLogAsString(collMeta, true);
+	
+	// work out some commonly used variables such as lang and url_prefix
+	String lang = request.getAttribute("lang");
+	if(lang.equals("")) {
+	    lang = "en";
+	}
+	//Get baseUrl from request 
+	String baseURL = request.getAttribute("baseURL");
+	
+	// url_prefix is of the form http://domain/greenstone3/library/collection/_colname_/
+	//String url_prefix = GlobalProperties.getFullGSDL3WebAddress()+"/"+this.library_name+"/collection/"+this.cluster_name;
+	String url_prefix = baseURL+"library"+"/"+this.library_name+"/collection/"+this.cluster_name;
+
+	// generate the header and footer
+	Document rssDoc = XMLConverter.newDOM();
+	
+	Element rssNode = rssDoc.createElement("rss"); // rootnode
+	rssNode.setAttribute("version", "2.0");
+
+	String namespace_url = "http://www.w3.org/2000/xmlns/";
+	rssNode.setAttributeNS(namespace_url, "xmlns:content", "http://purl.org/rss/1.0/modules/content/");
+	rssNode.setAttributeNS(namespace_url, "xmlns:taxo", "http://purl.org/rss/1.0/modules/taxonomy/");
+	rssNode.setAttributeNS(namespace_url, "xmlns:dc", "http://purl.org/dc/elements/1.1/");
+	rssNode.setAttributeNS(namespace_url, "xmlns:syn", "http://purl.org/rss/1.0/modules/syndication/");
+	rssNode.setAttributeNS(namespace_url, "xmlns:admin", "http://webns.net/mvcb/");
+	rssDoc.appendChild(rssNode);
+
+	// Setting the preproccessing header line (Utf-8) will be done in web/interfaces' rss.xsl
+	//ProcessingInstruction procInstruction = doc.createProcessingInstruction("xml","version=\"1.0\"");
+	//rssDoc.appendChild(procInstruction);
+	
+	Element channelNode = rssDoc.createElement("channel");
+	rssNode.appendChild(channelNode);
+	
+	Element childNode = rssDoc.createElement("title");
+	GSXML.setNodeText(childNode, this.cluster_name); //_collectionname_
+	channelNode.appendChild(childNode);
+
+	// _httppageabout_: of form http://domain/greenstone3/library/collection/_colname_/page/about
+	childNode = rssDoc.createElement("link");	
+	GSXML.setNodeText(childNode, url_prefix+"/page/about"); // _httppageabout_
+	channelNode.appendChild(childNode);
+
+	// get the description string for the requested language, else fallback on en description if present
+	childNode = rssDoc.createElement("description");
+	NodeList descriptions = GSXML.getNamedElements(collMeta, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, GSXML.DISPLAY_TEXT_DESCRIPTION);
+	//Element descriptEl = GSXML.getNamedElement(collMeta, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, GSXML.DISPLAY_TEXT_DESCRIPTION);
+	Element descriptEl = null;
+	if(descriptions != null) {
+	    for (int i = 0; i < descriptions.getLength(); i++) {
+		Element e = (Element) descriptions.item(i);
+		if(e.getAttribute("lang").equals(lang)) {
+		    descriptEl = e;
+		    break; // found the description for the requested language, finish loop
+		} else if(e.getAttribute("lang").equals("en")) {
+		    descriptEl = e; // at least found english fall-back description, continue loop
+		}
+	    }
+	}
+	String description = (descriptEl == null) ? "none" : GSXML.getNodeText(descriptEl);
+	GSXML.setNodeText(childNode, description); //_collectionextra_
+	channelNode.appendChild(childNode);
+
+	childNode = rssDoc.createElement("language");
+	GSXML.setNodeText(childNode, lang); //_cgiargl_
+	channelNode.appendChild(childNode);
+
+
+	// RSS specification: http://cyber.law.harvard.edu/rss/rss.html
+	// pubDate is date of first publication of the item. Use collection.getEarliestDatestamp()
+	// lastBuildDate is the date the item was last modified. Use collection.getLastmodified()
+
+	//HashMap<String, ModuleInterface> module_map = this.router.getModuleMap();
+	//Collection coll = (Collection)module_map.get(this.cluster_name);
+	Collection coll = (Collection)serviceCluster;
+	SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z");
+
+	childNode = rssDoc.createElement("pubDate");
+	Date date = new Date(coll.getEarliestDatestamp()); // "Thu, 23 Aug 1999 07:00:00 GMT"
+	GSXML.setNodeText(childNode, dateFormat.format(date));
+	channelNode.appendChild(childNode);
+
+	childNode = rssDoc.createElement("lastBuildDate");
+	date = new Date(coll.getLastmodified()); // "Thu, 23 Aug 1999 16:20:26 GMT"	
+	GSXML.setNodeText(childNode, dateFormat.format(date));
+	channelNode.appendChild(childNode);
+
+	childNode = rssDoc.createElement("managingEditor");
+	Element e = GSXML.getNamedElement(collMeta, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, "creator");
+	String value = (e == null) ? "unknown" : GSXML.getNodeText(e);
+	GSXML.setNodeText(childNode, value); //_creator_
+	channelNode.appendChild(childNode);
+
+	childNode = rssDoc.createElement("webMaster");
+	e = GSXML.getNamedElement(collMeta, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, "maintainer");
+	value = (e == null) ? "unknown" : GSXML.getNodeText(e);
+	GSXML.setNodeText(childNode, value); //_maintainer_
+	channelNode.appendChild(childNode);
+
+
+	// <image> child of <channel> has title, url, link and description children
+	Element collIcon = GSXML.getNamedElement(collMeta, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, "icon");
+	if(collIcon != null) { // since there is a collection image, create an imageNode
+	    
+	    Node imageNode = rssDoc.createElement("image");
+	    channelNode.appendChild(imageNode);	
+
+	    childNode = rssDoc.createElement("title");
+	    GSXML.setNodeText(childNode, this.cluster_name); //_collectionname_
+	    imageNode.appendChild(childNode);
+
+	    // need full URL for collection icon. Its name is in <displayItem name="icon_name.ext"/>
+	    // URL is of the form domain/servlet/sites/localsite/collect/lucene-jdbm-demo/images/icon_name.ext
+	    childNode = rssDoc.createElement("url");	    
+	    String domain = GlobalProperties.getFullGSDL3WebAddress(); // remove servlet as it's included in site_http_address
+	    domain = domain.substring(0, domain.lastIndexOf("/"));
+	    String image_url = GSXML.getNodeText(collIcon); // name of image file
+	    image_url = domain+"/"+this.site_http_address+"/"+"/collect/"+this.cluster_name+"/images/"+image_url;
+	    GSXML.setNodeText(childNode, image_url); // _iconcollection_
+	    imageNode.appendChild(childNode);
+
+	    childNode = rssDoc.createElement("link");
+	    GSXML.setNodeText(childNode, url_prefix+"/page/about"); // _httppageabout_
+	    imageNode.appendChild(childNode);
+
+	    childNode = rssDoc.createElement("description");
+	    GSXML.setNodeText(childNode, description); //_collectionextra_
+	    imageNode.appendChild(childNode);
+	}
+
+
+	// now add the contents of rss-items.rdf as a child of channel, 
+	// passing in url_prefix for url resolution
+	Element rss_raw_data = loadDocument("rss-items.rdf", url_prefix);
+	if(rss_raw_data != null) {
+	    NodeList rss_items = rss_raw_data.getElementsByTagName("item");
+	    for(int i = 0; i < rss_items.getLength(); i++) {
+		channelNode.appendChild(rssDoc.importNode(rss_items.item(i), true));
+	    }
+	}
+
+	// generate the GS3 response message containing the RSS xml
+	Element result = rssDoc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, RSS_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	result.appendChild(rssNode); // body of <response> is simply the <rss> root element of the RSS feed
+	return result;
+	
+    }
+
+    // load contents of rss-items.rdf file into an XML doc in memory, after doing the url_prefix replacements
+    protected Element loadDocument(String doc_name, String url_prefix) {
+	String document_encoding = "UTF-8";
+
+	// try to find the document
+	File doc_file = new File(GSFile.collectionIndexDir(this.site_home, this.cluster_name)+File.separator+doc_name);
+	    
+	if (!doc_file.exists()) {
+	    logger.info("couldn't find file in coll "+this.cluster_name +", file "+doc_name);
+	    return null;
+	}
+
+	// the rss-items.rdf file has no root element, only multiple <item> elements (and subelements)
+	// Without a root element, it can't be read into a DOM object. So we read it into a regular String,
+	// then bookend the contents with temporary <rssrawdata></rssrawdata> elements to provide a root 
+	// element and can read in that String as a DOM object.
+
+	StringBuffer contents = new StringBuffer("<rssrawdata>\n");
+	try {
+	    BufferedReader in = new BufferedReader(new FileReader(doc_file));	    
+	    String line = null;
+	    while((line = in.readLine()) != null) {
+		//line = line.replace("_httpcollection_", "/"+this.cluster_name);
+		line = line.replace("_httpdomain__httpcollection_", url_prefix);
+		contents.append(line);
+	    }
+	    contents.append("</rssrawdata>");
+	    in.close(); // close the fileread handle
+
+	} catch (Exception e) {
+	    e.printStackTrace();
+	    contents.append("couldn't read ");
+	    contents.append(doc_file);
+	    contents.append("\n</rssrawdata>");
+	}
+
+
+	Document the_doc = null;
+	try {
+	    the_doc = this.converter.getDOM(contents.toString()); // String input converted to DOM
+	} catch (Exception e) {
+	    logger.error("couldn't create a DOM from file "+doc_file.getPath());
+	    return null;
+	}
+	
+	return the_doc.getDocumentElement();
+
+    }
+
+    
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceRack.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceRack.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceRack.java	(revision 31634)
@@ -0,0 +1,632 @@
+/*
+ *    ServiceRack.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+// greenstone classes
+import java.lang.reflect.Method;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.collection.ServiceCluster;
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.core.ModuleInterface;
+import org.greenstone.gsdl3.util.CustomClassLoader;
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSPath;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.io.StringWriter;
+import java.io.PrintWriter;
+
+/**
+ * ServiceRack - abstract base class for services
+ * 
+ * A ServiceRack provides one or more Services. This base class implements the
+ * process method. Each service is invoked by a method called process<service
+ * name> which takes one parameter - the xml request Element, and returns an XML
+ * response Element. for example, the TextQuery service would be invoked by
+ * processTextQuery(Element request)
+ * 
+ * @author Katherine Don
+ */
+public abstract class ServiceRack implements ModuleInterface
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.ServiceRack.class.getName());
+
+	/** the absolute address of the site home */
+	protected String site_home = null;
+	/** the http address of the site home */
+	protected String site_http_address = null;
+
+	protected String library_name = null;
+	/**
+	 * the name of the cluster (or collection) that this service belongs to - if
+	 * any
+	 */
+	protected String cluster_name = null;
+
+	/** some services can talk back to the message router */
+	protected MessageRouter router = null;
+
+        /** Access to the collection/serviceCluster object for service subclasses */
+        protected ServiceCluster serviceCluster = null;
+
+	/** a converter class to create Documents etc */
+	protected XMLConverter converter = null;
+
+	/** the original config info - if need to store it */
+	protected Element config_info = null;
+
+  /** XML element for stored description XML */
+  protected Document desc_doc = null;
+
+	/**
+	 * XML element for describe requests - list of supported services - this is
+	 * static
+	 */
+	protected Element short_service_info = null;
+
+	/**
+	 * XML element for stylesheet requests - map of service name to format elem
+	 */
+	protected HashMap<String, Node> format_info_map = null;
+
+	protected Element _globalFormat = null;
+
+	/**
+	 * A class loader that knows about the collection resources directory can
+	 * put properties files, dtds etc in here
+	 */
+	CustomClassLoader class_loader = null;
+
+	/** sets the cluster name */
+	public void setClusterName(String cluster_name)
+	{
+		this.cluster_name = cluster_name;
+	}
+
+    	/** sets the serviceCluster/Collection object */
+	public void setServiceCluster(ServiceCluster serviceCluster)
+	{
+		this.serviceCluster = serviceCluster;
+	}
+
+	/** sets the collect name */
+	public void setCollectionName(String coll_name)
+	{
+		setClusterName(coll_name);
+	}
+
+	public void cleanUp()
+	{
+	}
+
+	public void setGlobalFormat(Element globalFormat)
+	{
+	  _globalFormat = GSXML.duplicateWithNewName(this.desc_doc, globalFormat, GSXML.GLOBAL_FORMAT_ELEM, false);
+	}
+
+	/** sets the site home */
+	public void setSiteHome(String site_home)
+	{
+		this.site_home = site_home;
+	}
+
+	/** sets the site http address */
+	public void setSiteAddress(String site_address)
+	{
+		this.site_http_address = site_address;
+	}
+
+	public void setLibraryName(String library_name)
+	{
+		this.library_name = library_name;
+	}
+
+	public String getLibraryName()
+	{
+		return this.library_name;
+	}
+
+	/** sets the message router */
+	public void setMessageRouter(MessageRouter m)
+	{
+		this.router = m;
+		setLibraryName(m.getLibraryName());
+	}
+
+	/** the no-args constructor */
+	public ServiceRack()
+	{
+		this.converter = new XMLConverter();
+		this.desc_doc = XMLConverter.newDOM();
+		this.short_service_info = this.desc_doc.createElement(GSXML.SERVICE_ELEM + GSXML.LIST_MODIFIER);
+		this.format_info_map = new HashMap<String, Node>();
+	}
+
+	/**
+	 * configure the service module
+	 * 
+	 * @param info
+	 *            the XML node <serviceRack name="XXX"/> with name equal to the
+	 *            class name (of the subclass)
+	 * 
+	 *            must configure short_service_info_ and service_info_map_
+	 * @return true if configured ok must be implemented in subclasses
+	 */
+	public boolean configure(Element info)
+	{
+		return configure(info, null);
+	}
+
+	public boolean configure(Element info, Element extra_info)
+	{
+		// set up the class loader
+	  // this needs modifying if we ever have serviceracks at siteconfig level that want to use class loader
+	  this.class_loader = new CustomClassLoader(this.getClass().getClassLoader(), GSFile.collectionResourceDir(this.site_home, this.cluster_name));
+		return true;
+	}
+
+	/**
+	 * Process an XML document - convenience method that uses Strings rather
+	 * than Elements. just calls process(Element).
+	 * 
+	 * @param xml_in
+	 *            the Document to process - a string
+	 * @return the resultant document as a string - contains any error messages
+	 * @see String
+	 */
+	public String process(String xml_in)
+	{
+
+		Document doc = this.converter.getDOM(xml_in);
+		if (doc == null)
+		{
+			logger.error("Couldn't parse request");
+			logger.error(xml_in);
+			return null;
+		}
+		Node res = process(doc);
+		return this.converter.getString(res);
+
+	}
+
+	/**
+	 * process an XML request in DOM form
+	 * 
+	 * @param message
+	 *            the Node node containing the request should be <message>
+	 * @return an Node with the result XML
+	 * @see Node/Element
+	 */
+	public Node process(Node message_node)
+	{
+		Element message = GSXML.nodeToElement(message_node);
+
+		NodeList requests = message.getElementsByTagName(GSXML.REQUEST_ELEM);
+		Document result_doc = XMLConverter.newDOM();
+		Element mainResult = result_doc.createElement(GSXML.MESSAGE_ELEM);
+		if (requests.getLength() == 0)
+		{
+			// no requests
+			return mainResult; // empty message for now
+		}
+
+		for (int i = 0; i < requests.getLength(); i++)
+		{
+			Element request = (Element) requests.item(i);
+
+			String type = request.getAttribute(GSXML.TYPE_ATT);
+			if (type.equals(GSXML.REQUEST_TYPE_DESCRIBE))
+			{
+			  Element response = processDescribe(request);
+				if (response != null)
+				{
+				  mainResult.appendChild(result_doc.importNode(response, true));
+				}
+
+			}
+			else if (type.equals(GSXML.REQUEST_TYPE_FORMAT))
+			{
+			  Element response = processFormat(request);
+			  mainResult.appendChild(result_doc.importNode(response, true));
+			}
+			else
+			{
+				// other type of request, must be processed by the subclass - 
+				// send to the service method
+				StringBuffer error_string = new StringBuffer();
+				String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+				Element response = null;
+				try
+				{
+					Class c = this.getClass();
+					Class[] params = { Class.forName("org.w3c.dom.Element") };
+
+					String method_name = "process" + to;
+					Method m = null;
+					while (c != null)
+					{
+
+						try
+						{
+							m = c.getDeclaredMethod(method_name, params);
+							// if this has worked, break
+							break;
+						}
+						catch (NoSuchMethodException e)
+						{
+							c = c.getSuperclass();
+						}
+						catch (SecurityException e)
+						{
+							logger.error("security exception for finding method " + method_name);
+							error_string.append("ServiceRack.process: security exception for finding method " + method_name);
+						}
+					} // while
+					if (m != null)
+					{
+						Object[] args = { request };
+						try
+						{
+							response = (Element) m.invoke(this, args);
+
+						}
+						catch (Exception e)
+						{
+							logger.error("Trying to call a processService type method (process" + to + ") on a subclass(" + this.getClass().getName() + "), but an exception happened:" + e.toString(), e);
+							// for debugging, it's useful to see what's really causing this supposed "invocationTargetException"
+							// When the error message is displayed in the "Net" tab in the browser's web inspector, scroll down to "Caused by"
+							StringWriter errors = new StringWriter();
+							e.printStackTrace(new PrintWriter(errors));
+							String errStr = errors.toString();
+							errStr += XMLConverter.getString(request);
+
+
+							error_string.append("Trying to call a processService type method (process" + to + ") on a subclass(" + this.getClass().getName() + "), but an exception happened:" + e.toString() + "\n" + errStr );
+						}
+					}
+					else
+					{
+						logger.error("method " + method_name + " not found for class " + this.getClass().getName());
+						error_string.append("ServiceRack.process: method " + method_name + " not found for class " + this.getClass().getName());
+					}
+
+				}
+				catch (ClassNotFoundException e)
+				{
+					logger.error("Element class not found");
+					error_string.append("Element class not found");
+				}
+				if (response != null)
+				{
+					mainResult.appendChild(result_doc.importNode(response, true));
+				}
+				else
+				{
+					// add in a dummy response
+					logger.error("adding in an error element\n");
+					response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+					GSXML.addError(response, error_string.toString());
+					mainResult.appendChild(response);
+
+				}
+
+			} // else process request
+		} // for each request
+
+		return mainResult;
+
+	}
+
+	/**
+	 * process method for describe requests
+	 */
+	protected Element processDescribe(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_DESCRIBE);
+
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+		if (to.equals(""))
+		{ // return the service list
+		  response.appendChild(result_doc.importNode(getServiceList(result_doc, lang), true));
+			return response;
+		}
+		response.setAttribute(GSXML.FROM_ATT, to);
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		Element description = null;
+		if (param_list == null)
+		{
+		  description = getServiceDescription(result_doc, to, lang, null);
+		}
+		else
+		{
+			NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
+			for (int i = 0; i < params.getLength(); i++)
+			{
+
+				Element param = (Element) params.item(i);
+				// Identify the structure information desired
+				if (param.getAttribute(GSXML.NAME_ATT).equals(GSXML.SUBSET_PARAM))
+				{
+					String info = param.getAttribute(GSXML.VALUE_ATT);
+					if (description == null)
+					{
+					  description = getServiceDescription(result_doc, to, lang, info);
+					}
+					else
+					{
+					  Element temp = getServiceDescription(result_doc, to, lang, info);
+						GSXML.mergeElements(description, temp);
+					}
+				}
+			}
+		}
+		if (description != null)
+		{ // may be null if non-existant service
+		  response.appendChild(description);
+		}
+		return response;
+
+	}
+
+	/**
+	 * process method for stylesheet requests
+	 */
+	protected Element processFormat(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_FORMAT);
+
+		String to = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT));
+
+		if (to.equals(""))
+		{ // serviceRack query - is this appropriate??
+			return response;
+		}
+
+		// describe a particular service	
+		if (this.format_info_map.containsKey(to))
+		{
+		  response.appendChild(getServiceFormat(result_doc, to));
+		  if (_globalFormat != null)
+			{
+			  response.appendChild(result_doc.importNode(this._globalFormat, true));
+			}
+			response.setAttribute(GSXML.FROM_ATT, to);
+			return response;
+		}
+
+		// else no format info
+		logger.error("ServiceRack describe request: no format info for " + to + ".");
+		return response;
+	}
+
+	/** returns the service list for the subclass */
+  protected Element getServiceList(Document doc, String lang)
+	{
+		// for now, it is static and has no language stuff
+	  return (Element)doc.importNode(this.short_service_info, true);
+	}
+
+	/** returns a specific service description */
+  abstract protected Element getServiceDescription(Document doc, String service, String lang, String subset);
+
+  protected Element getServiceFormat(Document doc, String service)
+	{
+	  Element format = (Element) this.format_info_map.get(service);
+	  if (format != null) {
+	    return (Element)doc.importNode(format, true);
+	  }
+	  return null;
+	}
+
+
+	/**
+	 * Returns the appropriate language element from a display elem, display is
+	 * the containing element, name is the name of the element to look for, lang
+	 * is the preferred language, lang_default is the fall back lang, if neither
+	 * lang is found will return the first one it finds
+	 */
+  public String getDisplayText(Element display, String name, String lang, String lang_default) {
+    return getDisplayText(display, name, lang, lang_default, null);
+  }
+
+  public String getDisplayText(Element display, String name, String lang, String lang_default, String default_dictionary) {
+	
+
+		String def = null;
+		String first = null;
+		String key = null;
+		String dictionary_name = default_dictionary;
+		NodeList elems = display.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		if (elems.getLength() == 0)
+			return "";
+		for (int i = 0; i < elems.getLength(); i++)
+		{
+			Element e = (Element) elems.item(i);
+			String n = e.getAttribute(GSXML.NAME_ATT);
+			if (name.equals(n))
+			{
+				String l = e.getAttribute(GSXML.LANG_ATT);
+				String k = e.getAttribute(GSXML.KEY_ATT);
+				// if we have a specific lang value, return that
+				if (!l.equals("") && lang.equals(l))
+				{
+					return GSXML.getNodeText(e);
+				}
+				else if (!l.equals("") && lang_default.equals(l))
+				{
+					def = GSXML.getNodeText(e);
+				}
+				else if (!k.equals("")) {
+				  if ( key == null) {
+				  // a key specified. only allowed one spec with key
+				    key = k;
+				    String d =  e.getAttribute(GSXML.DICTIONARY_ATT);
+				    if (!d.equals("")) {
+				      dictionary_name = d;
+				    }
+				  }
+				}
+				// we have no key and we don't match the languages
+				else if (first == null)
+				{
+				  // but we are the first one, so we remember the value
+					first = GSXML.getNodeText(e);
+				}
+			}
+			else
+			{
+				continue;
+			}
+		}
+
+		
+		if (key != null) {
+		  String s = getTextString(key, lang, dictionary_name);
+		  // only use this one if a value was actually found
+		  if (s!= null) {
+		    return s;
+		  }
+		  //if (!s.equals( "_"+key +"_")) {
+		  //  return s;
+		  //}
+		}
+
+		if (def != null)
+		{
+			return def;
+		}
+		if (first != null)
+		{
+			return first;
+		}
+		return "";
+	}
+
+	/** overloaded version for no args case */
+	protected String getTextString(String key, String lang)
+	{
+		return getTextString(key, lang, null, null);
+	}
+
+	protected String getTextString(String key, String lang, String dictionary)
+	{
+		return getTextString(key, lang, dictionary, null);
+	}
+
+	protected String getTextString(String key, String lang, String[] args)
+	{
+		return getTextString(key, lang, null, args);
+	}
+
+	/**
+	 * getTextString - retrieves a language specific text string for the given
+	 * key and locale, from the specified resource_bundle (dictionary)
+	 */
+	protected String getTextString(String key, String lang, String dictionary, String[] args)
+	{
+
+		// we want to use the collection class loader in case there are coll specific files
+		if (dictionary != null)
+		{
+			// just try the one specified dictionary
+			Dictionary dict = new Dictionary(dictionary, lang, this.class_loader);
+			String result = dict.get(key, args);
+			if (result == null)
+			{ // not found
+			  //return "_" + key + "_";
+			  return null;
+			}
+			return result;
+		}
+
+		// now we try class names for dictionary names
+		String original_class_name = this.getClass().getName();
+		original_class_name = original_class_name.substring(original_class_name.lastIndexOf('.') + 1);
+		Dictionary dict = new Dictionary(original_class_name, lang, this.class_loader);
+		String result = dict.get(key, args);
+		if (result != null)
+		{
+			return result;
+		}
+
+		// we have to try super classes
+		String class_name;
+		Class c = this.getClass().getSuperclass();
+		while (result == null && c != null)
+		{
+			class_name = c.getName();
+			class_name = class_name.substring(class_name.lastIndexOf('.') + 1);
+			if (class_name.equals("ServiceRack"))
+			{
+				// this is as far as we go
+				break;
+			}
+			dict = new Dictionary(class_name, lang, this.class_loader);
+			result = dict.get(key, args);
+			c = c.getSuperclass();
+		}
+		if (result == null)
+		{
+		  // try the ServiceRack properties
+		  // at the moment we look for original_class_name.key, then just key
+		  // if there is lots of inheritance, may want to try down the list of superclasses too...?
+		  class_name = "ServiceRack";
+		  dict = new Dictionary(class_name, lang, this.class_loader);
+		  String full_key = original_class_name+"."+key;
+		  result = dict.get(full_key, args);
+		  if(result == null) {
+		    result = dict.get(key, args);
+		  }
+		  if(result == null) {
+		    //return "_" + key + "_";
+		    return null;
+		  }
+		}
+		return result;
+	
+	}
+
+	protected String getMetadataNameText(String key, String lang)
+	{
+
+		String properties_name = "metadata_names";
+		Dictionary dict = new Dictionary(properties_name, lang);
+
+		String result = dict.get(key);
+		if (result == null)
+		{ // not found
+			return null;
+		}
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceUtil.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceUtil.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/ServiceUtil.java	(revision 31634)
@@ -0,0 +1,147 @@
+package org.greenstone.gsdl3.service;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public class ServiceUtil extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.ServiceUtil.class.getName());
+	
+	/**********************************************************
+	 * The list of services the utility service rack supports *
+	 *********************************************************/
+	protected static final String GET_ALL_IMAGES_IN_COLLECTION = "GetAllImagesInCollection";
+	/*********************************************************/
+	
+	String[] services = { GET_ALL_IMAGES_IN_COLLECTION };
+	
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring ServiceUtil...");
+		this.config_info = info;
+
+		for (int i = 0; i < services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, services[i]);
+			this.short_service_info.appendChild(service);
+		}
+
+		return true;
+	}
+
+	protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < services.length; i++)
+		{
+			if (service_id.equals(services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+	
+	protected Element processGetAllImagesInCollection(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, GET_ALL_IMAGES_IN_COLLECTION);
+
+		if (request == null)
+		{
+			GSXML.addError(result, GET_ALL_IMAGES_IN_COLLECTION + ": Request is null", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		
+		String lang = request.getAttribute(GSXML.LANG_ATT);
+		String uid = request.getAttribute(GSXML.USER_ID_ATT);
+		
+		// Get the parameters of the request
+		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+
+		if (param_list == null) {
+			GSXML.addError(result, GET_ALL_IMAGES_IN_COLLECTION + ": No param list specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;  // Return the empty result
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+		
+		String regex = (String)params.get("extregex");
+		if(regex == null)
+		{
+			GSXML.addError(result, GET_ALL_IMAGES_IN_COLLECTION + ": No file name extensions specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		regex = ".*" + regex + ".*";
+		
+		String collection = (String)params.get("c");
+		if(collection == null)
+		{
+			GSXML.addError(result, GET_ALL_IMAGES_IN_COLLECTION + ": No collection specified", GSXML.ERROR_TYPE_SYNTAX);
+			return result;
+		}
+		
+		File indexDir = new File(this.site_home + File.separator + "collect" + File.separator + collection + File.separator + "index" + File.separator + "assoc");
+		ArrayList<String> images = new ArrayList<String>();
+		getImagesRecursive(indexDir, regex, images);
+		
+		Element imageListElem = result_doc.createElement("imageList");
+		result.appendChild(imageListElem);
+		for(String i : images)
+		{
+			Element imageElem = result_doc.createElement("image");
+			imageElem.appendChild(result_doc.createTextNode(i));
+			imageListElem.appendChild(imageElem);
+		}
+		return result;
+	}
+	
+	/********************
+	 * Helper functions *
+	 *******************/
+	
+	protected void getImagesRecursive(File current, String regex, ArrayList<String> images)
+	{
+		if(current.isDirectory())
+		{
+			File[] files = current.listFiles();
+			for(File f : files)
+			{
+				getImagesRecursive(f, regex, images);
+			}
+		}
+		else
+		{
+			String path = current.getAbsolutePath();
+			String filename = path.substring(path.lastIndexOf(File.separator) + File.separator.length());
+
+			if(filename.matches(regex))
+			{
+				images.add(path);
+			}
+			else
+			{
+		
+			}
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/SharedSoleneGS2FieldSearch.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/SharedSoleneGS2FieldSearch.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/SharedSoleneGS2FieldSearch.java	(revision 31634)
@@ -0,0 +1,244 @@
+/*
+ *    SharedSoleneGS2FieldSearch.java -- shared base code for Solr and Lucene
+ *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+// Greenstone classes
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+import org.greenstone.LuceneWrapper4.SharedSoleneQuery;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+// Shared code for Solr and Lucene GS2FieldSearch
+
+public abstract class SharedSoleneGS2FieldSearch extends AbstractGS2FieldSearch
+{
+
+  protected static String RANK_PARAM_RANK = "rank";
+  protected static String RANK_PARAM_NONE = "none";
+
+  protected static final String SORT_ELEM = "sort";
+  protected static final String DEFAULT_SORT_ELEM = "defaultSort";
+
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.SharedSoleneGS2FieldSearch.class.getName());
+
+  protected SharedSoleneQuery solene_src = null;
+  protected String default_sort = "";
+  
+	public SharedSoleneGS2FieldSearch()
+	{
+		does_paging = true;
+		does_chunking = false; // actually it can but we don't want to do a max docs, as we are already paging results
+
+		// Lucene/Solr uses double operators, not single
+		//AND_OPERATOR = "&&";
+		//OR_OPERATOR = "||";
+		AND_OPERATOR = "AND";
+		OR_OPERATOR = "OR";
+		NOT_OPERATOR = "NOT";
+
+	}
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		// the search element
+		Element config_search = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);
+		Document owner = info.getOwnerDocument();
+		// find the sort fields in serviceRack xml, and add in the deisplayItems if any
+		NodeList sort_nodes = info.getElementsByTagName(SORT_ELEM);
+
+		for (int i = 0; i < sort_nodes.getLength(); i++)
+		  {
+		    Element sort = (Element) sort_nodes.item(i);
+		    String name = sort.getAttribute(GSXML.NAME_ATT);
+		    Element node_extra = GSXML.getNamedElement(config_search, SORT_ELEM, GSXML.NAME_ATT, name);
+		    if (node_extra == null)
+		      {
+			logger.error("haven't found extra info for sort field named " + name);
+			continue;
+		      }
+		    
+		    // get the display elements if any - displayName
+		    NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+		    if (display_names != null)
+		      {
+			for (int j = 0; j < display_names.getLength(); j++)
+			  {
+			    Element e = (Element) display_names.item(j);
+			    sort.appendChild(owner.importNode(e, true));
+			  }
+		      }
+		  } // for each sortfield
+
+		// get the default sort field
+		Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_SORT_ELEM);
+		if (def != null)
+		{
+			this.default_sort = def.getAttribute(GSXML.SHORTNAME_ATT);
+		}
+		
+		// Lucene/Solr doesn't do case folding or stemming or accent folding at the 
+		// moment
+		does_case = false;
+		does_stem = false;
+		does_accent = false;
+
+		return true;
+	}
+
+	/** add in the Lucene/Solr specific params to TextQuery */
+	protected void addCustomQueryParams(Element param_list, String lang)
+	{
+		super.addCustomQueryParams(param_list, lang);
+		/** Lucene's/Solr's rank (sort) param is based on sort fields, not ranked/not */
+		createParameter(RANK_PARAM, param_list, lang);
+	}
+  /** add in Lucene/SOLR specific params for AdvancedFieldQuery */
+  protected void addCustomQueryParamsAdvField(Element param_list, String lang)
+	{
+		super.addCustomQueryParamsAdvField(param_list, lang);
+		createParameter(RANK_PARAM, param_list, lang);
+		
+	}
+
+	/** create a param and add to the list */
+	/** we override this to do a special rank param */
+  protected void createParameter(String name, Element param_list, String lang)
+	{
+	  Document doc = param_list.getOwnerDocument();
+		Element param = null;
+		String param_default = paramDefaults.get(name);
+		if (name.equals(RANK_PARAM))
+		{
+			// get the fields
+			ArrayList<String> fields = new ArrayList<String>();
+			ArrayList<String> field_names = new ArrayList<String>();
+			param_default = default_sort;
+			if (!getSortData(fields, field_names, lang)) {
+			  fields.add(RANK_PARAM_RANK);
+			  fields.add(RANK_PARAM_NONE);
+			  field_names.add(getTextString("param." + RANK_PARAM + "." + RANK_PARAM_RANK, lang));
+			  field_names.add(getTextString("param." + RANK_PARAM + "." + RANK_PARAM_NONE, lang));
+			  param_default = RANK_PARAM_RANK;
+			}
+			else {
+			  if (param_default == null) {
+			    param_default = fields.get(0);
+			  }
+			}
+			param = GSXML.createParameterDescription2(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, fields, field_names);
+			
+		} 
+
+		if (param != null)
+		{
+			param_list.appendChild(param);
+		}
+		else
+		{
+			super.createParameter(name, param_list, lang);
+		}
+
+	}
+
+  protected boolean getSortData(ArrayList<String> sort_ids, ArrayList<String> sort_names, String lang) {
+
+    	Element sort_list = (Element) GSXML.getChildByTagName(this.config_info, SORT_ELEM + GSXML.LIST_MODIFIER);
+	if (sort_list == null) return false;
+	NodeList sorts = sort_list.getElementsByTagName(SORT_ELEM);
+	int len = sorts.getLength();
+	if (len == 0) return false;
+	for (int i = 0; i < len; i++)
+	  {
+	    Element sort = (Element) sorts.item(i);
+	    String shortname = sort.getAttribute(GSXML.SHORTNAME_ATT);
+	    sort_ids.add(shortname);
+	    String display_name = getDisplayText(sort, GSXML.DISPLAY_TEXT_NAME, lang, "en", "metadata_names");
+	    if (display_name.equals(""))
+	      {
+		// use the sort name and look up in dictionary
+		display_name = sort.getAttribute(GSXML.NAME_ATT);
+		if (display_name.equals(""))
+		  {
+		    display_name = shortname;
+		  }
+		else {
+		  display_name = getTextString("param.sortBy." + display_name, lang);
+		}
+	      }
+	    sort_names.add(display_name);
+	    
+	  }
+	return true;
+  }
+
+	protected String addFieldInfo(String query, String field)
+	{
+		// currently, allfields (ZZ) is stored as a extra field for Lucene
+		if (field.equals(""))
+		{ // || field.equals("ZZ")) {
+			return query;
+		}
+		return field + ":(" + query + ")";
+	}
+
+	protected void addQueryElem(StringBuffer s, String q, String f, String c)
+	{
+
+		String combine = "";
+		if (s.length() > 0)
+		{
+			combine = " " + c + " ";
+		}
+		s.append(combine + addFieldInfo(q, f));
+	}
+
+	/** Lucene/Solr doesn't use these options at the moment */
+	protected String addStemOptions(String query, String stem, String casef, String accent)
+	{
+		return query;
+	}
+
+	/**
+	 * Lucene/Solr does not use internal ids. It just uses hash ids. So we need
+	 * to override these methods so no conversion is done.
+	 */
+	/** convert indexer internal id to Greenstone oid */
+	protected String internalNum2OID(long docnum)
+	{
+		return Long.toString(docnum);
+	}
+
+	protected String internalNum2OID(String docnum)
+	{
+		return docnum;
+
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/TEIRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/TEIRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/TEIRetrieve.java	(revision 31634)
@@ -0,0 +1,53 @@
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+
+// General Java classes
+
+// just needs to override a few methods that are TEI specific
+public class TEIRetrieve extends XMLRetrieve {
+
+  protected Element extractTitleMeta(Document doc, Element section) {
+	
+	Element meta_elem = doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+	
+	Element section_head = (Element)GSXML.getChildByTagName(section, "head");
+	if (section_head == null || !section_head.hasChildNodes()) {
+	    // there is no head element, so take the type attribute and make that the title
+	    String title = "("+section.getAttribute("type")+")";
+	    Text t = doc.createTextNode(title);
+	    meta_elem.appendChild(t);
+	} else {
+	    // add the head element as the metadata content
+	    meta_elem.appendChild(doc.importNode(section_head, true));
+	}
+	return meta_elem;
+    }
+
+    // this assumes that the scope refers to a top level node - this may be overwritten if the scope bit in the id is a shorthand of some sort
+    protected String translateScope(String scope) {
+	if (scope.equals("front")) {
+	    return "text/front";
+	}
+	if (scope.equals("body")) {
+	    return "text/body";
+	}
+	if (scope.equals("back")) {
+	    return "text/back";
+	}
+	if (scope.equals("full")) { // is this ever used???
+	    return "";
+	}
+	return scope;
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/UserTracker.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/UserTracker.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/UserTracker.java	(revision 31634)
@@ -0,0 +1,131 @@
+package org.greenstone.gsdl3.service;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.log4j.Logger;
+import org.greenstone.gsdl3.util.DerbyWrapper;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public class UserTracker extends ServiceRack
+{
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.DocXMLUtil.class.getName());
+
+	/**************************************************
+	 * The list of services the User Tracker supports *
+	 *************************************************/
+	protected static final String RECORD_USER_ACTION = "RecordUserAction";
+	protected static final String GET_ACTIVITY_ON_PAGE = "GetActivityOnPage";
+	/*************************************************/
+
+	String[] services = { RECORD_USER_ACTION, GET_ACTIVITY_ON_PAGE };
+
+	/** configure this service */
+	public boolean configure(Element info, Element extra_info)
+	{
+		if (!super.configure(info, extra_info))
+		{
+			return false;
+		}
+
+		logger.info("Configuring DocXMLUtil...");
+		this.config_info = info;
+
+		for (int i = 0; i < services.length; i++)
+		{
+			Element service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+			service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+			service.setAttribute(GSXML.NAME_ATT, services[i]);
+			this.short_service_info.appendChild(service);
+		}
+
+		return true;
+	}
+
+	protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
+	{
+		for (int i = 0; i < services.length; i++)
+		{
+			if (service_id.equals(services[i]))
+			{
+				Element service_elem = doc.createElement(GSXML.SERVICE_ELEM);
+				service_elem.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+				service_elem.setAttribute(GSXML.NAME_ATT, services[i]);
+				return service_elem;
+			}
+		}
+
+		return null;
+	}
+
+	protected synchronized Element processRecordUserAction(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, RECORD_USER_ACTION);
+
+		Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+		if (paramList == null)
+		{
+			GSXML.addError(result, "Request has no parameter list");
+			return result;
+		}
+
+		HashMap<String, Serializable> params = GSXML.extractParams(paramList, true);
+		String username = (String) params.get("username");
+		String collection = (String) params.get("collection");
+		String site = (String) params.get("site");
+		String oid = (String) params.get("oid");
+		String action = (String) params.get("action");
+
+		DerbyWrapper database = new DerbyWrapper(GlobalProperties.getGSDL3Home() + File.separatorChar + "etc" + File.separatorChar + "usersDB");
+		database.addUserAction(username, site, collection, oid, action);
+		database.closeDatabase();
+
+		return result;
+	}
+
+	protected synchronized Element processGetActivityOnPage(Element request)
+	{
+	  Document result_doc = XMLConverter.newDOM();
+		Element result = GSXML.createBasicResponse(result_doc, GET_ACTIVITY_ON_PAGE);
+		try
+		{
+
+			Element paramList = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+			if (paramList == null)
+			{
+				GSXML.addError(result, "Request has no parameter list");
+				return result;
+			}
+
+			HashMap<String, Serializable> params = GSXML.extractParams(paramList, true);
+			String collection = (String) params.get("collection");
+			String site = (String) params.get("site");
+			String oid = (String) params.get("oid");
+
+			DerbyWrapper database = new DerbyWrapper(GlobalProperties.getGSDL3Home() + File.separatorChar + "etc" + File.separatorChar + "usersDB");
+			ArrayList<HashMap<String, String>> userActions = database.getMostRecentUserActions(site, collection, oid);
+
+			Element userList = result_doc.createElement("userList");
+			for (HashMap<String, String> userAction : userActions)
+			{
+				Element user = result_doc.createElement("user");
+				user.setAttribute("username", userAction.get("username"));
+				user.setAttribute("action", userAction.get("action"));
+				userList.appendChild(user);
+			}
+			result.appendChild(userList);
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XMLRetrieve.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XMLRetrieve.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XMLRetrieve.java	(revision 31634)
@@ -0,0 +1,472 @@
+/*
+ *    ServiceRack.java
+ *    Copyright (C) 2014 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.service;
+
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.*;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.Attr;
+import org.w3c.dom.Text;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+
+// General Java classes
+import java.io.File;
+import java.io.Serializable;
+import java.util.Vector;
+import java.util.HashMap;
+
+import org.apache.log4j.*;
+
+public class XMLRetrieve extends ServiceRack {
+
+      static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.XMLRetrieve.class.getName());
+    protected static final String CONTENT_SERVICE = "DocumentContentRetrieve";
+    protected static final String METADATA_SERVICE = "DocumentMetadataRetrieve";
+    protected static final String STRUCTURE_SERVICE = "DocumentStructureRetrieve";
+    
+    protected String toc_xsl_name = "";
+    protected String document_encoding = "";
+    protected String document_root_tag = "";
+    
+    protected Element collection_doc_list = null;
+    
+    protected boolean provide_content = true;
+    protected boolean provide_structure = true;
+    protected boolean provide_metadata = true;
+    
+  protected GSEntityResolver entity_resolver = null;
+
+    public boolean configure(Element info, Element extra_info) {
+	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+	logger.info("configuring XMLRetrieve...");
+	// look for the parameters
+	Element param_list = (Element)GSXML.getChildByTagName(info, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	HashMap<String, Serializable> params;
+	String services_to_provide = "";
+	if (param_list != null) {
+	    params = GSXML.extractParams(param_list, false);
+	    this.toc_xsl_name = (String)params.get("tocXSLT");
+	    this.document_encoding = (String)params.get("documentEncoding");
+	    this.document_root_tag = (String)params.get("documentRootTag");
+	    services_to_provide = (String)params.get("provideServices");
+	}
+	if (this.toc_xsl_name == null || this.toc_xsl_name.equals("")) {
+	    this.toc_xsl_name = "default_toc";
+	}
+	this.toc_xsl_name  = this.toc_xsl_name+".xsl";
+	
+	if (this.document_encoding == null || this.document_encoding.equals("")) {
+	    this.document_encoding = "UTF-8";
+	}
+	
+	if (services_to_provide != null && !services_to_provide.equals("")) {
+	    if (services_to_provide.indexOf("content")==-1) {
+		provide_content = false;
+	    }
+	    if (services_to_provide.indexOf("metadata")==-1) {
+		provide_metadata = false;
+	    }
+	    if (services_to_provide.indexOf("structure")==-1) {
+		provide_structure = false;
+	    }
+	    
+	}
+	
+	// set up short_service_info_ - for now just has name and type
+	Element retrieve_service;
+	if (provide_content) {
+	    retrieve_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, CONTENT_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	if (provide_metadata) {
+	    retrieve_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, METADATA_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	if (provide_structure) {
+	    retrieve_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
+	    retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	    retrieve_service.setAttribute(GSXML.NAME_ATT, STRUCTURE_SERVICE);
+	    this.short_service_info.appendChild(retrieve_service);
+	}
+	// find the doc list from the extra_info and keep it - should this be in collect.cfg or build.cfg??
+	collection_doc_list = (Element)GSXML.getChildByTagName(extra_info, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+
+	entity_resolver = new GSEntityResolver();
+	entity_resolver.setClassLoader(this.class_loader);
+	//this.converter.setEntityResolver(resolver);
+	return true;
+    }
+
+    // this may get called but is not useful in the case of retrieve services
+    protected Element getServiceDescription(Document doc, String service_id, String lang, String subset) {
+
+	Element retrieve_service = doc.createElement(GSXML.SERVICE_ELEM);
+	retrieve_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_RETRIEVE);
+	retrieve_service.setAttribute(GSXML.NAME_ATT, service_id);
+	return retrieve_service;
+    }
+    
+    protected Element processDocumentContentRetrieve(Element request) {
+      Document result_doc = XMLConverter.newDOM();
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, CONTENT_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    return result;
+	}
+	Element result_doc_list = (Element)result_doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	for (int i=0; i<docs.getLength(); i++) {
+
+	    Element doc = (Element)docs.item(i);
+	    Element content = result_doc.createElement(GSXML.NODE_CONTENT_ELEM);
+	    doc.appendChild(content);
+
+	    String node_id = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    String doc_name = getWorkName(node_id);
+	    
+	    Element doc_elem = loadDocument(doc_name); // should perhaps cache the read in docs??
+	    if (doc_elem == null) {
+		continue;
+	    } 
+
+	    
+	    // if we have asked for the whole doc, just append it
+	    if (doc_name.equals(node_id)) {
+		content.appendChild(result_doc.importNode(doc_elem, true));
+		continue;
+	    }
+		   
+	    // else we only want a sub section
+	    
+	    Element section = getSection(doc_elem, node_id);
+	    if (section != null) {
+		content.appendChild(result_doc.importNode(section, true));
+	    }
+	    
+	} // for each doc
+
+	return result;
+	
+    }
+    
+    protected Element processDocumentStructureRetrieve(Element request) {
+      Document result_doc = XMLConverter.newDOM();
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, STRUCTURE_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    logger.error("no documents specified in the request. ");
+	    return result;
+	}
+
+	Element result_doc_list = (Element)result_doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+	// first look for the stylesheet in the collection
+	File stylesheet = new File(GSFile.collStylesheetFile(this.site_home, this.cluster_name, this.toc_xsl_name));
+	if (!stylesheet.exists()) {
+	    // now try in the site
+	    stylesheet = new File(GSFile.siteStylesheetFile(this.site_home, this.toc_xsl_name));
+	}
+	if (!stylesheet.exists()) {
+	    logger.error("couldn't find the stylesheet file to produce the table of contents:"+stylesheet.getPath());
+	    return result;
+	}
+
+	// for now, we dont have any params, and we always return the structure of the whole document
+	
+	XMLTransformer transformer = new XMLTransformer();
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+
+	for (int i=0; i<docs.getLength(); i++) {
+
+	    Element doc = (Element)docs.item(i);
+	    
+	    Element structure = result_doc.createElement(GSXML.NODE_STRUCTURE_ELEM);
+	    doc.appendChild(structure);
+	    String doc_name = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    // make sure we are at the top level
+	    doc_name = getWorkName(doc_name);
+	    
+	    File doc_file = new File(GSFile.collectionIndexDir(this.site_home, this.cluster_name)+File.separator+"text"+File.separatorChar+doc_name+".xml");
+	    
+	    if (!doc_file.exists()) {
+		logger.error("couldn't find file in coll "+this.cluster_name +", file "+doc_name+".xml");
+	    } else {
+		try {
+		    Node toc = transformer.transform(stylesheet, doc_file);
+		    structure.appendChild(result_doc.importNode(toc, true));
+		} catch (Exception e) {
+		    logger.error("couldn't transform the document to get the toc");
+		}
+	    }
+	    
+	}
+	
+	return result;
+	
+    }
+    
+    // this just extracts a bit of text from the section to use as the Title
+    // this should be overwritten for any format that  has something more suitable
+    protected Element processDocumentMetadataRetrieve(Element request) {
+      Document result_doc = XMLConverter.newDOM();
+	Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
+	result.setAttribute(GSXML.FROM_ATT, METADATA_SERVICE);
+	result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+	
+	Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	if (doc_list == null) {
+	    logger.error("no documents in the request");
+	    return result;
+	}
+
+	Element result_doc_list = (Element)result_doc.importNode(doc_list, true);
+	result.appendChild(result_doc_list);
+
+	Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	if (param_list == null) {
+	    logger.error("no metadata in the request");
+	    return result;
+	}
+	
+	Vector<String> meta_name_list = new Vector<String>();
+	boolean all_metadata = false;
+	// Process the request parameters
+	Element param = GSXML.getFirstElementChild(param_list);//(Element) param_list.getFirstChild();
+	while (param != null) {
+	    // Identify the metadata information desired
+	    if (param.getAttribute(GSXML.NAME_ATT).equals("metadata")) {
+		String metadata = GSXML.getValue(param);
+		if (metadata.equals("all")) {
+		    all_metadata = true;
+		    break;
+		}
+		meta_name_list.add(metadata);
+	    }
+	    param = (Element) param.getNextSibling();
+	}
+		
+	NodeList docs = result_doc_list.getElementsByTagName(GSXML.DOC_NODE_ELEM);
+	for (int i=0; i<docs.getLength(); i++) {
+	    Element doc = (Element)docs.item(i);
+	    String node_id = doc.getAttribute(GSXML.NODE_ID_ATT);
+	    String doc_name = getWorkName(node_id);
+	    
+	    Element metadata_list = getMetadata(result_doc, node_id, all_metadata, meta_name_list); 
+	    doc.appendChild(metadata_list);
+	}
+	
+	return result;
+    }
+    
+    protected Element loadDocument(String doc_name) {
+	// try to find the document
+	File doc_file = new File(GSFile.collectionIndexDir(this.site_home, this.cluster_name)+File.separator+"text"+File.separatorChar+doc_name+".xml");
+	    
+	if (!doc_file.exists()) {
+	    logger.info("couldn't find file in coll "+this.cluster_name +", file "+doc_name+".xml");
+	    return null;
+	}
+	
+	Document the_doc = null;
+	try {
+	  the_doc = this.converter.getDOM(doc_file, this.document_encoding, this.entity_resolver);
+	} catch (Exception e) {
+	    logger.error("couldn't create a DOM from file "+doc_file.getPath());
+	    return null;
+	}
+	
+	return the_doc.getDocumentElement();
+
+    }
+
+    
+    protected Element getSection(Element doc_elem, String node_id) {
+	String [] bits = node_id.split("\\.");
+	if (bits.length > 4) {
+	    logger.error("badly formatted node id ("+node_id +"), cant retrieve the section");
+	    return null;
+	}
+	    
+	String id="";
+	String tagname = "";
+	String scope = "";
+	if (bits.length==2) {
+	    tagname = bits[1];
+	} else { 
+	    scope = bits[1];
+	    tagname = bits[2];
+	    
+	    if (bits.length == 4) {
+		id = bits[3];
+	    }
+	}
+	scope = translateScope(scope);
+	Element top=null;
+	if (!scope.equals("")) {
+	    top = (Element)GSXML.getNodeByPath(doc_elem, scope);
+	    if (top == null) {
+		// something gone wrong
+		return null;
+	    }
+	} else {
+	    top = doc_elem;
+	}
+	
+	NodeList elements = top.getElementsByTagName(tagname);
+	if (elements.getLength() == 0) {
+	    return null;
+	}
+	// no id, just return the first one
+	if (id.equals("")) {
+	    return (Element)elements.item(0);
+	}
+	// have an id, need to check and find the right one.
+	for (int i=0; i<elements.getLength();i++) {
+	    Element e = (Element)elements.item(i);
+	    if (e.getAttribute("gs3:id").equals(id)) {
+		return e;
+	    }
+	}
+	return null;
+	
+    }
+
+  protected Element getMetadata(Document result_doc, String node_id, boolean all, Vector<String> meta_name_list) {
+
+	// our default strategy here is to only return Title and root:Title
+	// ignore all others 
+	// the title of a section is just a little bit of the text inside it.
+	// the root_Title is the title from the doc info in the config file
+	Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM+ GSXML.LIST_MODIFIER);
+	String doc_name = getWorkName(node_id);
+	boolean node_is_root = false;
+	if (doc_name.equals(node_id)) {
+	    node_is_root = true;
+	}
+	
+	Element this_doc = GSXML.getNamedElement(this.collection_doc_list, GSXML.DOCUMENT_ELEM, GSXML.NAME_ATT, doc_name);
+	Element doc_meta_list  = (Element) GSXML.getChildByTagName(this_doc, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+	
+	boolean get_section_title = false;
+	
+	if (all) {
+	    if (node_is_root) {
+	    return (Element)result_doc.importNode(doc_meta_list, true);
+	    } else {
+		get_section_title = true;
+	    }
+	    
+	} else {
+	    // have to process metadata one by one
+	    for (int i=0; i<meta_name_list.size(); i++) {
+		String meta_name = meta_name_list.elementAt(i);
+		String actual_meta_name = meta_name;
+		if (meta_name.startsWith("root_")) {
+		    actual_meta_name = meta_name.substring(5);
+		} else {
+		    // its a section level one - check to see if doc is root
+		    if (!node_is_root) {
+			if (meta_name.equals("Title")) {
+			    get_section_title = true;
+			}
+			continue; // move on to teh next metadata
+		    }
+		}
+		
+		// here, we look for the specific meta elem in doc_meta_list
+		Element meta_item = GSXML.getNamedElement(doc_meta_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, actual_meta_name);
+		if (meta_item != null) {
+		    meta_item = (Element)result_doc.importNode(meta_item, true);
+		    meta_item.setAttribute(GSXML.NAME_ATT, meta_name);
+		    metadata_list.appendChild(meta_item);
+		}
+	    } // for each metadata
+	}
+
+	// now we have processed all teh doc metadata, just have section one to go, if needed
+	if (get_section_title) {
+	    
+	    Element doc_elem = loadDocument(doc_name);
+	    if (doc_elem != null) {
+		Element section = getSection(doc_elem, node_id);
+		if (section != null) {
+		  Element title_meta = extractTitleMeta(result_doc, section);
+		    if (title_meta != null) {
+			metadata_list.appendChild(title_meta);
+		    }
+		}
+	    }
+		
+	}
+	return metadata_list;
+    }
+    
+  protected Element extractTitleMeta(Document result_doc, Element section) {
+	Element meta_elem = result_doc.createElement(GSXML.METADATA_ELEM);
+	meta_elem.setAttribute(GSXML.NAME_ATT, "Title");
+
+	String title = "dummy title";
+	Text t = result_doc.createTextNode(title);
+	meta_elem.appendChild(t);
+	return meta_elem;
+
+    }
+    // some methods for handling nodeIDs - they may be different for different colls, so they can be overwritten
+
+    // the full default nodeID looks like work.scope.tag.id
+    // the shorter versions are work, work.tag, work.scope.tag
+    protected String getWorkName(String node_id) {
+	int pos = node_id.indexOf('.');
+	if (pos == -1) {
+	    return node_id;
+	}
+	return node_id.substring(0, pos);
+    }
+    
+    // this assumes that the scope refers to a top level node - this may be overwritten if the scope bit in the id is a shorthand of some sort
+    protected String translateScope(String scope) {
+	if (this.document_root_tag != null) {
+	    return GSPath.appendLink(this.document_root_tag, scope);
+	}
+	return scope;
+    }
+    
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XSLTServices.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XSLTServices.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/service/XSLTServices.java	(revision 31634)
@@ -0,0 +1,266 @@
+/*
+ *    XSLTServices.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.service;
+
+import org.greenstone.gsdl3.util.*;
+import org.greenstone.gsdl3.selfContained.*;
+
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import org.apache.xerces.dom.DocumentImpl;
+import org.apache.xerces.dom.ElementImpl;
+import org.apache.xerces.dom.TextImpl;
+import org.w3c.dom.Document; 
+import org.w3c.dom.Node; 
+import org.w3c.dom.Text; 
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList; 
+
+import java.util.HashMap;
+import java.util.Vector;
+import java.util.Set;
+import java.util.Map;
+import java.util.Iterator;
+import java.util.Locale;
+import java.io.*;
+import java.net.URLEncoder;
+import java.net.URLDecoder;
+
+import org.apache.log4j.*;
+
+/**
+ * A ServiceRack class for XSLT query. 
+ *
+ * Document ids are formed by encoding the document using standard URLEncoding
+ *
+ * @author Stuart Yeates
+ * @see ServiceRack
+ * @see java.net.URLEncoder
+ * @see java.net.URLDecoder
+ */
+
+public class XSLTServices 
+  extends ServiceRack {
+  
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.XSLTServices.class.getName());
+
+  // these strings must match what is found in the properties file
+  // the services on offer
+  private static final String XSLT_QUERY_SERVICE        = "XSLTQuery";
+  private static final String RESOURCE_RETRIEVE_SERVICE = "ResourceRetrieve";
+  // params used
+  private static final String QUERY_PARAM        = "query";
+  private static final String ALPHABET_PARAM     = "alphabet";
+  
+  private static final int RESULT_SET_SIZE        = 10;
+
+  // alphabets
+  private static final String MINIMUM_ALPHABET   = "minimum";
+  private static final String ASCII_ALPHABET     = "ascii";
+  private static final String UNICODE_ALPHABET   = "unicode";
+
+  private static final String DEFAULT_QUERY_STRING   =
+    "<xsl:stylesheet version=\"1.0\""+
+    "                xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">"+
+    "    <xsl:template match=\"*|/\">"+
+    "         <xsl:apply-templates/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"*|/\" mode=\"m\">"+
+    "         <xsl:apply-templates mode=\"m\"/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"text()|@*\">"+
+    "         <xsl:value-of select=\".\"/>"+
+    "    </xsl:template>"+
+    "    <xsl:template match=\"processing-instruction()|comment()\"/>"+
+    "</xsl:stylesheet>";
+  
+
+  private String alphabet = MINIMUM_ALPHABET;
+  private Element config_info_ = null;
+  
+
+  public XSLTServices() {
+  }
+  
+    /** configure this service */
+    public boolean configure(Element info, Element extra_info) {
+      	if (!super.configure(info, extra_info)){
+	    return false;
+	}
+
+      logger.info("configuring XSLTServices");
+      this.config_info = info;
+    try {
+      logger.info("called with:");
+      TransformerFactory transformerFactory = TransformerFactory.newInstance();
+      Element doc = GSXML.getFirstElementChild(info);//(Element)info.getFirstChild();
+      Transformer transformer = transformerFactory.newTransformer();
+      StreamResult result = new StreamResult(System.out);
+      Source source = new DOMSource(doc);
+      transformer.transform(source,result);
+    } catch (Throwable t) {
+      logger.error("Error printing XML in XSLTService::configure()");
+    }
+    return true;    
+  }
+
+    protected Element getServiceDescription(Document doc, String service, String lang, String subset) {
+
+	return null;
+    }
+  
+  /** process method for specific services - must be implemented by all
+   * subclasses
+   * should implement all services supported by the servicesImpl except
+   * for describe, which is implemented by this base class
+   *
+   */
+  protected Element processService(String service, Element request){
+    if (service.equals(XSLT_QUERY_SERVICE)) {
+      return processXSLTQuery(request);
+    } else if (service.equals(RESOURCE_RETRIEVE_SERVICE)) {
+      return processResourceRetrieve(request);
+    }
+    // create the response so we can report the error
+    Document result_doc = XMLConverter.newDOM();
+    Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, RESOURCE_RETRIEVE_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+    
+    logger.error("should never get here. service type wrong:"+service);
+    GSXML.addError(response,"XSLTServices:should never get here. service type wrong:"+service);
+    return response;
+  }
+
+  /** process a document resquest query */
+  protected Element processResourceRetrieve(Element request) {
+    Document result_doc = XMLConverter.newDOM();
+    // create the result and set the path so we know where we are
+    Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, RESOURCE_RETRIEVE_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+    // get param list 
+    Element param_elem=null;
+    Node n = request.getFirstChild();
+    while (n!=null) {
+	String node_name = n.getNodeName();
+	if (node_name.equals(GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER)) {
+	    param_elem = (Element)n;
+	    break;
+	}
+	n = n.getNextSibling();
+    }
+
+    if (param_elem==null) { 
+	logger.error("bad query request");
+	GSXML.addError(response,"bad query request in XSLTServices");
+	return response; 
+    }
+    
+    // Documents are just the ids decoding using standard URL decoding
+    
+    Element doc_list = (Element)GSXML.getChildByTagName(request, GSXML.DOCUMENT_ELEM+GSXML.LIST_MODIFIER);
+    String []ids = GSXML.getAttributeValuesFromList(doc_list, GSXML.NAME_ATT);
+    for (int j=0; j<ids.length; j++) {
+      String document = null;
+      try {
+	document = URLDecoder.decode(ids[j],"UTF-8");
+      } catch (UnsupportedEncodingException e) {
+	throw new Error(e.toString());
+      }
+      // something funny with the doc - 
+      Element new_doc = result_doc.createElement(GSXML.DOCUMENT_ELEM);
+      new_doc.setAttribute(GSXML.NAME_ATT, ids[j]); 
+      GSXML.addDocText(new_doc, document);
+      response.appendChild(new_doc);
+    }
+    
+    return response;
+  }
+  
+  /** process a XSLT query */
+  protected Element processXSLTQuery(Element request) {
+    Document result_doc = XMLConverter.newDOM();
+    // create the result and set the path so we know where we are
+    Element response = result_doc.createElement(GSXML.RESPONSE_ELEM);
+    String from = GSPath.appendLink(this.cluster_name, XSLT_QUERY_SERVICE);
+    response.setAttribute(GSXML.FROM_ATT, from);
+    response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+
+    // get the parameter list
+    Element param_list = 
+      (Element)GSXML.getChildByTagName(request, GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+
+    // extract the only parameter we care about
+    HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
+    String query = (String)params.get(QUERY_PARAM);
+    
+    if (query == null || query.equals(""))
+      query = DEFAULT_QUERY_STRING;
+    
+    
+    DocumentStream stream = null;
+    if (alphabet.equals(MINIMUM_ALPHABET)){
+      stream = new GeneratedDocumentStream('a','c');
+    } else if(alphabet.equals(ASCII_ALPHABET)){
+      stream = new GeneratedDocumentStream();
+    } else if(alphabet.equals(UNICODE_ALPHABET)){
+      stream = new GeneratedDocumentStream(Character.MIN_VALUE,
+					   Character.MAX_VALUE);
+    } else {
+      logger.error("bad alphabet name : "+ alphabet);
+      GSXML.addError(response,"XSLTServices: bad alphabet name : "+ alphabet);
+      stream = new GeneratedDocumentStream();
+    }
+
+    Element resource_list = result_doc.createElement(GSXML.RESOURCE_ELEM+GSXML.LIST_MODIFIER);
+    response.appendChild(resource_list);
+
+    // Framework to stringise the document
+    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+
+    // add each resource
+    for (int d=0; d<RESULT_SET_SIZE; d++) {
+      try {
+	Document document = stream.nextDocument();
+	Element doc = GSXML.getFirstElementChild(document);//(Element)document.getFirstChild();
+	Transformer transformer = transformerFactory.newTransformer();
+	StringWriter writer = new StringWriter();
+	StreamResult result = new StreamResult(writer);
+	Source source = new DOMSource(doc);
+	transformer.transform(source,result);
+	String id = writer.toString();
+	Element e = result_doc.createElement(GSXML.DOCUMENT_ELEM);
+	e.setAttribute(GSXML.NAME_ATT, id); 
+	resource_list.appendChild(e);
+      } catch (Throwable t) {
+	GSXML.addError(response, "Error in XSLTServices finding results:" + t.toString());
+      }
+    }
+    return response; 
+  }
+}  
+ 
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/DatabaseFactory.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/DatabaseFactory.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/DatabaseFactory.java	(revision 31634)
@@ -0,0 +1,111 @@
+package org.greenstone.gsdl3.sql;
+
+/**
+ * Returns a sql server according to the user's selection
+ *
+ */
+
+import org.apache.log4j.*;
+
+import java.util.HashMap;
+
+public class DatabaseFactory{
+    private static HashMap<String, SQLServer> serverMap = new HashMap<String, SQLServer>();
+    private static Logger logger = Logger.getLogger(org.greenstone.gsdl3.sql.DatabaseFactory.class.getName());
+
+
+    //upcase the first letter
+    public static String properDBName(String name_in){
+	String name_out = "";
+	name_in = name_in.toLowerCase();
+	if (name_in.length() >= 1){
+	    String firstLetter = name_in.substring(0,1).toUpperCase();
+	    name_out = firstLetter +  name_in.substring(1);
+	}
+	return name_out;
+    }
+
+
+    public static final SQLServer getDatabaseServer(String dbname){
+	dbname = properDBName(dbname);
+	if (serverMap.containsKey(dbname)) return serverMap.get(dbname);
+	String fullName = "org.greenstone.gsdl3.sql." + dbname.toLowerCase()+ "." + dbname +"SQLServer";
+	try {
+	    SQLServer server = (SQLServer)Class.forName(fullName).newInstance();
+	    serverMap.put(dbname, server);
+	    return server;
+	} catch (Exception e) {
+	    logger.debug("Couldn't load the database server "+ fullName);
+	}
+	
+	return null; 
+    }
+
+   public static final SQLStatements getSQLStatements(String sqlstate){
+       String fullsqlstate = "org.greenstone.gsdl3.sql." + sqlstate;
+	//load SQLStatements class
+       try{
+	   SQLStatements state = (SQLStatements)Class.forName(fullsqlstate).newInstance(); 
+	   return state;
+       }
+       catch (Exception e) {
+	   // if falied use sqlstate as a full path
+	   try{
+	       SQLStatements state = (SQLStatements)Class.forName(sqlstate).newInstance(); 
+	       return state;
+	   }
+	   catch (Exception e2) {
+	       // failed again, give up
+	       logger.debug("Couldn't load the sql statement  "+ sqlstate);
+	   }
+       }
+		
+       return null; 
+   }
+
+    public static final MetadataDBWrapper getMetadataDBWrapper(String dbname, String sqlstate){
+	dbname = properDBName(dbname);
+
+	String fullName = "org.greenstone.gsdl3.sql." + dbname.toLowerCase()+ "." + dbname +"DBWrapper";
+	try {
+	    MetadataDBWrapper wrapper = (MetadataDBWrapper)Class.forName(fullName).newInstance();
+	    if (wrapper != null){
+		//load SQLServer class
+		SQLServer server = getDatabaseServer(dbname);
+		if (server != null){
+		    wrapper.setSQLServer(server);
+		}
+
+		String fullsqlstate = "org.greenstone.gsdl3.sql." + sqlstate;
+		//load SQLStatements class
+		try{
+		   SQLStatements state = (SQLStatements)Class.forName(fullsqlstate).newInstance(); 
+		   if (state != null){
+		       wrapper.setSQLStatements(state); 
+		       return wrapper;
+		   }
+		}
+		catch (Exception e) {
+		    // if falied use sqlstate as a full path
+		    try{
+			SQLStatements state = (SQLStatements)Class.forName(sqlstate).newInstance(); 
+			if (state != null){
+			    wrapper.setSQLStatements(state);
+			    return wrapper;
+			}
+		    }
+		    catch (Exception e2) {
+			// failed again, give up
+			logger.debug("Couldn't load the sql statement  "+ sqlstate);
+		    }
+		}
+	    }
+	} catch (Exception e) {
+	    logger.debug("Couldn't load the database wrapper "+ fullName);
+	}
+	
+	return null; 
+
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/MetadataDBWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/MetadataDBWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/MetadataDBWrapper.java	(revision 31634)
@@ -0,0 +1,37 @@
+package org.greenstone.gsdl3.sql;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.sql.SQLServer;
+
+public interface MetadataDBWrapper{
+
+    public DBInfo getInfo(String id);
+
+    public void setSQLServer(SQLServer server);
+
+    public void setSQLStatements(SQLStatements sqlstate);
+
+    public boolean openConnection(String databasepath);
+
+    public boolean openAndCreateConnection(String databasepath);
+
+     //return a list of rows
+    public ArrayList<HashMap<String, Object>> executeQuery(String query_statement);
+
+    public ResultSet queryResultSet(String query_statement);
+
+    public boolean execute(String stat);
+
+    public boolean executeUpdate(String stat);
+
+    public void check4Table(String stat) throws SQLException;
+
+    public void closeConnection(String databasepath);
+   
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLServer.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLServer.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLServer.java	(revision 31634)
@@ -0,0 +1,21 @@
+package org.greenstone.gsdl3.sql;
+
+import java.sql.Connection;
+
+/*
+ * this class provide an interface for connecting and querying a database system. In order to not tie Greenstone to a particalur database system 
+ * every database action should go through this interface 
+ * 
+ */
+public interface SQLServer{
+ 
+    /** connect to a database without creating the database if it doesn't exist */
+    public Connection connect(String databasePath);
+
+    /** connect to a database and create the database if it doesn't exist */
+    public Connection connectAndCreate(String databasePath); 
+
+    public boolean disconnect(String databasePath);
+
+   	
+ }
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLStatements.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLStatements.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/SQLStatements.java	(revision 31634)
@@ -0,0 +1,7 @@
+package org.greenstone.gsdl3.sql;
+
+public interface SQLStatements{
+
+    public String getDBInfoStatement(String node_id);
+    public String undoDBSafe(Object text);
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbyDBWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbyDBWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbyDBWrapper.java	(revision 31634)
@@ -0,0 +1,228 @@
+package org.greenstone.gsdl3.sql.derby;
+
+import  org.greenstone.gsdl3.sql.*;
+
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.Statement;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+
+
+import org.greenstone.gsdl3.util.DBInfo;
+
+import org.apache.log4j.*;
+
+public class DerbyDBWrapper implements MetadataDBWrapper{
+    protected SQLServer sqlServer = null;
+    protected SQLStatements sqlState = null;
+    protected Connection connection = null; 
+    protected static Logger logger = Logger.getLogger(org.greenstone.gsdl3.sql.derby.DerbyDBWrapper.class.getName());
+
+
+    public DBInfo getInfo(String id){
+	if(sqlState == null) return null;
+        String dbInfoState = sqlState.getDBInfoStatement(id);
+        ArrayList<HashMap<String, Object>> result = executeQuery(dbInfoState);
+	if (result.size() == 0) return null;
+	DBInfo info = new DBInfo();
+	for(int i=0;  i<result.size();i++){
+	    HashMap arow = result.get(i);
+	    Iterator ite = arow.keySet().iterator();
+	    while(ite.hasNext()){
+		String key = (String)ite.next();
+		Object value = arow.get(key);
+		//logger.info(key + "=>" +value);
+		if (value == null || value.equals("")) continue;
+		info.addInfo(key.toLowerCase(),sqlState.undoDBSafe(value));
+	    }
+	}
+
+	return info;
+    }
+
+    public void setSQLServer(SQLServer server){
+	sqlServer = server;
+    }
+
+    public void setSQLStatements(SQLStatements state){
+	sqlState = state;
+    }
+
+    public boolean openConnection(String databasepath){
+	if(connection != null) return true;
+
+	connection = sqlServer.connect(databasepath);
+	if (connection == null) return false;
+	try{
+	    connection.setAutoCommit(true);	
+	}
+	catch(SQLException sqle){
+	    logger.debug("Database Error occured when creating a statement object",sqle);
+	    return false;
+	}
+	return true;
+    }
+
+    public boolean openAndCreateConnection(String databasepath){
+	if(connection != null) return true;
+    	connection = sqlServer.connectAndCreate(databasepath);
+	
+    	if (connection == null) {
+    		logger.error("sql connection is null. database path="+databasepath);
+    		return false;
+    	}
+
+    	try{
+	    connection.setAutoCommit(true);	
+	}
+    	catch(SQLException sqle){
+    		logger.debug("Database Error occured when creating a statement object",sqle);
+    		return false;
+    	}
+    	return true;
+    }
+
+    public synchronized ArrayList<HashMap<String, Object>> executeQuery(String query_statement){
+	//the database hasn't been correct yet
+    	ArrayList<HashMap<String, Object>> results = new ArrayList<HashMap<String, Object>>();
+	ResultSet rs = null;
+	try{	
+	    //by passing the two arguments, the ResultSet returned from executeQuery is updatable
+	    Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
+	   rs = statement.executeQuery(query_statement);
+	    ResultSetMetaData rsmd = rs.getMetaData();
+	    int numOfColumns = rsmd.getColumnCount();
+	    while(rs.next()){
+		HashMap<String, Object> arow = new HashMap<String, Object>();
+		for(int i = 1; i <= numOfColumns ; i++){
+		    arow.put(rsmd.getColumnName(i).toLowerCase(), rs.getObject(i));
+		}
+		results.add(arow);  
+	    }
+	    statement.close();
+	}
+	catch(SQLException sqle){
+	    logger.debug("Database Error occured when executeQuery " + query_statement,sqle);
+	    return results;
+	}
+	catch(Exception e){    		
+	    logger.debug(e);
+	    return null;
+	}
+	return results;
+    }
+    /**
+     * Same as executeQuery except the return type is ResultSet
+     * @param stat
+     * @return ResultSet of querying the statement 'stat'
+     */
+    public synchronized ResultSet queryResultSet(String stat){
+    	ResultSet rs = null;
+	//by passing the two arguments, the ResultSet returned from executeQuery is updatable
+
+    	try{	       	
+	    	Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
+    		if (statement == null){
+    			logger.info("Null sql statement provided.");
+    			return null;
+    		}
+    		logger.debug(stat);
+    		rs = statement.executeQuery(stat);
+   	      	logger.info("sql stat="+stat+ " result="+rs);
+    	}
+    	catch(SQLException sqle){
+    		logger.info("Database Error occured when execute query " + stat, sqle);
+    		return null;
+    	}
+    	catch(Exception e){    		
+    		logger.info("Exception="+e);
+    		return null;
+    	}
+    	logger.debug(" result="+rs);
+    	return rs;
+    }
+    /**
+     * Used by create (table)
+     */
+    public synchronized boolean execute(String stat) {
+    	boolean rs;
+
+    	try{	       	
+	    Statement statement = connection.createStatement();
+    		if (statement == null){
+    			logger.info("statement is null.");
+    			return false;
+    		}
+
+    		rs = statement.execute(stat);
+		statement.close();
+	}
+    	catch(SQLException sqle){    		
+    		logger.debug("Database Error occured when execute query " + stat, sqle);
+    		return false;
+    	}
+    	catch(Exception e){    		
+    		logger.debug(e);
+    		return false;
+    	}
+    	return rs;
+    }
+    /**
+     * Used by insert, update, and delete
+     */
+    public synchronized boolean executeUpdate(String stat) {
+    	int rs;
+	try{	       	
+	    Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
+    		if (statement == null){
+    			logger.info("statement is null.");
+    			return false;
+    		}
+    		
+    		//either the row count of INSERT, UPDATE OR DELETE statement, or 0 if the statement returns nothing
+    		rs = statement.executeUpdate(stat);
+		statement.close();
+		logger.debug("sql stat="+stat+ " result="+rs);
+    	}
+    	catch(SQLException sqle){    		
+    		logger.debug("Database Error occured when execute query " + stat, sqle);
+    		return false;
+    	}
+    	catch(Exception e){    		
+    		logger.debug("Exception="+e);
+    		return false;
+    	}
+    	logger.debug(" result="+rs);
+    	return (rs==-1)? false : true;
+    }
+    /**
+     * Used by checking the existence of table
+     * Same as the method 'execute' except throws SQLException
+     */
+    public void check4Table(String stat) throws SQLException {
+	Statement statement = connection.createStatement();
+   	statement.executeQuery(stat);
+	statement.close();
+    }
+
+    public void closeConnection(String databasepath){
+	try{
+	    if(connection != null){
+		connection.close();
+		connection = null;
+		sqlServer.disconnect(databasepath);
+	    }
+	}
+	catch(SQLException sqle){
+	    logger.debug("Database Error occured when close connection " + databasepath, sqle);
+	}
+    }
+  
+ }
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbySQLServer.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbySQLServer.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/sql/derby/DerbySQLServer.java	(revision 31634)
@@ -0,0 +1,85 @@
+package org.greenstone.gsdl3.sql.derby;
+
+
+import org.apache.log4j.*;
+import org.greenstone.gsdl3.sql.*;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import org.greenstone.util.GlobalProperties;
+
+public class DerbySQLServer implements SQLServer{
+   
+    static final String PORT = GlobalProperties.getProperty("derby.server.port", "1527");
+    static final String DERBYSERVER = GlobalProperties.getProperty("derby.server", "localhost");
+    static final String PROTOCOL = "jdbc:derby://"+DERBYSERVER+":"+PORT+"/"; // "jdbc:derby://localhost:1527";
+    static final String DRIVER = "org.apache.derby.jdbc.ClientDriver"; //"org.apache.derby.jdbc.EmbeddedDriver";
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.sql.derby.DerbySQLServer.class.getName());     
+
+    public DerbySQLServer(){
+	try{
+	    Class.forName(DRIVER).newInstance();
+	}
+        catch(Exception e){
+	    logger.error("Couldn't find derby driver: "+DRIVER,e);
+	}
+    }
+
+    public Connection connect(String databasePath){
+	try{
+	    String  protocol_str = PROTOCOL + databasePath; 
+	    Connection connection = DriverManager.getConnection(protocol_str);
+	    return connection;
+	}
+        catch(Exception e){
+	    logger.info("Connect to database "+databasePath + " failed!");
+	}
+
+	return null;
+	
+    }
+
+    public Connection connectAndCreate(String databasePath){
+	try{
+	    String  protocol_str = PROTOCOL + databasePath + ";create=true"; 
+	    Connection connection = DriverManager.getConnection(protocol_str);
+	    return connection;
+	}
+	catch(Exception e){
+	    logger.error("Connect to database "+databasePath + " failed!",e);
+	
+	}
+	return null;
+    } 
+
+    public boolean disconnect(String databasePath){
+
+	// Only shutdown if using embedded derby, 
+	// not if it's a networked derby server, which is what we now use
+
+	if(!DRIVER.equals("org.apache.derby.jdbc.EmbeddedDriver")) {
+	    return true;
+	}
+
+	// embedded derby driver
+	try{
+	    String protocol_str = PROTOCOL + databasePath + ";shutdown=true"; 	    
+	    DriverManager.getConnection(protocol_str);
+	    
+	}catch (SQLException se){
+	    String theError = (se).getSQLState();
+	    if (!theError.equals("08006")){
+//		logger.error("Database "+databasePath + " couldn't be shut down properly!",se);
+	    }
+	    else{
+		return true;
+	    }
+	}
+	
+	return false;
+    }
+
+
+   
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/AbstractBasicDocument.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/AbstractBasicDocument.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/AbstractBasicDocument.java	(revision 31634)
@@ -0,0 +1,147 @@
+/*
+ *    AbstractBasicDocument.java
+ *    Copyright (C) 2011 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+import org.apache.log4j.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public abstract class AbstractBasicDocument {
+
+  /** info types */
+	public static final String INFO_NUM_SIBS = "numSiblings";
+	public static final String INFO_NUM_CHILDREN = "numChildren";
+	public static final String INFO_SIB_POS = "siblingPosition";
+	public static final String INFO_DOC_TYPE = "documentType";
+
+    /** XML element for describe requests - the container doc */
+    //protected Document doc = null; // typically a shared reference to the one in ServiceRack
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.AbstractBasicDocument.class.getName());
+
+    public AbstractBasicDocument() 
+    {
+    }
+
+    /** create an element to go into a results list. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy'/>
+     */
+  public Element createDocNode(Document doc, String node_id) {
+    return createDocNode(doc, node_id, null);
+  }
+    /** create an element to go into a results list. A node element
+     * has the form
+     * <docNode nodeId='xxx' nodeType='leaf' docType='hierarchy' [rank='0.23']/>
+     */
+  public Element createDocNode(Document doc, String node_id, String rank) {
+	Element node = doc.createElement(GSXML.DOC_NODE_ELEM);
+	node.setAttribute(GSXML.NODE_ID_ATT, node_id);
+	if (rank != null) {
+	  node.setAttribute(GSXML.NODE_RANK_ATT, rank);
+	}
+	String doc_type = getDocType(node_id);
+	node.setAttribute(GSXML.DOC_TYPE_ATT, doc_type);
+	String node_type = getNodeType(node_id, doc_type);	
+	node.setAttribute(GSXML.NODE_TYPE_ATT, node_type);
+	return node;
+    }
+
+	/**
+	 * adds all the children of doc_id to the doc element, and if
+	 * recursive=true, adds all their children as well
+	 */
+  abstract public void addDescendants(Element doc, String doc_id, boolean recursive);
+  
+
+	/**
+	 * adds all the siblings of current_id to the parent element. returns the
+	 * new current element
+	 */
+  abstract public Element addSiblings(Element parent_node, String parent_id, String current_id);
+
+    /** returns the node type of the specified node.
+	should be one of 
+	GSXML.NODE_TYPE_LEAF, 
+	GSXML.NODE_TYPE_INTERNAL, 
+	GSXML.NODE_TYPE_ROOT
+    */
+    public String getNodeType(String node_id, String doc_type) {
+	if (doc_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+
+	if (!hasParent(node_id)) {
+	    return GSXML.NODE_TYPE_ROOT;
+	}
+	if (doc_type.equals(GSXML.DOC_TYPE_PAGED)) {
+	     return GSXML.NODE_TYPE_LEAF;
+	}
+	if (!hasChildren(node_id)) {
+	    return GSXML.NODE_TYPE_LEAF;
+	}
+	return GSXML.NODE_TYPE_INTERNAL;	
+	
+    }
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_PAGEDHIERARCHY, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    abstract public String getDocType(String node_id);
+    
+    /** returns true if the node has child nodes 
+     */
+    abstract public boolean hasChildren(String node_id);
+    
+	/**
+	 * returns the structural information asked for. info_type may be one of
+	 * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS, INFO_DOC_TYPE
+	 */
+	abstract public String getStructureInfo(String doc_id, String info_type);
+
+  abstract public int getNumChildren(String node_id) ;
+  /** returns a list of the child ids in order, null if no children
+   */
+  abstract public ArrayList<String> getChildrenIds(String node_id);
+    /** returns true if the node has a parent 
+     */
+    abstract public boolean hasParent(String node_id);
+
+	/**
+	 * returns the node id of the parent node, null if no parent 
+	 */
+  abstract public String getParentId(String node_id);
+
+  /** 
+   * returns the node id of the root node of the document containing node_id
+   */
+  abstract public String getRootId(String node_id);
+
+  /** returns the list of sibling ids, including the specified node_id */
+  abstract public ArrayList<String> getSiblingIds(String node_id);
+
+  /** get a piece of metadata about the document node */
+  abstract public String getMetadata(String node_id, String meta_name);
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocument.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocument.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocument.java	(revision 31634)
@@ -0,0 +1,142 @@
+/*
+ *    BasicDocument.java
+ *    Copyright (C) 2011 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+import org.apache.log4j.*;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+public class BasicDocument extends AbstractBasicDocument {
+
+
+    /** the default document type - use if all documents are the same type
+     */
+    protected String default_document_type = null;
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.BasicDocument.class.getName());
+
+  public BasicDocument(String default_document_type)
+    {
+	super();
+
+	this.default_document_type = default_document_type;
+    }
+
+	/**
+	 * returns the structural information asked for. info_type may be one of
+	 * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS, INFO_DOC_TYPE
+	 */
+  public String getStructureInfo(String doc_id, String info_type) {
+    if (info_type.equals(INFO_NUM_SIBS)) {
+      return "0";
+    }
+    if (info_type.equals(INFO_NUM_CHILDREN)) {
+      return "0";
+    }
+    if (info_type.equals(INFO_SIB_POS)) {
+      return "-1";
+    }
+    if (info_type.equals(INFO_DOC_TYPE)) {
+      return getDocType(doc_id);
+    }
+    return null;
+  }
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+	GSXML.DOC_TYPE_PAGEDHIERARCHY
+	default implementation returns GSXML.DOC_TYPE_SIMPLE, over ride 
+	if documents can be hierarchical
+    */
+    public String getDocType(String node_id) {
+      if (default_document_type != null) {
+	return default_document_type;
+      }
+      return GSXML.DOC_TYPE_SIMPLE;
+    }
+    
+    /** returns true if the node has child nodes 
+     * default implementation returns false, over ride if documents can be 
+     * hierarchical 
+     */
+    public boolean hasChildren(String node_id) {
+	return false;
+    }
+  public int getNumChildren(String node_id) {
+    return 0;
+  }
+  /** returns a list of the child ids in order, null if no children
+   * default implementation: return null
+   */
+  public ArrayList<String> getChildrenIds(String node_id) {
+    return null;
+  }
+    /** returns true if the node has a parent 
+     * default implementation returns false, over ride if documents can be 
+     * hierarchical*/
+    public boolean hasParent(String node_id) {
+	return false;
+    }
+
+	/**
+	 * returns the node id of the parent node, null if no parent 
+	 * default implementation return null
+	 */
+  public String getParentId(String node_id) {
+    return null;
+  }
+
+  /** 
+   * returns the node id of the root node of the document containing node_id
+   * default implementation: return node_id
+   */
+  public String getRootId(String node_id) {
+    return node_id;
+  }
+	/**
+	 * adds all the children of doc_id to the doc element, and if
+	 * recursive=true, adds all their children as well
+	 * default implementation: do nothing
+	 */
+  public void addDescendants(Element doc, String doc_id, boolean recursive) {
+    return;
+  }
+
+	/**
+	 * adds all the siblings of current_id to the parent element. returns the
+	 * new current element
+	 */
+  public Element addSiblings(Element parent_node, String parent_id, String current_id)
+  {
+    return null;
+  }
+
+  /** returns the list of sibling ids, including the specified node_id */
+  public ArrayList<String> getSiblingIds(String node_id) {
+    return null;
+  }
+
+  public String getMetadata(String node_id, String meta_name) {
+    return null;
+  }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocumentDatabase.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocumentDatabase.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/BasicDocumentDatabase.java	(revision 31634)
@@ -0,0 +1,367 @@
+/*
+ *    BasicDocumentDatabase.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *   the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+// Greenstone classes
+import org.greenstone.gsdl3.util.OID;
+import org.greenstone.gsdl3.util.DBInfo;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
+import org.greenstone.gsdl3.util.GSFile;
+
+// XML classes
+import org.w3c.dom.Document;
+import org.w3c.dom.Element; 
+import org.w3c.dom.NodeList;
+
+// java
+import java.util.Vector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.Iterator;
+import java.io.File;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.*;
+
+public class BasicDocumentDatabase extends AbstractBasicDocument
+{
+    // collection database
+    protected SimpleCollectionDatabase coll_db = null;
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.BasicDocumentDatabase.class.getName());
+
+
+    /** constructor */
+    public BasicDocumentDatabase(String database_type, String site_home,
+				  String cluster_name,  String index_stem)
+    {
+	super();
+
+	coll_db = new SimpleCollectionDatabase(database_type);
+	if (!coll_db.databaseOK()) {
+	  logger.error("Couldn't create the collection database of type "+database_type);
+	}
+
+	// Open database for querying
+	String coll_db_file = GSFile.collectionDatabaseFile(site_home, cluster_name, index_stem, database_type);
+	if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ)) {
+	    logger.error("Could not open collection database!");
+	    coll_db = null;
+	}
+    }
+
+    public boolean isValid() 
+    {
+	return (coll_db != null);
+    }
+	
+    public void cleanUp() {
+	this.coll_db.closeDatabase();
+    }
+
+
+    public DBInfo getInfo(String main_key) {
+	if (this.coll_db==null) {
+	    // Most likely cause is that this installation of Greenstone3 has not 
+	    // been compiled with support for this database type
+	    return null;
+	}
+
+	return coll_db.getInfo(main_key);
+    }
+
+
+    /** returns the document type of the doc that the specified node 
+	belongs to. should be one of 
+	GSXML.DOC_TYPE_SIMPLE, 
+	GSXML.DOC_TYPE_PAGED, 
+	GSXML.DOC_TYPE_HIERARCHY
+    */
+    public String getDocType(String node_id){
+	DBInfo info = this.coll_db.getInfo(node_id);
+	if (info == null) {
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+	String doc_type = info.getInfo("doctype");
+	if (!doc_type.equals("")&&!doc_type.equals("doc")) {
+	    return doc_type;
+	}
+
+	String top_id = OID.getTop(node_id);
+	boolean is_top = (top_id.equals(node_id) ? true : false);
+	
+	String children = info.getInfo("contains");
+	boolean is_leaf = (children.equals("") ? true : false);
+
+	if (is_top && is_leaf) { // a single section document
+	    return GSXML.DOC_TYPE_SIMPLE;
+	}
+
+	// now we just check the top node
+	if (!is_top) { // we need to look at the top info
+	    info = this.coll_db.getInfo(top_id);
+	}
+	if (info == null) {
+	    return GSXML.DOC_TYPE_HIERARCHY;
+	}
+ 
+	String childtype = info.getInfo("childtype");
+	if (childtype.equals("Paged")) {
+	    return GSXML.DOC_TYPE_PAGED;
+	}
+	if (childtype.equals("PagedHierarchy")) {
+	  
+	  return GSXML.DOC_TYPE_PAGED_HIERARCHY;
+	}
+	return GSXML.DOC_TYPE_HIERARCHY;
+
+    }
+    
+	/**
+	 * returns the structural information asked for. info_type may be one of
+	 * INFO_NUM_SIBS, INFO_NUM_CHILDREN, INFO_SIB_POS, INFO_DOC_TYPE
+	 */
+  public String getStructureInfo(String doc_id, String info_type){
+		String value = "";
+		if (info_type.equals(INFO_NUM_SIBS))
+		{
+			String parent_id = OID.getParent(doc_id);
+			if (parent_id.equals(doc_id))
+			{
+				value = "0";
+			}
+			else
+			{
+				value = String.valueOf(getNumChildren(parent_id));
+			}
+			return value;
+		}
+
+		if (info_type.equals(INFO_NUM_CHILDREN))
+		{
+			return String.valueOf(getNumChildren(doc_id));
+		}
+
+		if (info_type.equals(INFO_SIB_POS))
+		{
+			String parent_id = OID.getParent(doc_id);
+			if (parent_id.equals(doc_id))
+			{
+				return "-1";
+			}
+
+			DBInfo info = this.coll_db.getInfo(parent_id);
+			if (info == null)
+			{
+				return "-1";
+			}
+
+			String contains = info.getInfo("contains");
+			contains = StringUtils.replace(contains, "\"", parent_id);
+			String[] children = contains.split(";");
+			for (int i = 0; i < children.length; i++)
+			{
+				String child_id = children[i];
+				if (child_id.equals(doc_id))
+				{
+					return String.valueOf(i + 1); // make it from 1 to length
+
+				}
+			}
+
+			return "-1";
+		}
+		if (info_type.equals(INFO_DOC_TYPE))
+	
+		{
+		  return getDocType(doc_id);
+		}
+		return null;
+
+  }
+
+    /** returns true if the node has child nodes */
+    public boolean hasChildren(String node_id){
+	DBInfo info = this.coll_db.getInfo(node_id);
+	if (info == null) {
+	    return false;
+	}
+	String contains = info.getInfo("contains");
+	if (contains.equals("")) {
+	    return false;
+	}
+	return true;
+    }
+    
+  public int getNumChildren(String node_id) {
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return 0;
+		}
+		String contains = info.getInfo("contains");
+		if (contains.equals(""))
+		{
+			return 0;
+		}
+		String[] children = contains.split(";");
+		return children.length;
+
+
+  }
+  /** returns a list of the child ids in order, null if no children
+   */
+  public ArrayList<String> getChildrenIds(String node_id) {
+		DBInfo info = this.coll_db.getInfo(node_id);
+		if (info == null)
+		{
+			return null;
+		}
+
+		String contains = info.getInfo("contains");
+		if (contains.equals(""))
+		{
+			return null;
+		}
+		ArrayList<String> children = new ArrayList<String>();
+		StringTokenizer st = new StringTokenizer(contains, ";");
+		while (st.hasMoreTokens())
+		{
+			String child_id = StringUtils.replace(st.nextToken(), "\"", node_id);
+			children.add(child_id);
+		}
+		return children;
+
+
+  }
+
+  public String getMetadata(String node_id, String meta_name) {
+    DBInfo info = this.coll_db.getInfo(node_id);
+    if (info == null)
+      {
+  	return null;
+      }
+    return info.getInfo(meta_name);
+  }
+		
+    /** returns true if the node has a parent */
+    public boolean hasParent(String node_id){
+	String parent = OID.getParent(node_id);
+	if (parent.equals(node_id)) {
+	    return false;
+	}
+	return true;
+    }
+	/**
+	 * returns the node id of the parent node, null if no parent 
+	 */
+  public String getParentId(String node_id) {
+		String parent = OID.getParent(node_id);
+		if (parent.equals(node_id))
+		{
+			return null;
+		}
+		return parent;
+
+  }
+    
+  /** 
+   * returns the node id of the root node of the document containing node_id
+   */
+  public String getRootId(String node_id) {
+    return OID.getTop(node_id);
+  }
+
+   /** convert indexer internal id to Greenstone oid */
+    public String internalNum2OID(long docnum)
+    {
+	return this.coll_db.docnum2OID(docnum);
+    }
+
+    public String internalNum2OID(String docnum)
+    {
+	return this.coll_db.docnum2OID(docnum);
+    }
+
+
+	/**
+	 * adds all the children of doc_id to the doc element, and if
+	 * recursive=true, adds all their children as well
+	 */
+	public void addDescendants(Element doc, String doc_id, boolean recursive)
+	{
+		ArrayList<String> child_ids = getChildrenIds(doc_id);
+		if (child_ids == null)
+			return;
+		for (int i = 0; i < child_ids.size(); i++)
+		{
+			String child_id = child_ids.get(i);
+			Element child_elem = createDocNode(doc.getOwnerDocument(), child_id);
+			doc.appendChild(child_elem);
+			if (recursive && (!child_elem.getAttribute(GSXML.NODE_TYPE_ATT).equals(GSXML.NODE_TYPE_LEAF) || child_elem.getAttribute(GSXML.DOC_TYPE_ATT).equals(GSXML.DOC_TYPE_PAGED)))
+			{
+				addDescendants(child_elem, child_id, recursive);
+			}
+		}
+	}
+
+	/**
+	 * adds all the siblings of current_id to the parent element. returns the
+	 * new current element
+	 */
+	public Element addSiblings(Element parent_node, String parent_id, String current_id)
+	{
+		Element current_node = GSXML.getFirstElementChild(parent_node);//(Element)parent_node.getFirstChild();
+		if (current_node == null)
+		{
+			// create a sensible error message
+			logger.error(" there should be a first child.");
+			return null;
+		}
+		// remove the current child,- will add it in later in its correct place
+		parent_node.removeChild(current_node);
+
+		// add in all the siblings,
+		addDescendants(parent_node, parent_id, false);
+
+		// find the node that is now the current node
+		// this assumes that the new node that was created is the same as 
+		// the old one that was removed - we may want to replace the new one 
+		// with the old one.
+		Element new_current = GSXML.getNamedElement(parent_node, current_node.getNodeName(), GSXML.NODE_ID_ATT, current_id);
+		return new_current;
+	}
+
+  /** returns the list of sibling ids, including the specified node_id */
+  public ArrayList<String> getSiblingIds(String node_id){
+		String parent_id = getParentId(node_id);
+		if (parent_id == null)
+		{
+			return null;
+		}
+		return getChildrenIds(parent_id);
+
+  }
+
+}
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/CustomClassLoader.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/CustomClassLoader.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/CustomClassLoader.java	(revision 31634)
@@ -0,0 +1,64 @@
+/*
+ *    CustomClassLoader.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.net.URL;
+import java.util.Enumeration;
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Looks for classes/resources in the specified resources_dir directory
+ */
+public class CustomClassLoader extends ClassLoader
+{
+
+	String base_dir = null;
+
+  public CustomClassLoader(ClassLoader parent, String resources_dir)
+	{
+		super(parent);
+		this.base_dir = resources_dir; 
+	}
+  
+  // Resource Bundle loading with a class loader will call getResource(). 
+  // THe Java implementation of getResource tries using the parent class 
+  // loader to find the resource (or the system class loader if no parent). 
+  // If the resource not found, then it calls the local findResource method. 
+  // Ie parent:child ordering.
+  // Because we want local files to override default ones, we need 
+  // child:parent ordering, so change getResource to look locally first.
+  //public URL findResource(String name)
+	public URL getResource(String name)
+	{
+		File resource_path = new File(this.base_dir, name);
+		try
+		{
+			if (resource_path.exists())
+			{
+				return resource_path.toURI().toURL();
+			}
+		}
+		catch (Exception e)
+		{
+		}
+
+		return super.getResource(name);
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBHelper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBHelper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBHelper.java	(revision 31634)
@@ -0,0 +1,47 @@
+/*
+ *    DBHelper.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.HashMap;
+
+public class DBHelper
+{
+    protected static final HashMap<String, String> _extMap = new HashMap<String, String>();
+
+    public static String getDBExtFromDBType(String dbtype)
+    {
+	String lc_dbtype = dbtype.toLowerCase();
+	return _extMap.get(lc_dbtype);
+    }
+
+    /** @function registerDBTypeExt(String, String)
+     *
+     *  Given a database type name and the matching extension, register in
+     *  the static hashmap of mappings.
+     *
+     *  @param dbtype a String containing the database type's name
+     *  @param dbext a String containing the database type's extension
+     *
+     */
+    public static void registerDBTypeExt(String dbtype, String dbext)
+    {
+	_extMap.put(dbtype, dbext);
+    }
+    /** registerDBTypeExt(String, String) **/
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBInfo.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBInfo.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DBInfo.java	(revision 31634)
@@ -0,0 +1,99 @@
+/*
+ *    DBInfo.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.Vector;
+import java.util.HashMap;
+import java.util.Set;
+
+/**
+ * class to hold info from a gdbm (or equiv) query maps a String key to a list
+ * of Strings (values) at the moment, the user must know if something has a
+ * single value or not
+ */
+public class DBInfo
+{
+	protected HashMap<String, Vector<String>> info_map_;
+
+	public DBInfo()
+	{
+		info_map_ = new HashMap<String, Vector<String>>();
+	}
+
+	// methods for keys that can have a single value
+
+	/** set the value for a key - replaces any existing value */
+	public void setInfo(String key, String value)
+	{
+		Vector<String> v = new Vector<String>();
+		v.add(value);
+		info_map_.put(key, v);
+	}
+
+	/**
+	 * get the value - used for keys that have one value, otherwise just returns
+	 * the first one
+	 */
+	public String getInfo(String key)
+	{
+		Vector<String> items = info_map_.get(key);
+		if (items == null)
+		{
+			return "";
+		}
+		return items.firstElement();
+	}
+
+	// methods for keys that can have multiple values
+
+	/** add a value to a key - for keys that can have multiple values */
+	public void addInfo(String key, String value)
+	{
+		Vector<String> v = info_map_.get(key);
+		if (v == null)
+		{
+			v = new Vector<String>();
+		}
+		v.add(value);
+		info_map_.put(key, v);
+	}
+
+	/**
+	 * return a vector of all values associated with a key
+	 * 
+	 * @return Vector of Strings
+	 */
+	public Vector<String> getMultiInfo(String key)
+	{
+		return info_map_.get(key);
+	}
+
+	/** returns a list of all the keys in the info */
+	public Set<String> getKeys()
+	{
+		return info_map_.keySet();
+	}
+
+	/** returns the Info as a string */
+	public String toString()
+	{
+		return info_map_.toString();
+
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DerbyWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DerbyWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DerbyWrapper.java	(revision 31634)
@@ -0,0 +1,870 @@
+/*
+ *    DerbyWrapper.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.greenstone.gsdl3.service.Authentication;
+import org.greenstone.util.GlobalProperties;
+
+public class DerbyWrapper
+{
+    static final String PORT;
+    static final String DERBYSERVER;
+    static final String PROTOCOL;	
+
+    // static code block to initialise the above
+    static {
+	// GlobalProperties won't be loaded at this point if running ant config-admin or ant config-user 
+	// from the command line (both of which call ant update-userdb which in turn calls ModifyUsersDB.java)
+	// In such a case, the ant command will have set the system property (-Dgsdl3_writablehome) 
+	// and passed this to ModifyUsersDB.java. Use that to load the GlobalProperties at this point
+
+	if(GlobalProperties.getGSDL3Home() == null) { // testing whether GlobalProperties is already loaded
+	    String gsdl3_writablehome = System.getProperty("gsdl3.writablehome"); // set by 'ant update-userdb' cmd
+
+	    //System.err.println("@@@@@ writablehome: " + gsdl3_writablehome);
+	    GlobalProperties.loadGlobalProperties(gsdl3_writablehome);
+	}
+	
+	//System.err.println("@@@@@ GlobalProperties.getGSDL3Home(): " + GlobalProperties.getGSDL3Home()); //test
+	
+	// No more fallback values, use exactly what's propagated into global.properties from build.properties
+	String port = GlobalProperties.getProperty("derby.server.port");//, "1527");
+	if(port.indexOf("@") == -1) { // if there are still @placeholders@ in global.properties, then use fallback values
+	    PORT = port;
+	    DERBYSERVER = GlobalProperties.getProperty("derby.server");//, "localhost");
+	} else {
+	    PORT = "1527";
+	    DERBYSERVER = "localhost";
+	}
+	PROTOCOL = "jdbc:derby://"+DERBYSERVER+":"+PORT+"/"; // "jdbc:derby://localhost:1527"; // by default
+	//System.out.println("@@@ PROTOCOL:" + PROTOCOL); //check in installer
+    }
+   
+
+	static final String DRIVER = "org.apache.derby.jdbc.ClientDriver"; //"org.apache.derby.jdbc.EmbeddedDriver";
+	static final String USERSDB = "usersDB";
+	static final String USERS = "users";
+	static final String ROLES = "roles";
+	static final String DATA = "data";
+	private Connection conn = null;
+	private static String protocol_str;
+
+	public DerbyWrapper(String dbpath)
+	{
+		connectDatabase(dbpath, false);
+	}
+
+	public static void createDatabaseIfNeeded()
+	{
+		String usersDB_dir = GlobalProperties.getGSDL3Home() + File.separatorChar + "etc" + File.separatorChar + "usersDB";
+		protocol_str = PROTOCOL + usersDB_dir;
+		File usersDB_file = new File(usersDB_dir);
+		if (!usersDB_file.exists())
+		{
+			String etc_dir = GlobalProperties.getGSDL3Home() + File.separatorChar + "etc";
+			File etc_file = new File(etc_dir);
+			if (!etc_file.exists())
+			{
+				boolean success = etc_file.mkdir();
+				if (!success)
+				{
+					System.err.println("Couldn't create the etc dir under " + GlobalProperties.getGSDL3Home() + ".");
+				}
+			}
+			try
+			{
+				DerbyWrapper.createDatabase(DriverManager.getConnection(protocol_str + ";create=true"));
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+			}
+		}
+	}
+
+	public void connectDatabase(String dbpath, boolean create_database)
+	{
+		try
+		{
+			if (conn != null)
+			{
+				System.err.println("Connection already established, close the database first");
+				return;
+			}
+
+			Class.forName(DRIVER).newInstance();
+			protocol_str = PROTOCOL + dbpath;
+			if (create_database)
+			{
+				conn = DriverManager.getConnection(protocol_str + ";create=true");
+			}
+			else
+			{
+				conn = DriverManager.getConnection(protocol_str);
+			}
+			conn.setAutoCommit(false);
+		}
+		catch (Throwable e)
+		{
+			System.out.println("exception thrown:");
+			if (e instanceof SQLException)
+			{
+				printSQLError((SQLException) e);
+			}
+			else
+			{
+				e.printStackTrace();
+			}
+		}
+	}
+
+	public void closeDatabase()
+	{
+		try
+		{
+			conn.commit();
+			conn.close();
+			conn = null;
+		}
+		catch (SQLException e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public static void shutdownDatabaseServer()
+	{
+
+		// shutdown the server if we're using an embedded derby
+		// if we're a derby client using the derby network server
+		
+		if(!DRIVER.equals("org.apache.derby.jdbc.EmbeddedDriver")) {
+		    return;
+		}
+
+		boolean gotSQLExc = false;
+		try
+		{
+		    DriverManager.getConnection(PROTOCOL + ";shutdown=true");
+		}
+		catch (SQLException se)
+		{
+			// this is good (i.e. what Derby is designed to do on a successful shutdown)
+			gotSQLExc = true;
+			//System.out.println("Shutdown returned: " + se);
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+		}
+		if (!gotSQLExc)
+		{
+			System.err.println("Warning: Derby did not shut down normally");
+		}
+	}
+
+	public void clearUserData()
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("drop table data");
+			state.execute("create table data (username varchar(40) not null, name varchar(128) not null, value clob, primary key (username, name))");
+			conn.commit();
+			state.close();
+		}
+		catch (SQLException e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public void clearTrackerData()
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("drop table usertracker");
+			state.execute("create table usertracker (username varchar(40) not null, collection varchar(128) not null, site varchar(128) not null, oid varchar(128) not null, time varchar(128) not null, action varchar(128) not null, primary key (username, time))");
+			conn.commit();
+			state.close();
+		}
+		catch (SQLException e)
+		{
+			e.printStackTrace();
+		}
+	}
+
+	public static void createDatabase(Connection conn)
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("create table users (username varchar(40) not null, password varchar(40) not null, accountstatus varchar(10), comment varchar(100), email varchar(40), primary key(username))");
+			state.execute("create table roles (username varchar(40) not null, role varchar(40) not null, primary key (username, role))");
+			state.execute("create table data (username varchar(40) not null, name varchar(128) not null, value clob, primary key (username, name))");
+			state.execute("create table usertracker (username varchar(40) not null, collection varchar(128) not null, site varchar(128) not null, oid varchar(128) not null, time varchar(128) not null, action varchar(128) not null, primary key (username, time))");
+
+			state.execute("insert into " + USERS + " values ('admin', '" + Authentication.hashPassword("admin") + "', 'true', 'change the password for this account as soon as possible', '')");
+			state.execute("insert into " + ROLES + " values ('admin', 'administrator')");
+			state.execute("insert into " + ROLES + " values ('admin', 'all-collections-editor')");
+			conn.commit();
+			state.close();
+			conn.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+	}
+
+	public void addUserAction(String username, String site, String collection, String oid, String action)
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("INSERT INTO usertracker VALUES ('" + username + "', '" + collection + "', '" + site + "', '" + oid + "', '" + System.currentTimeMillis() + "', '" + action + "')");
+			conn.commit();
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+	}
+
+	public ArrayList<HashMap<String, String>> getMostRecentUserActions(String site, String collection, String oid)
+	{
+		ArrayList<HashMap<String, String>> actions = new ArrayList<HashMap<String, String>>();
+
+		try
+		{
+			String query = "SELECT username, action, time FROM usertracker WHERE site = '" + site + "' and collection = '" + collection + "' and oid = '" + oid + "' ORDER BY time";
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery(query);
+			conn.commit();
+
+			HashSet<String> usernamesSeen = new HashSet<String>();
+			while (rs.next())
+			{
+				String timeStr = rs.getString("time");
+				long time = Long.parseLong(timeStr);
+
+				if (System.currentTimeMillis() - time > 6000)
+				{
+					continue;
+				}
+
+				HashMap<String, String> action = new HashMap<String, String>();
+				if (!usernamesSeen.contains(rs.getString("username")))
+				{
+					action.put("username", rs.getString("username"));
+					action.put("action", rs.getString("action"));
+					actions.add(action);
+
+					usernamesSeen.add(rs.getString("username"));
+				}
+			}
+			state.close();
+
+			clearOldUserActions();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+		return actions;
+	}
+
+	public void clearOldUserActions()
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("DELETE FROM usertracker WHERE (CAST (time AS BIGINT)) < " + (System.currentTimeMillis() - 20000));
+			conn.commit();
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+	}
+
+	public UserQueryResult listAllUsers() throws SQLException
+	{
+		UserQueryResult userQueryResult = new UserQueryResult();
+		String sql_list_all_user = "SELECT username, password, accountstatus, email, comment FROM " + USERS;
+
+		ArrayList<HashMap<String, String>> users = new ArrayList<HashMap<String, String>>();
+		Statement state = conn.createStatement();
+		ResultSet rs = state.executeQuery(sql_list_all_user);
+		conn.commit();
+		state.close();
+
+		while (rs.next())
+		{
+			HashMap<String, String> user = new HashMap<String, String>();
+			user.put("username", rs.getString("username"));
+			user.put("password", rs.getString("password"));
+			user.put("as", rs.getString("accountstatus"));
+			user.put("comment", rs.getString("comment"));
+			user.put("email", rs.getString("email"));
+
+			users.add(user);
+		}
+
+		for (HashMap<String, String> user : users)
+		{
+			ResultSet gs = state.executeQuery("SELECT role FROM " + ROLES + " WHERE username = '" + user.get("username") + "'");
+			String group = "";
+			while (gs.next())
+			{
+				if (!group.equals(""))
+				{
+					group += ",";
+				}
+				group += gs.getString("role");
+			}
+			userQueryResult.addUserTerm(user.get("username"), user.get("password"), group, user.get("as"), user.get("comment"), user.get("email"));
+		}
+
+		if (userQueryResult.getSize() == 0)
+		{
+			System.out.println("couldn't find any users");
+			return null;
+		}
+		else
+		{
+			return userQueryResult;
+		}
+	}
+
+	public boolean addUserData(String username, String name, String value)
+	{
+		//Check if we already have a value under this name
+		boolean found = false;
+		try
+		{
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery("SELECT * FROM " + DATA + " WHERE username='" + username + "' AND name='" + name + "'");
+			conn.commit();
+			if (rs.next())
+			{
+				found = true;
+			}
+			else
+			{
+				found = false;
+			}
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			System.out.println("exception thrown:");
+			if (ex instanceof SQLException)
+			{
+				printSQLError((SQLException) ex);
+				ex.printStackTrace();
+			}
+			else
+			{
+				ex.printStackTrace();
+			}
+
+			System.out.println("Error:" + ex.getMessage());
+			return false;
+		}
+
+		try
+		{
+			PreparedStatement stmt = null;
+			if (!found)
+			{
+				stmt = conn.prepareStatement("INSERT INTO " + DATA + " VALUES (?, ?, ?)");
+				stmt.setString(1, username);
+				stmt.setString(2, name);
+				stmt.setString(3, value);
+				stmt.executeUpdate();
+			}
+			else
+			{
+				stmt = conn.prepareStatement("UPDATE " + DATA + " SET value=? WHERE username=? AND name=?");
+				stmt.setString(1, value);
+				stmt.setString(2, username);
+				stmt.setString(3, name);
+				stmt.executeUpdate();
+			}
+			conn.commit();
+			stmt.close();
+		}
+		catch (Exception ex)
+		{
+			System.out.println("exception thrown:");
+			if (ex instanceof SQLException)
+			{
+				printSQLError((SQLException) ex);
+			}
+			else
+			{
+				ex.printStackTrace();
+			}
+
+			System.out.println("Error:" + ex.getMessage());
+			return false;
+		}
+		return true;
+	}
+
+	public String getUserData(String username, String name)
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery("SELECT * FROM " + DATA + " WHERE username='" + username + "' AND name='" + name + "'");
+			conn.commit();
+			if (rs.next())
+			{
+				return rs.getString("value");
+			}
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			System.out.println("exception thrown:");
+			if (ex instanceof SQLException)
+			{
+				printSQLError((SQLException) ex);
+			}
+			else
+			{
+				ex.printStackTrace();
+			}
+
+			System.out.println("Error:" + ex.getMessage());
+		}
+		return null;
+	}
+
+	public boolean addUser(String username, String password, String groups, String accountstatus, String comment, String email)
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			String sql_insert_user = "insert into " + USERS + " values ('" + username + "', '" + password + "', '" + accountstatus + "', '" + comment + "', '" + email + "')";
+			state.execute(sql_insert_user);
+
+			String[] groupArray = groups.split(",");
+			for (String g : groupArray)
+			{
+				String sql_insert_group = "insert into " + ROLES + " values ('" + username + "', '" + g + "')";
+				state.execute(sql_insert_group);
+			}
+
+			conn.commit();
+			state.close();
+		}
+		catch (Throwable e)
+		{
+			System.out.println("exception thrown:");
+			if (e instanceof SQLException)
+			{
+				printSQLError((SQLException) e);
+			}
+			else
+			{
+				e.printStackTrace();
+			}
+
+			System.out.println("Error:" + e.getMessage());
+			return false;
+		}
+
+		return true;
+	}
+
+	public boolean deleteUser(String del_username)
+	{
+		try
+		{
+			String sql_delete_user = "delete from " + USERS + " where username='" + del_username + "'";
+			String sql_delete_groups = "delete from " + ROLES + " where username='" + del_username + "'";
+			Statement state = conn.createStatement();
+			state.execute(sql_delete_user);
+			state.execute(sql_delete_groups);
+			conn.commit();
+			state.close();
+		}
+		catch (Throwable e)
+		{
+			System.out.println("exception thrown:");
+			if (e instanceof SQLException)
+			{
+				printSQLError((SQLException) e);
+			}
+			else
+			{
+				e.printStackTrace();
+			}
+			return false;
+		}
+		return true;
+	}
+
+	public boolean deleteAllUser() throws SQLException
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("delete from " + USERS);
+			state.execute("delete from " + ROLES);
+			conn.commit();
+			state.close();
+		}
+		catch (Throwable e)
+		{
+			System.out.println("exception thrown:");
+			if (e instanceof SQLException)
+			{
+				printSQLError((SQLException) e);
+			}
+			else
+			{
+				e.printStackTrace();
+			}
+
+			return false;
+		}
+		return true;
+	}
+
+	public UserQueryResult findUser(String username, String password)
+	{
+		UserQueryResult userQueryResult = new UserQueryResult();
+
+		String sql_find_user = "SELECT  username, password, accountstatus, comment, email FROM " + USERS;
+		String append_sql = "";
+
+		if (username != null)
+		{
+			append_sql = " WHERE username = '" + username + "'";
+		}
+		if (password != null)
+		{
+			if (append_sql.equals(""))
+			{
+				append_sql = " WHERE password = '" + password + "'";
+			}
+			else
+			{
+				append_sql += " and password = '" + password + "'";
+			}
+		}
+		if (!append_sql.equals(""))
+		{
+			sql_find_user += append_sql;
+		}
+
+		try
+		{
+			ArrayList<HashMap<String, String>> users = new ArrayList<HashMap<String, String>>();
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery(sql_find_user);
+			while (rs.next())
+			{
+				HashMap<String, String> user = new HashMap<String, String>();
+				user.put("username", rs.getString("username"));
+				user.put("password", rs.getString("password"));
+				user.put("as", rs.getString("accountstatus"));
+				user.put("comment", rs.getString("comment"));
+				user.put("email", rs.getString("email"));
+
+				users.add(user);
+			}
+			conn.commit();
+
+			for (HashMap<String, String> user : users)
+			{
+				ResultSet gs = state.executeQuery("SELECT role FROM " + ROLES + " WHERE username = '" + user.get("username") + "'");
+
+				String group = "";
+				while (gs.next())
+				{
+					if (!group.equals(""))
+					{
+						group += ",";
+					}
+					group += gs.getString("role");
+				}
+
+				userQueryResult.addUserTerm(user.get("username"), user.get("password"), group, user.get("as"), user.get("comment"), user.get("email"));
+			}
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+			return null;
+		}
+
+		if (userQueryResult.getSize() > 0)
+		{
+			return userQueryResult;
+		}
+		else
+		{
+			return null;
+		}
+	}
+
+	// findUser(null) will return all users, which is why a UserQueryResult 
+	// (a vector of UserTermInfo) is returned
+	public UserQueryResult findUser(String username)
+	{
+		UserQueryResult userQueryResult = new UserQueryResult();
+
+		String sql_find_user = "SELECT  username, password, accountstatus, comment, email FROM " + USERS;
+		String append_sql = "";
+
+		if (username != null)
+		{
+			append_sql = " WHERE username = '" + username + "'";
+		}
+		if (!append_sql.equals(""))
+		{
+			sql_find_user += append_sql;
+		}
+
+		ArrayList<HashMap<String, String>> users = new ArrayList<HashMap<String, String>>();
+
+		try
+		{
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery(sql_find_user);
+			conn.commit();
+			while (rs.next())
+			{
+				HashMap<String, String> user = new HashMap<String, String>();
+				user.put("username", rs.getString("username"));
+				user.put("password", rs.getString("password"));
+				user.put("as", rs.getString("accountstatus"));
+				user.put("comment", rs.getString("comment"));
+				user.put("email", rs.getString("email"));
+
+				users.add(user);
+			}
+			state.close();
+
+			state = conn.createStatement();
+			for (HashMap<String, String> user : users)
+			{
+				ResultSet gs = state.executeQuery("SELECT role FROM " + ROLES + " WHERE username = '" + user.get("username") + "'");
+				conn.commit();
+
+				String group = "";
+				while (gs.next())
+				{
+					if (!group.equals(""))
+					{
+						group += ",";
+					}
+					group += gs.getString("role");
+				}
+
+				userQueryResult.addUserTerm(user.get("username"), user.get("password"), group, user.get("as"), user.get("comment"), user.get("email"));
+			}
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+
+		if (userQueryResult.getSize() > 0)
+		{
+			return userQueryResult;
+		}
+		else
+		{
+			System.out.println("couldn't find the user");
+			return null;
+		}
+	}
+
+	public String modifyUserInfo(String username, String new_password, String groups, String accountstatus, String comment, String email)
+	{
+		try
+		{
+			String sql_modify_user_info = "update " + USERS + " set ";
+
+			boolean needComma = false;
+			if (new_password != null && !new_password.equals(""))
+			{
+				sql_modify_user_info += "password='" + new_password + "'";
+				needComma = true;
+			}
+
+			if (accountstatus != null && comment != null)
+			{
+				sql_modify_user_info += (needComma ? "," : "") + " accountstatus='" + accountstatus + "'" + ", comment='" + comment + "'";
+				needComma = true;
+			}
+
+			if (email != null)
+			{
+				sql_modify_user_info += (needComma ? "," : "") + " email='" + email + "'";
+			}
+
+			sql_modify_user_info += " where username='" + username + "'";
+			Statement state = conn.createStatement();
+			state.execute(sql_modify_user_info);
+
+			String sql_delete_groups = "delete from " + ROLES + " where username='" + username + "'";
+			state.execute(sql_delete_groups);
+
+			String[] groupsArray = groups.split(",");
+			for (String g : groupsArray)
+			{
+				String sql_insert_group = "insert into " + ROLES + " values ('" + username + "', '" + g + "')";
+				state.execute(sql_insert_group);
+			}
+
+			conn.commit();
+			state.close();
+		}
+		catch (Throwable e)
+		{
+			System.out.println("exception thrown:");
+			if (e instanceof SQLException)
+			{
+				printSQLError((SQLException) e);
+			}
+			else
+			{
+				e.printStackTrace();
+			}
+
+			return "Error:" + e.getMessage();
+		}
+		return "succeed";
+	}
+
+	public void db2txt()
+	{
+		System.err.println(db2txtString());
+	}
+
+	public String db2txtString()
+	{
+		//String db2txt = "Error in converting db2txt string.";
+		String db2txt = "";
+		try
+		{
+			String sql_list_all_user = "select username, password, accountstatus, comment, email from " + USERS;
+
+			Statement state = conn.createStatement();
+			ResultSet rs = state.executeQuery(sql_list_all_user);
+
+			ArrayList<HashMap<String, String>> infoMap = new ArrayList<HashMap<String, String>>();
+
+			while (rs.next())
+			{
+				HashMap<String, String> userMap = new HashMap<String, String>();
+				userMap.put("username", rs.getString("username"));
+				userMap.put("password", rs.getString("password"));
+				userMap.put("status", rs.getString("accountstatus"));
+				userMap.put("comment", rs.getString("comment"));
+				userMap.put("email", rs.getString("email"));
+				infoMap.add(userMap);
+			}
+			conn.commit();
+
+			StringBuffer buffer = new StringBuffer();//("-------------------------------------");
+			for (HashMap<String, String> user : infoMap)
+			{
+				ResultSet groupsSet = state.executeQuery("SELECT role FROM " + ROLES + " WHERE username = '" + user.get("username") + "'");
+				String returnedGroups = "";
+				while (groupsSet.next())
+				{
+					if (!returnedGroups.equals(""))
+					{
+						returnedGroups += ",";
+					}
+					returnedGroups += groupsSet.getString("role");
+				}
+				conn.commit();
+
+				buffer.append("USERNAME = " + user.get("username"));
+				buffer.append("\nPASSWORD = " + user.get("password"));
+				buffer.append("\nGROUPS = " + returnedGroups);
+				buffer.append("\nSTATUS = " + user.get("status"));
+				buffer.append("\nCOMMENT = " + user.get("comment"));
+				buffer.append("\nEMAIL = " + user.get("email"));
+				buffer.append("\n-------------------------------------\n");
+			}
+			db2txt = buffer.toString();
+
+			conn.commit();
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+		finally
+		{
+			return db2txt;
+		}
+	}
+
+	static void printSQLError(SQLException e)
+	{
+		while (e != null)
+		{
+			System.out.println(e.toString());
+			e = e.getNextException();
+		}
+	}
+
+	public void clearUserDataWithPrefix(String username, String prefix)
+	{
+		try
+		{
+			Statement state = conn.createStatement();
+			state.execute("DELETE FROM data WHERE username = '" + username + "' AND SUBSTR(name, 1, " + prefix.length() + ") = '" + prefix + "'");
+			conn.commit();
+			state.close();
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Dictionary.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Dictionary.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Dictionary.java	(revision 31634)
@@ -0,0 +1,291 @@
+/*
+ *    Dictionary.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ResourceBundle;
+import java.util.Locale;
+import java.util.Enumeration;
+
+import org.apache.log4j.*;
+
+public class Dictionary
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.Dictionary.class.getName());
+
+	/** The locale of this dictionary. */
+	protected Locale locale = null;
+
+	/** The resource that has been loaded */
+	protected String resource = null;
+
+	/**
+	 * The ResourceBundle which contains the raw key-value mappings. Loaded from
+	 * a file named "resource_locale.properties
+	 */
+	private ResourceBundle raw = null;
+
+	/**
+	 * Constructs the Dictionary class by first creating a locale from the
+	 * specified language, (or using the default locale if this didn't work),
+	 * then loading a resource bundle based on this locale.
+	 */
+	public Dictionary(String resource, String lang)
+	{
+		// Initialize.
+
+		this.locale = new Locale(lang);
+		this.resource = resource;
+		if (this.locale == null)
+		{
+			this.locale = Locale.getDefault();
+		}
+		try
+		{
+			this.raw = ResourceBundle.getBundle(this.resource, this.locale);
+		}
+		catch (Exception e)
+		{
+			//logger.debug("Dictionary: couldn't locate a resource bundle for "+resource);
+		}
+	}
+
+	public Dictionary(String resource, Locale locale)
+	{
+		this.locale = locale;
+		this.resource = resource;
+		try
+		{
+			this.raw = ResourceBundle.getBundle(this.resource, this.locale);
+		}
+		catch (Exception e)
+		{
+			//logger.debug("Dictionary: couldn't locate a resource bundle for "+resource);
+		}
+	}
+
+	/**
+	 * Constructs the Dictionary class by first creating a locale from the
+	 * specified language, (or using the default locale if this didn't work),
+	 * then loading a resource bundle based on this locale. A classloader is
+	 * specified which can be used to find the resource.
+	 */
+	public Dictionary(String resource, String lang, ClassLoader loader)
+	{
+		// Initialize.
+		this.locale = new Locale(lang);
+		this.resource = resource;
+		if (this.locale == null)
+		{
+			this.locale = Locale.getDefault();
+		}
+		// try the specified class loader
+		try
+		{
+			this.raw = ResourceBundle.getBundle(this.resource, this.locale, loader);
+			return;
+		}
+		catch (Exception e)
+		{
+		}
+		;
+
+		try
+		{
+			this.raw = ResourceBundle.getBundle(this.resource, this.locale);
+		}
+		catch (Exception ex)
+		{
+			//logger.debug("Dictionary: couldn't locate a resource bundle for "+resource);
+		}
+	}
+
+	public Enumeration getKeys()
+	{
+		if (this.raw != null)
+		{
+			return this.raw.getKeys();
+		}
+		return null;
+	}
+
+	/**
+	 * Overloaded to call get with both a key and an empty argument array.
+	 * 
+	 * @param key
+	 *            A <strong>String</strong> which is mapped to a initial String
+	 *            within the ResourceBundle.
+	 * @return A <strong>String</strong> which has been referenced by the key
+	 *         String and that contains no argument fields.
+	 */
+	public String get(String key)
+	{
+		return get(key, null);
+	}
+
+        public static String processArgs(String initial, String args[])
+        {
+	    // If the string contains arguments we have to insert them.
+	    StringBuffer complete = new StringBuffer();
+	    // While we still have initial string left.
+	    while (initial.length() > 0 && initial.indexOf('{') != -1 && initial.indexOf('}') != -1)
+		{
+		    // Remove preamble
+		    int opening = initial.indexOf('{');
+		    int closing = initial.indexOf('}');
+		    int comment_mark = initial.indexOf('-', opening); // May not exist
+		    if (comment_mark > closing)
+			{ // May also be detecting a later comment
+			    comment_mark = -1;
+			}
+		    complete.append(initial.substring(0, opening));
+		    
+		    // Parse arg_num
+		    String arg_str = null;
+		    if (comment_mark != -1)
+			{
+			    arg_str = initial.substring(opening + 1, comment_mark);
+			}
+		    else
+			{
+			    arg_str = initial.substring(opening + 1, closing);
+			}
+		    if (closing + 1 < initial.length())
+			{
+			    initial = initial.substring(closing + 1);
+			}
+		    else
+			{
+			    initial = "";
+			}
+		    int arg_num = Integer.parseInt(arg_str);
+		    // Insert argument
+		    if (args != null && 0 <= arg_num && arg_num < args.length)
+			{
+			    complete.append(args[arg_num]);
+			}
+		}
+	    complete.append(initial);
+	    return complete.toString();	    
+	}
+    
+	/**
+	 * Used to retrieve a property value from the Locale specific
+	 * ResourceBundle, based upon the key and arguments supplied. If the key
+	 * cannot be found or if some other part of the call fails a default
+	 * (English) error message is returned. <BR>
+	 * Here the get recieves a second argument which is an array of Strings used
+	 * to populate argument fields, denoted {<I>n</I>}, within the value String
+	 * returned.
+	 * 
+	 * @param key
+	 *            A <strong>String</strong> which is mapped to a initial String
+	 *            within the ResourceBundle.
+	 * @param args
+	 *            A <strong>String[]</strong> used to populate argument fields
+	 *            within the complete String.
+	 * @return A <strong>String</strong> which has been referenced by the key
+	 *         String and that either contains no argument fields, or has had
+	 *         the argument fields populated with argument Strings provided in
+	 *         the get call.
+	 */
+	public String get(String key, String args[])
+	{
+	    // The following 'argsStr' doesn't appear to be used in rest of method, so
+	    // commenting out
+	    /*
+		String argsStr = "";
+		if (args != null)
+		{
+			for (String arg : args)
+			{
+				argsStr += arg + " ";
+			}
+		}
+	    */
+	    
+		if (this.raw == null)
+		{
+			return null;
+		}
+		try
+		{
+			String initial_raw = this.raw.getString(key);
+			// convert to unicode, copied from gatherer dictionary
+			String initial = new String(initial_raw.getBytes("ISO-8859-1"), "UTF-8");
+
+			// if we haven't been given any args, don't bother looking for them
+			if (args == null)
+			{
+				return initial;
+			}
+
+			return processArgs(initial,args);
+			/*
+			// If the string contains arguments we have to insert them.
+			StringBuffer complete = new StringBuffer();
+			// While we still have initial string left.
+			while (initial.length() > 0 && initial.indexOf('{') != -1 && initial.indexOf('}') != -1)
+			{
+				// Remove preamble
+				int opening = initial.indexOf('{');
+				int closing = initial.indexOf('}');
+				int comment_mark = initial.indexOf('-', opening); // May not exist
+				if (comment_mark > closing)
+				{ // May also be detecting a later comment
+					comment_mark = -1;
+				}
+				complete.append(initial.substring(0, opening));
+
+				// Parse arg_num
+				String arg_str = null;
+				if (comment_mark != -1)
+				{
+					arg_str = initial.substring(opening + 1, comment_mark);
+				}
+				else
+				{
+					arg_str = initial.substring(opening + 1, closing);
+				}
+				if (closing + 1 < initial.length())
+				{
+					initial = initial.substring(closing + 1);
+				}
+				else
+				{
+					initial = "";
+				}
+				int arg_num = Integer.parseInt(arg_str);
+				// Insert argument
+				if (args != null && 0 <= arg_num && arg_num < args.length)
+				{
+					complete.append(args[arg_num]);
+				}
+			}
+			complete.append(initial);
+			return complete.toString();
+			*/
+		}
+		catch (Exception e)
+		{
+			//logger.debug("Dictionary Error: couldn't find string for key:" + key +" in resource "+this.resource);
+			return null;
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DisplayItemUtil.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DisplayItemUtil.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/DisplayItemUtil.java	(revision 31634)
@@ -0,0 +1,132 @@
+/*
+ *    DisplayItemUtil.java
+ *    Copyright (C) 2016 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.apache.log4j.Logger;
+
+import org.greenstone.gsdl3.util.Dictionary;
+import org.greenstone.gsdl3.util.GSXML;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+
+/** various methods for handling displayItems and choosing the right one for the speicifed language */
+
+public class DisplayItemUtil
+{
+
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.DisplayItemUtil.class.getName());
+
+  /** looks for displayItems from config element and stores them in displayItemList in the form
+   
+      <displayItem name="X">
+        <displayItem name="X" lang="">value</displayItem>
+        <displayItem name="X" lang="">value</displayItem>
+      </displayItem>
+
+   */
+  public static boolean storeDisplayItems(Element display_item_list, Element config) {
+    if (config == null) {
+      logger.error("the config element is null, no displayItems to be found");
+      return false;
+    }
+    Document doc = display_item_list.getOwnerDocument();
+    NodeList displaynodes = config.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
+    if (displaynodes.getLength() > 0) {
+      
+      for (int k = 0; k < displaynodes.getLength(); k++) {
+	Element d = (Element) displaynodes.item(k);
+	String name = d.getAttribute(GSXML.NAME_ATT);
+	Element this_item = GSXML.getNamedElement(display_item_list, GSXML.DISPLAY_TEXT_ELEM, GSXML.NAME_ATT, name);
+	if (this_item == null) {
+	  this_item = doc.createElement(GSXML.DISPLAY_TEXT_ELEM);
+	  this_item.setAttribute(GSXML.NAME_ATT, name);
+	  display_item_list.appendChild(this_item);
+	}
+	
+	this_item.appendChild(doc.importNode(d, true));
+      }
+    }
+    
+    return true;
+     
+  }
+  
+  /** Finds the best language specific match for each displayItem in display_item_list and adds it to description */
+  public static boolean addLanguageSpecificDisplayItems(Element description, Element display_item_list, String lang, String default_lang, ClassLoader class_loader) {
+
+    Document doc = description.getOwnerDocument();
+    NodeList items = display_item_list.getChildNodes();
+    for (int i = 0; i < items.getLength(); i++)
+      { // for each key
+	Element m = (Element) items.item(i);
+	// is there one with the specified language?
+	Element new_m = GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, lang);
+	if (new_m == null) {
+	  // if not, have we got one with a key?
+	  new_m = GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.KEY_ATT, null);
+	  if (new_m != null) {
+	    // look up the dictionary
+	    String value = getTextString(new_m.getAttribute(GSXML.KEY_ATT), lang, new_m.getAttribute(GSXML.DICTIONARY_ATT), class_loader);
+	    if (value != null) {
+	      GSXML.setNodeText(new_m, value);
+	    }
+	    else {
+	      // haven't found the key in the dictionary, ignore this display item
+	      new_m = null;
+	    }
+	  }
+	}
+	if (new_m == null && lang != default_lang) {
+	  // still haven't got a value. can we use the default lang?
+	  new_m = GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, default_lang);
+	}
+	if (new_m == null)
+	  {
+	    // STILL haven't found one, lets use the first one with a lang att (so we don't just get the key one back
+	    new_m = (Element) GSXML.getNamedElement(m, GSXML.DISPLAY_TEXT_ELEM, GSXML.LANG_ATT, null);
+	  }
+	if (new_m != null) {
+	  description.appendChild(doc.importNode(new_m, true));
+	}
+      } // for each key
+    return true;
+    
+  }
+
+  protected static String getTextString(String key, String lang, String dictionary, ClassLoader class_loader) {
+    return getTextString(key, lang, dictionary, null, class_loader);
+  }
+
+  protected static String getTextString(String key, String lang, String dictionary, String[] args, ClassLoader class_loader)
+  {
+    Dictionary dict;
+    if (class_loader != null) {
+      dict = new Dictionary(dictionary, lang, class_loader);
+    } else {
+      dict = new Dictionary(dictionary, lang);
+    }
+    String result = dict.get(key, args);
+    return result;
+  }
+  
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FacetWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FacetWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FacetWrapper.java	(revision 31634)
@@ -0,0 +1,40 @@
+/**********************************************************************
+ *
+ * FacetWrapper.java 
+ *
+ * Copyright 2004 The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+package org.greenstone.gsdl3.util;
+
+import java.util.HashMap;
+
+public abstract class FacetWrapper
+{
+	protected String _name = null;
+	
+	public String getName()
+	{
+		return _name;
+	}
+	
+	public abstract HashMap<String, Long> getCounts();
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FlatDatabaseWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FlatDatabaseWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/FlatDatabaseWrapper.java	(revision 31634)
@@ -0,0 +1,55 @@
+/*
+ *    FlatDatabaseWrapper.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+public interface FlatDatabaseWrapper {
+
+  /* just read access, many readers can share a database */
+  public final static int READ = 0;
+  /* read/write, exclusive access */
+  public final static int WRITE = 1;
+
+  // do we want the other modes? 
+  // -- read/write - create if doesn't exist 
+  // -- read/write - create a new database
+  /** open database named filename, using mode */
+  public boolean openDatabase(String filename, int mode);
+
+  /** close the database associated with this wrapper */
+  public void closeDatabase();
+
+  /** returns the value associated with the key */
+  public String getValue(String key);
+  
+  /** sets the given key to the given value in the database */
+  public boolean setValue(String key, String value);
+  
+  /** deletes the given key from the database */
+  public boolean deleteKey(String key);
+
+  /** Returns all the keys of the database as String */
+  public ArrayList<String> getAllEntryKeys();
+
+  /** returns a string of key-value entries that can be 
+   *	printed for debugging purposes*/
+  public String displayAllEntries();
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GDBMWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GDBMWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GDBMWrapper.java	(revision 31634)
@@ -0,0 +1,323 @@
+/*
+ *    GDBMWrapper.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import au.com.pharos.gdbm.GdbmFile;
+import au.com.pharos.packing.*;
+import au.com.pharos.gdbm.GdbmException;
+
+import org.apache.log4j.Logger;
+
+import java.io.UnsupportedEncodingException;
+import java.io.File;
+import java.util.ArrayList;
+
+/**
+ * java wrapper class for gdbm - uses Java-GDBM written by Martin Pool replaces
+ * gdbmclass in the old version
+ */
+
+public class GDBMWrapper implements FlatDatabaseWrapper
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GDBMWrapper.class.getName());
+
+    static
+    {
+	/** Register this Wrapper with the DBHelper */
+	DBHelper.registerDBTypeExt("gdbm",".gdb");
+    }
+
+	/*
+	 * GdbmFile modes: READER - read access, many readers may share the database
+	 * WRITER - read/write access, exclusive access WRCREAT - read/write access,
+	 * create db if doesn't exist NEWDB - read/write access, db should be
+	 * replaced if exists
+	 */
+
+	protected GdbmFile db_ = null;
+
+	/**
+	 * open the database filename, with mode mode - uses the constants above, eg
+	 * GdbmFile.WRITER
+	 */
+	public boolean openDatabase(String filename, int mode)
+	{
+		// need to convert mode to GdbmFile mode
+		if (mode == READ)
+		{
+			mode = GdbmFile.READER;
+		}
+		else if (mode == WRITE)
+		{
+			mode = GdbmFile.WRITER;
+		}
+		else
+		{
+			logger.error("invalid mode, " + mode + ", opening db for reading only");
+			mode = GdbmFile.READER;
+		}
+		try
+		{
+			if (db_ != null)
+			{
+				db_.close();
+			}
+
+			// The java version of the C++ code in common-src/src/lib/gdbmclass.cpp
+			if (mode == GdbmFile.READER)
+			{
+				// Looking to read in the database
+				// we now use gdb extension. Check for ldb/bdb in case of legacy collection
+				// if not (first time) then generate using txt2db
+				if (!new File(filename).exists())
+				{
+					logger.warn("Database file " + filename + " does not exist. Looking for ldb/bdb version");
+					int extension = filename.lastIndexOf('.');
+					String filename_head = filename.substring(0, extension);
+					filename = filename_head + ".ldb";
+					if (!new File(filename).exists())
+					{
+						filename = filename_head + ".bdb";
+
+						if (!new File(filename).exists())
+						{
+							logger.warn("ldb/bdb version of database file " + filename + " does not exist. Looking for txtgz version of db file.");
+							// put the filename back to gdb
+							filename = filename_head + ".gdb";
+							// need to generate architecture native GDBM file using txt2db
+
+							// replace sought after gdbm filename ext with ".txt.gz"
+
+							String txtgzFilename = filename_head + ".txt.gz";
+							if (new File(txtgzFilename).exists())
+							{
+								// Test to make sure Perl is on the path
+								// On Linux, the output of the test goes to STDOUT so redirect it to STDERR
+								String cmdTest = "perl -v 2>&1";
+								//String cmdTest = "echo %PATH%";
+								int returnValue = Processing.runProcess(cmdTest);
+								if (returnValue != 0)
+								{
+									logger.error("Tried to find Perl. Return exit value of running " + cmdTest + ": " + returnValue + ", (expected this to be 0)");
+									logger.error("Check that Perl is set in your PATH environment variable.");
+									//log.error("At present, PATH=" + System.getenv("PATH"));
+								}
+
+								String cmd = "perl -S txtgz-to-gdbm.pl \"" + txtgzFilename + "\" \"" + filename + "\"";
+								returnValue = Processing.runProcess(cmd);
+								// For some reason, launching this command with gsdl_system() still returns 1
+								// even when it returns 0 when run from the command-line. We can check whether
+								// we succeeded by looking at whether the output database file was created.
+								if (returnValue != 0)
+								{
+									logger.warn("Warning, non-zero return value on running command \"" + cmd + "\": " + returnValue);
+									if (!new File(filename).exists())
+									{
+										logger.error("Tried to run command \"" + cmd + "\", but it failed");
+									}
+								}
+							}
+						}
+					}
+
+				}
+			}
+
+			db_ = new GdbmFile(filename, mode);
+		}
+		catch (GdbmException e)
+		{ // the database wasn't opened or created
+			logger.error("couldn't open database " + filename);
+			return false;
+		}
+		db_.setKeyPacking(new StringPacking());
+		db_.setValuePacking(new StringPacking());
+		return true;
+	}
+
+	/** close the database associated with this wrapper */
+	public void closeDatabase()
+	{
+		try
+		{
+			if (db_ != null)
+			{
+				db_.close();
+				db_ = null;
+				if (System.getProperty("os.name").startsWith("Windows"))
+				{
+					//Hack: force Windows to let go of the gdb file
+					System.gc();
+				}
+			}
+		}
+		catch (GdbmException e)
+		{
+			// should never get here - close never actually throws an exception
+			logger.error("error on close()");
+		}
+	}
+
+	public String getValue(String key)
+	{
+		if (db_ == null)
+		{
+			return null;
+		}
+		String s_info = null;
+		try
+		{
+			try
+			{
+				// The key is UTF8: do db lookup using the UTF8 version of key
+				if (db_.exists(key.getBytes("UTF-8")))
+				{
+					s_info = (String) db_.fetch(key.getBytes("UTF-8"));
+				}
+			}
+			catch (UnsupportedEncodingException e)
+			{
+				logger.warn("utf8 key for " + key + " unrecognised. Retrying with default encoding.");
+				// retry db lookup using default encoding of key instead of UTF8
+				s_info = (String) db_.fetch(key);
+			}
+		}
+		catch (GdbmException e)
+		{
+			logger.error("couldn't get record");
+			return null;
+		}
+		if (s_info == null)
+		{
+			// record not present
+			logger.error("key " + key + " not present in db");
+			return null;
+		}
+		return s_info;
+	}
+
+	/**
+	 * sets the given key to the given value in the database
+	 */
+	public boolean setValue(String key, String value)
+	{
+		if (db_ == null || !db_.isWritable())
+		{
+			logger.error("GDBM database is either null or not writable");
+			return false;
+		}
+
+		try
+		{
+			db_.store(key, value);
+		}
+		catch (GdbmException ex)
+		{
+			logger.error("Error storing " + key + " = " + value + " in GDBM database");
+			logger.error("Error message is: " + ex.getMessage());
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * deletes the entry for given key
+	 */
+	public boolean deleteKey(String key)
+	{
+		if (db_ == null)
+		{
+			logger.error("GDBM database is null");
+			return false;
+		}
+
+		try
+		{
+			db_.delete(key);
+		}
+		catch (GdbmException ex)
+		{
+			logger.error("Error deleting key " + key + " from the database");
+			logger.error("Error message is: " + ex.getMessage());
+			return false;
+		}
+
+		return true;
+	}
+
+    /** Returns all the keys of the database as Strings */
+    public ArrayList<String> getAllEntryKeys() {
+
+	ArrayList<String> keys = new ArrayList<String>();
+	
+	try {
+	    java.util.Enumeration e = db_.keys();
+	    while (e.hasMoreElements()) {
+		
+		Object key = e.nextElement();
+		keys.add((String)key);
+		
+	    }	 
+	} catch (Exception e) {
+	    logger.error("Exception encountered when trying to GDBMWrapper.getAllEntries():" + e);
+	}
+	
+	return keys;
+    }
+    
+
+	/**
+	 * returns a string of key-value entries that can be printed for debugging
+	 * purposes.
+	 */
+	public String displayAllEntries()
+	{
+		StringBuffer output = new StringBuffer();
+		try
+		{
+			java.util.Enumeration e = db_.keys();
+			while (e.hasMoreElements())
+			{
+				Object key = e.nextElement();
+				Object value = db_.fetch(key);
+
+				output.append("key href: ");
+				output.append((String) key);
+				output.append("\tvalue ID: ");
+				output.append((String) value);
+				output.append("\n");
+				//logger.warn("key: " + key + "\tvalue: " + value);
+
+				String urlkey = java.net.URLEncoder.encode((String) key, "UTF8");
+				output.append("URL encoded key: " + urlkey);
+				//logger.warn("URL encoded key: " + urlkey);
+			}
+		}
+		catch (UnsupportedEncodingException e)
+		{
+			logger.warn("Trouble converting key to UTF-8.");
+		}
+		catch (Exception e)
+		{
+			logger.warn("Exception encountered when trying to displayAllEntries():" + e);
+		}
+		return output.toString();
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2MacroResolver.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2MacroResolver.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2MacroResolver.java	(revision 31634)
@@ -0,0 +1,214 @@
+/*
+ *    GS2MacroResolver.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.net.URLDecoder;
+
+// Apache Commons
+import org.apache.commons.lang3.*;
+import org.greenstone.gsdl3.util.MacroResolver.Macro;
+
+import java.util.Stack;
+
+public class GS2MacroResolver extends MacroResolver
+{
+
+	protected SimpleCollectionDatabase coll_db = null;
+  protected ClassLoader class_loader = null;
+	private static Pattern p_back_slash = Pattern.compile("\\\"");// create a pattern "\\\"", but it matches both " and \"
+
+	// need to make it not add macros if they are already present
+  public GS2MacroResolver(SimpleCollectionDatabase db, ClassLoader class_loader)
+	{
+		super();
+		coll_db = db;
+		this.class_loader = class_loader;
+	}
+
+	public GS2MacroResolver()
+	{
+		super();
+	}
+
+	public void setDB(SimpleCollectionDatabase db)
+	{
+		this.coll_db = db;
+	}
+  public void setClassLoader(ClassLoader class_loader) {
+    this.class_loader = class_loader;
+  }
+  
+  public String resolve(String text, String lang, String scope, String doc_oid)
+	{
+		if (text == null || text.equals(""))
+			return text;
+		if (scope.equals(SCOPE_TEXT) && text_macros.size() == 0)
+			return text;
+		if (scope.equals(SCOPE_META) && metadata_macros.size() == 0)
+			return text;
+		DBInfo node_info = null;
+		DBInfo root_info = null;
+		boolean new_lang = false;
+		if (this.lang == null || !this.lang.equals(lang))
+		{
+			new_lang = true;
+			this.lang = lang;
+		}
+
+		Stack<Macro> macros = new Stack<Macro>();//ArrayList macros;
+		if (scope.equals(SCOPE_TEXT))
+		{
+			macros.addAll(text_macros);
+
+			//Two helpful runtime macros
+			Macro docIDMacro = new Macro();
+			docIDMacro.macro = "[DocOID]";
+			docIDMacro.text = doc_oid;
+			docIDMacro.type = TYPE_TEXT;
+
+			Macro docTopIDMacro = new Macro();
+			docTopIDMacro.macro = "[DocTopOID]";
+			docTopIDMacro.text = OID.getTop(doc_oid);
+			docTopIDMacro.type = TYPE_TEXT;
+
+			macros.add(docIDMacro);
+			macros.add(docTopIDMacro);
+		}
+		else
+		{
+			macros.addAll(metadata_macros);
+		}
+
+		//for (int i=0; i<macros.size(); i++) {
+		while (!macros.empty())
+		{
+			String new_text = null;
+			Macro m = macros.pop();//.get(i);
+			switch (m.type)
+			{
+			case TYPE_DICT:
+			  if (text.contains(m.macro))
+
+			    {
+			  // if we change the lang, then do a metadata resolve, then a text resolve, the lang hasn't changed, but the text might be leftover from the last language.
+			  // if (m.text == null || new_lang)
+			  // {
+				  Dictionary dict = new Dictionary(m.bundle, lang, this.class_loader);
+				  m.text = dict.get(m.key, null);
+					//	}
+				// we assume that dictionary entries will contain no macros
+				// otherwise we can't cache the answer because it might be 
+				// document specific
+				  text = StringUtils.replace(text, m.macro, m.text);}
+				break;
+			case TYPE_TEXT:
+				// make sure we resolve any macros in the text
+				// the (?s) treats the string as a single line, cos . 
+				// doesn't necessarily match line breaks
+				//if (text.matches("(?s).*"+m.macro+".*")) {
+
+				/*
+				 * Pattern p_text = Pattern.compile(".*" + m.macro +
+				 * ".*",Pattern.DOTALL); Matcher match_text =
+				 * p_text.matcher(text);
+				 */
+
+				// sm252
+				// String.contains is far faster than regex!
+				if (text.contains(m.macro))
+				{ //match_text.matches()) { //text.matches("(?s).*"+m.macro+".*")) {
+					if (m.resolve)
+					{
+					  new_text = this.resolve(m.text, lang, scope, doc_oid);
+					}
+					else
+					{
+						new_text = m.text;
+					}
+					text = StringUtils.replace(text, m.macro, new_text);
+					if (m.macro.endsWith("\\\\"))
+					{ // to get rid of "\" from the string likes: "src="http://www.greenstone.org:80/.../mw.gif\">"
+
+						Matcher m_slash = p_back_slash.matcher(text);
+						String clean_str = "";
+						int s = 0;
+						while (m_slash.find())
+						{
+							if (!text.substring(m_slash.end() - 2, m_slash.end() - 1).equals("\\"))
+							{
+								clean_str = clean_str + text.substring(s, m_slash.end() - 1); // it matches ", so get a substring before "
+							}
+							else
+							{
+								clean_str = clean_str + text.substring(s, m_slash.end() - 2);// it matches \", so get a substring before \
+							}
+							s = m_slash.end();// get the index of the last match
+							clean_str = clean_str + "\"";
+						}
+						text = clean_str + text.substring(s, text.length());
+					}
+				}
+				break;
+			case TYPE_META:
+				//Pattern p = Pattern.compile(".*" + m.macro + ".*",Pattern.DOTALL);
+				//Matcher match = p.matcher(text);
+				// sm252
+				if (text.contains(m.macro))
+				{ //(match.matches()) { //text.matches("(?s).*"+m.macro+".*")) {
+					if (node_info == null)
+					{
+						node_info = coll_db.getInfo(doc_oid);
+						if (node_info == null)
+						{
+							break;
+						}
+					}
+					new_text = node_info.getInfo(m.text);
+					if (new_text == null || new_text.equals(""))
+					{
+						// try the root node
+						if (root_info == null && !OID.isTop(doc_oid))
+						{
+							root_info = coll_db.getInfo(OID.getTop(doc_oid));
+						}
+						if (root_info == null)
+							break;
+						new_text = root_info.getInfo(m.text);
+					}
+					if (new_text != null)
+					{
+						if (m.resolve)
+						{
+						  new_text = this.resolve(new_text, lang, scope, doc_oid);
+						}
+						text = StringUtils.replace(text, m.macro, new_text);
+					}
+				}
+
+				break;
+			} // switch
+		}
+		return text;
+
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2Params.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2Params.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GS2Params.java	(revision 31634)
@@ -0,0 +1,44 @@
+/*
+ *    GS2Params.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+public class GS2Params extends GSParams {
+
+    public GS2Params() {
+	super();
+
+	// now add in all the service ones we will be using
+	addParameter("stem",  true);	
+	addParameter("case",  true);
+	addParameter("accent", true);
+	addParameter("index", true);
+	addParameter("matchMode",  true);
+	addParameter("maxDocs",  true);
+	addParameter("query", true);
+	addParameter("level",  true);
+	addParameter("qb",  true); // queryBox
+	addParameter("qm",  true); // queryMode
+	addParameter("qt",  true); // query type
+	addParameter("qfm",  true); // query form mode
+	addParameter("qfn",  true); // query form number of boxes
+	addParameter("ct",  true); // collection type - mg or mgpp
+	
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSConstants.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSConstants.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSConstants.java	(revision 31634)
@@ -0,0 +1,38 @@
+/*
+ *    GSConstants.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+/** these constants are used for servlet args, and config variables */
+public class GSConstants
+{
+	public static final String GSDL3_HOME = "gsdl3_home";
+	public static final String INTERFACE_NAME = "interface_name";
+	public static final String SITE_NAME = "site_name";
+	public static final String LIBRARY_NAME = "library_name";
+	public static final String BASE_INTERFACES = "base_interfaces";
+	public static final String DEFAULT_LANG = "default_lang";
+	public static final String USE_CLIENT_SIDE_XSLT = "use_client_side_xslt";
+        //public static final String CLIENT_SIDE_XSLT_INTERFACE_SUFFIX = "-client-xslt";
+
+	public static final String META_RELATION_SEP = "_";
+	public static final String META_SEPARATOR_SEP = "*";
+	public static final String META_FIRST = "first";
+	public static final String META_LAST = "last";
+	public static final String META_POS = "pos";
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSDocumentModel.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSDocumentModel.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSDocumentModel.java	(revision 31634)
@@ -0,0 +1,2086 @@
+package org.greenstone.gsdl3.util;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Vector;
+
+import org.apache.log4j.*;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Result;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class GSDocumentModel
+{
+  //static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSDocumentModel.class.getName());
+	//The two archive databases
+	protected static final String ARCHIVEINFSRC = "archiveinf-src";
+	protected static final String ARCHIVEINFDOC = "archiveinf-doc";
+
+	//Set operations
+	public static final int OPERATION_REPLACE = 1;
+	public static final int OPERATION_INSERT_BEFORE = 2;
+	public static final int OPERATION_INSERT_AFTER = 3;
+	public static final int OPERATION_APPEND = 4;
+
+	//Duplicate and move operations
+	public static final int OPERATION_TYPE_DOC_TO_DOC = 1;
+	public static final int OPERATION_TYPE_DOC_TO_SEC = 2;
+	public static final int OPERATION_TYPE_SEC_TO_DOC = 3;
+	public static final int OPERATION_TYPE_SEC_TO_SEC = 4;
+
+	//Error codes
+	public static final int NO_ERROR = 0;
+	public static final int ERROR_OID_NOT_SPECIFIED = -1;
+	public static final int ERROR_COLLECTION_NOT_SPECIFIED = -2;
+	public static final int ERROR_SOURCE_DOCUMENT_OR_SECTION_DOES_NOT_EXIST = -3;
+	public static final int ERROR_DESTINATION_DOCUMENT_OR_SECTION_DOES_NOT_EXIST = -4;
+	public static final int ERROR_DESTINATION_DOCUMENT_OR_SECTION_ALREADY_EXISTS = -5;
+	public static final int ERROR_COULD_NOT_DUPLICATE = -6;
+	public static final int ERROR_COULD_NOT_MOVE = -7;
+	public static final int ERROR_DOC_XML_COULD_NOT_BE_CREATED = -8;
+	public static final int ERROR_EXCEPTION_CREATING_DOC_XML_FILE = -9;
+	public static final int ERROR_METADATA_NAME_NOT_SPECIFIED = -10;
+	public static final int ERROR_METADATA_VALUE_NOT_SPECIFIED = -11;
+	public static final int ERROR_COULD_NOT_RETRIEVE_DOC_XML = -12;
+	public static final int ERROR_COULD_NOT_WRITE_TO_DOC_XML = -13;
+	public static final int ERROR_COULD_NOT_RETRIEVE_SECTION = -14;
+	public static final int ERROR_COULD_NOT_OPEN_DATABASE = -15;
+	public static final int ERROR_DATA_NOT_FOUND_IN_DATABASE = -16;
+	public static final int ERROR_COULD_NOT_DELETE = -16;
+	public static final int ERROR_OID_INCORRECT_FORMAT = -17;
+	public static final int ERROR_INVALID_MERGE = -18;
+	public static final int ERROR_INVALID_METADATA_POSITION = -19;
+	public static final int ERROR_INVALID_SPLIT = -20;
+	public static final int ERROR_DESTINATION_OID_NOT_SPECIFIED = -21;
+	public static final HashMap<Integer, String> _errorMessageMap;
+
+	static
+	{
+		//Corresponding error messages
+		HashMap<Integer, String> errorMessageMap = new HashMap<Integer, String>();
+		errorMessageMap.put(ERROR_OID_NOT_SPECIFIED, "OID not specified");
+		errorMessageMap.put(ERROR_COLLECTION_NOT_SPECIFIED, "Collection not specified");
+		errorMessageMap.put(ERROR_SOURCE_DOCUMENT_OR_SECTION_DOES_NOT_EXIST, "The specified source document or section does not exist");
+		errorMessageMap.put(ERROR_DESTINATION_DOCUMENT_OR_SECTION_DOES_NOT_EXIST, "The specified destination document or section does not exist");
+		errorMessageMap.put(ERROR_DESTINATION_DOCUMENT_OR_SECTION_ALREADY_EXISTS, "The specified destination document or section already exists");
+		errorMessageMap.put(ERROR_COULD_NOT_DUPLICATE, "There was an error duplicating document or section");
+		errorMessageMap.put(ERROR_COULD_NOT_MOVE, "There was an error moving document or section");
+		errorMessageMap.put(ERROR_DOC_XML_COULD_NOT_BE_CREATED, "The doc.xml file already exists or could not be created");
+		errorMessageMap.put(ERROR_EXCEPTION_CREATING_DOC_XML_FILE, "There was an exception while creating the doc.xml file");
+		errorMessageMap.put(ERROR_METADATA_NAME_NOT_SPECIFIED, "The name of the requested metadata was not specified");
+		errorMessageMap.put(ERROR_METADATA_VALUE_NOT_SPECIFIED, "The new value for this metadata was not specified");
+		errorMessageMap.put(ERROR_COULD_NOT_RETRIEVE_DOC_XML, "Could not retrieve the necessary doc.xml file");
+		errorMessageMap.put(ERROR_COULD_NOT_WRITE_TO_DOC_XML, "There was an error writing to the doc.xml file");
+		errorMessageMap.put(ERROR_COULD_NOT_RETRIEVE_SECTION, "There was an error retrieving the specified section from the doc.xml file");
+		errorMessageMap.put(ERROR_COULD_NOT_OPEN_DATABASE, "There was an error opening the archive database");
+		errorMessageMap.put(ERROR_DATA_NOT_FOUND_IN_DATABASE, "The specified information could not be found in the database");
+		errorMessageMap.put(ERROR_COULD_NOT_DELETE, "There was an error deleting this document");
+		errorMessageMap.put(ERROR_OID_INCORRECT_FORMAT, "The given OID was not in the correct format");
+		errorMessageMap.put(ERROR_INVALID_MERGE, "Merge can only be performed on two sections of the same level or a section and it's parent. Also the destination section cannot have any child sections");
+		errorMessageMap.put(ERROR_INVALID_METADATA_POSITION, "There is no metadata at the given position");
+		errorMessageMap.put(ERROR_INVALID_SPLIT, "A split at the given location is not possible, either the section does not have text or the split point does not exist");
+		errorMessageMap.put(ERROR_DESTINATION_OID_NOT_SPECIFIED, "The destination OID was not specified");
+		_errorMessageMap = errorMessageMap;
+	}
+
+	protected int _errorStatus = NO_ERROR;
+
+	protected String _siteHome;
+ 	protected MessageRouter _router;
+
+  // When we are doing document editing through browser, metadata changes are currently handled by metadata_server.pl. This edits files directly. If I then come to edit text via this class, the doc xml gets read from the cache and my external changes are lost. I am disabling it for now...
+  //protected HashMap<String, Document> _docCache = new HashMap<String, Document>();
+
+	public GSDocumentModel(String siteHome, MessageRouter router)
+	{
+		_siteHome = siteHome;
+		_router = router;
+	}
+
+	/**
+	 * Can be used to create a document or create a section of a document
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section to create.
+	 * @param collection
+	 *            is the collection we want to create the document/section in.
+	 */
+	public void documentCreate(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		//If the collection is not specified then we cannot continue
+		if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return;
+		}
+
+		//If the collection is not specified then we cannot continue
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return;
+		}
+
+		if (archiveCheckDocumentOrSectionExists(oid, collection, userContext))
+		{
+			_errorStatus = ERROR_DESTINATION_DOCUMENT_OR_SECTION_ALREADY_EXISTS;
+			return;
+		}
+
+		//Check if the OID is the OID for a section
+		boolean section = false;
+		if (oid.contains("."))
+		{
+			section = true;
+		}
+
+		if (!section)
+		{
+			//Create a basic doc.xml file to go in the new folder
+			documentXMLCreateDocXML(oid, collection, userContext);
+		}
+		else
+		{
+			documentXMLCreateSection(oid, collection, userContext);
+		}
+	}
+
+	/**
+	 * Can be used to delete a document or section
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section to delete.
+	 * @param collection
+	 *            is the collection to delete the document/section from.
+	 */
+	public void documentDelete(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return;
+		}
+		else if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return;
+		}
+
+		if (!archiveCheckDocumentOrSectionExists(oid, collection, userContext))
+		{
+			_errorStatus = ERROR_SOURCE_DOCUMENT_OR_SECTION_DOES_NOT_EXIST;
+			return;
+		}
+
+		//Check if the OID is the OID for a section
+		boolean section = false;
+		if (oid.contains("."))
+		{
+			section = true;
+		}
+
+		if (!section)
+		{
+			String archivesFile = archiveGetDocumentFilePath(oid, collection, userContext);
+			String archivesFolder = archivesFile.substring(0, archivesFile.lastIndexOf(File.separator));
+			File dirToDelete = new File(archivesFolder);
+
+			if (!dirToDelete.exists() || !dirToDelete.isDirectory() || !deleteDirectory(dirToDelete))
+			{
+				_errorStatus = ERROR_COULD_NOT_DELETE;
+				return;
+			}
+
+			//Remove the entry from the archive database 
+			archiveRemoveEntryFromDatabase(oid, collection, userContext);
+		}
+		else
+		{
+			documentXMLDeleteSection(oid, collection, userContext);
+		}
+	}
+
+	/**
+	 * Can be used to copy a document or section from one place to another.
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section that is to be
+	 *            copied.
+	 * @param collection
+	 *            is the collection the source document resides in.
+	 * @param newOID
+	 *            is the new identifier for the document/section (it cannot
+	 *            already exist).
+	 * @param newCollection
+	 *            is the collection the new document/section will be copied to.
+	 *            If this is null then the collection parameter will be used
+	 *            instead.
+	 */
+	public void documentMoveOrDuplicate(String oid, String collection, String newOID, String newCollection, int operation, boolean move, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		//If a new collection is not specified then assume the collection of the original document
+		if (newCollection == null || newCollection.equals(""))
+		{
+			newCollection = collection;
+		}
+
+		//Generate an OID for the duplicate if we are not given one
+		if (newOID == null || newOID.equals(""))
+		{
+			_errorStatus = ERROR_DESTINATION_OID_NOT_SPECIFIED;
+			return;
+		}
+
+		boolean requiresDatabaseEntry = false;
+		int operationType = getOperation(oid, newOID);
+		switch (operationType)
+		{
+		case OPERATION_TYPE_DOC_TO_DOC:
+		{
+			String archiveDir = archiveGetDocumentFilePath(oid, collection, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+
+			//Remove doc.xml from the file name
+			archiveDir = archiveDir.substring(0, archiveDir.lastIndexOf(File.separator));
+			File dirToDuplicate = new File(archiveDir);
+
+			//This is possibly not the best way to name the new directory
+			File newDir = new File(archiveDir.substring(0, archiveDir.lastIndexOf(File.separator) + File.separator.length()) + newOID);
+
+			if (dirToDuplicate.exists() && dirToDuplicate.isDirectory() && !newDir.exists())
+			{
+				if (!copyDirectory(dirToDuplicate, newDir))
+				{
+					_errorStatus = ERROR_COULD_NOT_DUPLICATE;
+					return;
+				}
+			}
+			else
+			{
+				_errorStatus = ERROR_COULD_NOT_DUPLICATE;
+				return;
+			}
+
+			if (move)
+			{
+				deleteDirectory(dirToDuplicate);
+			}
+
+			requiresDatabaseEntry = true;
+			break;
+		}
+		case OPERATION_TYPE_DOC_TO_SEC:
+		{
+			Document originalDocument = getDocXML(oid, collection, userContext);
+			Element originalSection = getTopLevelSectionElement(originalDocument);
+
+			documentXMLCreateSection(newOID, newCollection, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+
+			documentXMLSetSection(newOID, newCollection, originalSection, operation, userContext);
+
+			if (move)
+			{
+				String archiveDirStr = archiveGetDocumentFilePath(oid, collection, userContext);
+				if (_errorStatus != NO_ERROR)
+				{
+					return;
+				}
+
+				File archiveDir = new File(archiveDirStr);
+
+				if (archiveDir.exists() && archiveDir.isDirectory())
+				{
+					deleteDirectory(archiveDir);
+				}
+			}
+			break;
+		}
+		case OPERATION_TYPE_SEC_TO_DOC:
+		{
+			Document originalDocument = getDocXML(oid, collection, userContext);
+			Element originalSection = getSectionBySectionNumber(originalDocument, getSectionFromOID(oid));
+
+			documentCreate(newOID, newCollection, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+
+			documentXMLCreateSection(newOID, newCollection, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+
+			documentXMLSetSection(newOID, newCollection, originalSection, operation, userContext);
+
+			if (move)
+			{
+				originalDocument = getDocXML(oid, collection, userContext);
+				originalSection.getParentNode().removeChild(originalSection);
+
+				//Write the new change back into the file
+				if (!writeXMLFile(originalDocument, oid, collection, userContext))
+				{
+					_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+					return;
+				}
+			}
+
+			requiresDatabaseEntry = true;
+			break;
+		}
+		case OPERATION_TYPE_SEC_TO_SEC:
+		{
+			Document originalDocument = getDocXML(oid, collection, userContext);
+			Element originalSection = getSectionBySectionNumber(originalDocument, getSectionFromOID(oid));
+
+			if (operation == OPERATION_REPLACE)
+			{
+				documentXMLCreateSection(newOID, newCollection, userContext);
+				if (_errorStatus != NO_ERROR)
+				{
+					return;
+				}
+			}
+
+			documentXMLSetSection(newOID, newCollection, originalSection, operation, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+
+			if (move)
+			{
+				originalDocument = getDocXML(oid, collection, userContext);
+				originalSection.getParentNode().removeChild(originalSection);
+
+				//Write the new change back into the file
+				if (!writeXMLFile(originalDocument, oid, collection, userContext))
+				{
+					_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+					return;
+				}
+			}
+
+			break;
+		}
+		}
+
+		if (requiresDatabaseEntry)
+		{
+			HashMap<String, ArrayList<String>> entries = new HashMap<String, ArrayList<String>>();
+			ArrayList<String> values = new ArrayList<String>();
+			values.add(newOID + "/doc.xml");
+			entries.put("doc-file", values);
+
+			//Write the new entry to the archive database 
+			archiveWriteEntryToDatabase(newOID, newCollection, entries, userContext);
+		}
+	}
+
+	/**
+	 * Can be used to acquire information about a given document or section.
+	 * 
+	 * @param oid
+	 *            is the identifier of the document or section.
+	 * @param collection
+	 *            is the collection the document or section resides in.
+	 * @param requestedInfo
+	 *            is an array containing the various requests.
+	 * @return This returns an array containing the requested information.
+	 */
+	public String[] documentGetInformation(String oid, String collection, String[] requestedInfo, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return null;
+		}
+
+		for (int j = 0; j < requestedInfo.length; j++)
+		{
+			String currentRequest = requestedInfo[j];
+			//TODO: Decide what info requests are valid (e.g. number of sections etc.)
+			//-How many child/sibling sections
+			//-Metadata keys
+		}
+		//TODO: Implement
+		return null;
+	}
+
+	/**
+	 * Can be used to merge two parts of a document together. The sections must
+	 * be in the same document at the same level (e.g. D11.1.2 and D11.1.3) or a
+	 * section and it's parent (e.g. D11.1.2 and D11.1). Also, the destination
+	 * section cannot have any child sections.
+	 * 
+	 * @param oid
+	 *            the identifier of the section that is to be merged.
+	 * @param collection
+	 *            the collection the section resides in.
+	 * @param mergeOID
+	 *            the identifier of the section that the source section will be
+	 *            merged into.
+	 */
+	public void documentMerge(String oid, String collection, String mergeOID, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		if ((_errorStatus = checkOIDandCollection(mergeOID, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		int op = getOperation(oid, mergeOID);
+		if (op != OPERATION_TYPE_SEC_TO_SEC && op != OPERATION_TYPE_SEC_TO_DOC) //We allow SEC_TO_DOC in the case that someone wants to merge D11.1 with D11 (for example)
+		{
+			_errorStatus = ERROR_INVALID_MERGE;
+			return;
+		}
+
+		String[] sourceLevels = oid.split("\\.");
+		String[] destinationLevels = mergeOID.split("\\.");
+
+		if (destinationLevels.length > sourceLevels.length)
+		{
+			_errorStatus = ERROR_INVALID_MERGE;
+			return;
+		}
+
+		for (int i = 0; i < sourceLevels.length - 1; i++)
+		{
+			if (i >= destinationLevels.length || !sourceLevels[i].equals(destinationLevels[i]))
+			{
+				_errorStatus = ERROR_INVALID_MERGE;
+				return;
+			}
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element sourceSection = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		Element destinationSection = getSectionBySectionNumber(docXML, getSectionFromOID(mergeOID));
+
+		//Make sure the destination Section does not have any child Sections.
+		NodeList childSections = GSXML.getChildrenByTagName(destinationSection, GSXML.DOCXML_SECTION_ELEM);
+		if (childSections.getLength() != 0 && sourceLevels.length == destinationLevels.length)
+		{
+			_errorStatus = ERROR_INVALID_MERGE;
+			return;
+		}
+
+		//Get the children of the destination section so we can copy them to the source section before we overwrite the destination
+		NodeList childrenToKeep = destinationSection.getChildNodes();
+		ArrayList<Node> childList = new ArrayList<Node>();
+		for (int i = 0; i < childrenToKeep.getLength(); i++)
+		{
+			//Need to put these in a list to make them easier to loop through as using a NodeList is messy
+			childList.add(childrenToKeep.item(i));
+		}
+
+		//for(int i = 0; i < childrenToKeep.getLength(); i++)
+		for (int i = 0; i < childList.size(); i++)
+		{
+			Node currentChild = childList.get(i);
+			//If the child is not a <Content> node then add it to source section
+			if (!currentChild.getNodeName().equals(GSXML.DOCXML_CONTENT_ELEM))
+			{
+				sourceSection.appendChild(currentChild);
+				continue;
+			}
+
+			//Get the destination Section's Content node's text node, if it's empty then we don't need to worry about appending it
+			Node destinationTextNode = currentChild.getFirstChild();
+			if (destinationTextNode == null || destinationTextNode.getNodeValue() == null || destinationTextNode.getNodeValue().equals(""))
+			{
+				continue;
+			}
+
+			//If the source Section does not have a content then directly append the destination content
+			Element sourceContent = (Element) GSXML.getChildByTagName(sourceSection, GSXML.DOCXML_CONTENT_ELEM);
+			if (sourceContent == null)
+			{
+				sourceSection.appendChild(currentChild);
+				continue;
+			}
+
+			//If the source Section's Content is empty then again we can directly append the destination content
+			Node sourceTextNode = sourceContent.getFirstChild();
+			if (sourceTextNode == null || sourceTextNode.getNodeValue() == null || sourceTextNode.getNodeValue().equals(""))
+			{
+				sourceSection.appendChild(currentChild);
+				continue;
+			}
+
+			//Otherwise, set the new content to be destination + source text in that order.
+			sourceTextNode.setNodeValue(destinationTextNode.getNodeValue() + " " + sourceTextNode.getNodeValue());
+		}
+
+		documentXMLSetSection(mergeOID, collection, sourceSection, OPERATION_REPLACE, userContext);
+		if (_errorStatus != NO_ERROR)
+		{
+			return;
+		}
+
+		documentXMLDeleteSection(oid, collection, userContext);
+	}
+
+	/**
+	 * Can be used to split a section into two sections (e.g. D11.2.1 will
+	 * become D11.2.1 and D11.2.2). Any child section will belong to the second
+	 * section (D11.2.2 in the example).
+	 * 
+	 * @param oid
+	 *            is the identifer of the section to be split.
+	 * @param collection
+	 *            is the collection the section resides in.
+	 * @param splitPoint
+	 *            is the point in the text we want to split at.
+	 */
+	public void documentSplit(String oid, String collection, int splitPoint, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element sectionToSplit = null;
+		if (!oid.contains("."))
+		{
+			sectionToSplit = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			sectionToSplit = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		Element content = (Element) GSXML.getChildByTagName(sectionToSplit, GSXML.DOCXML_CONTENT_ELEM);
+		if (content == null)
+		{
+			_errorStatus = ERROR_INVALID_SPLIT;
+			return;
+		}
+
+		Node textNode = content.getFirstChild();
+		if (textNode == null)
+		{
+			_errorStatus = ERROR_INVALID_SPLIT;
+			return;
+		}
+
+		String text = textNode.getNodeValue();
+		if (splitPoint > text.length() - 2 || splitPoint < 1) //-1 would be the index of the last character, so the last valid split point is -2
+		{
+			_errorStatus = ERROR_INVALID_SPLIT;
+			return;
+		}
+
+		String firstPart = text.substring(0, splitPoint);
+		String secondPart = text.substring(splitPoint);
+
+		Element newSection = docXML.createElement(GSXML.DOCXML_SECTION_ELEM);
+		Element newContent = docXML.createElement(GSXML.DOCXML_CONTENT_ELEM);
+		Node newTextNode = docXML.createTextNode(firstPart);
+		newContent.appendChild(newTextNode);
+		newSection.appendChild(newContent);
+
+		documentXMLSetSection(oid, collection, newSection, OPERATION_INSERT_BEFORE, userContext);
+		if (_errorStatus != NO_ERROR)
+		{
+			return;
+		}
+		textNode.setNodeValue(secondPart);
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+	}
+
+	/**
+	 * Creates a basic doc.xml file with minimal contents.
+	 * 
+	 * @param oid
+	 *            is the identifier of the document to be created.
+	 * @param collection
+	 *            is the collection the new document will reside in.
+	 */
+	public void documentXMLCreateDocXML(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		try
+		{
+			String s = File.separator;
+
+			String docFolderPath = _siteHome + s + "collect" + s + collection + s + "import" + s + oid;
+			File docFolder = new File(docFolderPath);
+
+			if (!docFolder.exists())
+			{
+				if (!docFolder.mkdirs())
+				{
+					_errorStatus = ERROR_DOC_XML_COULD_NOT_BE_CREATED;
+					return;
+				}
+			}
+
+			File docFile = new File(docFolderPath + s + "doc.xml");
+			if (!docFile.exists() && !docFile.createNewFile())
+			{
+				_errorStatus = ERROR_DOC_XML_COULD_NOT_BE_CREATED;
+				return;
+			}
+
+			BufferedWriter bw = new BufferedWriter(new FileWriter(docFile));
+			bw.write("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n");
+			bw.write("<!DOCTYPE Archive SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n");
+			bw.write("<Archive>\n");
+			bw.write("  <Section>\n");
+			bw.write("    <Description>\n");
+			bw.write("      <Metadata name=\"Identifier\">" + oid + "</Metadata>\n");
+			bw.write("      <Metadata name=\"dc.Title\">UNTITLED DOCUMENT</Metadata>\n");
+			bw.write("    </Description>\n");
+			bw.write("    <Content>\n");
+			bw.write("    </Content>\n");
+			bw.write("  </Section>\n");
+			bw.write("</Archive>\n");
+			bw.close();
+
+			Document docXML = null;
+			DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+			DocumentBuilder db = dbf.newDocumentBuilder();
+			docXML = db.parse(docFile);
+
+			//_docCache.put(oid + "__" + collection, docXML);
+		}
+		catch (Exception ex)
+		{
+			ex.printStackTrace();
+			_errorStatus = ERROR_EXCEPTION_CREATING_DOC_XML_FILE;
+			return;
+		}
+	}
+
+	/**
+	 * Gets a metadata value from a document or section.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section or document to get metadata
+	 *            from.
+	 * @param collection
+	 *            is the collection the section or document resides in.
+	 * @param metadataName
+	 *            is the name of metadata to retrieve
+	 * @return an array of metadata elements containing the resquested metadata
+	 */
+	public ArrayList<Element> documentXMLGetMetadata(String oid, String collection, String metadataName, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return null;
+		}
+		else if (metadataName == null || metadataName.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_NAME_NOT_SPECIFIED;
+			return null;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return null;
+		}
+
+		return getMetadataElementsFromSection(docXML, oid, metadataName);
+	}
+
+	/**
+	 * 
+	 * @param oid
+	 *            is the identifier of the document or section that is to have
+	 *            it's metadata set.
+	 * @param collection
+	 *            is the collection the document/section resides in.
+	 * @param metadataName
+	 *            is the name of the metadata value that is to be set.
+	 * @param newMetadataValue
+	 *            is the new value of the metadata.
+	 * @param position
+	 *            specifies the position of the value to set.
+	 * @param operation
+	 *            can be one of OPERATION_REPLACE, OPERATION_INSERT_BEFORE,
+	 *            OPERATION_INSERT_AFTER or OPERATION_APPEND.
+	 */
+	public void documentXMLSetMetadata(String oid, String collection, String metadataName, String newMetadataValue, int position, int operation, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+		else if (metadataName == null || metadataName.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_NAME_NOT_SPECIFIED;
+			return;
+		}
+		else if (newMetadataValue == null || newMetadataValue.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_VALUE_NOT_SPECIFIED;
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		ArrayList<Element> metadataElems = getMetadataElementsFromSection(docXML, oid, metadataName);
+
+		if (operation != OPERATION_APPEND && metadataElems.get(position) == null)
+		{
+			_errorStatus = ERROR_INVALID_METADATA_POSITION;
+			return;
+		}
+
+		if (operation == OPERATION_REPLACE)
+		{
+			metadataElems.get(position).setNodeValue(newMetadataValue);
+		}
+		else if (operation == OPERATION_INSERT_BEFORE)
+		{
+			Element newMetadata = createElementWithValue(docXML, GSXML.DOCXML_METADATA_ELEM, metadataName, newMetadataValue);
+			Element existingMetadata = metadataElems.get(position);
+
+			existingMetadata.getParentNode().insertBefore(newMetadata, existingMetadata);
+		}
+		else if (operation == OPERATION_INSERT_AFTER)
+		{
+			Element newMetadata = createElementWithValue(docXML, GSXML.DOCXML_METADATA_ELEM, metadataName, newMetadataValue);
+			Element existingMetadata = metadataElems.get(position + 1);
+
+			if (existingMetadata != null)
+			{
+				existingMetadata.getParentNode().insertBefore(newMetadata, existingMetadata);
+			}
+			else
+			{
+				existingMetadata = metadataElems.get(position);
+				existingMetadata.getParentNode().appendChild(newMetadata);
+			}
+		}
+		else
+		{
+			Element section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+			Element description = (Element) GSXML.getChildByTagName(section, GSXML.DOCXML_DESCRIPTION_ELEM);
+			if (description == null)
+			{
+				description = docXML.createElement(GSXML.DOCXML_DESCRIPTION_ELEM);
+				section.appendChild(description);
+			}
+			Element newMetadata = createElementWithValue(docXML, GSXML.DOCXML_METADATA_ELEM, metadataName, newMetadataValue);
+			description.appendChild(newMetadata);
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+	}
+
+	/**
+	 * Can be used to delete metadata at a specific position in a document or
+	 * section (such as the third author of a document).
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section to delete metadata
+	 *            from.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @param metadataName
+	 *            is the name of the metadata that is to have an item deleted.
+	 * @param position
+	 *            is position of the item that is to be deleted.
+	 */
+	public void documentXMLDeleteMetadata(String oid, String collection, String metadataName, int position, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+		else if (metadataName == null || metadataName.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_NAME_NOT_SPECIFIED;
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		ArrayList<Element> metadataElems = getMetadataElementsFromSection(docXML, oid, metadataName);
+
+		if (metadataElems.get(position) != null)
+		{
+			metadataElems.get(position).getParentNode().removeChild(metadataElems.get(position));
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+
+	}
+
+	/**
+	 * Can be used to delete all the metadata with a specific name from a
+	 * document or section (e.g. all of the authors).
+	 * 
+	 * @param oid
+	 *            is the identifier of the document or section to delete the
+	 *            metadata from.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @param metadataName
+	 *            is the name of the metadata to delete.
+	 */
+	public void documentXMLDeleteMetadata(String oid, String collection, String metadataName, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+		else if (metadataName == null || metadataName.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_NAME_NOT_SPECIFIED;
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		ArrayList<Element> metadataElems = getMetadataElementsFromSection(docXML, oid, metadataName);
+
+		for (Element elem : metadataElems)
+		{
+			elem.getParentNode().removeChild(elem);
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+
+	}
+
+	/**
+	 * Can be used to replace a specific metadata item of a given name, given
+	 * it's value.
+	 * 
+	 * @param oid
+	 *            is the document/section of the metadata that is to be
+	 *            replaced.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @param metadataName
+	 *            is the name of the metadata to be replaced.
+	 * @param oldMetadataValue
+	 *            is the old value of the metadata (the value that will be
+	 *            replaced).
+	 * @param newMetadataValue
+	 *            is the new value of the metadata that will replace the old
+	 *            value.
+	 */
+	public void documentXMLReplaceMetadata(String oid, String collection, String metadataName, String oldMetadataValue, String newMetadataValue, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+		else if (metadataName == null || metadataName.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_NAME_NOT_SPECIFIED;
+			return;
+		}
+		else if (newMetadataValue == null || newMetadataValue.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_VALUE_NOT_SPECIFIED;
+			return;
+		}
+		else if (oldMetadataValue == null || oldMetadataValue.equals(""))
+		{
+			_errorStatus = ERROR_METADATA_VALUE_NOT_SPECIFIED;
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		ArrayList<Element> metadataElems = getMetadataElementsFromSection(docXML, oid, metadataName);
+
+		for (Element elem : metadataElems)
+		{
+			Node textNode = elem.getFirstChild();
+
+			if (textNode != null && textNode.getNodeValue().equals(oldMetadataValue))
+			{
+				textNode.setNodeValue(newMetadataValue);
+			}
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+	}
+
+	/**
+	 * Can be used to create a blank section.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to be created.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 */
+	public void documentXMLCreateSection(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return;
+		}
+		else if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return;
+		}
+
+		if (oid.contains(".") && !archiveCheckDocumentOrSectionExists(oid.substring(0, oid.indexOf(".")), collection, userContext))
+		{
+			documentCreate(oid.substring(0, oid.indexOf(".")), collection, userContext);
+			if (_errorStatus != NO_ERROR)
+			{
+				return;
+			}
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element topLevel = docXML.getDocumentElement();
+		if (!oid.contains("."))
+		{
+			if (GSXML.getChildByTagName(topLevel, GSXML.DOCXML_SECTION_ELEM) == null)
+			{
+				Element newSection = docXML.createElement(GSXML.DOCXML_SECTION_ELEM);
+				topLevel.appendChild(newSection);
+			}
+		}
+		else
+		{
+			int[] intLevels = null;
+			try
+			{
+				intLevels = oidToSectionNumberArray(oid);
+			}
+			catch (Exception ex)
+			{
+				_errorStatus = ERROR_OID_INCORRECT_FORMAT;
+				return;
+			}
+
+			Element current = (Element) GSXML.getChildByTagName(topLevel, GSXML.DOCXML_SECTION_ELEM);
+			if (current == null)
+			{
+				Element topLevelSection = docXML.createElement(GSXML.DOCXML_SECTION_ELEM);
+				topLevel.appendChild(topLevelSection);
+				current = topLevelSection;
+			}
+
+			for (int currentLevelValue : intLevels)
+			{
+				NodeList sections = GSXML.getChildrenByTagName(current, GSXML.DOCXML_SECTION_ELEM);
+
+				if (sections.item(currentLevelValue - 1) == null)
+				{
+					Element latest = null;
+					for (int j = 0; j < currentLevelValue - sections.getLength(); j++)
+					{
+						Element blankSection = docXML.createElement(GSXML.DOCXML_SECTION_ELEM);
+						current.appendChild(blankSection);
+						latest = blankSection;
+					}
+					current = latest;
+				}
+				else
+				{
+					current = (Element) sections.item(currentLevelValue - 1);
+				}
+			}
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+	}
+
+	/**
+	 * Can be used to delete an entire section.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to be deleted.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 */
+	public void documentXMLDeleteSection(String oid, String collection, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		if (section == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_DELETE;
+			return;
+		}
+
+		section.getParentNode().removeChild(section);
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+		}
+	}
+
+	/**
+	 * Can be used to get a section from a document.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to get.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @return the requested section.
+	 */
+	public Element documentXMLGetSection(String oid, String collection, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return null;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return null;
+		}
+
+		Element section = null;
+		if (!oid.contains("."))
+		{
+			section = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		if (section == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_SECTION;
+			return null;
+		}
+
+		return section;
+	}
+
+	/**
+	 * Can be used to set an OID to the given section element.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to be set.
+	 * @param collection
+	 *            is the collection the section will reside in.
+	 * @param newSection
+	 *            is the new section element.
+	 * @param operation
+	 *            can be one of OPERATION_REPLACE, OPERATION_INSERT_BEFORE,
+	 *            OPERATION_INSERT_AFTER or OPERATION_APPEND.
+	 * @throws IOException
+	 */
+	public void documentXMLSetSection(String oid, String collection, Element newSection, int operation, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element existingSection = null;
+		if (!oid.contains("."))
+		{
+			existingSection = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			existingSection = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		if (existingSection == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_SECTION;
+			return;
+		}
+
+		Element importedSection = (Element) docXML.importNode(newSection.cloneNode(true), true);
+		Node sectionParent = existingSection.getParentNode();
+
+		if (operation == OPERATION_APPEND)
+		{
+			existingSection.appendChild(importedSection);
+		}
+		else
+		{
+			//Remove the attributes that are only there to help us find the section
+			importedSection.removeAttribute(GSXML.NODE_ID_ATT);
+			importedSection.removeAttribute(GSXML.COLLECTION_ATT);
+
+			if (operation == OPERATION_INSERT_BEFORE || operation == OPERATION_REPLACE)
+			{
+				sectionParent.insertBefore(importedSection, existingSection);
+			}
+			else if (operation == OPERATION_INSERT_AFTER)
+			{
+				Node siblingNode = existingSection.getNextSibling();
+				while (siblingNode != null && siblingNode.getNodeType() != Node.ELEMENT_NODE)
+				{
+					siblingNode = siblingNode.getNextSibling();
+				}
+
+				if (siblingNode != null)
+				{
+					sectionParent.insertBefore(importedSection, siblingNode);
+				}
+				else
+				{
+					sectionParent.appendChild(importedSection);
+				}
+			}
+
+			if (operation == OPERATION_REPLACE)
+			{
+				sectionParent.removeChild(existingSection);
+			}
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+			return;
+		}
+	}
+
+	/**
+	 * Gets the text of a given section as a string.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to get the text from.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @return the text from the section.
+	 */
+	public String documentXMLGetText(String oid, String collection, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return null;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return null;
+		}
+
+		Element section = null;
+		if (!oid.contains("."))
+		{
+			section = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		if (section == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_SECTION;
+			return null;
+		}
+
+		Element contentNode = (Element) GSXML.getChildByTagName(section, GSXML.DOCXML_CONTENT_ELEM);
+		if (contentNode == null)
+		{
+			return null;
+		}
+
+		Node textNode = contentNode.getFirstChild();
+		if (textNode == null)
+		{
+			return null;
+		}
+
+		return textNode.getNodeValue();
+	}
+
+	/**
+	 * Sets the text of a given section using an element.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to set the text of.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @param newContent
+	 *            is the new content element for the section.
+	 */
+	public void documentXMLSetText(String oid, String collection, Element newContent, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element section = null;
+		if (!oid.contains("."))
+		{
+			section = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		if (section == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_SECTION;
+			return;
+		}
+
+		Element existingContent = (Element) GSXML.getChildByTagName(section, GSXML.DOCXML_CONTENT_ELEM);
+
+		//Remove the attributes that are only there to help us find the content
+		newContent.removeAttribute(GSXML.NODE_ID_ATT);
+		newContent.removeAttribute(GSXML.COLLECTION_ATT);
+
+		Element importedContent = (Element) docXML.importNode(newContent, true);
+		//If the current section does not have content then just add it, otherwise replace the existing one
+		if (existingContent == null)
+		{
+			section.appendChild(importedContent);
+		}
+		else
+		{
+			Node contentParent = existingContent.getParentNode();
+
+			//Replace the old node in the document tree
+			contentParent.insertBefore(importedContent, existingContent);
+			contentParent.removeChild(existingContent);
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+			return;
+		}
+	}
+
+	/**
+	 * Sets the text of a given section using a string.
+	 * 
+	 * @param oid
+	 *            is the identifier of the section to set the text of.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @param newContent
+	 *            is the new text for the section.
+	 */
+	public void documentXMLSetText(String oid, String collection, String newContent, UserContext userContext)
+	{
+		if ((_errorStatus = checkOIDandCollection(oid, collection, userContext)) != NO_ERROR)
+		{
+			return;
+		}
+
+		Document docXML = getDocXML(oid, collection, userContext);
+		if (docXML == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_DOC_XML;
+			return;
+		}
+
+		Element section = null;
+		if (!oid.contains("."))
+		{
+			section = getTopLevelSectionElement(docXML);
+		}
+		else
+		{
+			section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+		}
+
+		if (section == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_RETRIEVE_SECTION;
+			return;
+		}
+
+		Element existingContent = (Element) GSXML.getChildByTagName(section, GSXML.DOCXML_CONTENT_ELEM);
+
+		//If the current section does not have content then just add it, otherwise replace the existing one
+		if (existingContent == null)
+		{
+			Element newContentElem = docXML.createElement(GSXML.DOCXML_CONTENT_ELEM);
+			Node textNode = docXML.createTextNode(newContent);
+			newContentElem.appendChild(textNode);
+		}
+		else
+		{
+			Node textNode = existingContent.getFirstChild();
+			if (textNode != null)
+			{
+				textNode.setNodeValue(newContent);
+			}
+			else
+			{
+				existingContent.appendChild(docXML.createTextNode(newContent));
+			}
+		}
+
+		//Write the new change back into the file
+		if (!writeXMLFile(docXML, oid, collection, userContext))
+		{
+			_errorStatus = ERROR_COULD_NOT_WRITE_TO_DOC_XML;
+			return;
+		}
+	}
+
+	/**
+	 * Can be used to get the file path of the doc.xml file containing the given
+	 * OID.
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section to get the doc.xml
+	 *            of.
+	 * @param collection
+	 *            is the collection the document resides in.
+	 * @return the file path to the doc.xml file.
+	 */
+	public String archiveGetDocumentFilePath(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+
+		if (oid.contains("."))
+		{
+			oid = oid.substring(0, oid.indexOf("."));
+		}
+
+		String assocFilePath = getDocFilePathFromDatabase(oid, collection, userContext);
+		if (assocFilePath == null)
+		{
+			_errorStatus = ERROR_DATA_NOT_FOUND_IN_DATABASE;
+			return null;
+		}
+
+		if (File.separator.equals("\\"))
+		{
+			assocFilePath = assocFilePath.replace("/", "\\");
+		}
+
+		String docFilePath = _siteHome + File.separatorChar + "collect" + File.separatorChar + collection + File.separatorChar + "archives" + File.separatorChar + assocFilePath;
+		return docFilePath;
+	}
+
+	/**
+	 * Can be used to find the document that a specific source file is used in.
+	 * 
+	 * @param srcFile
+	 *            is the name of the source file.
+	 * @param collection
+	 *            is the collection the source file resides in.
+	 * @return the OID of the document that the source file is used in.
+	 */
+	public String archiveGetSourceFileOID(String srcFile, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFSRC, SimpleCollectionDatabase.READ, userContext);
+		if (coll_db == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_OPEN_DATABASE;
+			return null;
+		}
+
+		DBInfo info = coll_db.getInfo(srcFile);
+		if (info == null)
+		{
+			_errorStatus = ERROR_DATA_NOT_FOUND_IN_DATABASE;
+			coll_db.closeDatabase();
+			return null;
+		}
+
+		String oid = info.getInfo("oid");
+
+		coll_db.closeDatabase();
+		return oid;
+	}
+
+	/**
+	 * Checks to see if a document or section at a given OID exists.
+	 * 
+	 * @param oid
+	 *            is the identifier of the document/section to check.
+	 * @param collection
+	 *            is the collection to search in.
+	 * @return true if the document/section exists, false otherwise.
+	 */
+	public boolean archiveCheckDocumentOrSectionExists(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFDOC, SimpleCollectionDatabase.READ, userContext);
+		if (coll_db == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_OPEN_DATABASE;
+			return false;
+		}
+
+		boolean section = false;
+		if (oid.contains("."))
+		{
+			section = true;
+		}
+
+		DBInfo info = null;
+		if (section)
+		{
+			info = coll_db.getInfo(oid.substring(0, oid.indexOf(".")));
+		}
+		else
+		{
+			info = coll_db.getInfo(oid);
+		}
+		boolean exists = (info != null);
+
+		coll_db.closeDatabase();
+		if (section && exists)
+		{
+			Document docXML = getDocXML(oid, collection, userContext);
+			if (getSectionBySectionNumber(docXML, getSectionFromOID(oid)) == null)
+			{
+				return false;
+			}
+			return true;
+		}
+
+		return exists;
+	}
+
+	/**
+	 * Can be used to write a series of entries to the document database.
+	 * 
+	 * @param oid
+	 *            is the key that the entries will be written to.
+	 * @param collection
+	 *            is the collection whose database will be written to.
+	 * @param infoList
+	 *            is the list of entries to write.
+	 */
+	public void archiveWriteEntryToDatabase(String oid, String collection, HashMap<String, ArrayList<String>> infoList, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return;
+		}
+		else if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return;
+		}
+
+		DBInfo info = new DBInfo();
+
+		for (String s : infoList.keySet())
+		{
+			for (String v : infoList.get(s))
+			{
+				info.addInfo(s, v);
+			}
+		}
+
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFDOC, SimpleCollectionDatabase.WRITE, userContext);
+		if (coll_db == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_OPEN_DATABASE;
+			return;
+		}
+
+		coll_db.setInfo(oid, info);
+		coll_db.closeDatabase();
+	}
+
+	/**
+	 * Can be used to remove an entry from the document database.
+	 * 
+	 * @param oid
+	 *            is the key of the entry to erase.
+	 * @param collection
+	 *            is the collection whose database will have the entry removed.
+	 */
+	public void archiveRemoveEntryFromDatabase(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return;
+		}
+		else if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return;
+		}
+
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFDOC, SimpleCollectionDatabase.WRITE, userContext);
+		if (coll_db == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_OPEN_DATABASE;
+			return;
+		}
+		coll_db.deleteKey(oid);
+		coll_db.closeDatabase();
+	}
+
+	/**
+	 * Gets the list of associated files for a given document.
+	 * 
+	 * @param oid
+	 *            is the identifier that will be used to search for associated
+	 *            documents.
+	 * @param collection
+	 *            is the collection whose database will be searched.
+	 * @return the list of associated files.
+	 */
+	public ArrayList<String> archiveGetAssociatedImportFiles(String oid, String collection, UserContext userContext)
+	{
+		_errorStatus = NO_ERROR;
+		if (oid == null || oid.equals(""))
+		{
+			_errorStatus = ERROR_OID_NOT_SPECIFIED;
+			return null;
+		}
+		else if (collection == null || collection.equals(""))
+		{
+			_errorStatus = ERROR_COLLECTION_NOT_SPECIFIED;
+			return null;
+		}
+
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFDOC, SimpleCollectionDatabase.READ, userContext);
+		if (coll_db == null)
+		{
+			_errorStatus = ERROR_COULD_NOT_OPEN_DATABASE;
+			return null;
+		}
+
+		DBInfo info = coll_db.getInfo(oid);
+		if (info == null)
+		{
+			_errorStatus = ERROR_DATA_NOT_FOUND_IN_DATABASE;
+			coll_db.closeDatabase();
+			return null;
+		}
+
+		String srcFile = info.getInfo("src-file");
+		Vector<String> data = info.getMultiInfo("assoc-file");
+
+		ArrayList<String> assocFiles = new ArrayList<String>();
+		assocFiles.add(srcFile);
+		for (String d : (Vector<String>) data)
+		{
+			assocFiles.add(d);
+		}
+
+		coll_db.closeDatabase();
+		return assocFiles;
+	}
+
+	/********************
+	 * Helper functions *
+	 *******************/
+
+	public boolean checkError(Element elem, String methodName)
+	{
+		if (_errorMessageMap.get(_errorStatus) != null)
+		{
+			GSXML.addError(elem, methodName + ": " + _errorMessageMap.get(_errorStatus), GSXML.ERROR_TYPE_SYNTAX);
+			return true;
+		}
+
+		return false;
+	}
+
+	public String getSectionFromOID(String oid)
+	{
+		if (!oid.contains("."))
+		{
+			return null;
+		}
+		return oid.substring(oid.indexOf(".") + 1);
+	}
+
+	public Element createElementWithValue(Document doc, String nodeName, String name, String value)
+	{
+		Element metadataElem = doc.createElement(nodeName);
+		metadataElem.setAttribute(GSXML.NAME_ATT, name);
+		Node textNode = doc.createTextNode(value);
+		metadataElem.appendChild(textNode);
+		return metadataElem;
+	}
+
+	public int getOperation(String to, String from)
+	{
+		int op;
+		if (!to.contains(".") && !from.contains("."))
+		{
+			op = OPERATION_TYPE_DOC_TO_DOC;
+		}
+		else if (!to.contains(".") && from.contains("."))
+		{
+			op = OPERATION_TYPE_DOC_TO_SEC;
+		}
+		else if (to.contains(".") && !from.contains("."))
+		{
+			op = OPERATION_TYPE_SEC_TO_DOC;
+		}
+		else
+		{
+			op = OPERATION_TYPE_SEC_TO_SEC;
+		}
+		return op;
+	}
+
+	public int[] oidToSectionNumberArray(String oid) throws Exception
+	{
+		String[] strLevels = oid.split("\\.");
+		int[] intLevels = new int[strLevels.length - 1];
+
+		for (int i = 1; i < strLevels.length; i++) //Start at 1 to avoid the document identifier part of the OID
+		{
+			intLevels[i - 1] = Integer.parseInt(strLevels[i]);
+		}
+
+		return intLevels;
+	}
+
+	public String getDocFilePathFromDatabase(String oid, String collection, UserContext userContext)
+	{
+		SimpleCollectionDatabase coll_db = openDatabase(collection, ARCHIVEINFDOC, SimpleCollectionDatabase.WRITE, userContext);
+		if (coll_db == null)
+		{
+			return null;
+		}
+		DBInfo info = coll_db.getInfo(oid);
+		if (info == null)
+		{
+			return null;
+		}
+
+		String docFile = info.getInfo("doc-file");
+		coll_db.closeDatabase();
+		return docFile;
+	}
+
+	public boolean deleteDirectory(File current)
+	{
+		try
+		{
+			if (current == null || !current.exists())
+			{
+				return false;
+			}
+
+			if (!current.isDirectory())
+			{
+				current.delete();
+				return true;
+			}
+
+			for (File f : current.listFiles())
+			{
+				if (f.isDirectory())
+				{
+					deleteDirectory(f);
+				}
+				else
+				{
+					f.delete();
+				}
+			}
+			current.delete();
+		}
+		catch (Exception ex)
+		{
+			return false;
+		}
+
+		return true;
+	}
+
+	public int checkOIDandCollection(String oid, String collection, UserContext userContext)
+	{
+		if (oid == null || oid.equals(""))
+		{
+			return ERROR_OID_NOT_SPECIFIED;
+		}
+
+		if (collection == null || collection.equals(""))
+		{
+			return ERROR_COLLECTION_NOT_SPECIFIED;
+		}
+
+		if (!archiveCheckDocumentOrSectionExists(oid, collection, userContext))
+		{
+			return ERROR_SOURCE_DOCUMENT_OR_SECTION_DOES_NOT_EXIST;
+		}
+		return NO_ERROR;
+	}
+
+	public boolean copyDirectory(File src, File dest)
+	{
+		if (src.isDirectory())
+		{
+			//If the destination directory does not exist then create it
+			if (!dest.exists())
+			{
+				dest.mkdir();
+			}
+
+			//Get all the files in the directory
+			String files[] = src.list();
+			for (String file : files)
+			{
+				File srcFile = new File(src, file);
+				File destFile = new File(dest, file);
+
+				if (!copyDirectory(srcFile, destFile))
+				{
+					return false;
+				}
+			}
+		}
+		else
+		{
+			try
+			{
+				FileChannel in = new FileInputStream(src).getChannel();
+				FileChannel out = new FileOutputStream(dest).getChannel();
+
+				in.transferTo(0, in.size(), out);
+
+				in.close();
+				out.close();
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+				return false;
+			}
+		}
+		return true;
+	}
+
+	public Element getTopLevelSectionElement(Document docXML)
+	{
+		return (Element) GSXML.getChildByTagName(docXML.getDocumentElement(), GSXML.DOCXML_SECTION_ELEM);
+	}
+
+	public boolean writeXMLFile(Document doc, String oid, String collection, UserContext userContext)
+	{
+		try
+		{
+			DOMSource source = new DOMSource(doc);
+
+			String test = archiveGetDocumentFilePath(oid, collection, userContext);
+			File xmlFile = new File(test);
+			Result result = new StreamResult(xmlFile);
+
+			Transformer transformer = TransformerFactory.newInstance().newTransformer();
+			transformer.transform(source, result);
+		}
+		catch (Exception ex)
+		{
+			return false;
+		}
+		return true;
+	}
+
+	public Document getDocXML(String oid, String collection, UserContext userContext)
+	{
+		if (oid.contains("."))
+		{
+			oid = oid.substring(0, oid.indexOf("."));
+		}
+
+		Document docXML = null;
+		//if ((docXML = _docCache.get(oid + "__" + collection)) == null)
+		//	{
+			String filePath = archiveGetDocumentFilePath(oid, collection, userContext);
+			File docFile = new File(filePath);
+
+			if (!docFile.exists())
+			{
+				return null;
+			}
+
+			try
+			{
+				DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+				DocumentBuilder db = dbf.newDocumentBuilder();
+				docXML = db.parse(docFile);
+
+				//_docCache.put(oid + "__" + collection, docXML);
+			}
+			catch (Exception ex)
+			{
+				return null;
+			}
+			//	}
+		return docXML;
+	}
+
+	public ArrayList<Element> getMetadataElementsFromSection(Document docXML, String oid, String metadataName)
+	{
+		if (oid.contains("."))
+		{
+			Element section = getSectionBySectionNumber(docXML, getSectionFromOID(oid));
+			return getMetadataElementsFromSection(section, metadataName);
+		}
+		else
+		{
+			return getMetadataElementsFromSection(getTopLevelSectionElement(docXML), metadataName);
+		}
+	}
+
+	public ArrayList<Element> getMetadataElementsFromSection(Element section, String metadataName)
+	{
+		Element description = (Element) GSXML.getChildByTagName(section, GSXML.DOCXML_DESCRIPTION_ELEM);
+		if (description == null)
+		{
+			return null;
+		}
+
+		ArrayList<Element> elemList = new ArrayList<Element>();
+		NodeList metadataNodes = GSXML.getChildrenByTagName(description, GSXML.DOCXML_METADATA_ELEM);
+		for (int j = 0; j < metadataNodes.getLength(); j++)
+		{
+			//If this is a metadata element with the requested name then we have found what we are looking for
+			if (((Element) metadataNodes.item(j)).getAttribute(GSXML.NAME_ATT).equals(metadataName))
+			{
+				elemList.add((Element) metadataNodes.item(j));
+			}
+		}
+
+		return elemList;
+	}
+
+	public Element getSectionBySectionNumber(Document docXML, String sectionNum)
+	{
+		return getSectionBySectionNumber(getTopLevelSectionElement(docXML), sectionNum);
+	}
+
+	public Element getSectionBySectionNumber(Element current, String sectionNum)
+	{
+		if (sectionNum == null || sectionNum.equals(""))
+		{
+			return current;
+		}
+
+		try
+		{
+			String[] levels = sectionNum.split("\\.");
+			int currentSectionNum = Integer.parseInt(levels[0]);
+
+			NodeList sections = GSXML.getChildrenByTagName(current, GSXML.DOCXML_SECTION_ELEM);
+			if (levels.length > 1)
+			{
+				return getSectionBySectionNumber((Element) sections.item(currentSectionNum - 1), sectionNum.substring(sectionNum.indexOf(".") + 1));
+			}
+			else
+			{
+				return (Element) sections.item(currentSectionNum - 1);
+			}
+		}
+		catch (Exception ex)
+		{
+			return null;
+		}
+	}
+
+	public String getDatabaseTypeFromCollection(String collection, UserContext userContext)
+	{
+		//Find out what kind of database we have
+	  Document doc = XMLConverter.newDOM();
+		Element dbTypeMessage = doc.createElement(GSXML.MESSAGE_ELEM);
+		Element dbTypeRequest = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_DESCRIBE, collection, userContext);
+		dbTypeMessage.appendChild(dbTypeRequest);
+		Element dbTypeResponse = (Element) _router.process(dbTypeMessage);
+
+		String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.COLLECTION_ELEM);
+		Element collectionElem = (Element) GSXML.getNodeByPath(dbTypeResponse, path);
+
+		if (collectionElem != null)
+		{
+			return collectionElem.getAttribute(GSXML.DB_TYPE_ATT);
+		}
+		return "gdbm"; //The default collection database type
+	}
+
+	public SimpleCollectionDatabase openDatabase(String collection, String dbName, int readWrite, UserContext userContext)
+	{
+		//Find out what kind of database we have
+		String databaseType = getDatabaseTypeFromCollection(collection, userContext);
+		String dbExt = DBHelper.getDBExtFromDBType(databaseType);
+
+		SimpleCollectionDatabase coll_db = new SimpleCollectionDatabase(databaseType);
+		if (!coll_db.databaseOK())
+		{
+			System.err.println("Couldn't create the collection database of type " + databaseType);
+			return null;
+		}
+
+		coll_db.openDatabase(GSFile.collectionArchiveDir(_siteHome, collection) + File.separatorChar + dbName + dbExt, readWrite);
+
+		return coll_db;
+	}
+
+	public int operationStringToInt(String operation)
+	{
+		if (operation.equals("insertBefore"))
+		{
+			return OPERATION_INSERT_BEFORE;
+		}
+		else if (operation.equals("insertAfter"))
+		{
+			return OPERATION_INSERT_AFTER;
+		}
+		else if (operation.equals("append"))
+		{
+			return OPERATION_APPEND;
+		}
+		else
+		{
+			return OPERATION_REPLACE;
+		}
+	}
+
+	public int getErrorStatus()
+	{
+		return _errorStatus;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSEntityResolver.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSEntityResolver.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSEntityResolver.java	(revision 31634)
@@ -0,0 +1,96 @@
+/*
+ *    GSEntityResolver.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.util.Misc;
+
+import org.xml.sax.InputSource;
+import org.xml.sax.EntityResolver;
+import java.io.File;
+import java.net.URL;
+
+import org.apache.log4j.*;
+
+// uses a class loader to find entities
+// The class loader to use can be set by setClassLoader(), otherwise it will use the class loader that loaded itself. For the Tomcat webapp, this will be the webapp class loader, not the system class loader. the webapp classloader knows about the classes in the WEB-INF/classes dir, the system classloader knows about the ones on the classpath. The system class loader is a parent to web app classloader, so this will be used as well.
+
+public class GSEntityResolver implements EntityResolver {
+
+    ClassLoader class_loader = null;
+    File baseFilepath = null;
+    
+     static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSEntityResolver.class.getName());
+
+    public GSEntityResolver() {}
+
+    public GSEntityResolver(File baseFilepath) {
+	this.baseFilepath = baseFilepath;
+    }
+
+    /* Methods with the ClassLoader parameter are unused at present */
+    public GSEntityResolver(ClassLoader loader) {
+	this.class_loader = loader;
+    }
+
+    public void setClassLoader(ClassLoader loader) {
+	this.class_loader = loader;
+    }
+    
+    public InputSource resolveEntity (String public_id, String system_id) {
+	
+	logger.debug("entity resolver for "+system_id);
+	String temp_id = system_id;
+	if (temp_id.startsWith("file://")) {
+	    File f = new File(system_id);
+	    if (f.exists()) {
+		return new InputSource(system_id);
+	    } else {
+		temp_id = f.getName(); //temp_id = temp_id.substring(temp_id.lastIndexOf("/")+1);
+	    }
+	} else { // not a file
+	    if (temp_id.indexOf("/")!= -1) {
+		temp_id = temp_id.substring(temp_id.lastIndexOf("/")+1);
+	    }
+	}
+	
+	// use the baseFilepath, if one was provided
+	if(this.baseFilepath != null) {
+	
+		// The file protocol for windows starts with file:/// and has URL slashes
+		// http://en.wikipedia.org/wiki/File_URI_scheme#Windows
+	    String newpath = this.baseFilepath + File.separator + temp_id;		
+		if(Misc.isWindows()) {			
+			newpath = "file:///" + newpath.replace("\\", "/");			
+		} else { // linux version, file protocol starts with file:// and slashes are already URL-style
+			newpath = "file://" + newpath;
+		}
+		return new InputSource(newpath);
+	}
+
+	// try using a class loader
+	if (this.class_loader==null) {
+	    this.class_loader = this.getClass().getClassLoader();
+	}
+	URL url = class_loader.getResource(temp_id);
+	if (url == null) {
+	    return null;
+	}
+	return new InputSource("file://"+url.getFile());
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFile.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFile.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFile.java	(revision 31634)
@@ -0,0 +1,634 @@
+/*
+ *    GSFile.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+
+import org.apache.axis.encoding.Base64;
+import org.apache.log4j.Logger;
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.DBHelper;
+
+/**
+ * GSFile - utility class for Greenstone.
+ * 
+ * all file paths are created here also has file utility methods
+ * 
+ * @author Katherine Don
+ * @version $Revision$
+ * @see File
+ */
+
+public class GSFile
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSFile.class.getName());
+
+	/** site config file path */
+	static public String siteConfigFile(String site_home)
+	{
+		return site_home + File.separatorChar + "siteConfig.xml";
+
+	}
+	
+	/** site config file path */
+	static public String groupConfigFile(String site_home)
+	{
+		return site_home + File.separatorChar + "groupConfig.xml";
+
+	}
+	
+	/** site images file path */
+	static public String imagesFilePath(String site_home)
+	{
+		return site_home + File.separatorChar + "images";
+
+	}
+
+	/** interface config file path */
+	static public String interfaceConfigFile(String interface_home)
+	{
+		return interface_home + File.separatorChar + "interfaceConfig.xml";
+
+	}
+
+	/** collection directory path */
+	static public String collectDir(String site_home)
+	{
+		return site_home + File.separatorChar + "collect";
+	}
+
+	/** collection config file path */
+	static public String collectionConfigFile(String site_home, String collection_name)
+	{
+		return collectionConfigFile(collectionBaseDir(site_home, collection_name));
+	}
+
+	static public String collectionConfigFile(String collection_home)
+	{
+		return collectionEtcDir(collection_home) + File.separatorChar + "collectionConfig.xml";
+
+	}
+
+	/** collection init file path */
+	static public String collectionInitFile(String site_home, String collection_name)
+	{
+		return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "etc" + File.separatorChar + "collectionInit.xml";
+
+	}
+
+	/** collection build config file path */
+	static public String collectionBuildConfigFile(String site_home, String collection_name)
+	{
+		return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "index" + File.separatorChar + "buildConfig.xml";
+	}
+
+	/** collection build config file path */
+	static public String collectionBuildConfigFileBuilding(String site_home, String collection_name)
+	{
+		return collectionBuildConfigFileBuilding(collectionBaseDir(site_home, collection_name));
+	}
+
+	static public String collectionBuildConfigFileBuilding(String collection_home)
+	{
+		return collection_home + File.separatorChar + "building" + File.separatorChar + "buildConfig.xml";
+	}
+
+	/** XML Transform directory path */
+	static public String xmlTransformDir(String interface_home)
+	{
+		return interface_home + File.separatorChar + "transform";
+	}
+
+	/** collection base directory path */
+	static public String collectionBaseDir(String site_home, String collection_name)
+	{
+		return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name;
+	}
+
+	/** collection archive directory path */
+	static public String collectionArchiveDir(String site_home, String collection_name)
+	{
+		return collectionArchiveDir(collectionBaseDir(site_home, collection_name));
+	}
+
+	static public String collectionArchiveDir(String collection_home)
+	{
+		return collection_home + File.separatorChar + "archives";
+	}
+
+	/** collection building directory path */
+	static public String collectionBuildDir(String site_home, String collection_name)
+	{
+		return collectionBuildDir(collectionBaseDir(site_home, collection_name));
+	}
+
+	static public String collectionBuildDir(String collection_home)
+	{
+		return collection_home + File.separator + "building";
+	}
+
+	/** collection building directory path */
+	static public String collectionEtcDir(String site_home, String collection_name)
+	{
+		return collectionEtcDir(collectionBaseDir(site_home, collection_name));
+	}
+
+	static public String collectionEtcDir(String collection_home)
+	{
+		return collection_home + File.separator + "etc";
+	}
+
+	/** collection building directory path */
+	static public String collectionImportDir(String site_home, String collection_name)
+	{
+		return collectionImportDir(collectionBaseDir(site_home, collection_name));
+
+	}
+
+	static public String collectionImportDir(String collection_home)
+	{
+		return collection_home + File.separatorChar + "import";
+	}
+
+	/** collection building directory path */
+	static public String collectionIndexDir(String site_home, String collection_name)
+	{
+		return collectionIndexDir(collectionBaseDir(site_home, collection_name));
+
+	}
+
+	static public String collectionIndexDir(String collection_home)
+	{
+		return collection_home + File.separatorChar + "index";
+	}
+
+	/** text path (for doc retrieval) relative to collectionBaseDir */
+	static public String collectionTextPath(String index_stem)
+	{
+		return "index" + File.separatorChar + "text" + File.separatorChar + index_stem;
+	}
+
+	/** index path (for querying) relative to collectionBaseDir */
+	static public String collectionIndexPath(String index_stem, String index_name)
+	{
+		return "index" + File.separatorChar + index_name + File.separatorChar + index_stem;
+	}
+
+	/** collection resources directory path */
+	static public String collectionResourceDir(String site_home, String collection_name)
+	{
+		return collectionResourceDir(collectionBaseDir(site_home, collection_name));
+
+	}
+
+	static public String collectionResourceDir(String collection_home)
+	{
+		return collection_home + File.separatorChar + "resources";
+	}
+
+  static public String siteResourceDir(String site_home) {
+    return site_home+File.separatorChar+"resources";
+  }
+	/** absolute path for an associated file */
+	static public String assocFileAbsolutePath(String site_home, String collection_name, String assoc_file_path, String filename)
+	{
+		return collectionBaseDir(site_home, collection_name) + File.separatorChar + "index" + File.separatorChar + "assoc" + File.separatorChar + assoc_file_path + File.separatorChar + filename;
+	}
+
+	static public String extHome(String gsdl3_home, String ext_name)
+	{
+		return gsdl3_home + File.separatorChar + "ext" + File.separatorChar + ext_name;
+	}
+
+	static public String siteHome(String gsdl3_home, String site_name)
+	{
+		return gsdl3_home + File.separatorChar + "sites" + File.separatorChar + site_name;
+	}
+
+	static public String interfaceHome(String gsdl3_home, String interface_name)
+	{
+		return gsdl3_home + File.separatorChar + "interfaces" + File.separatorChar + interface_name;
+	}
+
+	static public String interfaceStylesheetFile(String gsdl3_home, String interface_name, String filename)
+	{
+		return gsdl3_home + File.separatorChar + "interfaces" + File.separatorChar + interface_name + File.separatorChar + "transform" + File.separatorChar + filename;
+	}
+
+	static public String siteStylesheetFile(String site_home, String filename)
+	{
+		return site_home + File.separatorChar + "transform" + File.separatorChar + filename;
+	}
+
+	static public String collStylesheetFile(String site_home, String coll_name, String filename)
+	{
+		return collectionBaseDir(site_home, coll_name) + File.separatorChar + "transform" + File.separatorChar + filename;
+	}
+
+	/**
+	 * returns the absolute path to a stylesheet. Stylesheets are looked for in
+	 * the following places, in the following order: current collection, current
+	 * site, current interface, base interfaces returns null if the file cannot
+	 * be found
+	 * 
+	 * this is not so good because sites may be on a different computer
+	 */
+	static public String stylesheetFile(String gsdl3_home, String site_name, String collection, String interface_name, ArrayList<String> base_interfaces, String filename)
+	{
+
+		String site_home = siteHome(gsdl3_home, site_name);
+		// try collection first
+		File stylesheet = null;
+		if (!collection.equals(""))
+		{
+
+			String coll_home = collectionBaseDir(site_home, collection);
+			stylesheet = new File(coll_home + File.separatorChar + "transform" + File.separatorChar + filename);
+			if (stylesheet.exists())
+			{
+				return stylesheet.getPath();
+			}
+		}
+
+		// try site one next	
+		stylesheet = new File(site_home + File.separatorChar + "transform" + File.separatorChar + filename);
+		if (stylesheet.exists())
+		{
+			return stylesheet.getPath();
+		}
+
+		// try current interface
+		String interface_home = interfaceHome(gsdl3_home, interface_name);
+		stylesheet = new File(interface_home + File.separatorChar + "transform" + File.separatorChar + filename);
+		if (stylesheet.exists())
+		{
+			return stylesheet.getPath();
+		}
+		// try base interface
+		if (base_interfaces == null || base_interfaces.size() == 0)
+		{
+			return null; // no base interfaces to look for
+		}
+		for (int i = 0; i < base_interfaces.size(); i++)
+		{
+			interface_home = interfaceHome(gsdl3_home, base_interfaces.get(i));
+			stylesheet = new File(interface_home + File.separatorChar + "transform" + File.separatorChar + filename);
+			if (stylesheet.exists())
+			{
+				return stylesheet.getPath();
+			}
+		}
+
+		// still can't find it and we have looked everywhere
+		return null;
+	}
+
+	static public ArrayList<File> getStylesheetFiles(String gsdl3_home, String site_name, String collection, String interface_name, ArrayList<String> base_interfaces, String filename)
+	{
+		ArrayList<File> stylesheets = new ArrayList<File>();
+		String site_home = siteHome(gsdl3_home, site_name);
+		// try collection first
+		File stylesheet = null;
+		if (!collection.equals(""))
+		{
+			String coll_home = collectionBaseDir(site_home, collection);
+			stylesheet = new File(coll_home + File.separatorChar + "transform" + File.separatorChar + filename);
+			if (stylesheet.exists())
+			{
+				stylesheets.add(stylesheet);
+			}
+		}
+
+		// try site one next	
+		stylesheet = new File(site_home + File.separatorChar + "transform" + File.separatorChar + filename);
+		if (stylesheet.exists())
+		{
+			stylesheets.add(stylesheet);
+		}
+
+		// try current interface
+		String interface_home = interfaceHome(gsdl3_home, interface_name);
+		stylesheet = new File(interface_home + File.separatorChar + "transform" + File.separatorChar + filename);
+		if (stylesheet.exists())
+		{
+			stylesheets.add(stylesheet);
+		}
+		// try base interface
+		if (base_interfaces != null && base_interfaces.size() != 0)
+		{
+			for (int i = 0; i < base_interfaces.size(); i++)
+			{
+				interface_home = interfaceHome(gsdl3_home, base_interfaces.get(i));
+				stylesheet = new File(interface_home + File.separatorChar + "transform" + File.separatorChar + filename);
+				if (stylesheet.exists())
+				{
+					stylesheets.add(stylesheet);
+				}
+			}
+		}
+
+		return stylesheets;
+	}
+
+	/** base directory for phind data */
+	public static String phindBaseDir(String site_home, String coll_name, String phind_index)
+	{
+		return site_home + File.separatorChar + "collect" + File.separatorChar + coll_name + File.separatorChar + "index" + File.separatorChar + "phind" + phind_index;
+	}
+
+    /** the collection database file - */
+    static public String collectionDatabaseFile(String site_home, String collection_name, String index_stem, String database_type)
+    {
+	String db_ext = DBHelper.getDBExtFromDBType(database_type);
+	if (null == db_ext || db_ext.equals("")) {
+	    logger.warn("Could not recognise database type \"" + database_type + "\", defaulting to GDBM and extension \".gdb\"");
+	    // assume gdbm
+	    db_ext = ".gdb";
+	}
+	return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "index" + File.separatorChar + "text" + File.separatorChar + index_stem + db_ext;
+    }
+    
+    /** the oai-inf database file in the collection's etc folder */
+    static public String OAIInfoDatabaseFile(String site_home, String collection_name, String db_tailname, String database_type)
+    {
+	String db_ext = DBHelper.getDBExtFromDBType(database_type);
+	if (null == db_ext || db_ext.equals("")) {
+	    logger.warn("Could not recognise database type \"" + database_type + "\", defaulting to GDBM and extension \".gdb\"");
+	    // assume gdbm
+	    db_ext = ".gdb";
+	}
+	return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "etc" + File.separatorChar + db_tailname + db_ext; // db tailname should be oai-inf
+    }
+
+
+    /** the archives database file - */
+    static public String archivesDatabaseFile(String site_home, String collection_name, String database_type)
+    {
+	String db_ext = DBHelper.getDBExtFromDBType(database_type);
+	if (null == db_ext || db_ext.equals("")) {
+	    logger.warn("Could not recognise database type \"" + database_type + "\", defaulting to GDBM and extension \".gdb\"");
+	    // assume gdbm
+	    db_ext = ".gdb";
+	}
+	return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "archives" + File.separatorChar + "archiveinf-doc" + db_ext;
+    }
+	// some file utility methods
+
+	/**
+	 * read in a file and encode it using base64 encoded data returned as a
+	 * String
+	 */
+	static public String base64EncodeFromFile(String in_filename)
+	{
+		byte[] data = null;
+		try
+		{
+			data = readFile(in_filename);
+		}
+		catch (Exception e)
+		{
+			logger.error("couldn't read the file");
+		}
+		String encodedString = Base64.encode(data);
+		return encodedString;
+
+	}
+
+	/** decode some base64 data, and write it to the specified file */
+	static public boolean base64DecodeToFile(String data, String out_filename)
+	{
+		try
+		{
+			byte[] buffer = Base64.decode(data);
+			writeFile(buffer, out_filename);
+
+		}
+		catch (Exception e)
+		{
+			logger.error("file opening/closing errors" + e.getMessage());
+			return false;
+		}
+		return true;
+
+	}
+
+	/** read in a file to a byte array */
+	public static byte[] readFile(String filename) throws IOException
+	{
+		File file = new File(filename);
+		BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
+		int bytes = (int) file.length();
+		byte[] buffer = new byte[bytes];
+		int readBytes = bis.read(buffer);
+		bis.close();
+		return buffer;
+	}
+
+	/** write a byte array to a file */
+	public static void writeFile(byte[] buffer, String filename) throws IOException
+	{
+		File file = new File(filename);
+		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file));
+		bos.write(buffer);
+		bos.close();
+	}
+
+	public static boolean deleteFile(File f)
+	{
+
+		if (f.isDirectory())
+		{
+			File[] files = f.listFiles();
+			for (int i = 0; files != null && i < files.length; i++)
+			{
+				deleteFile(files[i]);
+			}
+
+		}
+		// delete the file or directory
+		return f.delete();
+	}
+
+	public static boolean copyFile(File source, File destination)
+	{
+		if (!source.isFile())
+		{
+			logger.error(source.getPath() + " is not a file!");
+			return false;
+		}
+		try
+		{
+			destination.getParentFile().mkdirs();
+			FileInputStream in = new FileInputStream(source);
+			FileOutputStream out = new FileOutputStream(destination);
+			int value = 0;
+			while ((value = in.read()) != -1)
+			{
+				out.write(value);
+			}
+			in.close();
+			out.close();
+		}
+		catch (Exception e)
+		{
+			logger.error("something went wrong copying " + source.getPath() + " to " + destination.getPath());
+			logger.error("Exception: " + e.getMessage());
+			return false;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Recursively moves the contents of source file to destination, maintaining
+	 * paths.
+	 * 
+	 * @param source
+	 *            A File representing the directory whose contents you wish to
+	 *            move.
+	 * @param destination
+	 *            A File representing the directory you wish to move files to.
+	 * @return true if successful
+	 */
+	public static boolean moveDirectory(File source, File destination)
+	{
+
+		// first try rename
+		if (source.renameTo(destination))
+		{
+			return true;
+		}
+
+		// john T. said that this sometimes doesn't work, so you have to copy files manually
+
+		// copied from gatherer Utility class
+		File input[] = source.listFiles();
+		for (int i = 0; i < input.length; i++)
+		{
+			File output = new File(destination, input[i].getName());
+			if (input[i].isDirectory())
+			{
+				moveDirectory(input[i], output);
+			}
+			else
+			{
+				// Copy the file
+				try
+				{
+					output.getParentFile().mkdirs();
+					FileInputStream in = new FileInputStream(input[i]);
+					FileOutputStream out = new FileOutputStream(output);
+
+					FileChannel inC = in.getChannel();
+					FileChannel outC = out.getChannel();
+
+					System.err.println(inC.transferTo(0, inC.size(), outC));
+
+					in.close();
+					out.close();
+
+					// Delete file
+					input[i].delete();
+				}
+				catch (Exception e)
+				{
+					logger.error("exception: " + e.getMessage());
+					return false;
+				}
+
+			}
+		}
+		return true;
+	}
+
+	public static ArrayList<File> getAllXSLFiles(String siteName)
+	{
+		ArrayList<File> filesToReturn = new ArrayList<File>();
+
+		String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), siteName);
+
+		//Add XSL files from the site transform directory
+		File siteTransformDir = new File(siteHome + File.separator + "transform");
+		if (siteTransformDir.exists() && siteTransformDir.isDirectory())
+		{
+			filesToReturn.addAll(getXSLFilesFromDirectoryRecursive(siteTransformDir));
+		}
+
+		//Add XSL files from collection transform directories
+		File siteCollectionDir = new File(siteHome + File.separator + "collect");
+		if (siteCollectionDir.exists() && siteCollectionDir.isDirectory())
+		{
+			File[] collections = siteCollectionDir.listFiles();
+
+			for (File collection : collections)
+			{
+				if (collection.isDirectory())
+				{
+					File collectionTranformDir = new File(collection.getAbsolutePath() + File.separator + "transform");
+					if (collectionTranformDir.exists() && collectionTranformDir.isDirectory())
+					{
+						filesToReturn.addAll(getXSLFilesFromDirectoryRecursive(collectionTranformDir));
+					}
+				}
+			}
+		}
+
+		//Add XSL files from the interface transform directory
+		File interfaceDir = new File(GlobalProperties.getGSDL3Home() + File.separator + "interfaces");
+		if (interfaceDir.exists() && interfaceDir.isDirectory())
+		{
+			filesToReturn.addAll(getXSLFilesFromDirectoryRecursive(interfaceDir));
+		}
+
+		return filesToReturn;
+	}
+
+	protected static ArrayList<File> getXSLFilesFromDirectoryRecursive(File directory)
+	{
+		ArrayList<File> filesToReturn = new ArrayList<File>();
+
+		if (!directory.isDirectory())
+		{
+			return filesToReturn;
+		}
+
+		File[] currentFiles = directory.listFiles();
+		for (File current : currentFiles)
+		{
+			if (current.isDirectory())
+			{
+				filesToReturn.addAll(GSFile.getXSLFilesFromDirectoryRecursive(current));
+			}
+			else if (current.getName().endsWith(".xsl"))
+			{
+				filesToReturn.add(current);
+			}
+		}
+
+		return filesToReturn;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFileTest.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFileTest.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSFileTest.java	(revision 31634)
@@ -0,0 +1,67 @@
+/*
+ *    GSFileTest.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import junit.framework.*; 
+import java.io.File;
+
+public class GSFileTest extends TestCase {
+
+    /** what type of test - used by the testing framework */
+    public static final String TEST_ALL_TEST_TYPE="UNIT";
+
+    public GSFileTest(String name) {
+	super(name);
+    }
+
+    /** test suite that dynamically runs all the tests */
+    public static Test suite() {
+	return new TestSuite(GSFileTest.class);
+    }
+
+    public void testBase64Conversion() {
+	
+	// start off with reverse conversion - decode then encode
+	String data = "sdkfjho9w4385khrf9ow834rfia8ufd38rhf9812";
+	File outfile;
+	try {
+	    outfile = File.createTempFile("b64convert_test", "");
+	} catch (java.io.IOException e) {
+	    fail("couldn't create a temporary file in default temp dir");
+	    return;
+	}
+	String filename = outfile.getAbsolutePath();
+	
+	String newdata;
+	if (GSFile.base64DecodeToFile(data, filename)) {
+	    newdata = GSFile.base64EncodeFromFile(filename);
+
+	    assertEquals("decoded/encoded string does not match the origianl",
+			 data, newdata);
+	    // clean up
+	    outfile.delete();
+	} else {
+	    fail("couldn't decode the data string");
+	}
+
+	//try with a real file???
+	
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSHTML.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSHTML.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSHTML.java	(revision 31634)
@@ -0,0 +1,82 @@
+/*
+ *    GSHTML.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+/** GSHTML - provides some convenience methods for dealing
+ * with html
+ */
+public class GSHTML {
+
+    /** make a string html safe */
+    public static String htmlSafe(String input) {
+
+	StringBuffer filtered = new StringBuffer(input.length());
+	char c;
+	for (int i=0; i<input.length(); i++) {
+	    c = input.charAt(i);
+	    if (c == '<') {
+		filtered.append("&lt;");
+	    } else if (c == '>') {
+		filtered.append("&gt;");
+	    } else if (c == '"') {
+		filtered.append("&quot;");
+	    } else if (c == '&') {
+		filtered.append("&amp;");
+	    } else {
+		filtered.append(c);
+	    }
+	}
+	return(filtered.toString());
+    }
+    /** undo the html safe action */
+    public static String htmlUnsafe(String input) {
+	StringBuffer filtered = new StringBuffer(input.length());
+	char c;
+	for (int i=0; i<input.length(); i++) {
+	    c = input.charAt(i);
+	    if (c =='&') {
+		int j=input.indexOf(';', i);
+		String entity = input.substring(i, j);
+		i=j;
+		if (entity.equals("&amp")) {
+		    filtered.append('&');
+		} else if (entity.equals("&lt")){
+		    filtered.append('<');
+		} else if (entity.equals("&gt")){
+		    filtered.append('>');
+		} else if (entity.equals("&quot")) {
+		    filtered.append('"');
+		} // else just ignore it.
+	    } else {
+		filtered.append(c);
+	    }
+
+	}
+	return(filtered.toString());
+    }
+    
+    /** produce a default error page */
+    public static String  errorPage(String error) {
+	String page = "<html><head><Title>GSDL3 Error!</Title></head>\n"+
+	    "<body><h1>Greenstone Error!</h1>"+
+	    "<p/>"+error+"</body></html>";
+	return page;
+    }
+	
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSParams.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSParams.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSParams.java	(revision 31634)
@@ -0,0 +1,190 @@
+/*
+ *    GSParams.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.HashMap;
+
+/** keeps track of the servlet parameters, and their defaults */
+public class GSParams
+{
+
+	// cgi parameter names
+	public static final String ACTION = "a"; // the major type of action- eg query or browse or process
+	public static final String SUBACTION = "sa"; // subtype of action if we want different processing than the default
+	public static final String REQUEST_TYPE = "rt"; // whether the request is just to display the service form, or to actually do a request to the service
+	public static final String RESPONSE_ONLY = "ro"; // if == 1 do the request and pass back the response xml - no page formatting
+	public static final String OUTPUT = "o"; // if processing is to be done, what type of output - html/xml/other??
+	public static final String SERVICE = "s"; // the name of the service
+	
+	public static final String UN = "un"; // username for authenticated-ping
+	public static final String PW = "pw"; // pwd for authenticated-ping
+
+	public static final String CLUSTER = "c"; // these two are the same
+	public static final String COLLECTION = "c";
+	public static final String COLLECTION_TYPE = "ct"; // collection type - mg, mgpp, lucene etc
+
+	public static final String LANGUAGE = "l";
+	public static final String DOCUMENT = "d";
+	public static final String DOCUMENT_TYPE = "dt";
+	public static final String START_PAGE = "startPage";
+	public static final String S_START_PAGE = "s1.startPage";
+	public static final String HREF = "href"; // url. might be an external url, or a relative one that needs translating
+	public static final String RELATIVE_LINK = "rl"; // whether the href url is relative to the collection or not.
+	public static final String EXTERNAL_LINK_TYPE = "el"; // for an external link, go direct to the page or frame it in the collection
+	public static final String PROCESS_ID = "pid"; // if a request wasn't completed, this identifies the request - used when asking for a status update
+
+	public static final String HTTP_HEADER_FIELDS = "hhf";
+	public static final String QUICK_SEARCH = "qs";
+
+	// internal configure args
+	public static final String SYSTEM_SUBSET = "ss";
+	public static final String SYSTEM_CLUSTER = "sc";
+	public static final String SYSTEM_MODULE_NAME = "sn";
+	public static final String SYSTEM_MODULE_TYPE = "st";
+
+	// used for filtering out a piece of the final page
+	public static final String EXCERPT_ID = "excerptid";
+	public static final String EXCERPT_TAG = "excerpttag";
+
+	public static final String INLINE_TEMPLATE = "ilt";
+	public static final String DISPLAY_METADATA = "dmd";
+	public static final String FILE_LOCATION = "fl";
+	public static final String DOC_EDIT = "docEdit";
+	public static final String AJAX_LOAD_BYPASS = "alb";
+
+	//Administration
+	public static final String PASSWORD = "password";
+	public static final String S_PASSWORD = "s1.password";
+	public static final String S_NEW_PASSWORD = "s1.newPassword";
+	public static final String S_OLD_PASSWORD = "s1.oldPassword";
+	
+	//Facets
+	public static final String S_FACETS = "s1.facets";
+	public static final String S_FACETS_QUERIES = "s1.facetQueries";
+
+	// some standard arg values
+	public static final String SYSTEM_ACTION = "s";
+
+	// rss feeds
+	public static final String RSS_ACTION = "rss";
+
+	public static final String EXTERNAL_LINK_TYPE_DIRECT = "direct";
+	public static final String EXTERNAL_LINK_TYPE_FRAMED = "frame";
+	
+	public static final String DEBUG = "debug";
+
+	protected HashMap<String, Param> param_map = null;
+
+	public GSParams()
+	{
+		this.param_map = new HashMap<String, Param>(30);
+
+		// add in all the standard params
+		addParameter(ACTION, false);
+		addParameter(SUBACTION, false);
+		addParameter(REQUEST_TYPE, false);
+		addParameter(RESPONSE_ONLY, false);
+		addParameter(CLUSTER, false); // we don't want to save cluster/collection 
+		addParameter(LANGUAGE, true);
+		addParameter(DOCUMENT, true);
+		addParameter(DOCUMENT_TYPE, false);
+		addParameter(QUICK_SEARCH, false);
+		addParameter(START_PAGE, false);
+		addParameter(S_START_PAGE, false);
+		// should the following two just be in doc action??
+		addParameter(HREF, false);
+		addParameter(RELATIVE_LINK, false);
+		addParameter(OUTPUT, false);
+		addParameter(SERVICE, false);
+		addParameter(PROCESS_ID, true);
+		addParameter(SYSTEM_SUBSET, false);
+		addParameter(SYSTEM_CLUSTER, false);
+		addParameter(SYSTEM_MODULE_NAME, false);
+		addParameter(SYSTEM_MODULE_TYPE, false);
+		addParameter(INLINE_TEMPLATE, false);
+		addParameter(DISPLAY_METADATA, false);
+		addParameter(AJAX_LOAD_BYPASS, false);
+		addParameter(DOC_EDIT, false);
+		addParameter(PASSWORD, false);
+		addParameter(S_PASSWORD, false);
+		addParameter(S_NEW_PASSWORD, false);
+		addParameter(S_OLD_PASSWORD, false);
+		
+		addParameter(S_FACETS, false);
+		addParameter(S_FACETS_QUERIES, false);
+
+		addParameter(COLLECTION_TYPE, true);
+		addParameter(EXTERNAL_LINK_TYPE, false);
+		// filtering args must be specified each time
+		addParameter(EXCERPT_ID, false);
+		addParameter(EXCERPT_TAG, false);
+		
+		addParameter(DEBUG, true);
+	}
+
+	public boolean addParameter(String name, boolean save)
+	{
+		return addParameter(name, "", save);
+	}
+
+	public boolean addParameter(String name, String default_value, boolean save)
+	{
+		if (this.param_map.containsKey(name))
+		{
+			// already there so could not add
+			return false;
+		}
+
+		this.param_map.put(name, new Param(default_value, save));
+		return true;
+	}
+
+	public boolean setParamDefault(String name, String default_value)
+	{
+		Param p = this.param_map.get(name);
+		if (p == null)
+			return false;
+		p.default_value = default_value;
+		return true;
+	}
+
+	public boolean shouldSave(String name)
+	{
+		// p. is used to store previous settings
+		if (name.startsWith("p.") || name.startsWith("md___"))
+			return false;
+		Param p = this.param_map.get(name);
+		if (p == null)
+			return true; // if things are not in here, always save.
+		return p.save;
+	}
+
+	private class Param
+	{
+
+		public String default_value = null;
+		public boolean save = true;
+
+		public Param(String default_value, boolean save)
+		{
+			this.default_value = default_value;
+			this.save = save;
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSPath.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSPath.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSPath.java	(revision 31634)
@@ -0,0 +1,150 @@
+/*
+ *    GSPath.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+/**
+ * GSPath - utility class for greenstone
+ *
+ * modifies and examines message address paths and names
+ *
+ * @author Katherine Don
+ * @version $Revision$
+ *
+ */
+
+public class GSPath {
+
+    /** adds new_link to the end of path
+     *
+     * @return the new path
+     */
+    static public String appendLink(String path, String new_link) {
+	if (path.equals("")) {
+	    return new_link;
+	}
+	return path +"/"+new_link;
+    }
+
+    /** adds new_link to the front of path
+     *
+     * @return the new path
+     */
+    static public String prependLink(String path, String new_link) {
+	if (path.equals("")) {
+	    return new_link;
+	}
+	return  new_link+"/"+path;
+    }
+
+    /** replaces the first link in the path with the specified name
+     */
+    static public String replaceFirstLink(String path, String new_first_link) {
+	path = removeFirstLink(path);
+	path = prependLink(path, new_first_link);
+	return path;
+
+    }
+    /** returns the first name in the path
+     *
+     */
+    static public String getFirstLink(String path) {
+
+	int i = path.indexOf("/");
+	if (i==0) { // have a '/' on the front
+	    path = path.substring(1);
+	    i= path.indexOf("/");
+	}
+	if (i==-1) { // '/' not found
+	    return path;
+	}
+	
+	return path.substring(0,i);
+    }
+
+    /** removes the first name in the path
+     *
+     * @return the modified path
+     */
+    static public String removeFirstLink(String path) {
+	int i = path.indexOf("/");
+	if (i==0) { // have a '/' on the front
+	    path = path.substring(1);
+	    i= path.indexOf("/");
+	}
+	if (i==-1) { // '/' not found - remove the whole path
+	    return "";
+	}
+	return path.substring(i+1); // remove the '/' char
+    }
+
+    /** removes the last name in the path
+     *
+     * @return the modified path
+     */
+    static public String removeLastLink(String path) {
+	int i = path.lastIndexOf("/");
+	if (i==-1) { // '/' not found - remove the whole path
+	    return "";
+	}
+	return path.substring(0, i); // remove the '/' char
+    }
+    
+    static public String getLastLink(String path) {
+	int i = path.lastIndexOf("/");
+	if (i==-1) { // '/' not found - return the whole path
+	    return path;
+	}
+	return path.substring(i+1); // remove the last link
+    }
+
+    
+    static public String createPath(String [] links) {
+	String path = links[0];
+	for (int i=1; i<links.length; i++) {
+	    path = appendLink(path, links[i]);
+	}
+	return path;
+    }
+
+    static public int getIndex(String link) {
+	int i = link.indexOf("[");
+	if (i==-1) {
+	    return -1;
+	}
+	int j = link.indexOf("]");
+	String num = link.substring(i+1,j);
+	try {
+	    int index = Integer.parseInt(num);
+	    return index;
+	} catch (Exception e) {
+	    return -1;
+	}
+    }
+
+    static public String removeIndex(String link) {
+	int i = link.indexOf("[");
+	if (i==-1) {
+	    return link;
+	}
+	return link.substring(0, i);
+    }
+
+}
+
+    
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSStatus.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSStatus.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSStatus.java	(revision 31634)
@@ -0,0 +1,54 @@
+/*
+ *    GSStatus.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+public class GSStatus {
+
+    // responses for initial request
+    public static final int SUCCESS = 1; // request succeeded
+    public static final int ACCEPTED = 2; // request accepted but not completed
+    public static final int ERROR = 3; // error and process stopped
+
+    // responses for status requests
+    public static final int CONTINUING = 10; // request still continuing
+    public static final int COMPLETED = 11; // request finished
+    public static final int HALTED = 12; // process stopped
+	    
+    // other
+    public static final int INFO = 20; // just an info message that doesnt mean anything
+
+    /** returns true if teh code indicates that a process is no longer running 
+     */
+    public static boolean isCompleted(int code) {
+	if (code == SUCCESS || code == ERROR || code == COMPLETED || code == HALTED) {
+	    return true;
+	} 
+	return false;
+    }
+
+    // there may be other types of error codes
+    public static boolean isError(int code) {
+	if (code == ERROR) {
+	    return true;
+	}
+	return false;
+    }
+
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXML.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXML.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXML.java	(revision 31634)
@@ -0,0 +1,1665 @@
+/*
+ *    GSXML.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Vector;
+
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.log4j.Logger;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+
+import org.greenstone.gsdl3.util.MyNodeList;
+/** various functions for extracting info out of GS XML */
+public class GSXML
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSXML.class.getName());
+
+	// greenstone message xml elements
+	public static final String MESSAGE_ELEM = "message";
+	public static final String REQUEST_ELEM = "request";
+	public static final String RESPONSE_ELEM = "response";
+	public static final String COLLECTION_ELEM = "collection";
+	public static final String SERVICE_ELEM = "service";
+	public static final String CLUSTER_ELEM = "serviceCluster";
+	public static final String SITE_ELEM = "site";
+	public static final String PARAM_ELEM = "param";
+	public static final String PARAM_OPTION_ELEM = "option";
+        public static final String LIBRARY_PARAM_ELEM = "libraryParam";
+	public static final String CONTENT_ELEM = "content";
+	public static final String RESOURCE_ELEM = "resource";
+	public static final String DOCUMENT_ELEM = "document";
+	public static final String METADATA_ELEM = "metadata";
+	public static final String SERVICE_CLASS_ELEM = "serviceRack";
+	public static final String CLASSIFIER_ELEM = "classifier";
+	public static final String APPLET_ELEM = "applet";
+	public static final String APPLET_DATA_ELEM = "appletData";
+	public static final String CONFIGURE_ELEM = "configure";
+	public static final String STATUS_ELEM = "status";
+	public static final String ERROR_ELEM = "error";
+	public static final String DEFAULT_ELEM = "default";
+	public static final String STYLESHEET_ELEM = "format";//"stylesheet"; // any additional stylesheet stuff is carried in the message inside this elem
+	public static final String FORMAT_ELEM = "format"; // config files use format - should we use this instead of stylesheet??
+	public static final String GLOBAL_FORMAT_ELEM = "globalFormat";
+	public static final String TERM_ELEM = "term";
+	public static final String STOPWORD_ELEM = "stopword";
+	public static final String FACET_ELEM = "facet";
+	public static final String COUNT_ELEM = "count";
+	public static final String SYSTEM_ELEM = "system";
+	public static final String FORMAT_STRING_ELEM = "formatString";
+	public static final String EXTRA_METADATA = "extraMetadata";
+	public static final String PATH_ELEM = "path";
+
+	//config file elems
+	public static final String COLLECTION_CONFIG_ELEM = "collectionConfig";
+	public static final String COLLECTION_BUILD_ELEM = "buildConfig";
+	public static final String COLLECTION_INIT_ELEM = "collectionInit";
+	public static final String RECOGNISE_ELEM = "recognise";
+	public static final String DOC_TYPE_ELEM = "docType";
+	public static final String SEARCH_ELEM = "search";
+	public static final String INFODB_ELEM = "infodb";
+	public static final String INDEX_ELEM = "index";
+	public static final String INDEX_STEM_ELEM = "indexStem";
+	public static final String INDEX_OPTION_ELEM = "indexOption";
+	public static final String PARAM_DEFAULT_ELEM = "paramDefault";
+	public static final String BROWSE_ELEM = "browse";
+	public static final String DISPLAY_ELEM = "display";
+	public static final String LEVEL_ELEM = "level";
+	public static final String REPLACE_ELEM = "replace";
+  public static final String EXTRA_INFO_ELEM = "extraInfo";
+	public static final String DBINFO_ELEM = "dbInfo";
+	public static final String DBNAME_ATT = "dbname";
+	public static final String DBPATH_ATT = "dbpath";
+	public static final String SQLSTATE_ATT = "sqlstate";
+	public static final String DATABASE_TYPE_ELEM = "databaseType";
+	public static final String SHORTNAME_ATT = "shortname";
+	public static final String NOTIFY_ELEM = "notify";
+	public static final String NOTIFY_HOST_ATT = "host";
+
+	//doc.xml file elems
+	public static final String DOCXML_SECTION_ELEM = "Section";
+	public static final String DOCXML_DESCRIPTION_ELEM = "Description";
+	public static final String DOCXML_METADATA_ELEM = "Metadata";
+	public static final String DOCXML_CONTENT_ELEM = "Content";
+
+	// elems for the pages to be processed by xslt
+	public final static String PAGE_ELEM = "page";
+	public final static String CONFIGURATION_ELEM = "config";
+	public final static String PAGE_REQUEST_ELEM = "pageRequest";
+	public final static String PAGE_RESPONSE_ELEM = "pageResponse";
+	public final static String PAGE_EXTRA_ELEM = "pageExtra";
+
+	//public final static String DESCRIPTION_ELEM = "description";
+
+	public static final String ACTION_ELEM = "action";
+	public static final String SUBACTION_ELEM = "subaction";
+
+	// add on to another elem type to get a list of that type
+	public static final String LIST_MODIFIER = "List";
+	// used to refer back to another element type
+	public static final String REF_MODIFIER = "Ref";
+	// greenstone xml attributes
+	public static final String COLLECTION_ATT = "collection";
+	public static final String NAME_ATT = "name";
+	public static final String MATCH_ATT = "match";
+	public static final String TO_ATT = "to";
+	public static final String USER_ID_ATT = "uid";
+	public static final String FROM_ATT = "from";
+	public static final String LANG_ATT = "lang";
+  public static final String KEY_ATT = "key";
+  public static final String DICTIONARY_ATT = "dictionary";
+	public static final String HREF_ATT = "href";
+	public static final String TYPE_ATT = "type";
+	public static final String DB_TYPE_ATT = "dbType";
+	public static final String VALUE_ATT = "value";
+	public static final String DEFAULT_ATT = "default";
+	public static final String INFO_ATT = "info";
+	public static final String ACTION_ATT = "action";
+	public static final String SUBACTION_ATT = "subaction";
+	public static final String OUTPUT_ATT = "output";
+	public static final String ADDRESS_ATT = "address";
+	public static final String LOCAL_SITE_ATT = "localSite";
+	public static final String LOCAL_SITE_NAME_ATT = "localSiteName";
+	public static final String STATUS_ERROR_CODE_ATT = "code";
+	public static final String STATUS_PROCESS_ID_ATT = "pid";
+	public static final String PARAM_SHORTNAME_ATT = "shortname";
+	public static final String PARAM_IGNORE_POS_ATT = "ignore";
+	public static final String CLASSIFIER_CONTENT_ATT = "content";
+	public static final String ERROR_TYPE_ATT = "type";
+	public static final String COLLECT_TYPE_ATT = "ct";
+	public static final String HIDDEN_ATT = "hidden";
+	public static final String FACET_ATT = "facet";
+	public static final String PATH_ATT = "path";
+
+	// document stuff
+	public static final String DOC_TYPE_ATT = "docType";
+	public static final String DOC_NODE_ELEM = "documentNode";
+	public static final String NODE_CONTENT_ELEM = "nodeContent";
+	public static final String NODE_STRUCTURE_ELEM = "nodeStructure";
+	public static final String NODE_ID_ATT = "nodeID";
+	public static final String HREF_ID_ATT = "hrefId"; // for ids that need translating
+	public static final String HL_SNIPPET_ELEM = "highlightSnippet";
+	public static final String ID_MOD_ATT = "idMod"; // might hold .rt etc for hrefIds
+	public static final String NODE_OID = "oid";
+	public static final String NODE_NAME_ATT = "nodeName";
+	public static final String NODE_TYPE_ATT = "nodeType";
+	public static final String NODE_RANK_ATT = "rank";
+	public static final String NODE_TYPE_ROOT = "root";
+	public static final String NODE_TYPE_INTERNAL = "internal";
+	public static final String NODE_TYPE_LEAF = "leaf";
+
+	public static final String DOC_TYPE_SIMPLE = "simple";
+	public static final String DOC_TYPE_PAGED = "paged";
+	public static final String DOC_TYPE_HIERARCHY = "hierarchy";
+	public static final String DOC_TYPE_PAGED_HIERARCHY = "pagedhierarchy";
+
+	public static final String SESSION_EXPIRATION = "session_expiration";
+	public static final String USER_SESSION_CACHE_ATT = "user_session_cache";
+
+	// classifier stuff
+	public static final String CLASS_NODE_ELEM = "classifierNode";
+	public static final String CLASS_NODE_ORIENTATION_ATT = "orientation";
+
+	// parameter types
+	public static final String PARAM_TYPE_INTEGER = "integer";
+	public static final String PARAM_TYPE_BOOLEAN = "boolean";
+	public static final String PARAM_TYPE_ENUM_START = "enum";
+	public static final String PARAM_TYPE_ENUM_SINGLE = "enum_single";
+	public static final String PARAM_TYPE_ENUM_MULTI = "enum_multi";
+	public static final String PARAM_TYPE_STRING = "string";
+	public static final String PARAM_TYPE_TEXT = "text";
+	public static final String PARAM_TYPE_MULTI = "multi";
+	public static final String PARAM_TYPE_FILE = "file";
+	public static final String PARAM_TYPE_INVISIBLE = "invisible";
+	public static final String PARAM_TYPE_CHECKBOX_LIST = "checkbox_list";
+	// stuff for text strings
+	public static final String DISPLAY_TEXT_ELEM = "displayItem";
+	// the following are used for the name attributes
+	public static final String DISPLAY_TEXT_NAME = "name";
+	public static final String DISPLAY_TEXT_SUBMIT = "submit";
+	public static final String DISPLAY_TEXT_DESCRIPTION = "description";
+
+	// request types
+	// get the module description
+	public static final String REQUEST_TYPE_DESCRIBE = "describe";
+	// startup a process
+	public static final String REQUEST_TYPE_PROCESS = "process";
+	// get the status of an ongoing process
+	public static final String REQUEST_TYPE_STATUS = "status";
+	// system type request - eg reload a collection
+	public static final String REQUEST_TYPE_SYSTEM = "system";
+	// page requests to the Receptionist/Actions
+	public static final String REQUEST_TYPE_PAGE = "page"; // used to be cgi
+	// get any format info for a service
+	public static final String REQUEST_TYPE_FORMAT = "format";
+	// modify the requests
+	public static final String REQUEST_TYPE_MESSAGING = "messaging";
+	// save the format string
+	public static final String REQUEST_TYPE_FORMAT_STRING = "formatString";
+	// check credentials
+	public static final String REQUEST_TYPE_SECURITY = "security";
+
+	// service types
+	public static final String SERVICE_TYPE_QUERY = "query";
+	public static final String SERVICE_TYPE_RETRIEVE = "retrieve";
+	public static final String SERVICE_TYPE_BROWSE = "browse";
+	public static final String SERVICE_TYPE_GROUPINFO = "groupinfo";
+	public static final String SERVICE_TYPE_APPLET = "applet";
+	public static final String SERVICE_TYPE_PROCESS = "process";
+	public static final String SERVICE_TYPE_ENRICH = "enrich";
+	public static final String SERVICE_TYPE_OAI = "oai";
+	public static final String FLAX_PAGE = "flaxPage";
+	public static final String FLAX_PAGE_GENERATION = "FlaxPageGeneration";
+
+	// system command types and attributes
+	public static final String SYSTEM_TYPE_CONFIGURE = "configure";
+	public static final String SYSTEM_TYPE_ACTIVATE = "activate";
+	public static final String SYSTEM_TYPE_DEACTIVATE = "deactivate";
+	public static final String SYSTEM_TYPE_PING = "ping";
+	public static final String SYSTEM_TYPE_AUTHENTICATED_PING = "authenticated-ping";
+	//public static final String SYSTEM_TYPE_ISPERSISTENT = "is-persistent";
+
+	public static final String SYSTEM_SUBSET_ATT = "subset";
+	public static final String SYSTEM_MODULE_TYPE_ATT = "moduleType";
+	public static final String SYSTEM_MODULE_NAME_ATT = "moduleName";
+
+	// communicator types
+	public static final String COMM_TYPE_SOAP_JAVA = "soap";
+
+	// error types
+	public static final String ERROR_TYPE_SYNTAX = "syntax";
+	public static final String ERROR_TYPE_SYSTEM = "system";
+	public static final String ERROR_TYPE_INVALID_ID = "invalid_id";
+	public static final String ERROR_TYPE_OTHER = "other";
+
+	// some system wide param names
+	public static final String SUBSET_PARAM = "subset";
+
+	//for plugin
+	public static final String PLUGIN_ELEM = "plugin";
+	public static final String IMPORT_ELEM = "import";
+
+	//for authentication
+	public static final String AUTHEN_NODE_ELEM = "authenticationNode";
+	public static final String USER_NODE_ELEM = "userNode";
+
+	//for configure action results
+	public static final String SUCCESS = "success";
+	public static final String ERROR = "error";
+
+	//security tags and attributes
+	public static final String SECURITY_ELEM = "security";
+        public static final String DISABLED_ATT = "disabled";
+	public static final String SCOPE_ATT = "scope";
+	public static final String DEFAULT_ACCESS_ATT = "default_access";
+	public static final String EXCEPTION_ELEM = "exception";
+	public static final String DOCUMENT_SET_ELEM = "documentSet";
+	public static final String GROUP_ELEM = "group";
+	public static final String MATCH_ELEM = "match";
+	public static final String FIELD_ATT = "field";
+	public static final String USER_INFORMATION_ELEM = "userInformation";
+	public static final String USERNAME_ATT = "username";
+	public static final String GROUPS_ATT = "groups";
+	public static final String BASE_URL = "baseURL";
+
+	// only for authenticated-ping
+	public static final String PASSWORD_ATT = "password";
+	
+	//for classifiers
+	public static final String CHILD_TYPE_ATT = "childType";
+	public static final String CLASSIFIER_STYLE_ATT = "classifierStyle";
+	public static final String HLIST = "HList";
+	public static final String VLIST = "VList";
+
+	//for namespaces
+	public static final String XSL_NAMESPACE = "http://www.w3.org/1999/XSL/Transform";
+	public static final String GSF_NAMESPACE = "http://www.greenstone.org/greenstone3/schema/ConfigFormat";
+  public static final String GSLIB_NAMESPACE = "http://www.greenstone.org/skinning";
+	public static final String JAVA_NAMESPACE = "http://xml.apache.org/xslt/java";
+	public static final String UTIL_NAMESPACE = "xalan://org.greenstone.gsdl3.util.XSLTUtil";
+
+  // all the namespaces in a string for easy addition to a string
+  public  static final String ALL_NAMESPACES_ATTS = "xmlns:gsf='" + GSF_NAMESPACE + "' xmlns:xsl='" + XSL_NAMESPACE + "' xmlns:gslib='" + GSLIB_NAMESPACE + "' xmlns:java='" + JAVA_NAMESPACE + "' xmlns:util='" + UTIL_NAMESPACE +"' ";
+  public static final String STD_NAMESPACES_ATTS = "xmlns:gsf='" + GSF_NAMESPACE + "' xmlns:xsl='" + XSL_NAMESPACE + "' xmlns:gslib='" + GSLIB_NAMESPACE +"' ";
+
+	//Groups configuration
+	public static final String HIERARCHY_ELEM = "hierarchy";
+	public static final String GROUP_DESC_ELEM = "groupDescriptions";
+	public static final String TITLE_ELEM = "title";
+	public static final String DESCRIPTION_ELEM = "description";
+	public static final String SHORT_DESCRIPTION_ELEM = "shortDescription";
+	public static final String BACKGROUND_IMAGE_ELEM = "backgroundImage";
+	public static final String POSITION_ATT = "position";
+	
+	/**
+	 * takes a list of elements, and returns an array of strings of the values
+	 * of attribute att_name
+	 */
+	public static String[] getAttributeValuesFromList(Element list, String att_name)
+	{
+
+		NodeList children = list.getChildNodes();
+
+		int num_nodes = children.getLength();
+		String[] ids = new String[num_nodes];
+		for (int i = 0; i < num_nodes; i++)
+		{
+			Element e = (Element) children.item(i);
+			String id = e.getAttribute(att_name);
+			ids[i] = id;
+		}
+
+		return ids;
+	}
+
+      public static HashMap<String, String> getParamMap(NodeList params) {
+      HashMap<String, String> map = new HashMap<String, String>();
+      for(int i=0; i<params.getLength(); i++) {
+        Element param = (Element)params.item(i);
+        String param_name = param.getAttribute(NAME_ATT);
+        String param_value = param.getAttribute(VALUE_ATT);
+        map.put(param_name, param_value);
+      }
+      return map;
+    }
+
+	public static HashMap<String, Serializable> extractParams(Element xml, boolean deep)
+	{
+		return extractParams(xml, deep, null);
+	}
+
+	/**
+	 * takes a paramList element, and gets a HashMap of name-value pairs if
+	 * deep=true, extracts embedded params, otherwise just top level params
+	 */
+	public static HashMap<String, Serializable> extractParams(Element xml, boolean deep, String toFind)
+	{
+		if (xml == null)
+		{
+			return null;
+		}
+
+		if (!xml.getNodeName().equals(PARAM_ELEM + LIST_MODIFIER))
+		{
+			logger.error("paramList element should have been passed to extractParams, instead it was " + xml.getNodeName());
+			return null;
+		}
+
+		NodeList params = null;
+		if (deep)
+		{ // get all the nested ones
+			params = xml.getElementsByTagName(PARAM_ELEM);
+		}
+		else
+		{ // just get the top  level ones
+			params = xml.getChildNodes();
+		}
+		HashMap<String, Serializable> param_map = new HashMap<String, Serializable>();
+		for (int i = 0; i < params.getLength(); i++)
+		{
+			if (params.item(i).getNodeName().equals(PARAM_ELEM))
+			{
+				Element param = (Element) params.item(i);
+				String name = param.getAttribute(NAME_ATT);
+				String value = getValue(param); //att or content
+
+				// For only one parameter
+				if (toFind != null && name.equals(toFind))
+				{
+					param_map.put(name, value);
+					return param_map;
+				}
+				else if (toFind != null)
+					continue;
+
+				int pos = name.indexOf('.');
+				if (pos == -1)
+				{ // a base param
+					param_map.put(name, value);
+				}
+				else
+				{ // a namespaced param
+
+					String namespace = name.substring(0, pos);
+					name = name.substring(pos + 1);
+					HashMap<String, String> map = (HashMap<String, String>) param_map.get(namespace);
+					if (map == null)
+					{
+						map = new HashMap<String, String>();
+						param_map.put(namespace, map);
+					}
+					map.put(name, value);
+				}
+			}
+		}
+		return param_map;
+	}
+
+	/** gets the value att or the text content */
+	public static String getValue(Element e)
+	{
+		String val = e.getAttribute(VALUE_ATT);
+		if (val == null || val.equals(""))
+		{
+			// have to get it out of the text
+			val = getNodeText(e);
+
+		}
+		else
+		{
+			// unescape the xml stuff
+			val = unXmlSafe(val);
+		}
+		return val;
+	}
+
+	/** extracts the text out of a node */
+	public static Node getNodeTextNode(Element param)
+	{
+		param.normalize();
+		Node n = param.getFirstChild();
+		while (n != null && n.getNodeType() != Node.TEXT_NODE)
+		{
+			n = n.getNextSibling();
+		}
+		return n;
+	}
+
+	/** extracts the text out of a node */
+	public static String getNodeText(Element param)
+	{
+		Node text_node = getNodeTextNode(param);
+		if (text_node == null)
+		{
+			return "";
+		}
+		return text_node.getNodeValue();
+	}
+
+	public static void setNodeText(Element elem, String text)
+	{
+		Node old_text_node = getNodeTextNode(elem);
+		if (old_text_node != null)
+		{
+			elem.removeChild(old_text_node);
+		}
+		Text t = elem.getOwnerDocument().createTextNode(text);
+		elem.appendChild(t);
+	}
+
+	/** add text to a document/subsection element */
+	public static boolean addDocText(Element doc, String text)
+	{
+	  
+	  Element content = doc.getOwnerDocument().createElement(NODE_CONTENT_ELEM);
+		Text t = doc.getOwnerDocument().createTextNode(text);
+		content.appendChild(t);
+		doc.appendChild(content);
+		return true;
+	}
+
+	/** add an error message, unknown error type */
+  public static boolean addError(Element doc, String text)
+	{
+		return addError(doc, text, ERROR_TYPE_OTHER);
+	}
+
+	/** add an error message */
+	public static boolean addError(Element doc, String text, String error_type)
+	{
+
+		Element content = doc.getOwnerDocument().createElement(ERROR_ELEM);
+		content.setAttribute(ERROR_TYPE_ATT, error_type);
+		Text t = doc.getOwnerDocument().createTextNode(text);
+		content.appendChild(t);
+		doc.appendChild(content);
+		return true;
+	}
+
+	/** add an error message */
+	public static boolean addError(Element doc, Throwable error)
+	{
+		return addError(doc, error, ERROR_TYPE_OTHER);
+	}
+
+	/** add an error message */
+	public static boolean addError(Element doc, Throwable error, String error_type)
+	{
+		error.printStackTrace();
+		return addError(doc, error.toString(), error_type);
+	}
+
+  public static boolean addGSNamespaces(Element elem, boolean include_extension_ns) {
+    if (elem == null) {
+      return false;
+    }
+    elem.setAttribute("xmlns:gsf", GSF_NAMESPACE);
+    elem.setAttribute("xmlns:xsl", XSL_NAMESPACE);
+    elem.setAttribute("xmlns:gslib", GSLIB_NAMESPACE);
+    if (include_extension_ns) {
+      elem.setAttribute("xmlns:util", UTIL_NAMESPACE);
+      elem.setAttribute("xmlns:java", JAVA_NAMESPACE);
+    }
+    return true;
+  }
+	public static Element createMetadataParamList(Document owner, Vector meta_values)
+	{
+
+		Element meta_param_list = owner.createElement(PARAM_ELEM + LIST_MODIFIER);
+		Iterator i = meta_values.iterator();
+		while (i.hasNext())
+		{
+			String next = (String) i.next();
+			Element meta_param = owner.createElement(PARAM_ELEM);
+			meta_param_list.appendChild(meta_param);
+			meta_param.setAttribute(NAME_ATT, "metadata");
+			meta_param.setAttribute(VALUE_ATT, next);
+		}
+		return meta_param_list;
+	}
+
+	/** adds a metadata elem to a list */
+	public static boolean addMetadata(Element list, String meta_name, String meta_value)
+	{
+		if (meta_value == null || meta_value.equals(""))
+		{
+			return false;
+		}
+		Document owner = list.getOwnerDocument();
+		Element data = owner.createElement(METADATA_ELEM);
+		data.setAttribute(NAME_ATT, meta_name);
+		Text t = owner.createTextNode(meta_value);
+		data.appendChild(t);
+		list.appendChild(data);
+		return true;
+
+	}
+
+	/**
+	 * copies the metadata out of the metadataList of 'from' into the
+	 * metadataList of 'to'
+	 */
+  public static boolean mergeMetadataLists(Node to, Node from) {
+    return mergeSpecifiedLists(to, from, METADATA_ELEM);
+  }
+
+  /** copies metadata from meta_list into the metadataList of to **/
+  public static boolean mergeMetadataFromList(Node to, Node meta_list) {
+    return mergeSpecifiedFromList(to, meta_list, METADATA_ELEM);
+  }
+
+
+  public static boolean mergeSpecifiedLists(Node to, Node from, String element_name) {
+    // find the list in the from element
+    Node from_list = getChildByTagName(from, element_name+LIST_MODIFIER);
+    if (from_list == null) {
+      // nothing to copy
+      return false;
+    }
+   
+    return mergeSpecifiedFromList(to, from_list, element_name);
+  }
+    public static boolean mergeSpecifiedFromList(Node to, Node from_list, String element_name) {
+      if (from_list == null) {
+	return false;
+      }
+      Document to_owner = to.getOwnerDocument();
+      Node to_list = getChildByTagName(to, element_name+LIST_MODIFIER);
+      if (to_list == null) {
+	// just copy over the whole list
+	to.appendChild(to_owner.importNode(from_list, true));
+	return true;
+      }
+      // otherwise we copy all the elements
+      NodeList from_items = ((Element) from_list).getElementsByTagName(element_name);
+
+      for (int i = 0; i < from_items.getLength(); i++)
+	{
+	  to_list.appendChild(to_owner.importNode(from_items.item(i), true));
+	}
+      return true;
+      
+    }
+  
+  /** copies all the children from from to to */
+  public static boolean mergeElements(Element to, Element from)
+	{
+
+		Document owner = to.getOwnerDocument();
+		Node child = from.getFirstChild();
+		while (child != null)
+		{
+			to.appendChild(owner.importNode(child, true));
+			child = child.getNextSibling();
+		}
+		return true;
+	}
+
+	/** returns the (first) element child of the node n */
+	public static Element getFirstElementChild(Node n)
+	{
+
+		Node child = n.getFirstChild();
+		while (child != null)
+		{
+			if (child.getNodeType() == Node.ELEMENT_NODE)
+			{
+				return (Element) child;
+			}
+			child = child.getNextSibling();
+		}
+		return null; //no element child found
+	}
+
+	/** returns the (first) child element with the given name */
+	public static Node getChildByTagName(Node n, String name)
+	{
+		if (n != null)
+		{ // this line is an attempted solution to the NullPointerException mentioned 
+			// in trac bug ticket #225. If n is null can't do n.getFirstChild() below. As per bug #225: 
+			// GSXML.getNodeByPath() is called by GS2BrowseAction, which then calls this method.
+			// If n is null, null will be returned which GS2BrowseAction already checks for. It's here
+			// that the NullPointerException was thrown.
+
+			Node child = n.getFirstChild();
+			while (child != null)
+			{
+				if (child.getNodeName().equals(name))
+				{
+					return child;
+				}
+				child = child.getNextSibling();
+			}
+		}
+		return null; //not found
+	}
+
+	/**
+	 * returns the (nth) child element with the given name index numbers start
+	 * at 0
+	 */
+	public static Node getChildByTagNameIndexed(Node n, String name, int index)
+	{
+		if (index == -1)
+		{
+			return getChildByTagName(n, name);
+		}
+		int count = 0;
+		Node child = n.getFirstChild();
+		while (child != null)
+		{
+			if (child.getNodeName().equals(name))
+			{
+				if (count == index)
+				{
+					return child;
+				}
+				else
+				{
+					count++;
+				}
+			}
+			child = child.getNextSibling();
+		}
+		return null; //not found
+	}
+
+	/**
+	 * takes an xpath type expression of the form name/name/... and returns the
+	 * first node that matches, or null if not found
+	 */
+	public static Node getNodeByPath(Node n, String path)
+	{
+
+		String link = GSPath.getFirstLink(path);
+		path = GSPath.removeFirstLink(path);
+		while (!link.equals(""))
+		{
+			n = getChildByTagName(n, link);
+			if (n == null)
+			{
+				return null;
+			}
+			link = GSPath.getFirstLink(path);
+			path = GSPath.removeFirstLink(path);
+		}
+		return n;
+	}
+
+	/**
+	 * takes an xpath type expression of the form name/name/... and returns the
+	 * first node that matches, or null if not found can include [i] indices.
+	 * index numbers start at 0
+	 */
+	public static Node getNodeByPathIndexed(Node n, String path)
+	{
+
+		String link = GSPath.getFirstLink(path);
+		int index = GSPath.getIndex(link);
+		if (index != -1)
+		{
+			link = GSPath.removeIndex(link);
+		}
+		path = GSPath.removeFirstLink(path);
+		while (!link.equals(""))
+		{
+			n = getChildByTagNameIndexed(n, link, index);
+			if (n == null)
+			{
+				return null;
+			}
+			link = GSPath.getFirstLink(path);
+			index = GSPath.getIndex(link);
+			if (index != -1)
+			{
+				link = GSPath.removeIndex(link);
+			}
+			path = GSPath.removeFirstLink(path);
+		}
+		return n;
+	}
+
+	public static HashMap<String, Node> getChildrenMap(Node n)
+	{
+
+		HashMap<String, Node> map = new HashMap<String, Node>();
+		Node child = n.getFirstChild();
+		while (child != null)
+		{
+			String name = child.getNodeName();
+			map.put(name, child);
+			child = child.getNextSibling();
+		}
+		return map;
+	}
+
+	public static NodeList getChildrenByTagName(Node n, String name)
+	{
+		MyNodeList node_list = new MyNodeList();
+		Node child = n.getFirstChild();
+		while (child != null)
+		{
+			if (child.getNodeName().equals(name))
+			{
+				node_list.addNode(child);
+			}
+			child = child.getNextSibling();
+		}
+		return node_list;
+	}
+
+	public static NodeList getChildrenByTagNameNS(Node n, String namespace, String local_name)
+	{
+		MyNodeList node_list = new MyNodeList();
+		Node child = n.getFirstChild();
+		while (child != null)
+		{
+			if (child.getNodeType() == Node.ELEMENT_NODE && child.getNamespaceURI() != null && child.getNamespaceURI().equals(namespace) && child.getLocalName() != null && child.getLocalName().equals(local_name))
+			{
+				node_list.addNode(child);
+			}
+			child = child.getNextSibling();
+		}
+		return node_list;
+	}
+
+	/** Duplicates an element, but gives it a new name */
+	public static Element duplicateWithNewName(Document owner, Element element, String element_name, boolean with_attributes)
+	{
+		return duplicateWithNewNameNS(owner, element, element_name, null, with_attributes);
+	}
+
+	/** Duplicates an element, but gives it a new name */
+	public static Element duplicateWithNewNameNS(Document owner, Element element, String element_name, String namespace_uri, boolean with_attributes)
+	{
+		Element duplicate;
+		if (namespace_uri == null)
+		{
+			duplicate = owner.createElement(element_name);
+		}
+		else
+		{
+			duplicate = owner.createElementNS(namespace_uri, element_name);
+		}
+
+		// Copy element attributes
+		if (with_attributes)
+		{
+			NamedNodeMap attributes = element.getAttributes();
+			for (int i = 0; i < attributes.getLength(); i++)
+			{
+				Node attribute = attributes.item(i);
+				duplicate.setAttribute(attribute.getNodeName(), attribute.getNodeValue());
+			}
+		}
+
+		// Copy element children
+		NodeList children = element.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node child = children.item(i);
+			duplicate.appendChild(owner.importNode(child, true));
+		}
+
+		return duplicate;
+	}
+
+	public static void copyAllChildren(Element to, Element from)
+	{
+
+		Document to_doc = to.getOwnerDocument();
+		Node child = from.getFirstChild();
+		while (child != null)
+		{
+			to.appendChild(to_doc.importNode(child, true));
+			child = child.getNextSibling();
+		}
+	}
+
+  public static void copyNode(Element to_elem, Node original_element) {
+    to_elem.appendChild(to_elem.getOwnerDocument().importNode(original_element, true));
+
+  }
+
+   public static void copyNamedElement(Element to, Element from, String elem_name) {
+      
+      Document to_doc = to.getOwnerDocument();
+      Node child = from.getFirstChild();
+      while (child != null) {
+        if (child.getNodeName().equals(elem_name)) {
+          to.appendChild(to_doc.importNode(child, true));
+          return;
+        }
+        child = child.getNextSibling();
+      }
+    }
+
+ 	/**
+	 * Given a Node representing an Element or Document, will return the
+	 * Element/docroot Element. Returns null if the Node was not an element.
+	 */
+	public static Element nodeToElement(Node node)
+	{
+		if (node == null)
+		{
+			return null;
+		}
+		short nodeType = node.getNodeType();
+
+		if (nodeType == Node.DOCUMENT_NODE)
+		{
+			Document docNode = (Document) node;
+			return docNode.getDocumentElement();
+		}
+		else if (nodeType == Node.ELEMENT_NODE)
+		{
+			return (Element) node;
+		}
+		else
+		{
+			String message = "Expecting Document or Element node type but got " + node.getNodeName() + "\nReturning null";
+			System.err.println(message);
+			logger.warn(message);
+			return null;
+		}
+	}
+ 
+
+	/** returns a basic request message */
+	public static Element createBasicRequest(Document owner, String request_type, String to, UserContext userContext)
+	{
+		Element request = owner.createElement(REQUEST_ELEM);
+		request.setAttribute(TO_ATT, to);
+		request.setAttribute(TYPE_ATT, request_type);
+		if (userContext != null) { // should we allow this??
+		  request.setAttribute(LANG_ATT, userContext._lang);
+		  Element userContextElem = owner.createElement("userContext");
+		  request.appendChild(userContextElem);
+		  userContextElem.setAttribute(LANG_ATT, userContext._lang);
+		  userContextElem.setAttribute(USERNAME_ATT, userContext._username);
+		  userContextElem.setAttribute(USER_ID_ATT, userContext._userID);
+		  
+		  if (userContext._groups != null)
+		    {
+		      String groupString = "";
+		      for (int i = 0; i < userContext._groups.length; i++)
+			{
+			  groupString += userContext._groups[i];
+			  if (i != userContext._groups.length - 1)
+			    {
+			      groupString += ",";
+			    }
+			}
+		      
+		      if (groupString.length() > 0)
+			{
+			  userContextElem.setAttribute(GROUPS_ATT, groupString);
+			}
+		    }
+		}
+		return request;
+	}
+
+	public static Element createBasicResponse(Document owner, String from)
+	{
+		Element response = owner.createElement(GSXML.RESPONSE_ELEM);
+		response.setAttribute(GSXML.FROM_ATT, from);
+		response.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);
+		return response;
+	}
+
+	public static Element createMetadataElement(Document owner, String name, String value)
+	{
+		Element metaElem = owner.createElement(GSXML.METADATA_ELEM);
+		metaElem.setAttribute(GSXML.NAME_ATT, name);
+		metaElem.setAttribute(GSXML.VALUE_ATT, value);
+		return metaElem;
+	}
+
+	public static Element createTextElement(Document owner, String elem_name, String text)
+	{
+		Element e = owner.createElement(elem_name);
+		Text t = owner.createTextNode(text);
+		e.appendChild(t);
+		return e;
+
+	}
+
+	public static Element createTextElement(Document owner, String elem_name, String text, String att_name, String att_value)
+	{
+		Element e = owner.createElement(elem_name);
+		e.setAttribute(att_name, att_value);
+		Text t = owner.createTextNode(text);
+		e.appendChild(t);
+		return e;
+
+	}
+
+	public static Element createDisplayTextElement(Document owner, String text_name, String text)
+	{
+		Element e = owner.createElement(DISPLAY_TEXT_ELEM);
+		e.setAttribute(NAME_ATT, text_name);
+		Text t = owner.createTextNode(text);
+		e.appendChild(t);
+		return e;
+
+	}
+
+	public static Element createParameter(Document owner, String name, String value)
+	{
+		Element param = owner.createElement(PARAM_ELEM);
+		param.setAttribute(NAME_ATT, name);
+		param.setAttribute(VALUE_ATT, value);
+		return param;
+	}
+
+  public static void addParameterToList(Element param_list, String name, String value) {
+    Element p = createParameter(param_list.getOwnerDocument(), name, value);
+    param_list.appendChild(p);
+  }
+	public static void addParametersToList(Element param_list, HashMap params)
+	{
+		if (params == null)
+		{
+			return;
+		}
+
+		Set items = params.entrySet();
+		Iterator i = items.iterator();
+		Document owner = param_list.getOwnerDocument();
+		while (i.hasNext())
+		{
+			Map.Entry m = (Map.Entry) i.next();
+			param_list.appendChild(createParameter(owner, (String) m.getKey(), (String) m.getValue()));
+		}
+
+	}
+
+	public static Element createParameterDescription(Document owner, String id, String display_name, String type, String default_value, String[] option_ids, String[] option_names)
+	{
+
+		Element p = owner.createElement(PARAM_ELEM);
+		p.setAttribute(NAME_ATT, id);
+		p.setAttribute(TYPE_ATT, type);
+		p.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, display_name));
+
+		if (default_value != null)
+		{
+			p.setAttribute(DEFAULT_ATT, default_value);
+		}
+		if (option_ids != null && option_names != null)
+		{
+			for (int i = 0; i < option_ids.length; i++)
+			{
+				Element e = owner.createElement(PARAM_OPTION_ELEM);
+				e.setAttribute(NAME_ATT, option_ids[i]);
+				e.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, option_names[i]));
+				p.appendChild(e);
+			}
+		}
+		return p;
+	}
+
+	public static Element createParameterDescription2(Document owner, String id, String display_name, String type, String default_value, ArrayList<String> option_ids, ArrayList<String> option_names)
+	{
+
+		Element p = owner.createElement(PARAM_ELEM);
+		p.setAttribute(NAME_ATT, id);
+		p.setAttribute(TYPE_ATT, type);
+		p.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, display_name));
+		if (default_value != null)
+		{
+			p.setAttribute(DEFAULT_ATT, default_value);
+		}
+		if (option_ids != null && option_names != null)
+		{
+			for (int i = 0; i < option_ids.size(); i++)
+			{
+				Element e = owner.createElement(PARAM_OPTION_ELEM);
+				e.setAttribute(NAME_ATT, option_ids.get(i));
+				e.appendChild(createDisplayTextElement(owner, GSXML.DISPLAY_TEXT_NAME, option_names.get(i)));
+				p.appendChild(e);
+			}
+		}
+		return p;
+	}
+
+	/** returns the element parent/node_name[@attribute_name='attribute_value'] 
+	 if attribute_value is null, just returns the first one with that attribute */
+	public static Element getNamedElement(Element parent, String node_name, String attribute_name, String attribute_value)
+	{
+
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node child = children.item(i);
+			if (child.getNodeName().equals(node_name))
+			{
+			  String att_val = ((Element) child).getAttribute(attribute_name);
+			  if (!att_val.equals("") ) {
+			    if (attribute_value == null || att_val.equals(attribute_value)) {
+			      return (Element) child;
+			    }
+			  }
+			}
+		} // for each child
+		// not found
+		return null;
+	}
+
+	public static Element getNamedElementNS(Element parent, String namespace_uri, String node_local_name, String attribute_name, String attribute_value)
+	{
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node child = children.item(i);
+			if (child.getNodeType() == Node.ELEMENT_NODE && child.getNamespaceURI() != null && child.getNamespaceURI().equals(namespace_uri) && child.getLocalName().equals(node_local_name))
+			{
+				if (((Element) child).getAttribute(attribute_name).equals(attribute_value))
+					return (Element) child;
+			}
+		}
+		// not found
+		return null;
+	}
+
+	public static NodeList getNamedElementsNS(Element parent, String namespace_uri, String node_local_name, String attribute_name, String attribute_value)
+	{
+		MyNodeList result = new MyNodeList();
+		NodeList children = parent.getChildNodes();
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Node child = children.item(i);
+			if (child.getNodeType() == Node.ELEMENT_NODE && child.getNamespaceURI() != null && child.getNamespaceURI().equals(namespace_uri) && child.getLocalName().equals(node_local_name))
+			{
+				if (((Element) child).getAttribute(attribute_name).equals(attribute_value))
+					result.addNode(child);
+			}
+		}
+		return result;
+	}
+
+	public static NodeList getElementsWithAttributesNS(Element parent, String namespace_uri, String node_local_name, String[] attribute_names, String[] attribute_values)
+	{
+		if (attribute_names.length == 0 || attribute_names.length != attribute_values.length)
+		{
+			return new MyNodeList();
+		}
+
+		MyNodeList result = new MyNodeList();
+
+		NodeList matchingNodes = GSXML.getNamedElementsNS(parent, namespace_uri, node_local_name, attribute_names[0], attribute_values[0]);
+		for (int i = 0; i < matchingNodes.getLength(); i++)
+		{
+			Element current = (Element) matchingNodes.item(i);
+			boolean nodeMatches = true;
+			for (int j = 1; j < attribute_names.length; j++)
+			{
+				String currentName = attribute_names[j];
+				String currentValue = attribute_values[j];
+				if (!current.getAttribute(currentName).equals(currentValue))
+				{
+					nodeMatches = false;
+					break;
+				}
+			}
+
+			if (nodeMatches)
+			{
+				result.addNode(current);
+			}
+		}
+
+		return result;
+	}
+
+	public static void removeElementsWithAttributesNS(Element parent, String namespace_uri, String node_local_name, String[] attribute_names, String[] attribute_values)
+	{
+		NodeList matchingNodes = GSXML.getElementsWithAttributesNS(parent, namespace_uri, node_local_name, attribute_names, attribute_values);
+		for (int i = 0; i < matchingNodes.getLength(); i++)
+		{
+			parent.removeChild(matchingNodes.item(i));
+		}
+	}
+
+	// In element main, tries to find any previous occurrence of elements with xsl-template-name=templateName, 
+	// and whose named attribute (attributeName) has the same value as the same attribute in node.
+	// If this is the case, such a previous occurrence is removed from element main, since
+	// the new node will contain a more specific redefinition of this element.
+	public static void removeNamedElementNS(Element parent, String namespace_uri, String node_local_name, String attribute_name, String attribute_value)
+	{
+		if (attribute_value.equals(""))
+		{
+			// it has no identifying attribute, so we can't find any matches
+			return;
+		}
+
+		Element old_elem = GSXML.getNamedElementNS(parent, namespace_uri, node_local_name, attribute_name, attribute_value);
+		if (old_elem != null)
+		{
+			parent.removeChild(old_elem);
+		}
+	}
+
+	public static void removeNamedElementsNS(Element parent, String namespace, String node_local_name, String attribute_name, String attribute_value)
+	{
+		if (attribute_value.equals(""))
+		{
+			// it has no identifying attribute, so we can't find any matches
+			return;
+		}
+
+		NodeList children = parent.getChildNodes();
+		for (int i = children.getLength() - 1; i >= 0; i--)
+		{
+			Node child = children.item(i);
+			if (child.getNodeType() == Node.ELEMENT_NODE && child.getNamespaceURI() != null && child.getNamespaceURI().equals(namespace) && child.getLocalName() != null && child.getLocalName().equals(node_local_name))
+			{
+				if (((Element) child).getAttribute(attribute_name).equals(attribute_value))
+					parent.removeChild(child);
+			}
+		}
+	}
+
+	/**
+	 * returns a NodeList of elements:
+	 * ancestor/node_name[@attribute_name='attribute_value']
+	 */
+	public static NodeList getNamedElements(Element ancestor, String node_name, String attribute_name, String attribute_value)
+	{
+		MyNodeList node_list = new MyNodeList();
+		NodeList children = ancestor.getElementsByTagName(node_name);
+
+		if (children != null && children.getLength() > 0)
+		{
+
+			for (int i = 0; i < children.getLength(); i++)
+			{
+				Node child = children.item(i);
+				if (child.getNodeName().equals(node_name))
+				{
+					if (((Element) child).getAttribute(attribute_name).equals(attribute_value))
+						node_list.addNode(child);
+				}
+			}
+		}
+		return node_list;
+	}
+
+	public static Element getLastElementByTagNameNS(Element main, String namespace, String node_name)
+	{
+		NodeList nodes = main.getElementsByTagNameNS(namespace, node_name);
+		int len = nodes.getLength();
+		if (len == 0)
+		{
+			return null;
+		}
+		return (Element) nodes.item(len - 1);
+	}
+
+	public static int SORT_TYPE_STRING = 0;
+	public static int SORT_TYPE_INT = 1;
+	public static int SORT_TYPE_FLOAT = 2;
+
+	// sort type:
+	public static Element insertIntoOrderedList(Element parent_node, String node_name, Element start_from_elem, Element new_elem, String sort_att, boolean descending)
+	{
+		if (new_elem == null)
+			return null;
+		Element cloned_elem = (Element) parent_node.getOwnerDocument().importNode(new_elem, true);
+		if (start_from_elem == null)
+		{
+			parent_node.appendChild(cloned_elem);
+			return cloned_elem;
+		}
+
+		Node current_node = start_from_elem;
+		String insert_att = cloned_elem.getAttribute(sort_att);
+		String list_att = start_from_elem.getAttribute(sort_att);
+		while ((!descending && list_att.compareTo(insert_att) < 0) || (descending && list_att.compareTo(insert_att) > 0))
+		{
+			current_node = current_node.getNextSibling();
+			if (current_node == null)
+				break; // end of the list
+			if (!current_node.getNodeName().equals(node_name))
+			{
+				continue; // not a valid node
+			}
+			list_att = ((Element) current_node).getAttribute(sort_att);
+		}
+
+		parent_node.insertBefore(cloned_elem, current_node);
+		return cloned_elem;
+	}
+
+  public static String getMetadataValue(Element metadata_list, String name) {
+    Element meta = getNamedElement(metadata_list, METADATA_ELEM, NAME_ATT, name);
+    if (meta == null) {
+      return "";
+    }
+    String att_value = meta.getAttribute(VALUE_ATT);
+    if (att_value.equals("")) {
+      // try the text
+      att_value = getNodeText(meta);
+    }
+    return att_value;
+  }
+
+	// replaces < > " ' & in the original with their entities
+	public static String xmlSafe(String original)
+	{
+
+		StringBuffer filtered = new StringBuffer(original.length());
+		char c;
+		for (int i = 0; i < original.length(); i++)
+		{
+			c = original.charAt(i);
+			if (c == '>')
+			{
+				filtered.append("&gt;");
+			}
+			else if (c == '<')
+			{
+				filtered.append("&lt;");
+			}
+			else if (c == '"')
+			{
+				filtered.append("&quot;");
+			}
+			else if (c == '&')
+			{
+				filtered.append("&amp;");
+			}
+			else if (c == '\'')
+			{
+				filtered.append("&apos;");
+			}
+			else
+			{
+				filtered.append(c);
+			}
+		}
+		return filtered.toString();
+	}
+
+	// replaces < > " ' & entities with their originals
+	public static String unXmlSafe(String original)
+	{
+
+		StringBuffer filtered = new StringBuffer(original.length());
+		char c;
+		for (int i = 0; i < original.length(); i++)
+		{
+			c = original.charAt(i);
+			if (c == '&')
+			{
+				int pos = original.indexOf(";", i);
+				String entity = original.substring(i + 1, pos);
+				if (entity.equals("gt"))
+				{
+					filtered.append(">");
+				}
+				else if (entity.equals("lt"))
+				{
+					filtered.append("<");
+				}
+				else if (entity.equals("apos"))
+				{
+					filtered.append("'");
+				}
+				else if (entity.equals("amp"))
+				{
+					filtered.append("&");
+				}
+				else if (entity.equals("quot"))
+				{
+					filtered.append("\"");
+				}
+				else
+				{
+					filtered.append("&" + entity + ";");
+				}
+				i = pos;
+			}
+			else
+			{
+				filtered.append(c);
+			}
+		}
+		return filtered.toString();
+	}
+
+	public static void printXMLNode(Node e, boolean printText)
+	{
+		printXMLNode(e, 0, printText);
+	}
+
+	public static String xmlNodeToString(Node e)
+	{
+		StringBuffer sb = new StringBuffer("");
+		xmlNodeToString(sb, e, true, "\t", 0);
+		return sb.toString();
+	}
+
+	public static void xmlNodeToString(StringBuffer sb, Node e, boolean indent, String indentString, int depth)
+	{
+		if (e.getNodeType() == Node.TEXT_NODE)
+		{
+			if (e.getNodeValue() != "")
+			{
+				String text = e.getNodeValue();
+				if (text != null)
+				{
+					text = text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("^[\\n\\r\\t\\s]*", "").replaceAll("[\\n\\r\\t\\s]*$", "");
+					sb.append(text);
+				}
+			}
+			return;
+		}
+
+		if (e.getNodeType() == Node.COMMENT_NODE)
+		{
+			if (e.getNodeValue() != "")
+			{
+				sb.append("<!--" + e.getNodeValue() + "-->");
+			}
+			return;
+		}
+
+		if (indent)
+		{
+			for (int i = 0; i < depth; i++)
+			{
+				sb.append(indentString);
+			}
+		}
+
+		sb.append('<');
+		sb.append(e.getNodeName());
+		NamedNodeMap attrs = e.getAttributes();
+		if (attrs != null)
+		{
+			for (int i = 0; i < attrs.getLength(); i++)
+			{
+				Node attr = attrs.item(i);
+				sb.append(' ');
+				sb.append(attr.getNodeName());
+				sb.append("=\"");
+				sb.append(attr.getNodeValue().replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;"));
+				sb.append('"');
+			}
+		}
+
+		if (e.hasChildNodes())
+		{
+			boolean hasElements = false;
+			boolean indentSwapped = false;
+
+			Node child = e.getFirstChild();
+			do
+			{
+				if (child.getNodeType() == Node.ELEMENT_NODE)
+				{
+					hasElements = true;
+				}
+				if (child.getNodeType() == Node.TEXT_NODE && indent)
+				{
+					if (child.getNodeValue() != null && child.getNodeValue().trim().length() > 0)
+					{
+						indentSwapped = true;
+						indent = false;
+					}
+				}
+			} while ((child = child.getNextSibling()) != null);
+
+			sb.append(">");
+			if (hasElements && indent)
+			{
+				sb.append("\n");
+			}
+
+			child = e.getFirstChild();
+			do
+			{
+				xmlNodeToString(sb, child, indent, indentString, depth + 1);
+			} while ((child = child.getNextSibling()) != null);
+
+			if (indent)
+			{
+				for (int i = 0; i < depth; i++)
+				{
+					sb.append(indentString);
+				}
+			}
+
+			sb.append("</" + e.getNodeName() + ">");
+
+			if ((hasElements && indent) || indentSwapped)
+			{
+				sb.append("\n");
+			}
+		}
+		else
+		{
+			sb.append("/>");
+
+			if (indent)
+			{
+				sb.append("\n");
+			}
+		}
+
+		/*
+		boolean hasElements = false;
+		boolean indentSwapped = false;
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			if (children.item(i).getNodeType() == Node.ELEMENT_NODE)
+			{
+				hasElements = true;
+			}
+			if (children.item(i).getNodeType() == Node.TEXT_NODE && indent)
+			{
+				if (children.item(i).getNodeValue() != null && children.item(i).getNodeValue().trim().length() > 0)
+				{
+					indentSwapped = true;
+					indent = false;
+				}
+			}
+		}
+
+		if (children == null || children.getLength() == 0)
+		{
+			sb.append("/>");
+
+			if (indent)
+			{
+				sb.append("\n");
+			}
+		}
+		else
+		{
+			sb.append(">");
+			if (hasElements && indent)
+			{
+				sb.append("\n");
+			}
+
+			int len = children.getLength();
+			for (int i = 0; i < len; i++)
+			{
+				xmlNodeToString(sb, children.item(i), indent, indentString, depth + 1);
+			}
+
+			if (indent)
+			{
+				for (int i = 0; i < depth; i++)
+				{
+					sb.append(indentString);
+				}
+			}
+
+			sb.append("</" + e.getNodeName() + ">");
+
+			if ((hasElements && indent) || indentSwapped)
+			{
+				sb.append("\n");
+			}
+		}
+		*/
+	}
+
+	public static void printXMLNode(Node e, int depth, boolean printText)
+	{ //recursive method call using DOM API...
+
+		if (e == null)
+		{
+			return;
+		}
+
+		for (int i = 0; i < depth; i++)
+			System.out.print(' ');
+
+		if (e.getNodeType() == Node.TEXT_NODE)
+		{
+			if (printText)
+			{
+				System.out.println(e.getNodeValue());
+			}
+			else
+			{
+				System.out.println("text");
+			}
+			return;
+		}
+
+		System.out.print('<');
+		System.out.print(e.getNodeName());
+		NamedNodeMap attrs = e.getAttributes();
+
+		if (attrs != null)
+		{
+			for (int i = 0; i < attrs.getLength(); i++)
+			{
+				Node attr = attrs.item(i);
+				System.out.print(' ');
+				System.out.print(attr.getNodeName());
+				System.out.print("=\"");
+				System.out.print(attr.getNodeValue());
+				System.out.print('"');
+			}
+		}
+
+		NodeList children = e.getChildNodes();
+
+		if (children == null || children.getLength() == 0)
+			System.out.println("/>");
+		else
+		{
+
+			System.out.println('>');
+
+			int len = children.getLength();
+			for (int i = 0; i < len; i++)
+			{
+				printXMLNode(children.item(i), depth + 1, printText);
+			}
+
+			for (int i = 0; i < depth; i++)
+				System.out.print(' ');
+
+			System.out.println("</" + e.getNodeName() + ">");
+		}
+	}
+
+	public static void elementToLogAsString(Element e, boolean indent)
+	{
+		String str = elementToString(e, indent);
+		System.err.println(str);
+		logger.error(str);
+	}
+
+	public static String elementToString(Element e, boolean indent)
+	{
+		String str = "";
+		try
+		{
+			TransformerFactory tf = TransformerFactory.newInstance();
+			Transformer trans = tf.newTransformer();
+			StringWriter sw = new StringWriter();
+			if (indent)
+			{
+				trans.setOutputProperty(OutputKeys.INDENT, "yes");
+			}
+			else
+			{
+				trans.setOutputProperty(OutputKeys.INDENT, "no");
+			}
+			trans.transform(new DOMSource(e), new StreamResult(sw));
+			str += sw.toString();
+		}
+		catch (Exception ex)
+		{
+			str += "Exception: couldn't write " + e + " to log";
+		}
+		finally
+		{
+			return str;
+		}
+	}
+
+	public static ArrayList<String> getGroupsFromSecurityResponse(Element securityResponse)
+	{
+		ArrayList<String> groups = new ArrayList<String>();
+
+		Element groupList = (Element) GSXML.getChildByTagName(securityResponse, GSXML.GROUP_ELEM + GSXML.LIST_MODIFIER);
+		if (groupList == null)
+		{
+			return groups;
+		}
+
+		NodeList groupElems = GSXML.getChildrenByTagName(groupList, GSXML.GROUP_ELEM);
+
+		for (int i = 0; i < groupElems.getLength(); i++)
+		{
+			Element groupElem = (Element) groupElems.item(i);
+			groups.add(groupElem.getAttribute(GSXML.NAME_ATT));
+		}
+
+		return groups;
+	}
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXSLT.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXSLT.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/GSXSLT.java	(revision 31634)
@@ -0,0 +1,786 @@
+/*
+ *    GSXSLT.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Vector;
+
+import org.apache.log4j.Logger;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.UserDataHandler;
+
+/** various functions for manipulating Greenstone xslt */
+public class GSXSLT
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSXSLT.class.getName());
+
+	public static void mergeStylesheets(Document main_xsl, Element extra_xsl, boolean overwrite)
+	{
+		mergeStylesheetsDebug(main_xsl, extra_xsl, overwrite, false, null, null);
+	}
+
+	/**
+	 * takes a stylesheet Document, and adds in any child nodes from extra_xsl
+	 * named templates overwrite any existing one, while match templates are
+	 * just added to the end of the stylesheet
+	 * 
+	 * elements are added in following order, and added to preserve original
+	 * order with imported ones coming after existing ones import, include,
+	 * output, variable, template
+	 */
+	public static void mergeStylesheetsDebug(Document main_xsl, Element extra_xsl, boolean overwrite, boolean debug, String firstDocFileName, String secondDocFileName)
+	{
+		if (debug)
+		{
+			insertDebugElements(main_xsl, firstDocFileName);
+		}
+
+		Element main = main_xsl.getDocumentElement();
+		Node insertion_point = null;
+		Element last_import = GSXML.getLastElementByTagNameNS(main, GSXML.XSL_NAMESPACE, "import");
+		if (last_import != null)
+		{
+			insertion_point = last_import.getNextSibling();
+		}
+		else
+		{
+			insertion_point = main.getFirstChild();
+		}
+
+		// imports
+		NodeList children = extra_xsl.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "import");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// If the new xsl:import element is identical (in terms of href attr value) 
+			// to any in the merged document, don't copy it over
+			if (GSXML.getNamedElementNS(main, GSXML.XSL_NAMESPACE, "import", "href", node.getAttribute("href")) == null)
+			{
+				// Import statements should be the first children of an xsl:stylesheet element
+				// If firstchild is null, then this xsl:import element will be inserted at the "end"
+				// Although Node.insertBefore() will first remove identical nodes before inserting, we check
+				// only the href attribute to see if they're "identical" to any pre-existing <xsl:import>
+				//main.insertBefore(main_xsl.importNode(node, true), main.getFirstChild());
+				main.insertBefore(main_xsl.importNode(node, true), insertion_point);
+			}
+		}
+
+		// do we have a new insertion point??
+		Element last_include = GSXML.getLastElementByTagNameNS(main, GSXML.XSL_NAMESPACE, "include");
+		if (last_include != null)
+		{
+			insertion_point = last_include.getNextSibling();
+		}
+
+		// includes
+		children = extra_xsl.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "include");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// If the new xsl:include element is identical (in terms of href attr value)
+			// to any in the merged document, don't copy it over
+			// Although Node.appendChild() will first remove identical nodes before appending, we check
+			// only the href attribute to see if they're "identical" to any pre-existing <xsl:include>
+			if (GSXML.getNamedElementNS(main, GSXML.XSL_NAMESPACE, "include", "href", node.getAttribute("href")) == null)
+			{
+				//main.appendChild(main_xsl.importNode(node, true));
+				main.insertBefore(main_xsl.importNode(node, true), insertion_point);
+			}
+		} // for each include
+
+		if (main.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "output").getLength() == 0)
+		{
+			// outputs
+			children = extra_xsl.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "output");
+			for (int i = 0; i < children.getLength(); i++)
+			{
+				Element node = (Element) children.item(i);
+				// If the new xsl:output element is identical (in terms of the value for the method attr) 
+				// to any in the merged document, don't copy it over
+
+				main.insertBefore(main_xsl.importNode(node, true), insertion_point);
+			}
+		}
+
+		// variables - only top level ones!!
+		// append to end of document
+		children = GSXML.getChildrenByTagNameNS(extra_xsl, GSXML.XSL_NAMESPACE, "variable");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// If the new xsl:import element is identical (in terms of href attr value) 
+			// to any in the merged document, don't copy it over
+			if (GSXML.getNamedElementNS(main, GSXML.XSL_NAMESPACE, "variable", "name", node.getAttribute("name")) == null)
+			{
+				main.appendChild(main_xsl.importNode(node, true));
+			}
+		}
+
+		// params - only top level ones!!
+		// append to end of document
+		children = GSXML.getChildrenByTagNameNS(extra_xsl, GSXML.XSL_NAMESPACE, "param");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// If the new xsl:import element is identical (in terms of href attr value) 
+			// to any in the merged document, don't copy it over
+			if (GSXML.getNamedElementNS(main, GSXML.XSL_NAMESPACE, "param", "name", node.getAttribute("name")) == null)
+			{
+				main.appendChild(main_xsl.importNode(node, true));
+			}
+		}
+
+		// key -- xsl:key elements need to be defined at the top level
+		children = GSXML.getChildrenByTagNameNS(extra_xsl, GSXML.XSL_NAMESPACE, "key");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// If the new xsl:key element is identical (in terms of name attr value) 
+			// to any in the merged document, don't copy it over
+			if (GSXML.getNamedElementNS(main, GSXML.XSL_NAMESPACE, "key", "name", node.getAttribute("name")) == null)
+			{
+				main.appendChild(main_xsl.importNode(node, true));
+			}
+		}
+
+		// templates
+		// append to end of document
+		children = extra_xsl.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "template");
+		for (int i = 0; i < children.getLength(); i++)
+		{
+			Element node = (Element) children.item(i);
+			// remove any previous occurrences of xsl:template with the same value for name or match 
+			String template_match = node.getAttribute("match");
+			String template_name = node.getAttribute("name");
+			String template_mode = node.getAttribute("mode");
+
+			if (overwrite)
+			{
+				// if we have a name attribute, remove any other similarly named template
+				GSXML.removeElementsWithAttributesNS(main, GSXML.XSL_NAMESPACE, "template", new String[] { "name", "match", "mode" }, new String[] { template_name, template_match, template_mode });
+
+				// now add our good template in
+				main.appendChild(main_xsl.importNode(node, true));
+			}
+			else
+			{
+				// if overwrite is false, then we only add in templates if they don't match something else.
+				// In this case (eg from expanding imported stylesheets)
+				// there can't be any duplicate named templates, so just look for matches
+				// we already have the one with highest import precedence (from the top most level) so don't add any more in
+				if (GSXML.getElementsWithAttributesNS(main, GSXML.XSL_NAMESPACE, "template", new String[] { "name", "match", "mode" }, new String[] { template_name, template_match, template_mode }).getLength() == 0)
+				{
+					main.appendChild(main_xsl.importNode(node, true));
+				}
+			}
+		}
+
+		if (debug)
+		{
+			insertDebugElements(main_xsl, secondDocFileName);
+		}
+	}
+
+	public static void insertDebugElements(Document doc, String filename)
+	{
+		NodeList xslTemplates = doc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "template");
+		NodeList gsfTemplates = doc.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "template");
+
+		for (int i = 0; i < xslTemplates.getLength() + gsfTemplates.getLength(); i++)
+		{
+			boolean gsf = (i >= xslTemplates.getLength());
+			Element currentTemplate = (Element) (!gsf ? xslTemplates.item(i) : gsfTemplates.item(i - xslTemplates.getLength()));
+
+			NodeList childNodes = currentTemplate.getChildNodes();
+			boolean debugInformationAlreadyExists = false;
+			for (int j = 0; j < childNodes.getLength(); j++)
+			{
+				Node current = childNodes.item(j);
+				if (current instanceof Element && ((Element) current).getNodeName().equals("debug") && (!((Element) current).getAttribute("nodename").startsWith("gsf:") || ((Element) current).getAttribute("nodename").equals("gsf:template")))
+				{
+					debugInformationAlreadyExists = true;
+					break;
+				}
+			}
+
+			if (debugInformationAlreadyExists)
+			{
+				continue;
+			}
+
+			Element debugElement = doc.createElement("debug");
+			debugElement.setAttribute("filename", filename);
+			debugElement.setAttribute("nodename", gsf ? "gsf:template" : "xsl:template");
+
+			if (currentTemplate.getAttribute("match").length() > 0)
+			{
+				debugElement.setAttribute("match", currentTemplate.getAttribute("match"));
+			}
+			if (currentTemplate.getAttribute("name").length() > 0)
+			{
+				debugElement.setAttribute("name", currentTemplate.getAttribute("name"));
+			}
+
+			if (currentTemplate.getUserData("xpath") != null)
+			{
+				debugElement.setAttribute("xpath", (String) currentTemplate.getUserData("xpath"));
+			}
+
+			if (childNodes.getLength() > 0)
+			{
+				int paramCount = 0;
+				while (childNodes.getLength() > paramCount)
+				{
+					Node currentNode = childNodes.item(paramCount);
+					if (currentNode instanceof Element)
+					{
+						if (((Element) currentNode).getNodeName().equals("xsl:param") || ((Element) currentNode).getNodeName().equals("xslt:param") || (((Element) currentNode).getNodeName().equals("param") && ((Element) currentNode).getNamespaceURI().equals(GSXML.XSL_NAMESPACE)))
+						{
+							paramCount++;
+						}
+						else
+						{
+							debugElement.appendChild(currentNode);
+						}
+					}
+					else
+					{
+						debugElement.appendChild(currentNode);
+					}
+				}
+				currentTemplate.appendChild(debugElement);
+			}
+			else
+			{
+				currentTemplate.appendChild(debugElement);
+			}
+
+			Element textElement = doc.createElementNS(GSXML.XSL_NAMESPACE, "text");
+			textElement.appendChild(doc.createTextNode(" "));
+			debugElement.appendChild(textElement);
+		}
+	}
+
+	public static void inlineImportAndIncludeFiles(Document doc, String pathExtra, String site, String collection, String interface_name, ArrayList<String> base_interfaces)
+	{
+		inlineImportAndIncludeFilesDebug(doc, pathExtra, false, null, site, collection, interface_name, base_interfaces);
+	}
+
+	public static void inlineImportAndIncludeFilesDebug(Document doc, String pathExtra, boolean debug, String docFileName, String site, String collection, String interface_name, ArrayList<String> base_interfaces)
+	{
+		String path = (pathExtra == null) ? "" : pathExtra;
+
+		NodeList importList = doc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "import");
+		NodeList includeList = doc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "include");
+
+		for (int i = 0; i < importList.getLength() + includeList.getLength(); i++)
+		{
+			Element current = (Element) ((i < importList.getLength()) ? importList.item(i) : includeList.item(i - importList.getLength()));
+			String href = current.getAttribute("href");
+
+			try
+			{
+				//Document inlineDoc = converter.getDOM(new File(filePath), "UTF-8");
+				Document inlineDoc = mergedXSLTDocumentCascade(path + href, site, collection, interface_name, base_interfaces, debug);
+				String newPath = path;
+				int lastSepIndex = href.lastIndexOf("/");
+				if (lastSepIndex != -1)
+				{
+					newPath += href.substring(0, lastSepIndex + 1);
+				}
+
+				//Do this recursively
+				inlineImportAndIncludeFilesDebug(inlineDoc, newPath, debug, "merged " + href/* filePath */, site, collection, interface_name, base_interfaces);
+				GSXSLT.mergeStylesheetsDebug(doc, inlineDoc.getDocumentElement(), false, debug, docFileName, /* filePath */"merged " + href);
+			}
+			catch (Exception ex)
+			{
+				ex.printStackTrace();
+				return;
+			}
+		}
+
+		while (importList.getLength() > 0)
+		{
+			Element importElem = (Element) importList.item(0);
+			importElem.getParentNode().removeChild(importElem);
+		}
+		while (includeList.getLength() > 0)
+		{
+			Element includeElem = (Element) includeList.item(0);
+			includeElem.getParentNode().removeChild(includeElem);
+		}
+	}
+
+	public static Document mergedXSLTDocumentCascade(String xslt_filename, String site, String collection, String this_interface, ArrayList<String> base_interfaces, boolean debug)
+	{
+		XMLConverter converter = new XMLConverter();
+		// find the list of stylesheets with this name
+		ArrayList<File> stylesheets = GSFile.getStylesheetFiles(GlobalProperties.getGSDL3Home(), site, collection, this_interface, base_interfaces, xslt_filename);
+		if (stylesheets.size() == 0)
+		{
+			logger.error(" Can't find stylesheet for " + xslt_filename);
+			return null;
+		}
+		logger.debug("Stylesheet: " + xslt_filename);
+
+		Document finalDoc = converter.getDOM(stylesheets.get(stylesheets.size() - 1), "UTF-8");
+		if (finalDoc == null)
+		{
+			return null;
+		}
+
+		for (int i = stylesheets.size() - 2; i >= 0; i--)
+		{
+			Document currentDoc = converter.getDOM(stylesheets.get(i), "UTF-8");
+			if (currentDoc == null)
+			{
+				return null;
+			}
+
+			if (debug)
+			{
+				GSXSLT.mergeStylesheetsDebug(finalDoc, currentDoc.getDocumentElement(), true, true, stylesheets.get(stylesheets.size() - 1).getAbsolutePath(), stylesheets.get(i).getAbsolutePath());
+			}
+			else
+			{
+				GSXSLT.mergeStylesheets(finalDoc, currentDoc.getDocumentElement(), true);
+			}
+		}
+
+		if (stylesheets.size() == 1 && debug)
+		{
+			insertDebugElements(finalDoc, stylesheets.get(0).getAbsolutePath());
+		}
+
+		return finalDoc;
+	}
+
+	/**
+	 * takes any import or include nodes, and creates absolute path names for
+	 * the files
+	 */
+	public static void absoluteIncludePaths(Document stylesheet, String gsdl3_home, String site_name, String collection, String interface_name, ArrayList<String> base_interfaces)
+	{
+		Element base_node = stylesheet.getDocumentElement();
+		if (base_node == null)
+		{
+			return;
+		}
+		Node child = base_node.getFirstChild();
+		while (child != null)
+		{
+			String name = child.getNodeName();
+			if (name.equals("xsl:import") || name.equals("xsl:include"))
+			{
+				((Element) child).setAttribute("href", GSFile.stylesheetFile(gsdl3_home, site_name, collection, interface_name, base_interfaces, ((Element) child).getAttribute("href")));
+			}
+			child = child.getNextSibling();
+		}
+	}
+
+
+  public static void findExtraMetadataNames(Element xsl_elem, HashSet<String> meta_names) {
+
+    // gsf:metadata and gsf:foreach-metadata
+    NodeList metadata_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "metadata"); 
+    NodeList foreach_metadata_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "foreach-metadata");
+    int num_meta_nodes = metadata_nodes.getLength();
+    int total_nodes = num_meta_nodes +foreach_metadata_nodes.getLength();
+    for (int i = 0; i < total_nodes; i++) {
+      Element current;
+      if (i<num_meta_nodes) {
+	current = (Element) metadata_nodes.item(i);
+      } else {
+	current = (Element) foreach_metadata_nodes.item(i-num_meta_nodes);
+      }
+      String full_name = current.getAttribute("name");
+      String select = current.getAttribute("select");
+      
+      String [] names = full_name.split(",");
+      for(int j=0; j<names.length; j++) {
+	
+	String name = names[j];
+	if (!name.equals("")) {
+	  if (!select.equals("")) {
+	    name = select + GSConstants.META_RELATION_SEP + name;
+	  }
+	  meta_names.add(name);
+	}
+      }
+    }
+    
+    // gsf:link
+    boolean getEquivLinkMeta = false;
+    NodeList link_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "link");
+    for (int i = 0; i < link_nodes.getLength(); i++) {
+      
+      Element elem = (Element) link_nodes.item(i);
+      String type = elem.getAttribute("type");
+      if (type.equals("source"))
+	{
+	  meta_names.add("root_assocfilepath");
+	  meta_names.add("srclinkFile");
+	}
+      else if (type.equals("web"))
+	{
+	  meta_names.add("weblink");
+	  meta_names.add("webicon");
+	  meta_names.add("/weblink");
+	}
+      else if (type.equals("equivdoc"))
+	{
+	  getEquivLinkMeta = true; // equivalent to gsf:equivlinkgs3
+	}
+    }
+    // gsf:equivlinkgs3
+    link_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "equivlinkgs3");
+    if (getEquivLinkMeta || link_nodes.getLength() > 0) {
+      
+      String[] equivlink_metanames = { "equivDocIcon", "equivDocLink", "/equivDocLink" };
+
+      for (int i = 0; i < equivlink_metanames.length; i++)
+	{
+	  StringBuffer metadata = new StringBuffer();
+	  metadata.append("all"); // this means the attr multiple = true;
+	  metadata.append(GSConstants.META_RELATION_SEP);
+	  
+	  metadata.append(GSConstants.META_SEPARATOR_SEP);
+	  metadata.append(','); // attr separator = ","
+	  metadata.append(GSConstants.META_SEPARATOR_SEP);
+	  metadata.append(GSConstants.META_RELATION_SEP);
+	  
+	  // the name of the metadata we're retrieving
+	  metadata.append(equivlink_metanames[i]);
+	  meta_names.add(metadata.toString());
+	}
+    }
+    
+    // gsf:icon
+    NodeList icon_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "icon");
+    for (int i = 0; i < icon_nodes.getLength(); i++) {
+      Element current = (Element) icon_nodes.item(i);
+      String type = current.getAttribute(GSXML.TYPE_ATT);
+      if (type == null || type.length() == 0) {
+	continue;
+      }
+      if (type.equals("web")) {
+	meta_names.add("webicon");
+	break; // this is the only one we are looking for at the moment
+      }
+    }
+    
+    // gsf:image
+    NodeList image_nodes = xsl_elem.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "image");
+    for (int i = 0; i < image_nodes.getLength(); i++) {
+      Element current = (Element) image_nodes.item(i);
+      String type = current.getAttribute(GSXML.TYPE_ATT);
+      if (type == null || type.length() == 0) {
+	continue;
+      }
+      
+      if (type.equals("source")) {
+	
+	String[] standardSourceMeta = new String[] { "SourceFile", "ImageHeight", "ImageWidth", "ImageType", "srcicon" };
+	for (String meta : standardSourceMeta) {
+	  meta_names.add(meta);
+	}
+	
+      }
+      else if (type.equals("screen")) {
+	
+	String[] standardScreenMeta = new String[] { "Screen", "ScreenHeight", "ScreenWidth", "ScreenType", "screenicon" };
+	for (String meta : standardScreenMeta) {
+	  meta_names.add(meta);
+	}
+      }
+      else if (type.equals("thumb")) {
+	String[] standardThumbMeta = new String[] { "Thumb", "ThumbHeight", "ThumbWidth", "ThumbType", "thumbicon" };
+	for (String meta : standardThumbMeta) {
+	  meta_names.add(meta);
+	}
+      }
+      else if (type.equals("cover")) {
+	meta_names.add("hascover");
+	logger.error("adding hascover");
+      }
+    }  
+  }
+  
+	/**
+	 * looks through a stylesheet for <xxx:template match='template_name'>
+	 * inside this template it looks for any <xxx:value-of
+	 * select='metadataList/metadata[@name=yyy]> elements, and extracts the
+	 * metadata names into a Vector
+	 */
+	public static Vector<String> extractWantedMetadata(Document stylesheet, String template_name)
+	{
+
+		Vector<String> metadata = new Vector<String>();
+		Element base_node = stylesheet.getDocumentElement();
+		NodeList templates = base_node.getElementsByTagNameNS("*", "template");
+		for (int i = 0; i < templates.getLength(); i++)
+		{
+			Element template = (Element) templates.item(i);
+			String match_name = template.getAttribute("match");
+			if (!match_name.equals(template_name))
+			{
+				continue; // we're only looking for specific templates
+			}
+			String mode = template.getAttribute("mode");
+			if (!mode.equals(""))
+			{
+				continue; // we only want ones without modes - these are processing ones, not display ones
+			}
+			// we have one that we want to look through
+			NodeList values = template.getElementsByTagNameNS("*", "value-of");
+			for (int v = 0; v < values.getLength(); v++)
+			{
+				String select = ((Element) values.item(v)).getAttribute("select");
+				if (select.startsWith("metadataList/metadata[@name="))
+				{
+					String[] bits = select.split("'|\"");
+					// there should be two quotes in teh string, therefore 3 items, and the second one is teh one we want
+					String name = bits[1];
+					metadata.add(name);
+				}
+			}
+		}
+		return metadata;
+	}
+
+	public static void mergeFormatElements(Element mainFormat, Element secondaryFormat, boolean overwrite)
+	{
+		NodeList xslChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.XSL_NAMESPACE, "variable");
+		NodeList gsfChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.GSF_NAMESPACE, "variable");
+		for (int i = 0; i < xslChildren.getLength() + gsfChildren.getLength(); i++)
+		{
+			Element node = (Element) ((i < xslChildren.getLength()) ? xslChildren.item(i) : gsfChildren.item(i - xslChildren.getLength()));
+			if (GSXML.getNamedElementNS(mainFormat, "http://www.w3.org/1999/XSL/Transform", "variable", "name", node.getAttribute("name")) == null && GSXML.getNamedElementNS(mainFormat, "http://www.greenstone.org/greenstone3/schema/ConfigFormat", "variable", "name", node.getAttribute("name")) == null)
+			{
+				mainFormat.appendChild(node);
+			}
+		}
+
+		xslChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.XSL_NAMESPACE, "param");
+		gsfChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.GSF_NAMESPACE, "param");
+		for (int i = 0; i < xslChildren.getLength() + gsfChildren.getLength(); i++)
+		{
+			Element node = (Element) ((i < xslChildren.getLength()) ? xslChildren.item(i) : gsfChildren.item(i - xslChildren.getLength()));
+			if (GSXML.getNamedElementNS(mainFormat, "http://www.w3.org/1999/XSL/Transform", "param", "name", node.getAttribute("name")) == null && GSXML.getNamedElementNS(mainFormat, "http://www.greenstone.org/greenstone3/schema/ConfigFormat", "param", "name", node.getAttribute("name")) == null)
+			{
+				mainFormat.appendChild(node);
+			}
+		}
+
+		xslChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.XSL_NAMESPACE, "template");
+		gsfChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.GSF_NAMESPACE, "template");
+		for (int i = 0; i < xslChildren.getLength() + gsfChildren.getLength(); i++)
+		{
+			Element node = (Element) ((i < xslChildren.getLength()) ? xslChildren.item(i) : gsfChildren.item(i - xslChildren.getLength()));
+			// remove any previous occurrences of xsl:template with the same value for name or match 
+			String template_match = node.getAttribute("match");
+			String template_name = node.getAttribute("name");
+			String template_mode = node.getAttribute("mode");
+
+			String[] attributeNames = new String[] { "name", "match", "mode" };
+			String[] attributeValues = new String[] { template_name, template_match, template_mode };
+
+			if (overwrite)
+			{
+				// if we have a name attribute, remove any other similarly named template
+				GSXML.removeElementsWithAttributesNS(mainFormat, GSXML.XSL_NAMESPACE, "template", attributeNames, attributeValues);
+				GSXML.removeElementsWithAttributesNS(mainFormat, GSXML.GSF_NAMESPACE, "template", attributeNames, attributeValues);
+
+				// now add our good template in
+				mainFormat.appendChild(node);
+			}
+			else
+			{
+				// if overwrite is false, then we only add in templates if they don't match something else.
+				// In this case (eg from expanding imported stylesheets)
+				// there can't be any duplicate named templates, so just look for matches
+				// we already have the one with highest import precedence (from the top most level) so don't add any more in
+				if (GSXML.getElementsWithAttributesNS(mainFormat, "http://www.w3.org/1999/XSL/Transform", "template", attributeNames, attributeValues).getLength() == 0 && GSXML.getElementsWithAttributesNS(mainFormat, "http://www.greenstone.org/greenstone3/schema/ConfigFormat", "template", attributeNames, attributeValues).getLength() == 0)
+				{
+					mainFormat.appendChild(node);
+				}
+			}
+		}
+
+		gsfChildren = GSXML.getChildrenByTagNameNS(secondaryFormat, GSXML.GSF_NAMESPACE, "option");
+		for (int i = 0; i < gsfChildren.getLength(); i++)
+		{
+			Element node = (Element) gsfChildren.item(i);
+			if (GSXML.getNamedElementNS(mainFormat, GSXML.GSF_NAMESPACE, "option", "name", node.getAttribute("name")) == null)
+			{
+				mainFormat.appendChild(node);
+			}
+		}
+	}
+
+	public static void fixTables(Document doc)
+	{
+		NodeList debugElements = doc.getElementsByTagName("debug");
+
+		HashMap<Element, ArrayList<Element>> tracker = new HashMap<Element, ArrayList<Element>>();
+		for (int i = 0; i < debugElements.getLength(); i++)
+		{
+			Element currentElement = (Element) debugElements.item(i);
+
+			boolean hasChildElements = false;
+			NodeList children = currentElement.getChildNodes();
+			for (int j = 0; j < children.getLength(); j++)
+			{
+				Node current = children.item(j);
+				if (current instanceof Element)
+				{
+					hasChildElements = true;
+				}
+			}
+
+			if (hasChildElements && currentElement.getParentNode() != null && currentElement.getParentNode() instanceof Element)
+			{
+				Element parent = findNonDebugParent(currentElement);
+				if (parent == null)
+				{
+					continue;
+				}
+
+				if (parent.getNodeName().toLowerCase().equals("table") || parent.getNodeName().toLowerCase().equals("tr"))
+				{
+					if (tracker.get(parent) == null)
+					{
+						ArrayList<Element> debugElems = new ArrayList<Element>();
+						debugElems.add(currentElement);
+						tracker.put(parent, debugElems);
+					}
+					else
+					{
+						ArrayList<Element> debugElems = tracker.get(parent);
+						debugElems.add(currentElement);
+					}
+				}
+			}
+		}
+
+		for (Element tableElem : tracker.keySet())
+		{
+			ArrayList<Element> debugElems = tracker.get(tableElem);
+			ArrayList<String> attrNames = new ArrayList<String>();
+
+			for (Element debugElem : debugElems)
+			{
+				NamedNodeMap attributes = debugElem.getAttributes();
+				for (int i = 0; i < attributes.getLength(); i++)
+				{
+					attrNames.add(attributes.item(i).getNodeName());
+				}
+			}
+
+			for (String name : attrNames)
+			{
+				String attrValueString = "[";
+				for (int i = debugElems.size() - 1; i >= 0; i--)
+				{
+					Element current = debugElems.get(i);
+					attrValueString += "\'" + current.getAttribute(name).replace("\\", "\\\\").replace("'", "\\'") + "\'";
+					if (i != 0)
+					{
+						attrValueString += ",";
+					}
+				}
+				attrValueString += "]";
+
+				tableElem.setAttribute(name, attrValueString);
+			}
+			tableElem.setAttribute("debug", "true");
+			tableElem.setAttribute("debugSize", "" + debugElems.size());
+		}
+	}
+
+	private static Element findNonDebugParent(Element elem)
+	{
+		Node parent = elem.getParentNode();
+		while (parent instanceof Element && parent.getNodeName().equals("debug"))
+		{
+			parent = parent.getParentNode();
+		}
+
+		if (parent instanceof Element)
+		{
+			return (Element) parent;
+		}
+		return null;
+	}
+
+	public static void modifyCollectionConfigForDebug(Element coll_config_xml)
+	{
+		NodeList xslTemplates = coll_config_xml.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "template");
+		NodeList gsfTemplates = coll_config_xml.getElementsByTagNameNS(GSXML.GSF_NAMESPACE, "template");
+
+		for (int i = 0; i < xslTemplates.getLength() + gsfTemplates.getLength(); i++)
+		{
+			Element currentTemplate = (Element) ((i < xslTemplates.getLength()) ? xslTemplates.item(i) : gsfTemplates.item(i - xslTemplates.getLength()));
+			Element temp = currentTemplate;
+			String xPath = "";
+			while (!temp.getNodeName().toLowerCase().equals("collectionconfig"))
+			{
+				temp = (Element) temp.getParentNode();
+				String nodeName = temp.getNodeName();
+
+				int count = 1;
+				Node counter = temp.getPreviousSibling();
+				while (counter != null)
+				{
+					if (counter.getNodeType() == Node.ELEMENT_NODE && ((Element) counter).getNodeName().equals(nodeName))
+					{
+						count++;
+					}
+					counter = counter.getPreviousSibling();
+				}
+				xPath = nodeName + ((count > 1) ? ("[" + count + "]") : "") + "/" + xPath;
+			}
+
+			xPath = xPath.substring(0, xPath.length() - 1);
+			currentTemplate.setUserData("xpath", xPath, new DataTransferHandler());
+		}
+	}
+
+	static class DataTransferHandler implements UserDataHandler
+	{
+		public void handle(short operation, String key, Object data, Node src, Node dst)
+		{
+			if (operation == NODE_IMPORTED || operation == NODE_CLONED)
+			{
+				//Thread.dumpStack();
+				dst.setUserData(key, data, new DataTransferHandler());
+			}
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/JDBMWrapper.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/JDBMWrapper.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/JDBMWrapper.java	(revision 31634)
@@ -0,0 +1,280 @@
+/*
+ *    JDBMWrapper.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.apache.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+
+import java.util.Properties;
+import java.util.ArrayList;
+
+import jdbm.RecordManager;
+import jdbm.RecordManagerFactory;
+import jdbm.helper.FastIterator;
+import jdbm.htree.HTree;
+
+public class JDBMWrapper implements FlatDatabaseWrapper
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.JDBMWrapper.class.getName());
+
+	static String TNAME = "greenstone";
+
+	RecordManager recman_ = null;
+	HTree hashtable_ = null;
+
+	String db_filename_;
+
+	static private PrintWriter utf8out = null;
+
+	static
+	{
+		try
+		{
+			OutputStreamWriter osw = new OutputStreamWriter(System.out, "UTF-8");
+			utf8out = new PrintWriter(osw, true);
+		}
+		catch (UnsupportedEncodingException e)
+		{
+			System.out.println(e);
+		}
+		// Register this wrapper with DBHelper
+		DBHelper.registerDBTypeExt("jdbm",".jdb");
+	}
+
+	/** open database named filename, using mode */
+	public boolean openDatabase(String db_filename, int mode)
+	{
+	        String db_filename_with_ext = db_filename;
+
+		if (db_filename.endsWith(".jdb"))
+		{
+			// remove file extension as JDBM does not expect it
+			db_filename = db_filename.substring(0, db_filename.length() - 4);
+		}
+
+		// Map the mode value into equivalent JDBM 'must_exist' value
+		// currently this is very simple as there is only READ and WRITE
+		// (no WRITER_CREATE)
+		// => assume the database must exist
+		boolean must_exist = true; // default
+
+		if (recman_ != null)
+		{
+			String message = "openDatabase() called when the class already has a database open\n";
+			message += "  Use closeDatabase before opening the next one.\n";
+			message += "  Existing database file: " + db_filename_ + "\n";
+			message += "  New database file:      " + db_filename + "\n";
+			logger.warn(message);
+			// consider closing it automatically?
+		}
+
+		try
+		{
+			// create or open a record manager
+			Properties props = new Properties();
+			recman_ = RecordManagerFactory.createRecordManager(db_filename, props);
+			
+			// load existing table (if exists) otherwise create new one
+			long recid = recman_.getNamedObject(TNAME);
+
+			if (recid != 0)
+			{
+				logger.info("# Loading existing database table '" + TNAME + "' ...");
+				hashtable_ = HTree.load(recman_, recid);
+			}
+			else
+			{
+				if (must_exist)
+				{
+					recman_.close();
+					recman_ = null;
+					db_filename_ = null;
+
+					logger.error("Database table '" + TNAME + "' does not exist.");
+					throw new IOException();
+				}
+				else
+				{
+					logger.info("# No database table '" + TNAME + "' to set.  Creating new one");
+					hashtable_ = HTree.createInstance(recman_);
+					recman_.setNamedObject(TNAME, hashtable_.getRecid());
+				}
+			}
+		}
+		catch (IOException e)
+		{
+		    e.printStackTrace();
+
+			logger.error("couldn't open database " + db_filename_with_ext);
+			return false;
+		}
+
+		db_filename_ = db_filename;
+
+		return true;
+	}
+
+	/** close the database associated with this wrapper */
+	public void closeDatabase()
+	{
+		try
+		{
+			if (recman_ != null)
+			{
+				recman_.close();
+				recman_ = null;
+				db_filename_ = null;
+			}
+		}
+		catch (IOException e)
+		{
+			logger.error("Failed to close JDBM database");
+		}
+	}
+
+	/** returns the value associated with the key */
+	public String getValue(String key)
+	{
+		if (hashtable_ == null) {
+			logger.error("Database was not opened or not exist!");
+			return null;
+		}
+		
+		String val;
+
+		try
+		{
+			val = (String) hashtable_.get(key);
+			
+			recman_.commit();
+		}
+		catch (IOException e)
+		{
+			logger.error("Failed get key " + key + "from JDBM database");
+			return null;
+		}
+		
+		return val;
+	}
+	
+	/**
+	 * Sets the given key to the given value in the database
+	 */
+	public boolean setValue(String key, String value)
+	{
+		try
+		{
+			hashtable_.put(key, value);
+			recman_.commit();
+		}
+		catch (Exception ex)
+		{
+			logger.error("Failed to set " + key + " = " + value + " in the JDBM database");
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Deletes the given key (and corresponding value) from the database
+	 */
+	public boolean deleteKey(String key)
+	{
+		try
+		{
+			hashtable_.remove(key);
+			recman_.commit();
+		}
+		catch (Exception ex)
+		{
+			logger.error("Failed to delete key " + key + " in the JDBM database");
+			return false;
+		}
+		return true;
+	}
+
+    /** Returns all the keys of the database as Strings */
+    public ArrayList<String> getAllEntryKeys() {
+	
+	ArrayList<String> keys = new ArrayList<String>();
+	
+	try {
+	    FastIterator iter = hashtable_.keys();
+	    
+	    String key = (String) iter.next();
+	    
+	    while (key != null) {
+		keys.add(key);
+		key = (String) iter.next();
+	    }
+	    
+	    recman_.commit();
+	} catch (IOException e) {
+	    logger.error("Failed to get all keys from JDBM database");
+	    return null;
+	}
+	
+	return keys;
+    }
+    
+
+	/**
+	 * returns a string of key-value entries that can be printed for debugging
+	 * purposes
+	 */
+	public String displayAllEntries()
+	{
+
+		StringBuffer keys = new StringBuffer();
+
+		try
+		{
+			FastIterator iter = hashtable_.keys();
+			String key = (String) iter.next();
+
+			String nl = System.getProperty("line.separator");
+
+			while (key != null)
+			{
+				String val = (String) hashtable_.get(key);
+				keys.append("[" + key + "]" + nl);
+				keys.append(val + nl);
+				// 70 hypens
+				keys.append("----------------------------------------------------------------------" + nl);
+				key = (String) iter.next();
+			}
+
+			recman_.commit();
+		}
+		catch (IOException e)
+		{
+			logger.error("Failed to get all keys and values from JDBM database");
+			return null;
+		}
+
+		return keys.toString();
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MacroResolver.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MacroResolver.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MacroResolver.java	(revision 31634)
@@ -0,0 +1,146 @@
+/*
+ *    MacroResolver.java
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+
+abstract public class MacroResolver {
+
+    public static final String SCOPE_TEXT = "text";
+    public static final String SCOPE_META = "metadata";
+    public static final String SCOPE_ALL = "all";
+
+    public static final int TYPE_TEXT = 0;
+    public static final int TYPE_META = 1;
+    public static final int TYPE_DICT = 2;
+    
+    protected ArrayList<Macro> text_macros = null;
+    protected ArrayList<Macro> metadata_macros = null;
+
+    String lang = null;
+
+    public MacroResolver() {
+    	this.text_macros = new ArrayList<Macro>();
+	this.metadata_macros = new ArrayList<Macro>();
+    }
+
+    
+    public void setSiteDetails(String site_address, String cluster_name, String library_name) {
+	if (site_address != null) {
+	    addMacro(TYPE_TEXT, "_httpsite_", site_address, SCOPE_ALL, false);
+	}
+	if (cluster_name != null) {
+	    addMacro(TYPE_TEXT, "_clustername_", cluster_name, SCOPE_ALL, false);
+	}
+	if (library_name!=null){
+	    addMacro(TYPE_TEXT, "_libraryname_", library_name, SCOPE_ALL, false);
+	}
+    }
+
+    public void addMacro(int type, String macro, String text_or_metadata, String scope, boolean resolve) {
+	Macro m = new Macro();
+	m.type = type;
+	m.macro = macro;
+	m.text = text_or_metadata;
+	m.resolve = resolve;
+	addMacro(m, scope);
+    }
+
+    public void addMacro(int type, String macro, String bundle, String key, String scope, boolean resolve) {
+	Macro m = new Macro();
+	m.type = type;
+	m.macro = macro;
+	m.bundle = bundle;
+	m.key = key;
+	m.resolve = resolve;
+	addMacro(m, scope);
+    }
+    
+    public void addMacros(Element replace_list_elem) {
+	NodeList replaces = replace_list_elem.getElementsByTagName(GSXML.REPLACE_ELEM);
+	for (int i=0; i<replaces.getLength(); i++) {
+	    Element e = (Element)replaces.item(i);
+	    String scope = e.getAttribute("scope");
+	    if (scope.equals("")) {
+		scope = SCOPE_ALL;
+	    }
+	    boolean resolve = true;
+	    String resolve_str = e.getAttribute("resolve");
+	    if (resolve_str.equals("false")) {
+		resolve = false;
+	    }
+	    String from = e.getAttribute("macro");
+	    String to = e.getAttribute("text");
+	    if (!to.equals("")) {
+		addMacro(TYPE_TEXT, from, to, scope, resolve);
+	    } else {
+		String meta = e.getAttribute("metadata");
+		if (!meta.equals("")) {
+		    addMacro(TYPE_META, from, meta, scope, resolve);
+		} else {
+		    String key = e.getAttribute("key");
+		    String bundle = e.getAttribute("bundle");
+		    addMacro(TYPE_DICT, from, bundle, key, scope, resolve);
+		}
+	    } 
+	}
+	
+    }
+
+    protected void addMacro(Macro m, String scope) {
+
+	if (scope.equals(SCOPE_TEXT)) {
+	    this.text_macros.add(m);
+	}
+	else if (scope.equals(SCOPE_META)) {
+	    this.metadata_macros.add(m);
+	}
+	else if (scope.equals(SCOPE_ALL)) {
+	    this.text_macros.add(m);
+	    this.metadata_macros.add(m);
+	    
+	}
+
+    }
+    
+    abstract public String resolve(String text, String lang, String scope, 
+				   String doc_oid);
+    
+    protected class Macro {
+	
+	/** type of replacement: TEXT, METADATA, DICTIONARY */
+	public int type;
+	/** the macro to replace */
+	public String macro = null;
+	/** If text type, holds the text to replace with. If metadata type, holds the metadata name */
+	public String text = null; 
+	/** If dictionary type, holds the key to look up in the dictionary */
+	public String key = null;
+	/** If dictionary type, holds the resource bundle name */
+	public String bundle = null;
+	/** If true, will try to resolve macros in the replacement text */
+	public boolean resolve = true;
+	
+    }
+	
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ModifyUsersDB.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ModifyUsersDB.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ModifyUsersDB.java	(revision 31634)
@@ -0,0 +1,162 @@
+/*
+ *    txt2usersDB.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.sql.SQLException;
+import org.greenstone.gsdl3.service.Authentication;
+//import org.greenstone.admin.guiext.PropertiesStep;
+
+/** 
+    To run this from the command-line, first make sure that the networked derby server is running (ant start-derby),
+    then run:
+
+    java -Dgsdl3.writablehome=/Scratch/ak19/gs3-svn-2Sep2015/web -cp web/WEB-INF/lib/gsdl3.jar:web/WEB-INF/lib/gutil.jar:web/WEB-INF/lib/derby.jar:web/WEB-INF/lib/derbyclient.jar:web/WEB-INF/lib/log4j-1.2.8.jar:web/WEB-INF/lib/commons-codec-1.7.jar:web/WEB-INF/classes org.greenstone.gsdl3.util.ModifyUsersDB web/etc/usersDB/ <username> [options specifying user fields to change, e.g.: password=me!]
+    
+    Don't forget to stop the networked derby server again at the end, if you had started it: ant stop-derby
+
+    or if using embedded derby, ensure that tomcat is stopped, then run:
+    java -cp /full/path/to/GS3/web/WEB-INF/lib/gsdl3.jar:/full/path/to/GS3/web/WEB-INF/lib/derby.jar org.greenstone.gsdl3.util.ModifyUsersDB web/etc/usersDB/
+*/ 
+public class ModifyUsersDB
+{
+    public static int PWD_MIN_LENGTH = 3;
+    public static int PWD_MAX_LENGTH = 20;
+
+	public static void main(String[] args) throws SQLException
+	{
+
+		if (args.length < 3)
+		{ // at minimum one field belonging to a username has to be changed
+			System.out.println("Usage: java org.greenstone.gsdl3.ModifyUsersDB <full_path_of_the_usersDB> <username> [-noAdd] [password=pwd] [groups=grp] [accounstatus=status] [comment=cmt] [email=address]");
+			System.exit(0);
+		}
+
+		String usersDB = args[0];
+		String username = args[1];
+
+		String password = "";
+		String groups = "";
+		String accountstatus = "true";
+		String comment = "";
+		String email = "";
+
+		boolean noAdd = false;
+
+		// If the user specifically sets any of the fields on the cmdline, they'll be overwritten in the db, 
+		// even if the user had set them to empty. Except the password which must be between PWD_MIN_LENGTH and PWD_MAX_LENGTH characters.
+		for (int i = 2; i < args.length; i++)
+		{
+			if (args[i].startsWith("password="))
+			{
+				password = args[i].substring("password=".length());
+
+				if (password.length() < PWD_MIN_LENGTH || password.length() > PWD_MAX_LENGTH)
+				{
+					if (!password.equals(""))
+					{
+						System.out.println("Password not updated. It should be between " + PWD_MIN_LENGTH + " and " + PWD_MAX_LENGTH + " characters (inclusive).");
+					}
+				}
+				else
+				{
+					// Use the same encryption technique used by the Admin Authentication page
+					// This ensures that the password generated for a string remains consistent
+					//System.err.println("**** Password entered was: " + password);
+					password = Authentication.hashPassword(password);
+				}
+
+			}
+			else if (args[i].startsWith("groups="))
+			{
+				groups = args[i].substring("groups=".length());
+			}
+			else if (args[i].startsWith("accountstatus="))
+			{
+				accountstatus = args[i].substring("accountstatus=".length());
+			}
+			else if (args[i].startsWith("status="))
+			{
+				accountstatus = args[i].substring("status=".length());
+			}
+			else if (args[i].startsWith("comment="))
+			{
+				comment = args[i].substring("comment=".length());
+			}
+			else if (args[i].startsWith("email="))
+			{
+				email = args[i].substring("email=".length());
+			}
+			else if (args[i].equals("-noAdd"))
+			{
+				noAdd = true;
+			}
+		}
+
+		// find the user to modify
+		DerbyWrapper dw = new DerbyWrapper(usersDB);
+		UserQueryResult findUserResult = dw.findUser(username);
+
+		if (findUserResult == null)
+		{
+			if (noAdd)
+			{
+				System.out.println("Failed to update user. Cannot find user " + username + " in " + usersDB + " database.");
+			}
+			else
+			{ // add new user
+				//System.err.println("**** Trying to add user: ");
+				//System.err.println("**** " + username + " " + password + " " + groups + " " + accountstatus + " " + comment + " " + email);
+				dw.addUser(username, password, groups, accountstatus, comment, email);
+			}
+		}
+		else
+		{ // modify existing user data
+
+			// in case any of the other fields are not specified, get fallbacks from the database
+			UserTermInfo user = findUserResult.getUserTerms().get(0);
+
+			if (password.equals(""))
+			{
+				password = user.password; // already stored hashed-and-hexed in DB
+			}
+			if (groups.equals(""))
+			{
+				groups = user.groups;
+			}
+			if (accountstatus.equals(""))
+			{
+				accountstatus = user.accountstatus.equals("") ? "true" : user.accountstatus;
+			}
+			if (comment.equals(""))
+			{
+				comment = user.comment;
+			}
+			if (email.equals(""))
+			{
+				email = user.email;
+			}
+
+			//System.err.println("**** " + username + " " + password + " " + groups + " " + accountstatus + " " + comment + " " + email);
+			dw.modifyUserInfo(username, password, groups, accountstatus, comment, email); // other than username and pwd, remaining fields are allowed to be ""
+		}
+
+		dw.closeDatabase();
+
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MyNodeList.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MyNodeList.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/MyNodeList.java	(revision 31634)
@@ -0,0 +1,61 @@
+/*
+ *    MyNodeList.java
+ *   
+
+ *    Copyright (C) 2005 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.ArrayList;
+
+public class MyNodeList implements NodeList
+{
+	private ArrayList<Node> _nodeList = null;
+
+	public MyNodeList()
+	{
+		_nodeList = new ArrayList<Node>();
+	}
+
+	public int getLength()
+	{
+		return _nodeList.size();
+	}
+
+	public Node item(int i)
+	{
+		if (i < 0 || i >= _nodeList.size())
+			return null;
+		return _nodeList.get(i);
+	}
+
+	public void addNode(Node n)
+	{
+		_nodeList.add(n);
+	}
+
+	public void addNodeList(NodeList nl)
+	{
+		for(int i = 0; i < nl.getLength(); i++)
+		{
+			_nodeList.add(nl.item(i));
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIResumptionToken.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIResumptionToken.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIResumptionToken.java	(revision 31634)
@@ -0,0 +1,320 @@
+/*
+ *    OAIResumptionToken.java
+ *    Copyright (C) 2014 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.util.GlobalProperties;
+
+import org.w3c.dom.*;
+
+import java.io.File;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+
+// import file Logger.java
+import org.apache.log4j.*;
+
+/** */
+public class OAIResumptionToken {
+  
+
+
+  // Other fields we save
+  public static final String CURRENT_SET = "current_set";
+  public static final String CURRENT_CURSOR = "current_cursor";
+
+  // for token XML
+  public static final String TOKEN = "token";
+  public static final String TOKEN_LIST = "tokenList";
+  public static final String NAME = "name";
+  public static final String EXPIRATION = "expiration";
+  public static final String INITIAL_TIME = "initial";
+  //public static final String FROM
+  public static final String FILE_SEPARATOR = File.separator;
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSXML.class.getName());
+
+  public static XMLConverter converter = new XMLConverter();
+  public static Element resumption_token_elem = null;
+  //used when saving the token file
+  public static File resumption_token_file = null;
+
+  private static HashMap<String, HashMap<String,String>> stored_tokens = new HashMap<String, HashMap<String,String>>();
+
+  private static HashMap<String, Long> expiration_data = new HashMap<String, Long>();
+
+  /** initialize stored resumption tokens and clear any expired ones*/
+  public static void init() {
+    if (!findOrCreateTokenFile()) {
+      // can't read in old tokens
+      logger.error("Can't find token file, so we can't initialise tokens");
+      return;
+    }
+
+    Document token_doc = converter.getDOM(resumption_token_file, "utf-8");
+    Element token_elem = null;
+    if (token_doc != null) {
+      token_elem = token_doc.getDocumentElement();
+    } else {
+      logger.error("Failed to parse oai resumption token file OAIResumptionToken.xml.");
+      resetTokenFile();
+      return;
+    }
+      
+    // read in stored tokens
+    NodeList tokens = token_elem.getElementsByTagName(TOKEN);
+    for(int i=0; i<tokens.getLength(); i++) {
+      Element token = (Element)tokens.item(i);
+      String set = token.getAttribute(OAIXML.SET);
+      if (set.isEmpty()) {
+	set = null;
+      }
+      String meta_prefix = token.getAttribute(OAIXML.METADATA_PREFIX);
+      if (meta_prefix.isEmpty()) {
+	meta_prefix = null;
+      }
+      String from = token.getAttribute(OAIXML.FROM);
+      if (from.isEmpty()) {
+	from = null;
+      }
+      String until = token.getAttribute(OAIXML.UNTIL);
+      if (until.isEmpty()) {
+	until = null;
+      }
+      storeToken(token.getAttribute(NAME), set, meta_prefix, from, until, token.getAttribute(EXPIRATION));
+
+    }
+  }
+  // store the token read in from saved tokens file
+  protected static void storeToken(String name, String set, String metadata_prefix, String from, String until, String expiration) {
+    HashMap<String, String> data = new HashMap<String, String>();
+    data.put(OAIXML.FROM, from);
+    data.put(OAIXML.UNTIL, until);
+    data.put(OAIXML.SET, set);
+    data.put(OAIXML.METADATA_PREFIX, metadata_prefix);
+    stored_tokens.put(name, data);
+    expiration_data.put(name, new Long(expiration));
+  }
+
+  /** generate the token, and store all the data for it */
+  public static String createAndStoreResumptionToken(String set, String metadata_prefix, String from, String until, String cursor, String current_set, String current_cursor) {
+    HashMap<String, String> data = new HashMap<String, String>();
+    long expiration_date = System.currentTimeMillis();
+    String base_token_name = ""+expiration_date;
+    stored_tokens.put(base_token_name, data);   
+    expiration_date += (OAIXML.getTokenExpiration());
+    expiration_data.put(base_token_name, new Long(expiration_date));
+    String token_name = base_token_name+":" + cursor + ":" + current_set +":"+ current_cursor;
+    data.put(OAIXML.FROM, from);
+    data.put(OAIXML.UNTIL, until);
+    data.put(OAIXML.SET, set);
+    data.put(OAIXML.METADATA_PREFIX, metadata_prefix);
+    return token_name;
+
+  }
+
+  public static long getExpirationDate(String token) {
+    if (token.indexOf(":") != -1) {
+      token = token.substring(0, token.indexOf(":"));
+    }
+    return expiration_data.get(token).longValue();
+  }
+
+  public static String updateToken(String token, String cursor, String current_set, String current_cursor) {
+    if (token.indexOf(":") != -1) {
+      token = token.substring(0, token.indexOf(":"));
+    }
+
+    // when we are generating a new token, we update the expiration date to allow the new token to have the full length of time
+    long exp_date = System.currentTimeMillis();
+    exp_date += (OAIXML.getTokenExpiration());
+    expiration_data.put(token, exp_date);
+    
+    token = token + ":" + cursor + ":" + current_set + ":" + current_cursor;
+    return token;
+  }
+
+  /** check that this token is currently valid */
+  public static boolean isValidToken(String token) {
+    // we clear expired tokens each time we check one.
+    clearExpiredTokens();
+    // take off the set/cursor parts to check for the main key
+    if (token.indexOf(":") != -1) {
+      token = token.substring(0, token.indexOf(":"));
+      logger.error("looking up "+token);
+    }
+    if (stored_tokens.containsKey(token)) {
+      return true;
+    }
+    return false;
+  }
+
+  public static HashMap<String, String> getTokenData(String token) {
+    // find the base name
+    String base_name = token;
+    if (token.indexOf(":") != -1) {
+      base_name = token.substring(0, token.indexOf(":"));
+      logger.error("getting data for "+base_name);
+    }
+    HashMap<String, String> data = new HashMap<String, String>(stored_tokens.get(base_name));
+    if (data == null) { 
+      logger.error("data was null!!");
+      return null;
+    }
+    // add in cursor, etc from the token name    
+    if (base_name != token) {
+      String[] parts = token.split(":");
+      if (parts.length != 4) {
+	// something wrong!!
+      }
+      data.put(OAIXML.CURSOR, parts[1]);
+      data.put(CURRENT_SET, parts[2]);
+      data.put(CURRENT_CURSOR, parts[3]);
+      data.put(INITIAL_TIME, base_name);
+    }
+    return data;
+
+  }
+
+  // used to manually expire a particular token - eg when one of its collections has changed since token was issued.
+  public static void expireToken(String token) {
+    if (token.indexOf(":") != -1) {
+      token = token.substring(0, token.indexOf(":"));
+    }
+    expiration_data.remove(token);
+    stored_tokens.remove(token);
+  }
+  // read through all stored expiry dates, and delete any tokens that are too
+  // old
+  public static void clearExpiredTokens() {
+
+    Set<Map.Entry<String,Long>> token_set = expiration_data.entrySet();
+    Iterator<Map.Entry<String,Long>> i = token_set.iterator();
+    int size = expiration_data.size();
+    logger.error("start tokens "+size);
+    Long time_now = System.currentTimeMillis();
+    while (i.hasNext()==true) {
+      Map.Entry<String,Long> entry = i.next();
+      String key = entry.getKey();
+      Long exp = entry.getValue();
+      if (exp < time_now) {
+	logger.error("token "+key+" is expired, "+ OAIXML.getTime(exp));
+	i.remove();
+	// also remove the token from the stored tokens
+	stored_tokens.remove(key);
+      }
+    }
+    size = expiration_data.size();
+    logger.error("end tokens "+size);
+  }
+
+  protected static boolean findOrCreateTokenFile() {
+    
+    try {
+      URL token_file_url = Class.forName("org.greenstone.gsdl3.OAIServer").getClassLoader().getResource("OAIResumptionToken.xml");
+      if (token_file_url != null) {
+	resumption_token_file = new File(token_file_url.toURI());
+	if (resumption_token_file != null && resumption_token_file.exists()) {
+	  return true;
+	}
+	else {
+	  logger.error("Resumption token file found, "+ token_file_url.toURI()+" but couldn't create a file from it.");
+	}
+      }
+    } catch (Exception e) {
+      logger.error("couldn't find or ResumptionToken.xml "+e.getMessage());
+    }
+    // if we have got here, have't managed to load file via class loader -
+    // it may not exist yet.
+    // try and create a new empty one
+    if (resetTokenFile()) {
+      return true;
+    }
+    return false;
+  }
+
+  // If there was no file, or something went wrong with the file, use this to
+  // create a new one.
+  public static boolean resetTokenFile() {
+    resumption_token_file = new File(GlobalProperties.getGSDL3Home() + 
+				     FILE_SEPARATOR + "WEB-INF" + 
+				     FILE_SEPARATOR + "classes" +
+				     FILE_SEPARATOR + "OAIResumptionToken.xml");
+    try {
+      if (!resumption_token_file.exists()) {
+
+	if (!resumption_token_file.createNewFile()) {
+	  logger.error("Couldn't create a new resumption token file "+ resumption_token_file.getPath());
+	  resumption_token_file = null;
+	  return false;
+	}
+      }
+    } catch (Exception e) {
+      logger.error("Couldn't create a new resumption token file "+ resumption_token_file.getPath()+", "+e.getMessage());
+      resumption_token_file = null;
+      return false;
+    }
+    Document doc = converter.newDOM();
+    Element elem = doc.createElement(TOKEN_LIST);
+    if (converter.writeDOM(elem, resumption_token_file)) {
+      return true;
+    }
+    resumption_token_file = null;
+    return false;
+  }
+  
+
+  public static boolean saveTokensToFile() {
+    clearExpiredTokens();
+    if (resumption_token_file == null) {
+      logger.error("no available resumption token file, not storing tokens");
+      return false;
+    }
+    
+    // Create an XML representation of the stored tokens
+    Document doc = converter.newDOM();
+    Element token_list = doc.createElement(TOKEN_LIST);
+
+    if (stored_tokens.size() == 0) {
+      // nothing to store, store an empty file
+      converter.writeDOM(token_list, resumption_token_file);
+      return true;
+    }
+    Set<String> keys = stored_tokens.keySet();
+    Iterator<String> i = keys.iterator();
+    while(i.hasNext()) {
+      String token = i.next();
+      HashMap<String, String> data = stored_tokens.get(token);
+      Element token_elem = doc.createElement(TOKEN);
+      token_elem.setAttribute(NAME, token);
+      token_elem.setAttribute(OAIXML.FROM, data.get(OAIXML.FROM));
+      token_elem.setAttribute(OAIXML.UNTIL, data.get(OAIXML.UNTIL));
+      token_elem.setAttribute(OAIXML.SET, data.get(OAIXML.SET));
+      token_elem.setAttribute(OAIXML.METADATA_PREFIX, data.get(OAIXML.METADATA_PREFIX));
+      token_elem.setAttribute(EXPIRATION, ""+expiration_data.get(token));
+      token_list.appendChild(token_elem);
+    }
+    converter.writeDOM(token_list, resumption_token_file);
+    return true;
+  }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIXML.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIXML.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OAIXML.java	(revision 31634)
@@ -0,0 +1,628 @@
+/*
+ *    OAIXML.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.greenstone.util.GlobalProperties;
+
+import org.w3c.dom.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+// import file Logger.java
+import org.apache.log4j.*;
+
+/** these constants are used for the OAI service */
+public class OAIXML {
+  
+  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.GSXML.class.getName());
+
+  // the leading keyword of oai protocol
+  public static final String VERB = "verb";
+
+  // six valid oai verbs
+  public static final String GET_RECORD = "GetRecord";
+  public static final String LIST_RECORDS = "ListRecords";
+  public static final String LIST_IDENTIFIERS = "ListIdentifiers";
+  public static final String LIST_SETS = "ListSets";
+  public static final String LIST_METADATA_FORMATS = "ListMetadataFormats";
+  public static final String IDENTIFY = "Identify";
+
+  // oai request parameters
+  public static final String METADATA_PREFIX = "metadataPrefix";
+  public static final String FROM = "from";
+  public static final String UNTIL = "until";
+  public static final String SET = "set";
+  public static final String RESUMPTION_TOKEN = "resumptionToken";
+  public static final String IDENTIFIER = "identifier";
+
+  // Error element and code att
+  public static final String ERROR = "error";
+  public static final String CODE = "code";
+  
+  // OAI error codes
+  public static final String BAD_ARGUMENT = "badArgument";
+  public static final String BAD_RESUMPTION_TOKEN = "badResumptionToken";
+  public static final String BAD_VERB = "badVerb";
+  public static final String CANNOT_DISSEMINATE_FORMAT = "cannotDisseminateFormat";
+  public static final String ID_DOES_NOT_EXIST = "idDoesNotExist";
+  public static final String NO_METADATA_FORMATS = "noMetadataFormats";
+  public static final String NO_RECORDS_MATCH = "noRecordsMatch";
+  public static final String NO_SET_HIERARCHY = "noSetHierarchy";
+  
+
+  // words used to compose oai responses
+  // many of these used in OAIConfig too
+
+  // General
+  public static final String OAI_PMH = "OAI-PMH";
+  public static final String RESPONSE_DATE = "responseDate";
+  public static final String REQUEST = "request";
+  
+  // Identify data
+  public static final String ADMIN_EMAIL = "adminEmail";
+  public static final String BASE_URL = "baseURL";
+  public static final String COMPRESSION = "compression";
+  public static final String DELETED_RECORD = "deletedRecord";
+  public static final String DESCRIPTION = "description";
+  public static final String EARLIEST_DATESTAMP = "earliestDatestamp";
+  public static final String GRANULARITY = "granularity";
+  public static final String LAST_MODIFIED = "lastmodified";
+  public static final String PROTOCOL_VERSION = "protocolVersion";
+  public static final String REPOSITORY_NAME = "repositoryName";
+  public static final String OAI_IDENTIFIER = "oai-identifier";
+  public static final String SCHEME = "scheme";
+  public static final String REPOSITORY_IDENTIFIER = "repositoryIdentifier";
+  public static final String DELIMITER = "delimiter";
+  public static final String SAMPLE_IDENTIFIER = "sampleIdentifier";
+
+  // metadata formats
+  public static final String METADATA_FORMAT = "metadataFormat";
+  public static final String SCHEMA = "schema";
+  public static final String METADATA_NAMESPACE = "metadataNamespace";
+  public static final String META_FORMAT_DC = "oai_dc";
+
+  // record response data
+  // SET_SPEC
+  public static final String RECORD = "record";
+  public static final String HEADER = "header";
+  public static final String DATESTAMP = "datestamp";
+  public static final String METADATA = "metadata";
+
+  // list sets
+  // SET, 
+  public static final String SET_NAME = "setName";
+  public static final String SET_SPEC = "setSpec";
+  public static final String SET_DESCRIPTION = "setDescription";
+
+  // resumption token element
+  public static final String RESUMPTION_TOKEN_ELEM = "resumptionToken";
+  public static final String EXPIRATION_DATE = "expirationDate";
+  public static final String COMPLETE_LIST_SIZE = "completeListSize";
+  public static final String CURSOR = "cursor";
+  
+  // extra elements/attributes from OAIConfig
+  public static final String OAI_INFO = "oaiInfo";
+  public static final String USE_STYLESHEET = "useOAIStylesheet";
+  public static final String STYLESHEET = "OAIStylesheet";
+  public static final String RESUME_AFTER = "resumeAfter";
+  public static final String RESUMPTION_TOKEN_EXPIRATION = "resumptionTokenExpiration";
+  public static final String OAI_SUPER_SET = "oaiSuperSet";
+  public static final String ELEMENT = "element";
+  public static final String ELEMENTS = "elements";
+  public static final String MAPPING = "mapping";
+  public static final String SELECT = "select";
+  public static final String SELECT_SINGLE_VALUE = "firstvalue";
+  public static final String SELECT_FIRST_VALID_META = "firstvalidmetadata";
+  public static final String SELECT_ALL_VALUES = "allvalues";
+  // code constants
+  public static final String GS_OAI_DATE_STAMP = "gs.OAIDateStamp";
+   public static final String GS_OAI_RESOURCE_URL = "gs.OAIResourceURL";
+   public static final String ILLEGAL_OAI_VERB = "Illegal OAI verb";
+   public static final String LASTMODIFIED = "lastmodified";
+  // // The node id in the collection database, which contains all the OIDs in the database
+   public static final String BROWSELIST = "browselist";
+   public static final String OAI_LASTMODIFIED = "oailastmodified";
+   public static final String OAIPMH = "OAIPMH";
+   public static final String OAI_SET_LIST = "oaiSetList";
+   public static final String OAI_SERVICE_UNAVAILABLE = "OAI service unavailable";
+   public static final String OID = "OID";
+
+    // The node id in the oai-inf database of the collection, which should contain all the OIDs in the db
+    public static final String OAI_INF_STATUS = "status"; // D = Deleted; E = Existing (PD = Provisionally Deleted but PD should not be present at this stage in the oai-inf database)
+    public static final String OAI_INF_TIMESTAMP = "timestamp"; // the time of deletion/last (re)indexing of doc
+    public static final String OAI_INF_DATESTAMP = "datestamp"; // date version of timestamp    
+    // oai-inf db values for OAI_INF_STATUS
+    public static final String OAI_INF_DELETED = "D";
+    public static final String OAI_INF_EXISTS = "E";
+    public static final String OAI_INF_PROVISIONALLY_DELETED = "PD";
+    // header values for ListRecords/GetRecord and listIdentifiers
+    public static final String HEADER_STATUS_ATTR_DELETED = "deleted";
+
+  public static final String OAI_SERVICE_RACK = "OAIPMH";
+  //system-dependent file separator, maybe '/' or '\'
+  public static final String FILE_SEPARATOR = File.separator;
+  public static final String OAI_VERSION1 = "1.0";
+  public static final String OAI_VERSION2 = "2.0";
+  /*************************above are final values****************************/
+    
+    
+  //initialized in getOAIConfigXML()
+  public static Element oai_config_elem = null;
+    
+  //stores the date format "yyyy-MM-ddTHH:mm:ssZ"
+  // this is the granularity for datestamps
+  public static String granularity = "";
+
+  // http://www.openarchives.org/OAI/openarchivesprotocol.html#DatestampsRequests 
+  // specifies that all repositories must support YYYY-MM-DD (yyyy-MM-dd in Java)
+  // this would be in addition to the other (optional) granularity of above that 
+  // a repository may additionally choose to support.
+  public static final String default_granularity = "yyyy-MM-dd";
+
+  public static long token_expiration = 7200;
+  /** which version of oai that this oaiserver supports; default is 2.0 
+   *  initialized in getOAIConfigXML()
+   */
+  public static String oai_version = "2.0";
+  public static String baseURL = "";
+    
+  /** Converter for parsing files and creating Elements */
+  public static XMLConverter converter = new XMLConverter();
+  
+  public static String[] special_char = {"/", "?", "#", "=", "&", ":", ";", " ", "%", "+"};
+  public static String[] escape_sequence = {"%2F", "%3F", "%23", "%3D", "%26", "%3A", "%3B", "%20", "%25", "%2B"};
+
+  public static String getOAIVersion() {
+    return oai_version;
+  }
+	
+  public static String getBaseURL() {
+    return baseURL;
+  }
+	
+  /** Read in OAIConfig.xml (residing web/WEB-INF/classes/) and use it to configure the receptionist etc.
+   *  the oai_version and baseURL variables are also set in here. 
+   *  The init() method is also called in here. */ 
+  public static Element getOAIConfigXML() {
+     
+    File oai_config_file = null;
+
+    try {
+      URL oai_config_url = Class.forName("org.greenstone.gsdl3.OAIServer").getClassLoader().getResource("OAIConfig.xml");
+      if (oai_config_url == null) {
+	logger.error("couldn't find OAIConfig.xml via class loader");
+	return null;
+      }
+      oai_config_file = new File(oai_config_url.toURI());
+      if (!oai_config_file.exists()) {
+        logger.error(" oai config file: "+oai_config_file.getPath()+" not found!");
+        return null;
+      }
+    } catch(Exception e) {
+      logger.error("couldn't find OAIConfig.xml "+e.getMessage());
+      return null;
+    }
+
+    Document oai_config_doc = converter.getDOM(oai_config_file, "utf-8");
+    if (oai_config_doc != null) {
+      oai_config_elem = oai_config_doc.getDocumentElement();
+    } else {
+      logger.error("Failed to parse oai config file OAIConfig.xml.");
+      return null;
+    }
+      
+    //initialize oai_version
+    Element protocol_version = (Element)GSXML.getChildByTagName(oai_config_elem, PROTOCOL_VERSION);
+    oai_version = GSXML.getNodeText(protocol_version).trim();
+
+    // initialize baseURL
+    Element base_url_elem = (Element)GSXML.getChildByTagName(oai_config_elem, BASE_URL);
+    baseURL = GSXML.getNodeText(base_url_elem);
+	  
+    //initialize token_expiration
+    Element expiration = (Element)GSXML.getChildByTagName(oai_config_elem, RESUMPTION_TOKEN_EXPIRATION);
+    String expire_str = GSXML.getNodeText(expiration).trim();
+    if (expiration != null && !expire_str.equals("")) {
+      token_expiration = Long.parseLong(expire_str);
+    }
+      
+    // read granularity from the config file
+    Element granu_elem = (Element)GSXML.getChildByTagName(oai_config_elem, GRANULARITY);
+    //initialize the granu_str which might be used by other methods (eg, getDate())
+    granularity = GSXML.getNodeText(granu_elem).trim();
+
+    //change "yyyy-MM-ddTHH:mm:ssZ" to "yyyy-MM-dd'T'HH:mm:ss'Z'"
+    granularity = granularity.replaceAll("T", "'T'");
+    granularity = granularity.replaceAll("Z", "'Z'");
+    granularity = granularity.replaceAll("YYYY", "yyyy").replaceAll("DD", "dd").replaceAll("hh", "HH");
+    return oai_config_elem;
+  }
+
+  public static String[] getMetadataMapping(Element metadata_format) {
+
+    if (metadata_format == null) {
+      return null;
+    }
+    NodeList mappings = metadata_format.getElementsByTagName(MAPPING);
+    int size = mappings.getLength();
+    if (size == 0) {
+      logger.info("No metadata mappings are provided in OAIConfig.xml."); 
+      return null;
+    }
+    String[] names = new String[size];
+    for (int i=0; i<size; i++) {
+      names[i] = GSXML.getNodeText((Element)mappings.item(i)).trim();
+    }
+    return names;      
+    
+  }
+  
+  public static String[] getGlobalMetadataMapping(String prefix) {
+    Element list_meta_formats = (Element)GSXML.getChildByTagName(oai_config_elem, LIST_METADATA_FORMATS);
+    if(list_meta_formats == null) {
+      return null;
+    }
+    Element metadata_format = GSXML.getNamedElement(list_meta_formats, METADATA_FORMAT, METADATA_PREFIX, prefix);
+    if(metadata_format == null) {
+      return null;
+    }
+    return getMetadataMapping(metadata_format);
+  }
+
+  /** Copies out the main info from a metadataFormat element, leaving behind the mapping stuff. This gets the bit needed for OAI response */
+  public static Element getMetadataFormatShort(Document doc, Element meta_format_long) {
+    Element meta_fmt = doc.createElement(OAIXML.METADATA_FORMAT);
+    // Copy in the elements that we want, and ignore the rest
+    meta_fmt.appendChild(doc.importNode(GSXML.getChildByTagName(meta_format_long, OAIXML.METADATA_PREFIX), true));
+	meta_fmt.appendChild(doc.importNode(GSXML.getChildByTagName(meta_format_long, OAIXML.SCHEMA), true));
+	meta_fmt.appendChild(doc.importNode(GSXML.getChildByTagName(meta_format_long, OAIXML.METADATA_NAMESPACE), true));
+
+	return meta_fmt;
+  }
+
+  public static long getTokenExpiration() {
+    return token_expiration*1000; // in milliseconds
+  }
+
+  /** TODO: returns a basic response for appropriate oai version
+   *  
+   */
+  public static  Element createBasicResponse(Document doc, String verb, String[] pairs) {
+
+    Element response = createResponseHeader(doc, verb);
+      
+    //set the responseDate and request elements accordingly
+    Element request_elem = (Element)GSXML.getChildByTagName(response, REQUEST);
+    if (verb.equals("")) {
+      request_elem.setAttribute(VERB, verb);
+    }
+    int num_pairs = (pairs==null)? 0 : pairs.length;
+    for (int i=num_pairs - 1; i>=0; i--) {
+      int index = pairs[i].indexOf("=");
+      if (index != -1) {
+	String[] strs = pairs[i].split("=");
+	if(strs != null && strs.length == 2) {
+	  request_elem.setAttribute(strs[0], oaiDecode(strs[1]));
+	}
+      }
+    }//end of for()
+      
+    GSXML.setNodeText(request_elem, baseURL);
+      
+    Node resp_date = GSXML.getChildByTagName(response, RESPONSE_DATE);
+    if (resp_date != null) {
+      GSXML.setNodeText((Element)resp_date, getCurrentUTCTime());
+    }
+
+    return response;
+  }
+  /** @param error_code the value of the code attribute
+   *  @param error_text the node text of the error element
+   *  @return an oai error <message><response><error>
+   */
+  public static Element createErrorMessage(String error_code, String error_text) {
+    Document doc = converter.newDOM();
+    Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element resp = doc.createElement(GSXML.RESPONSE_ELEM);
+    message.appendChild(resp);
+    Element error = createErrorElement(doc, error_code, error_text);
+    resp.appendChild(error);
+    return message;
+  }
+    
+  /** @param error_code the value of the code attribute
+   *  @param error_text the node text of the error element
+   *  @return an oai error <response><error>
+   */
+  public static Element createErrorResponse(String error_code, String error_text) {
+    Document doc = converter.newDOM();
+    Element resp = doc.createElement(GSXML.RESPONSE_ELEM);
+    Element error = createErrorElement(doc, error_code, error_text);
+    resp.appendChild(error);
+    return resp;
+  }
+    
+  /** @param error_code the value of the code attribute
+   *  @param error_text the node text of the error element
+   *  @return an oai error <error>
+   */
+  public static Element createErrorElement(Document doc, String error_code, String error_text) {
+    Element error = doc.createElement(ERROR);
+    error.setAttribute(CODE, error_code);
+    GSXML.setNodeText(error, error_text);
+    return error;
+  }
+
+  public static Element createResetResponse(boolean success) {
+    Document doc = converter.newDOM();
+    Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+    if (success) {
+      response.setAttribute("status", "OK");
+      GSXML.setNodeText(response, "Reset OAIServer successfully");
+    } else {
+      response.setAttribute("status", "FAIL");
+      GSXML.setNodeText(response, "Failed to reset oaiserver");
+    }
+    return response;
+  }
+  /** convert the escaped sequences (eg, '%3A') of those special characters back to their
+   *  original form (eg, ':'). 
+   */
+  public static String oaiDecode(String escaped_str) {
+    logger.info("oaiDecode() " +escaped_str);
+    for (int i=0; i<special_char.length; i++) {
+      if (escaped_str.indexOf(escape_sequence[i]) != -1) {
+	escaped_str = escaped_str.replaceAll(escape_sequence[i], special_char[i]);
+      }
+    }
+    return escaped_str;        
+  }
+  /** convert those special characters (eg, ':') to their
+   *  escaped sequences (eg, '%3A'). 
+   */
+  public static String oaiEncode(String original_str) {
+    logger.info("oaiEncode() " + original_str);      
+    for (int i=0; i<special_char.length; i++) {
+      if (original_str.indexOf(special_char[i]) != -1) {
+	original_str = original_str.replaceAll(special_char[i], escape_sequence[i]);
+      }
+    }
+    return original_str;  
+  }
+  /** convert YYYY-MM_DDThh:mm:ssZ to yyyy-MM-ddTHH:mm:ssZ
+   */
+  public static String convertToJava(String oai_format) {
+    oai_format = oai_format.replaceAll("YYYY", "yyyy").replaceAll("DD", "dd").replaceAll("hh", "HH");
+    return oai_format;
+  }
+  /** convert yyyy-MM-ddTHH:mm:ssZ to YYYY-MM_DDThh:mm:ssZ
+   */
+  public static String convertToOAI(String java_format) {
+    java_format = java_format.replaceAll("yyyy", "YYYY").replaceAll("dd", "DD").replaceAll("HH", "hh");
+    return java_format;      
+  }
+  public static String getCurrentUTCTime() {
+    Date current_utc = new Date(System.currentTimeMillis());
+    //granularity is in the form: yyyy-MM-dd'T'HH:mm:ss'Z '
+    DateFormat formatter = new SimpleDateFormat(granularity);
+    return formatter.format(current_utc);
+  }
+  /** get a Date object from a Date format pattern string
+   *
+   * @param pattern - in the form: 2007-06-14T16:48:25Z, for example.
+   * @return a Date object - null if the pattern is not in the specified form
+   */
+
+  public static Date getDate(String pattern) {
+    if (pattern == null || pattern.equals("")) {
+      return null;
+    }
+    Date date = null;
+    //      String str = pattern.replaceAll("T", " ");
+    //      str = str.replaceAll("Z", "");
+    SimpleDateFormat sdf = null;
+    try {
+      sdf = new SimpleDateFormat(granularity);
+      date = sdf.parse(pattern);
+    } catch(Exception e) {
+      if(!default_granularity.equals(granularity)) { // try validating against default granularity
+	try {
+	  date = null;
+	  sdf = null;
+	  sdf = new SimpleDateFormat(default_granularity);
+	  date = sdf.parse(pattern);
+	} catch(Exception ex) {
+	  logger.error("invalid date format: " + pattern);
+	  return null;
+	}
+      } else {
+	logger.error("invalid date format: " + pattern);
+	return null;
+      }
+    }
+    return date;
+  }
+  /** get the million second value from a string representing time in a pattern
+   * (eg, 2007-06-14T16:48:25Z)
+   */
+  public static long getTime(String pattern) {
+    if (pattern == null || pattern.equals("")) {
+      return -1;
+    }
+    Date date = null;
+    SimpleDateFormat sdf = null;
+    try {
+      //granularity is a global variable in the form: yyyy-MM-ddTHH:mm:ssZ 
+      sdf = new SimpleDateFormat(granularity);
+      date = sdf.parse(pattern);
+    } catch(Exception e) {
+      if(!default_granularity.equals(granularity)) { // try validating against default granularity
+	try {
+	  date = null;
+	  sdf = null;
+	  sdf = new SimpleDateFormat(default_granularity);
+	  date = sdf.parse(pattern);
+	} catch(Exception ex) {
+	  logger.error("invalid date format: " + pattern);
+	  return -1;
+	}
+      } else {
+	logger.error("invalid date format: " + pattern);
+	return -1;
+      }        
+    }
+    return date.getTime();
+  }    
+  /** get the string representation of a time from a long value(long type)
+   */
+  public static String getTime(long milliseconds) {
+    Date date = new Date(milliseconds);
+    SimpleDateFormat sdf = new SimpleDateFormat(granularity);
+    return sdf.format(date);
+  }    
+  public static Element createResponseHeader(Document response_doc, String verb) {
+    String tag_name = (oai_version.equals(OAI_VERSION2))? OAI_PMH : verb;
+    Element oai = response_doc.createElement(tag_name);
+    Element resp_date = response_doc.createElement(RESPONSE_DATE);
+    Element req = response_doc.createElement(REQUEST);
+    oai.appendChild(resp_date);
+    oai.appendChild(req);
+
+    if(oai_version.equals(OAI_VERSION2)) {
+      oai.setAttribute("xmlns", "http://www.openarchives.org/OAI/2.0/");
+      oai.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+      oai.setAttribute("xsi:schemaLocation", "http://www.openarchives.org/OAI/2.0/ \n http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd");
+    } else {
+      oai.setAttribute("xmlns", "http://www.openarchives.com/OAI/1.1/OAI_" + verb);
+      oai.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+      oai.setAttribute("xsi:schemaLocation", "http://www.openarchives.org/OAI/1.1/OAI_" + verb + "\n http://www.openarchives.org/OAI/1.1/OAI_" + verb + ".xsd");
+    }
+    return oai;
+  }
+
+  public static Element getMetadataPrefixElement(Document doc, String prefix, String version) {
+    if (prefix.equals(META_FORMAT_DC)) {
+      //examples of tag_name: dc, oai_dc:dc, etc.
+      String tag_name = getMetadataTagName(prefix, version);
+      Element oai = doc.createElement(tag_name);
+      if (version.equals(OAI_VERSION2)) {
+	oai.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/");
+	oai.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/");
+	oai.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+	oai.setAttribute("xsi:schemaLocation", "http://www.openarchives.org/OAI/2.0/oai_dc/ \n http://www.openarchives.org/OAI/2.0/oai_dc.xsd");
+      } else {
+	oai.setAttribute("xmlns", "http://www.openarchives.com/OAI/1.1/");
+	oai.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+	oai.setAttribute("xsi:schemaLocation", "http://www.openarchives.org/OAI/1.1/" + tag_name + ".xsd");        
+      }
+      return oai;
+    }
+    return null;
+    
+  }
+
+  public static String getMetadataTagName(String prefix, String oai_version) {
+    if (prefix.equals(META_FORMAT_DC)) {
+      if (oai_version.equals(OAI_VERSION2)) {
+	return "oai_dc:dc";
+      }
+      return "dc";
+    }
+    return prefix;
+  }
+
+  public static HashMap<String, Node> getChildrenMapByTagName(Node n, String tag_name) {
+	
+    HashMap<String, Node> map= new HashMap<String, Node>();
+    Node child = n.getFirstChild();
+    while (child!=null) {
+      String name = child.getNodeName();
+      if(name.equals(tag_name)) {
+	map.put(name, child);
+      }
+      child = child.getNextSibling();
+    }
+    return map;
+  }
+    
+  public static Element createOAIIdentifierXML(Document doc, String repository_id, String sample_collection, String sample_doc_id) {
+    String xml = "<oai-identifier xmlns=\"http://www.openarchives.org/OAI/2.0/oai-identifier\"\n xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/oai-identifier\n http://www.openarchives.org/OAI/2.0/oai-identifier.xsd\">\n <scheme>oai</scheme>\n<repositoryIdentifier>" + repository_id + "</repositoryIdentifier>\n<delimiter>:</delimiter>\n<sampleIdentifier>oai:"+repository_id+":"+sample_collection+":"+sample_doc_id+"</sampleIdentifier>\n</oai-identifier>";
+
+    Document xml_doc = converter.getDOM(xml);
+    return (Element)doc.importNode(xml_doc.getDocumentElement(), true);
+    
+
+  }
+
+  public static Element createGSDLElement(Document doc) {
+    String xml = "<gsdl xmlns=\"http://www.greenstone.org/namespace/gsdl_oaiinfo/1.0/gsdl_oaiinfo\"\n xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xsi:schemaLocation=\"http://www.greenstone.org/namespace/gsdl_oaiinfo/1.0/gsdl_oaiinfo\n   http://www.greenstone.org/namespace/gsdl_oaiinfo/1.0/gsdl_oaiinfo.xsd\"></gsdl>";
+    Document xml_doc = converter.getDOM(xml);
+    return (Element)doc.importNode(xml_doc.getDocumentElement(), true);
+    
+
+  }
+
+  public static Element createSet(Document doc, String spec, String name, String description) {
+
+    Element set_elem = doc.createElement(SET);
+    Element set_spec = doc.createElement(SET_SPEC);
+    GSXML.setNodeText(set_spec, spec);
+    set_elem.appendChild(set_spec);
+    Element set_name = doc.createElement(SET_NAME);
+    GSXML.setNodeText(set_name, name);
+    set_elem.appendChild(set_name);
+    if (description != null) {
+      Element set_description = doc.createElement(SET_DESCRIPTION);
+      GSXML.setNodeText(set_description, description);
+      set_elem.appendChild(set_description);
+    }
+    return set_elem;
+    
+  }
+
+  /** returns the resumptionToken element to go into an OAI response */
+  public static Element createResumptionTokenElement(Document doc, String token_name, int total_size, int cursor, long expiration_time) {
+    Element token = doc.createElement(OAIXML.RESUMPTION_TOKEN);
+    if (total_size != -1) {
+      token.setAttribute(OAIXML.COMPLETE_LIST_SIZE, "" + total_size);
+    }
+    if (cursor != -1) {
+      token.setAttribute(OAIXML.CURSOR, "" + cursor);
+    }
+    if(expiration_time !=-1) {
+      token.setAttribute(OAIXML.EXPIRATION_DATE, getTime(expiration_time));
+    }
+   
+    if (token != null) {
+      GSXML.setNodeText(token, token_name);
+    }
+    return token;
+  }
+  
+}
+
+
+
+
+
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OID.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OID.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OID.java	(revision 31634)
@@ -0,0 +1,205 @@
+/*
+ *    OID.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import org.apache.log4j.*;
+
+/** utility class to handle greenstone OIDs
+ *
+ * based around OIDtools.h from gsdl
+ */
+public class OID {
+
+    static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.OID.class.getName());
+
+    public interface OIDTranslatable {
+	public String processOID(String doc_id, String top, String suffix, int sibling_num);
+    }
+    
+    /** returns everything up to the first dot 
+      if no dot, returns oid */
+    public static String getTop(String oid) {
+      int pos = oid.indexOf('.');
+      if (pos == -1) {
+          return oid;
+      }
+      return oid.substring(0, pos);
+      
+    }
+    /** returns true is oid is top level (ie has no dots) 
+     returns false for an empty oid */
+    public static boolean isTop(String oid) {
+      if (oid.equals("")) {
+          return false;
+      }
+      return (oid.indexOf('.')==-1);
+    }
+      
+    /** returns the parent of oid (everything up to last dot) 
+     returns oid if oid has no parent */
+    public static String getParent(String oid) {
+      int pos = oid.lastIndexOf('.');
+      if (pos == -1) {
+          return oid;
+      }
+      return oid.substring(0, pos);
+    }
+
+    /** returns the full name - replaces all " with parent */
+    public static String translateParent(String oid, String parent) {
+      return oid.replaceAll("\"", parent); 
+    }
+    /** does the opposite to translate_parent */
+    public static String shrinkParent(String oid) {
+      int pos = oid.lastIndexOf('.');
+      if (pos==-1) return oid;
+      return "\""+oid.substring(pos);
+    }
+    /** returns true if oid uses .fc, .lc, .pr, .ns, .ps .rt (root) .ss (specified sibling)*/
+    public static boolean needsTranslating(String oid) {
+      if (oid.length()<4) return false;
+      String tail = oid.substring(oid.length()-3);
+      return (tail.equals(".fc") || tail.equals(".lc") || 
+            tail.equals(".pr") || tail.equals(".ns") || 
+            tail.equals(".ps") || tail.equals(".rt") || 
+            tail.equals(".ss") || tail.equals(".np") ||
+            tail.equals(".pp"));
+    }
+    /** strips suffix from end */
+    public static String stripSuffix(String oid) {
+      String tail = oid.substring(oid.length()-3);
+      while (tail.equals(".fc") || tail.equals(".lc") || 
+             tail.equals(".pr") || tail.equals(".ns") || 
+             tail.equals(".ps") || tail.equals(".ss") || 
+             tail.equals(".np") || tail.equals(".pp") ||
+             tail.equals(".rt") ) {
+          if (tail.equals(".ss")) { // have doc.sibnum.ss
+            oid = oid.substring(0, oid.length()-3);
+            int pos = oid.lastIndexOf('.');
+            //strip that too
+            oid = oid.substring(0, pos);
+          } 
+          oid = oid.substring(0, oid.length()-3);
+          tail = oid.substring(oid.length()-3);
+      }
+      
+      return oid;
+    }
+    /** returns true if child is a child of parent 
+     an oid is not a child of itself */
+    public static boolean isChildOf(String parent, String child) {
+      if (parent.equals(child)) {
+          return false;
+      }
+      return child.startsWith(parent);
+    }
+
+    
+  /** translates relative oids into proper oids:
+   * .pr (parent), .rt (root) .fc (first child), .lc (last child),
+   * .ns (next sibling), .ps (previous sibling) 
+   * .np (next page), .pp (previous page) : links sections in the order that you'd read the document
+   * a suffix is expected to be present so test before using
+   * Other classes can implement the method processOID of interface OIDTranslatable
+   * to process the oid further to handle siblings.
+   */
+    public static String translateOID(OIDTranslatable translator, String oid) {
+    int p = oid.lastIndexOf('.');
+    if (p != oid.length()-3) {
+      logger.info("translateoid error: '.' is not the third to last char!!");
+      return oid;
+    }
+	
+    String top = oid.substring(0, p);
+    String suff = oid.substring(p+1);
+
+    // just in case we have multiple extensions, we must translate
+    // we process inner ones first
+    if (OID.needsTranslating(top)) {
+	top = OID.translateOID(translator, top);
+    }
+    if (suff.equals("pr")) {
+      return OID.getParent(top);
+    } 
+    if (suff.equals("rt")) {
+      return OID.getTop(top);
+    } 
+    if (suff.equals("np")) {
+      // try first child
+
+      String node_id = OID.translateOID(translator, top+".fc");
+      if (!node_id.equals(top)) {
+	  return node_id;
+      }
+
+      // try next sibling
+      node_id = OID.translateOID(translator, top+".ns");
+      if (!node_id.equals(top)) {
+	  return node_id;
+      }
+      // otherwise we keep trying parents sibling
+      String child_id = top;
+      String parent_id = OID.getParent(child_id);
+      while(!parent_id.equals(child_id)) {
+	node_id = OID.translateOID(translator, parent_id+".ns");
+	if (!node_id.equals(parent_id)) {
+	  return node_id;
+	}
+	child_id = parent_id;
+	parent_id = OID.getParent(child_id);
+      }
+      return top; // we couldn't get a next page, so just return the original
+    } 
+    if (suff.equals("pp")) {
+      String prev_sib = OID.translateOID(translator, top+".ps");
+      if (prev_sib.equals(top)) {
+	// no previous sibling, so return the parent
+	return OID.getParent(top);
+      }
+      // there is a previous sibling, so it's either this section, or the last child of the last child
+      String last_child = OID.translateOID(translator, prev_sib+".lc");
+      while (!last_child.equals(prev_sib)) {
+	prev_sib = last_child;
+	last_child = OID.translateOID(translator, prev_sib+".lc");
+      }
+      return last_child;
+    } 
+	
+    int sibling_num = 0;
+    if (suff.equals("ss")) {
+      // we have to remove the sib num before we get top
+      p = top.lastIndexOf('.');
+      sibling_num = Integer.parseInt(top.substring(p+1));
+      top = top.substring(0, p);
+    }
+	
+    // need to get info out of Fedora
+    String doc_id = top;
+    if (suff.endsWith("s")) {
+      doc_id = OID.getParent(top);
+      if (doc_id.equals(top)) {
+	// i.e. we are already at the top
+	return top;
+      }
+    }
+    
+    return translator.processOID(doc_id, top, suff, sibling_num);
+
+  }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OIDTest.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OIDTest.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/OIDTest.java	(revision 31634)
@@ -0,0 +1,108 @@
+/*
+ *    OIDTest.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import junit.framework.*; 
+
+public class OIDTest extends TestCase {
+
+    /** what type of test - used by the testing framework */
+    public static final String TEST_ALL_TEST_TYPE="UNIT";
+
+    private static final String empty = "";
+    private static final String oid1="HASH123";
+    private static final String oid2="HASH123.1";
+    private static final String oid3="HASH123.1.2";
+    private static final String oidfc = oid1+".fc";
+    private static final String oidlc = oid1+".lc";
+    private static final String oidpr = oid1+".pr";
+    private static final String oidns = oid1+".ns";
+    private static final String oidps = oid1+".ps";
+
+    public OIDTest(String name) {
+      super(name);
+    }
+
+    /** test suite that dynamically runs all the tests */
+    public static Test suite() {
+      return new TestSuite(OIDTest.class);
+    }
+
+    public void testGetTop() {
+      assertEquals(oid1, OID.getTop(oid1));
+      assertEquals(oid1, OID.getTop(oid2));
+      assertEquals(empty, OID.getTop(empty));
+    }
+  
+    public void testIsTop() {
+      assertTrue(!OID.isTop(empty));
+      assertTrue(OID.isTop(oid1));
+      assertTrue(!OID.isTop(oid2));
+    }
+
+    public void testGetParent() {
+      assertEquals(oid1, OID.getParent(oid1));
+      assertEquals(oid1, OID.getParent(oid2));
+      assertEquals(empty, OID.getParent(empty));
+    }
+
+    public void testTranslateParent() {
+      String short_id = "\".1";
+      assertEquals(oid2, OID.translateParent(short_id, oid1));
+      // should return the original oid if its not a shortened form
+      assertEquals(oid2, OID.translateParent(oid2, oid1));
+    }
+
+    public void testShrinkParent() {
+      String short_oid2 = "\".1";
+      String short_oid3 = "\".2";
+
+      assertEquals(oid1, OID.shrinkParent(oid1));
+      assertEquals(short_oid2, OID.shrinkParent(oid2));
+      assertEquals(short_oid3, OID.shrinkParent(oid3));
+    }
+
+    public void testNeedsTranslating() {
+      assertTrue(!OID.needsTranslating(oid1));
+      assertTrue(!OID.needsTranslating(oid2));
+      assertTrue(OID.needsTranslating(oidfc));
+      assertTrue(OID.needsTranslating(oidlc));
+      assertTrue(OID.needsTranslating(oidpr));
+      assertTrue(OID.needsTranslating(oidns));
+      assertTrue(OID.needsTranslating(oidps));
+
+    }
+
+    public void testStripSuffix() {
+      assertEquals(oid1, OID.stripSuffix(oidfc));
+      assertEquals(oid1, OID.stripSuffix(oidlc));
+      assertEquals(oid1, OID.stripSuffix(oidpr));
+      assertEquals(oid1, OID.stripSuffix(oidns));
+      assertEquals(oid1, OID.stripSuffix(oidps));
+    }
+
+    public void testIsChildOf() {
+      assertTrue(OID.isChildOf(oid1, oid2));
+      assertTrue(OID.isChildOf(oid1, oid3));
+      assertTrue(OID.isChildOf(oid2, oid3));
+      assertTrue(!OID.isChildOf(oid3, oid1));
+      assertTrue(!OID.isChildOf(oid1, oid1));
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Processing.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Processing.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/Processing.java	(revision 31634)
@@ -0,0 +1,120 @@
+/*
+ *    Processing.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.InputStreamReader;
+
+import org.apache.log4j.*;
+
+public class Processing {
+
+      static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.Processing.class.getName());
+ 
+      /** Determine if the given process is still executing. It does this by attempting to throw an exception - not the most efficient way, but the only one as far as I know
+     * @param process the Process to test
+     * @return true if it is still executing, false otherwise
+     */
+    static public boolean processRunning(Process process) {
+	boolean process_running = false;
+
+	try {
+	    process.exitValue(); // This will throw an exception if the process hasn't ended yet.
+	}
+	catch(IllegalThreadStateException itse) {
+	    process_running = true;
+	}
+	catch(Exception exception) {
+	    exception.printStackTrace();	    
+	}
+	return process_running;
+    }
+    
+    static public int runProcess(String  command) {
+	
+	logger.error("executing command "+command);
+	int exit_value = -1;
+	try {
+	    Process prcs = Runtime.getRuntime().exec(command);
+	    
+	    InputStreamReader error_stream = new InputStreamReader( prcs.getErrorStream(), "UTF-8" );
+	    InputStreamReader output_stream = new InputStreamReader( prcs.getInputStream(), "UTF-8"  );
+	    
+	    StringBuffer error_buffer = new StringBuffer();
+	    StringBuffer output_buffer = new StringBuffer();
+	    
+	    while(processRunning(prcs)) {
+		// Hopefully this doesn't block if the process is trying to write to STDOUT.
+		if((error_stream!=null)) {// && error_stream.ready()) { 
+		    while (error_stream.ready()) {
+			error_buffer.append((char)error_stream.read());
+		    }
+		    //error_buffer = get_stream_char(error_stream,error_buffer,bos);
+		}
+		//Hopefully this won't block if the process is trying to write to STDERR
+		/*else if*/ while (output_stream.ready()) {
+		    output_buffer.append((char)output_stream.read());
+		    //output_buffer = get_stream_char(output_stream,output_buffer,bos);
+		}
+		//else {
+		try {
+		    Thread.sleep(100);
+		}
+		catch(Exception exception) {
+		}
+		// }
+	    }
+	    
+	    // Of course, just because the process is finished doesn't
+	    // mean the incoming streams are empty. Unfortunately I've
+	    // got no chance of preserving order, so I'll process the
+	    // error stream first, then the out stream
+	    while(error_stream.ready()) {
+		error_buffer.append((char)error_stream.read());
+		//error_buffer = get_stream_char(error_stream,error_buffer,bos);
+	    }
+	    
+	    while(output_stream.ready()) {
+		output_buffer.append((char)output_stream.read());
+		//output_buffer = get_stream_char(output_stream,output_buffer,bos);
+	    }
+	    
+	    // do something with the messages
+	    logger.error("err>"+error_buffer.toString());
+	    logger.error("out>"+output_buffer.toString());
+	    // Ensure that any messages still remaining in the string buffers are fired off.
+	    // Now display final message based on exit value
+	    
+	    prcs.waitFor(); 
+	    error_stream.close();
+	    output_stream.close();
+	    error_buffer = null;
+	    output_buffer = null;
+	    
+	    exit_value =  prcs.exitValue();
+	} catch (Exception e) {
+	    logger.error(e);
+	    return 1;
+	}
+	return exit_value;
+	
+	
+    }
+   
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ServletRealmCheck.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ServletRealmCheck.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/ServletRealmCheck.java	(revision 31634)
@@ -0,0 +1,136 @@
+/*
+ *    ServletRealmCheck.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Properties;
+
+/**
+ * Commandline script that is used by gliserver.pl to authenticate a username and password and 
+ * return the user's groups, while the derby server is running. Because 2 JVM instances can't
+ * access the same embedded derby server at the same time, gliserver can't call usersDB2txt.java.
+ * If a collection parameter is additionally provided, this script will check the user's groups
+ * to see if any of these allow the user to edit that collection.
+ *
+ * Run as java org.greenstone.gsdl3.util.ServletRealmCheck <GS3SRCHOME> <un> <pwd> [colname]
+ *
+ * GS3\src\java>"C:\Program Files\Java\jdk1.6.0_22\bin\java" 
+ * 		-classpath "GS3\web\WEB-INF\lib\gsdl3.jar;GS3\web\WEB-INF\lib\derby.jar" 
+ * 		org.greenstone.gsdl3.util.ServletRealmCheck "GS3" admin <pw> 2>&1
+ *
+ * Tries URL: http://hostname:8383/greenstone3/library?a=s&sa=authenticated-ping&excerptid=gs_content&un=admin&pw=<pw>[&col=demo] 
+ * The &excerptid=gs_content in the URL will return just the <div id="gs_content" /> part of the 
+ * page that we're interested in.
+ *
+ * Result: either prints out an error message ("Authentication failed...") or a positive result, 
+ * which is the user's groups. For the admin user example: administrator,all-collections-editor.
+ *
+*/
+public class ServletRealmCheck
+{	
+    public static void main(String[] args) {
+	
+	if (args.length < 3 || args.length > 4){
+		System.out.println("Run with: <GSDL3SRCHOME> <un> <pwd> [collection-name]");	    
+	    System.exit(0);
+	}
+	
+	String gsdl3srchome = args[0];
+	String username = args[1];
+	String password = args[2];
+	String collection = (args.length > 3) ? args[3] : null;		
+	
+	//System.err.println("gsdl3srchome: " + gsdl3srchome);
+	//System.err.println("username: " + username);
+	//System.err.println("password: " + password);
+	//System.err.println("collection: " + collection);
+	
+	
+	// Load the build.properties file, get the GS3 server URL and send authenticated-ping and print the return result
+	
+	//http://www.mkyong.com/java/java-properties-file-examples/
+	Properties buildProps = new Properties();
+	InputStream input = null;
+ 
+	try {
+		File buildPropsFile = new File(gsdl3srchome, "build.properties");
+		input = new FileInputStream(buildPropsFile);
+ 
+		// load a properties file
+		buildProps.load(input);
+ 
+		// get the property value and print it out
+		String servername = buildProps.getProperty("tomcat.server");
+		String port = buildProps.getProperty("tomcat.port");
+		int portNum = Integer.parseInt(port);
+		
+		// Appending &excerptid=gs_content will get just the <div ... id="gs_content"/> from the final web page:
+		String urlSuffix = "/greenstone3/library?a=s&sa=authenticated-ping&excerptid=gs_content&un="+username+"&pw="+password;
+		if(collection != null) {
+			urlSuffix = urlSuffix + "&col="+collection;
+		}
+		URL authenticationUrl = new URL("http", servername, portNum, urlSuffix);
+		
+		HttpURLConnection conn = (HttpURLConnection)authenticationUrl.openConnection();		
+		BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+		String result = "";
+		String line = null;
+		
+		while((line = reader.readLine()) != null) {
+			result += line;
+		}
+		
+		//System.err.println("** Sent: " + authenticationUrl);
+		//System.err.println("** Got result:\n" + result);
+		
+		// Parse out the content nested inside <div ... id="gs_content"> </div>
+		int start = result.indexOf("id=\"gs_content\"");
+		if(start != -1) {
+			start = result.indexOf(">", start);
+			int end = result.indexOf("<", start);
+			result = result.substring(start+1, end);
+			result = result.trim();
+		}		
+		
+		// Now we finally have what we actually want to print out for the caller to use
+		System.out.print(result); // don't add newline to end
+		
+	} catch (IOException ex) {
+		ex.printStackTrace();
+	} finally {
+		if (input != null) {
+			try {
+				input.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+	
+  }	
+   
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleCollectionDatabase.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleCollectionDatabase.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleCollectionDatabase.java	(revision 31634)
@@ -0,0 +1,343 @@
+/*
+ *    SimpleCollectionDatabase.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.log4j.*;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class SimpleCollectionDatabase implements OID.OIDTranslatable
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.SimpleCollectionDatabase.class.getName());
+
+	/* just read access, many readers can share a database */
+	public final static int READ = FlatDatabaseWrapper.READ;
+	/* read/write, exclusive access */
+	public final static int WRITE = FlatDatabaseWrapper.WRITE;
+
+	protected FlatDatabaseWrapper coll_db = null;
+
+	public SimpleCollectionDatabase(String db_type)
+	{
+	    if (db_type.toLowerCase().endsWith("server")) {
+			db_type = db_type.substring(0, db_type.length() - 6);
+	       }
+
+		// Access databaseWrapper through reflection (forName) so code
+		// can be more dynamic as to the database backends that are
+		// supported for this installation of Greenstone
+
+		String dbwrap_name = db_type.toUpperCase() + "Wrapper";
+		Class dbwrap_class = null;
+
+		try
+		{
+			String full_dbwrap_name = "org.greenstone.gsdl3.util." + dbwrap_name;
+			dbwrap_class = Class.forName(full_dbwrap_name);
+		}
+		catch (ClassNotFoundException e)
+		{
+			try
+			{
+				//try the dbwrap_name alone in case the package name is
+				//already specified
+				dbwrap_class = Class.forName(dbwrap_name);
+			}
+			catch (ClassNotFoundException ae)
+			{
+				logger.error("Couldn't create SimpleCollectionDatabase of type " + db_type);
+				logger.info(ae.getMessage());
+			}
+		}
+		catch (Exception ae) {
+		    logger.error("Couldn't create SimpleCollectionDatabase of type " + db_type);
+		    logger.info(ae.getMessage());
+		}
+
+		try
+		{
+			this.coll_db = (FlatDatabaseWrapper) dbwrap_class.newInstance();
+		}
+		catch (Exception e)
+		{
+			logger.error("Failed to call the constructor " + dbwrap_name + "()");
+			logger.info(e.getMessage());
+		}
+
+	}
+
+	public boolean databaseOK()
+	{
+		// Previously failed to open database
+		// Most likely cause is that this installation of Greenstone3 has not 
+		// been compiled with support for this database type
+		return coll_db != null;
+	}
+
+	/**
+	 * open the database filename, with mode mode - uses the FlatDatabaseWrapper
+	 * modes
+	 */
+	public boolean openDatabase(String filename, int mode)
+	{
+		return this.coll_db.openDatabase(filename, mode);
+	}
+
+	/** close the database */
+	public void closeDatabase()
+	{
+		this.coll_db.closeDatabase();
+	}
+
+	/**
+	 * Returns a DBInfo structure of the key-value pairs associated with a
+	 * particular main key in the database
+	 */
+	public DBInfo getInfo(String main_key)
+	{
+	  //logger.warn("All the entries of the db are:");
+	  //this.coll_db.displayAllEntries();
+
+		if (this.coll_db == null)
+		{
+			// Most likely cause is that this installation of Greenstone3 has not 
+			// been compiled with support for this database type
+			return null;
+		}
+
+		String key_info = this.coll_db.getValue(main_key);
+		if (key_info == null || key_info.equals(""))
+		{
+			return null;
+		}
+		
+		DBInfo info = new DBInfo();
+		// add in the Identifier field // hack for OAI. useful for other things? or not???
+		info.addInfo("Identifier", main_key);
+		String[] lines = StringUtils.split(key_info, "\n");
+		String key;
+		String value;
+		for (int i = 0; i < lines.length; i++)
+		{
+		  //logger.debug("line:" + lines[i]);
+			int a = lines[i].indexOf('<');
+			int b = lines[i].indexOf('>');
+			if (a == -1 || b == -1)
+			{
+				logger.error("bad format in db");
+			}
+			else
+			{
+				key = lines[i].substring(a + 1, b);
+				value = lines[i].substring(b + 1);
+				//logger.debug("key=" + key + ", val=" + value);
+				info.addInfo(key, value);
+			}
+		}
+		
+		return info;
+	}
+	
+	public boolean setInfo(String mainKey, DBInfo info)
+	{
+		StringBuilder valueToAdd = new StringBuilder();
+		Iterator<String> i = info.getKeys().iterator();
+		while (i.hasNext())
+		{
+			String currentKey = i.next();
+			Vector<String> currentValues = (Vector<String>)info.getMultiInfo(currentKey);
+			
+			if(currentValues.size() == 0)
+			{
+				valueToAdd.append("<" + currentKey + ">\n");
+				continue;
+			}
+			
+			for(int j = 0; j < currentValues.size(); j++)
+			{
+				valueToAdd.append("<" + currentKey + ">" + currentValues.get(j) + "\n");
+			}
+		}
+		
+		//Remove the final \n
+		if (valueToAdd.length() > 0)
+		{
+			valueToAdd.delete(valueToAdd.length() - 1, valueToAdd.length());
+		}
+		
+		return this.coll_db.setValue(mainKey, valueToAdd.toString());
+	}
+	
+	public String getValue(String key)
+	{
+		return this.coll_db.getValue(key);
+	}
+	
+	public boolean setValue(String key, String value)
+	{
+		return this.coll_db.setValue(key, value);
+	}
+	
+	public boolean deleteKey(String key)
+	{
+		return this.coll_db.deleteKey(key);
+	}
+
+	/** converts a greenstone OID to internal docnum */
+	public String OID2Docnum(String OID)
+	{
+		DBInfo info = getInfo(OID);
+		if (info != null)
+		{
+			return info.getInfo("docnum");
+		}
+		return null;
+	}
+
+    public ArrayList<String> getAllKeys() {
+	return this.coll_db.getAllEntryKeys();
+    }
+
+	/**
+	 * converts a greenstone OID to an internal docnum, returning a Long -
+	 * convenience method
+	 */
+	public long OID2DocnumLong(String OID)
+	{
+		DBInfo info = getInfo(OID);
+		if (info != null)
+		{
+			long real_num = Long.parseLong(info.getInfo("docnum"));
+			return real_num;
+		}
+		return -1;
+	}
+
+	/** converts a docnum to greenstone OID */
+	public String docnum2OID(String docnum)
+	{
+		DBInfo info = getInfo(docnum);
+		if (info != null)
+		{
+			String oid = info.getInfo("section");
+			return oid;
+		}
+		else
+		{
+			return null;
+		}
+	}
+
+	/**
+	 * converts a docnum to greenstone OID - convenience method
+	 */
+	public String docnum2OID(long docnum)
+	{
+		return docnum2OID(Long.toString(docnum));
+	}
+
+	/** converts an external id to greenstone OID */
+	public String externalId2OID(String extid)
+	{
+		DBInfo info = getInfo(extid);
+		if (info != null)
+		{
+			String oid = info.getInfo("section");
+			return oid;
+		}
+		return null;
+	}
+
+	/**
+	 * After OID.translateOID() is through, this method processes OID further to
+	 * translate relative oids into proper oids: .pr (parent), .rt (root) .fc
+	 * (first child), .lc (last child), .ns (next sibling), .ps (previous
+	 * sibling) .np (next page), .pp (previous page) : links sections in the
+	 * order that you'd read the document a suffix is expected to be present so
+	 * test before using
+	 */
+	public String processOID(String doc_id, String top, String suff, int sibling_num)
+	{
+		DBInfo info = getInfo(doc_id);
+		if (info == null)
+		{
+			logger.info("info is null!!");
+			return top;
+		}
+
+		String contains = info.getInfo("contains");
+		if (contains.equals(""))
+		{
+			// something is wrong
+			return top;
+		}
+		contains = StringUtils.replace(contains, "\"", doc_id);
+		String[] children = StringUtils.split(contains, ";");
+		if (suff.equals("fc"))
+		{
+			return children[0];
+		}
+		else if (suff.equals("lc"))
+		{
+			return children[children.length - 1];
+		}
+		else
+		{
+			if (suff.equals("ss"))
+			{
+				return children[sibling_num - 1];
+			}
+			// find the position that we are at.
+			int i = 0;
+			while (i < children.length)
+			{
+				if (children[i].equals(top))
+				{
+					break;
+				}
+				i++;
+			}
+
+			if (suff.equals("ns"))
+			{
+				if (i == children.length - 1)
+				{
+					return children[i];
+				}
+				return children[i + 1];
+			}
+			else if (suff.equals("ps"))
+			{
+				if (i == 0)
+				{
+					return children[i];
+				}
+				return children[i - 1];
+			}
+		}
+
+		return top;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleMacroResolver.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleMacroResolver.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/SimpleMacroResolver.java	(revision 31634)
@@ -0,0 +1,57 @@
+/*
+ *    SimpleMacroResolver.java
+ *    Copyright (C) 2012 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+
+import org.apache.commons.lang3.*;
+import org.greenstone.gsdl3.util.MacroResolver.Macro;
+
+public class SimpleMacroResolver extends MacroResolver
+{
+
+  public SimpleMacroResolver()
+  { 
+    super();
+  }
+
+  public void addMacro(String macro, String text) {
+    Macro m = new Macro();
+    m.type = TYPE_TEXT;
+    m.macro = macro;
+    m.text = text;
+    m.resolve = false;
+    addMacro(m, SCOPE_TEXT);
+  }
+
+  // this ignores all other considerations, is a straight text swap
+  public String resolve(String text)
+  {
+    for (int i=0; i<text_macros.size(); i++) {
+      Macro m = text_macros.get(i);
+      text = StringUtils.replace(text, m.macro, m.text);
+    }
+    return text;
+  }
+  public String resolve(String text, String lang, String scope, String doc_oid)
+  {
+    return resolve(text);
+  }
+
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserContext.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserContext.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserContext.java	(revision 31634)
@@ -0,0 +1,94 @@
+package org.greenstone.gsdl3.util;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+public class UserContext
+{
+	protected String _userID = null;
+	protected String _username = null;
+	protected String _lang = null;
+	protected String[] _groups = null;
+
+	public UserContext()
+	{
+	}
+
+	public UserContext(Element xmlRequest)
+	{
+		NodeList elems = xmlRequest.getElementsByTagName("userContext");
+
+		if (elems.getLength() > 0)
+		{
+			Element userContext = (Element) elems.item(0);
+			_userID = userContext.getAttribute(GSXML.USER_ID_ATT);
+			_username = userContext.getAttribute(GSXML.USERNAME_ATT);
+			_lang = userContext.getAttribute(GSXML.LANG_ATT);
+			_groups = userContext.getAttribute(GSXML.GROUPS_ATT).split(",");
+		}
+	}
+
+	public UserContext(String lang, String username, String userID, String[] groups)
+	{
+		_userID = userID;
+		_username = username;
+		_lang = lang;
+		_groups = groups;
+	}
+
+	public void setUsername(String username)
+	{
+		_username = username;
+	}
+
+	public void setLanguage(String lang)
+	{
+		_lang = lang;
+	}
+
+	public void setUserID(String userID)
+	{
+		_userID = userID;
+	}
+
+	public void setGroups(String[] groups)
+	{
+		_groups = groups;
+	}
+
+	public String getLanguage()
+	{
+		if (_lang != null)
+		{
+			return _lang;
+		}
+		return "";
+	}
+
+	public String getUserID()
+	{
+		if (_userID != null)
+		{
+			return _userID;
+		}
+		return "";
+	}
+
+	public String getUsername()
+	{
+		if (_username != null)
+		{
+			return _username;
+		}
+		return "";
+	}
+
+	public String[] getGroups()
+	{
+		if (_groups != null)
+		{
+			return _groups;
+		}
+		return new String[0];
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserQueryResult.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserQueryResult.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserQueryResult.java	(revision 31634)
@@ -0,0 +1,71 @@
+/*
+ *    UserQueryResult.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.util.Vector;
+import org.greenstone.gsdl3.util.UserTermInfo;
+
+public class UserQueryResult
+{
+	/** the list of UserInfo */
+	public Vector<UserTermInfo> users = null;
+
+	UserQueryResult()
+	{
+		users = new Vector<UserTermInfo>();
+	}
+
+	public void clear()
+	{
+		users.clear();
+	}
+
+	public void addUserTerm(String username, String password, String groups, String accountstatus, String comment, String email)
+	{
+		UserTermInfo ui = new UserTermInfo();
+		ui.username = username;
+		ui.password = password;
+		ui.groups = groups;
+		ui.accountstatus = accountstatus;
+		ui.comment = comment;
+		ui.email = email;
+		users.add(ui);
+	}
+
+	public Vector<UserTermInfo> getUserTerms()
+	{
+		return users;
+	}
+
+	public String toString()
+	{
+		String result = "";
+		for (int i = 0; i < users.size(); i++)
+		{
+			result += users.elementAt(i).toString() + ", ";
+		}
+
+		return result;
+	}
+
+	public int getSize()
+	{
+		return users.size();
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserTermInfo.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserTermInfo.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/UserTermInfo.java	(revision 31634)
@@ -0,0 +1,41 @@
+/*
+ *    UserTermInfo.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.gsdl3.util;
+
+public class UserTermInfo
+{
+	public String username = null;
+	public String password = null;
+	public String groups = null;
+	public String accountstatus = null;
+	public String comment = null;
+	public String email = null;
+
+	public String toString()
+	{
+		String result = "";
+		result += "<username = " + username + ">";
+		result += "<password = " + password + ">";
+		result += "<groups = " + groups + ">";
+		result += "<enable = " + accountstatus + ">";
+		result += "<comment = " + comment + ">";
+		return result;
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLConverter.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLConverter.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLConverter.java	(revision 31634)
@@ -0,0 +1,611 @@
+/*
+ *    XMLConverter.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+// XML classes
+import org.w3c.dom.DOMImplementation;
+import org.w3c.dom.Document;
+import org.w3c.dom.DocumentType;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.NamedNodeMap;
+import org.xml.sax.InputSource;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXParseException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.apache.xerces.parsers.DOMParser;
+import org.apache.xerces.dom.DocumentImpl; // for new Documents
+import org.apache.xerces.dom.DocumentTypeImpl; 
+
+// other java classes
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.FileWriter;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.util.regex.*;
+
+import org.apache.log4j.*;
+
+// Apache Commons
+import org.apache.commons.lang3.*;
+
+import java.util.*;
+import java.lang.reflect.*;
+
+/**
+ * XMLConverter - utility class for greenstone
+ * 
+ * generates new Documents
+ * parses XML Strings into Documents, converts Nodes to Strings 
+ * different parsers have different behaviour - can experiment in here 
+ * at the moment we only use xerces
+ * all xerces specific code is in here
+ */
+public class XMLConverter
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.XMLConverter.class.getName());
+
+	/** the no-args constructor */
+	public XMLConverter()
+	{
+
+	}
+
+	/** returns a DOM Document */
+  public static Document getDOM(String in)
+	{
+
+		try
+		{
+			Reader reader = new StringReader(in);
+			InputSource xml_source = new InputSource(reader);
+			Document doc = getDOM(xml_source, null);
+			reader.close();
+			return doc;
+
+		}
+		catch (Exception e)
+		{
+			logger.error(e.getMessage());
+			logger.error("Input string was:\n" + in);
+			e.printStackTrace();
+		}
+		return null;
+	}
+
+	/** returns a DOM Document */
+  public static Document getDOM(String in, String encoding)
+	{
+		try
+		{
+			InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(in.getBytes(encoding)), encoding);
+			InputSource xml_source = new InputSource(reader);
+			Document doc = getDOM(xml_source, null);
+			reader.close();
+			return doc;
+
+		}
+		catch (Exception e)
+		{
+			logger.error(e.getMessage());
+			logger.error("Input string was:\n" + in);
+			e.printStackTrace();
+		}
+		return null;
+	}
+
+	/** returns a DOM Document */
+  public static Document getDOM(File in) {
+		try
+		{
+			FileReader reader = new FileReader(in);
+			InputSource xml_source = new InputSource(reader);
+			Document doc = getDOM(xml_source, null);
+			reader.close();
+			return doc;
+
+		}
+		catch (Exception e)
+		{
+			logger.error(e.getMessage(), e);
+			logger.error("File was:\n" + in.getPath());
+			e.printStackTrace();
+
+		}
+		return null;
+	}
+
+  public static Document getDOM(File in, String encoding) {
+    return getDOM(in, encoding, null);
+  }
+
+	/** returns a DOM document */
+  public static Document getDOM(File in, String encoding, EntityResolver er) {
+    
+    try {
+      
+      
+      InputStreamReader isr = new InputStreamReader(new FileInputStream(in), encoding);
+      InputSource xml_source = new InputSource(isr);
+      Document doc = getDOM(xml_source, er);
+      isr.close();
+      return doc;
+      
+    }
+    catch (Exception e)
+      {
+	logger.error(e.getMessage());
+	logger.error("File was:\n" + in.getPath());
+	e.printStackTrace();
+      }
+    return null;
+	}
+
+  public static Document getDOM(File in, EntityResolver er) {
+    
+    try {      
+      InputSource xml_source = new InputSource(new FileInputStream(in));
+      Document doc = getDOM(xml_source, er);      
+      return doc;      
+    }
+    catch (Exception e)
+      {
+	logger.error(e.getMessage());
+	logger.error("File was:\n" + in.getPath());
+	e.printStackTrace();
+      }
+    return null;
+  }
+
+  public static Document getDOM(InputSource source, EntityResolver er) {
+    
+    try {
+    DOMParser parser = new DOMParser();
+    parser.setFeature("http://xml.org/sax/features/validation", false);
+    // don't try and load external DTD - no need if we are not validating, and may cause connection errors if a proxy is not set up.
+    parser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+    // a performance test showed that having this on lead to increased 
+    // memory use for small-medium docs, and not much gain for large 
+    // docs.
+    // http://www.sosnoski.com/opensrc/xmlbench/conclusions.html
+    parser.setFeature("http://apache.org/xml/features/dom/defer-node-expansion", false);
+    // add an errorhandler to the parser which will output messages on encountering fatal errors, errors and warnings when parsing
+    parser.setErrorHandler(new ParseErrorHandler());
+    if (er != null) {
+      parser.setEntityResolver(er);
+    }
+    parser.parse(source);
+
+    Document doc = parser.getDocument();
+    return doc;
+
+    } catch (Exception e) {
+      
+      logger.error(e.getMessage());
+      logger.error("InputSource:\n");
+      e.printStackTrace();
+    }
+    return null;
+    
+  }
+
+	/** creates a new empty DOM Document */
+	public static Document newDOM()
+	{
+		Document doc = new DocumentImpl();
+		return doc;
+	}
+
+	/**
+	 * This method's parameters represent the parts of the Doctype of this
+	 * Document that is to be created. For more info see
+	 * http://xerces.apache.org
+	 * /xerces-j/apiDocs/org/apache/xerces/dom/DocumentTypeImpl
+	 * .html#DocumentTypeImpl
+	 * (org.apache.xerces.dom.CoreDocumentImpl,%20java.lang.String)
+	 * 
+	 * */
+	public static Document newDOM(String qualifiedName, String publicID, String systemID)
+	{
+		// create empty DOM document
+		DocumentImpl docImpl = new DocumentImpl();
+
+		// Need to use the document to create the docType for it
+		DocumentType myDocType = new DocumentTypeImpl(docImpl, qualifiedName, publicID, systemID);
+
+		// Although we have created the docType using the document, we need to still 
+		// put it into the empty document we just created 
+		try
+		{
+			docImpl.appendChild(myDocType);
+		}
+		catch (Exception e)
+		{
+			System.out.println("Could not append docType because: " + e);
+		}
+
+		// return the document containing a DocType
+		return docImpl;
+	}
+
+	/** returns the Node as a String */
+	public static String getString(Node xmlNode)
+	{
+		StringBuffer xmlRepresentation = new StringBuffer();
+		getString(xmlNode, xmlRepresentation, 0, false);
+		return xmlRepresentation.toString();
+	}
+
+	/**
+	 * returns the node as a nicely formatted String - this introduces extra
+	 * text nodes if the String is read back in as a DOM, so should only be used
+	 * for printing
+	 */
+	public static String getPrettyString(Node xmlNode)
+	{
+		StringBuffer xmlRepresentation = new StringBuffer();
+		getString(xmlNode, xmlRepresentation, 0, true);
+		return xmlRepresentation.toString();
+	}
+
+	/*
+	 * For the purposes of logger.debug statements, where this is called and
+	 * hence outputted, returns an empty string if debugging is not enabled
+	 */
+	public static String getPrettyStringLogger(Node xmlNode, Logger log)
+	{
+
+		if (log.isDebugEnabled())
+			return getPrettyString(xmlNode);
+
+		return "";
+
+	}
+
+	private static void getString(Node xmlNode, StringBuffer xmlRepresentation, int depth, boolean pretty)
+	{
+
+		if (xmlNode == null)
+		{
+			xmlRepresentation.append("<null>");
+			return;
+		}
+
+		short nodeType = xmlNode.getNodeType();
+		String nodeName = xmlNode.getNodeName();
+
+		if (nodeType == Node.DOCUMENT_NODE)
+		{
+			Document xmlDocNode = (Document) xmlNode;
+
+			//if (xmlDocNode.getDoctype() == null) {
+			//System.err.println("Doctype is null.");
+			//}
+			//else {
+			if (xmlDocNode.getDoctype() != null)
+			{
+				DocumentType dt = xmlDocNode.getDoctype();
+
+				String name = dt.getName();
+				String pid = dt.getPublicId();
+				String sid = dt.getSystemId();
+
+				// Use previously assigned name, not dt.getName() again
+				String doctype_str = "<!DOCTYPE " + name + " PUBLIC \"" + pid + "\" \"" + sid + "\">\n";
+
+				xmlRepresentation.append(doctype_str);
+			}
+			getString(xmlDocNode.getDocumentElement(), xmlRepresentation, depth, pretty);
+			return;
+		}
+		// Handle Element nodes
+		if (nodeType == Node.ELEMENT_NODE)
+		{
+			if (pretty)
+			{
+				xmlRepresentation.append("\n");
+				for (int i = 0; i < depth; i++)
+				{
+					xmlRepresentation.append("  ");
+				}
+			}
+
+			// Write opening tag
+			xmlRepresentation.append("<");
+			xmlRepresentation.append(nodeName);
+
+			// Write the node attributes
+			NamedNodeMap nodeAttributes = xmlNode.getAttributes();
+			for (int i = 0; i < nodeAttributes.getLength(); i++)
+			{
+				Node attribute = nodeAttributes.item(i);
+				xmlRepresentation.append(" ");
+				xmlRepresentation.append(attribute.getNodeName());
+				xmlRepresentation.append("=\"");
+				String attr_val = attribute.getNodeValue();
+				
+				attr_val = attr_val.replaceAll("&","&amp;");
+				attr_val = attr_val.replaceAll("<","&lt;");
+				attr_val = attr_val.replaceAll(">","&gt;");
+				attr_val = attr_val.replaceAll("\"","&quot;");
+				// assume that any of the above chars that was already entity escaped
+				// was already correct => return back to how they were
+				attr_val = attr_val.replaceAll("&amp;amp;","&amp;");
+				attr_val = attr_val.replaceAll("&amp;lt;","&lt;");
+				attr_val = attr_val.replaceAll("&amp;gt;","&gt;");
+				attr_val = attr_val.replaceAll("&amp;quot;","&quot;");
+				
+				
+				xmlRepresentation.append(attr_val);
+				xmlRepresentation.append("\"");
+			}
+
+			// If the node has no children, close the opening tag and return
+			if (xmlNode.hasChildNodes() == false)
+			{
+				// This produces somewhat ugly output, but it is necessary to compensate
+				// for display bugs in Netscape. Firstly, the space is needed before the
+				// closing bracket otherwise Netscape will ignore some tags (<br/>, for
+				// example). Also, a newline character would be expected after the tag,
+				// but this causes problems with the display of links (the link text
+				// will contain a newline character, which is displayed badly).
+				xmlRepresentation.append(" />");
+				return;
+			}
+
+			// Close the opening tag
+			xmlRepresentation.append(">");
+
+			// Process the children. We process text nodes here, but recursively process other nodes. 
+			// hack for nodes next to text nodes - dont make them pretty, ie don'e do any new lines or indenting
+			// Usually text nodes will be inside their own element. Sometimes we have eg span tags next to text nodes - don't want those indented.
+			// also if these are inside a pre tag then the space shows up in the page.
+			
+			NodeList children = xmlNode.getChildNodes();
+			boolean do_pretty = pretty;
+			boolean output_escaping = true; // record if we have encountered a disable-output-escaping instruction
+			for (int i = 0; i < children.getLength(); i++)
+			{
+			  Node child = children.item(i);
+			  short child_type = child.getNodeType();
+			  if (child_type == Node.PROCESSING_INSTRUCTION_NODE) {
+			    if (child.getNodeName().equals("javax.xml.transform.disable-output-escaping")) {
+			      output_escaping = false;
+			    }
+			    else if (child.getNodeName().equals("javax.xml.transform.enable-output-escaping")) {
+			      output_escaping = true;
+			    }
+			    else {
+			      logger.warn("Unhandled processing instruction " + child.getNodeName());
+			    }
+			  }
+			  else if (child_type == Node.TEXT_NODE) {
+			   do_pretty = false; // if there is a text node amongst the children, do all the following nodes in non-pretty mode - hope this doesn't stuff up something else
+			    // output the text
+			    String text = child.getNodeValue();
+
+			    // Perform output escaping, if required
+			    // Apache Commons replace method is far superior to String.replaceAll - very fast!
+			    if (output_escaping) {
+			      text = StringUtils.replace(text, "&", "&amp;");
+			      text = StringUtils.replace(text, "<", "&lt;");
+			      text = StringUtils.replace(text, ">", "&gt;");
+			      text = StringUtils.replace(text, "'", "&apos;");
+			      text = StringUtils.replace(text, "\"", "&quot;");
+			    }
+			    // Remove any control-C characters
+			    text = StringUtils.replace(text, "" + (char) 3, "");
+			    
+			    xmlRepresentation.append(text);
+			    
+			  }
+			  else {
+			    // recursively call getString
+			    getString(child, xmlRepresentation, depth + 1, do_pretty);
+			  }
+			} // foreach child of the element
+
+			// Write closing tag
+			if (pretty)
+			{
+				if (xmlRepresentation.charAt(xmlRepresentation.length() - 1) == '\n')
+				{
+					for (int i = 0; i < depth; i++)
+						xmlRepresentation.append("  ");
+				}
+			}
+			xmlRepresentation.append("</");
+			xmlRepresentation.append(nodeName);
+			xmlRepresentation.append(">");
+			if (pretty)
+			{
+				xmlRepresentation.append("\n");
+			}
+		} // ELEMENT_NODE
+
+		else if (nodeType == Node.COMMENT_NODE)
+		{
+			String text = xmlNode.getNodeValue();
+			xmlRepresentation.append("<!-- ");
+			xmlRepresentation.append(text);
+			xmlRepresentation.append(" -->");
+		}
+
+		// TEXT and PROCESSING_INSTRUCTION nodes are handled inside their containing element node
+		// A type of node that is not handled yet
+		else
+		{
+			logger.warn("Unknown node type: " + nodeType + " " + getNodeTypeString(nodeType));
+		}
+
+		return;
+	}
+
+	protected static String getNodeTypeString(short node_type)
+	{
+
+		String type = "";
+		switch (node_type)
+		{
+		case Node.ATTRIBUTE_NODE:
+			type = "ATTRIBUTE_NODE";
+			break;
+		case Node.CDATA_SECTION_NODE:
+			type = "CDATA_SECTION_NODE";
+			break;
+		case Node.COMMENT_NODE:
+			type = "COMMENT_NODE";
+			break;
+		case Node.DOCUMENT_FRAGMENT_NODE:
+			type = "DOCUMENT_FRAGMENT_NODE";
+			break;
+		case Node.DOCUMENT_NODE:
+			type = "DOCUMENT_NODE";
+			break;
+		case Node.DOCUMENT_TYPE_NODE:
+			type = "DOCUMENT_TYPE_NODE";
+			break;
+		case Node.ELEMENT_NODE:
+			type = "ELEMENT_NODE";
+			break;
+		case Node.ENTITY_NODE:
+			type = "ENTITY_NODE";
+			break;
+		case Node.ENTITY_REFERENCE_NODE:
+			type = "ENTITY_REFERENCE_NODE";
+			break;
+		case Node.NOTATION_NODE:
+			type = "NOTATION_NODE";
+			break;
+		case Node.PROCESSING_INSTRUCTION_NODE:
+			type = "PROCESSING_INSTRUCTION_NODE";
+			break;
+		case Node.TEXT_NODE:
+			type = "TEXT_NODE";
+			break;
+		default:
+			type = "UNKNOWN";
+		}
+
+		return type;
+	}
+
+	// returns null if there no error occurred during parsing, or else returns the error message
+
+	// public String getParseErrorMessage()
+	// {
+	// 	ParseErrorHandler errorHandler = (ParseErrorHandler) this.parser.getErrorHandler();
+	// 	return errorHandler.getErrorMessage();
+	// }
+
+	// Errorhandler for SAXParseExceptions that are errors, fatal errors or warnings. This class can be used to 
+	// register a handler for any fatal errors, errors and warnings that may occur when parsing an xml file. The
+	// errors are printed both to the greenstone.log and to the tomcat console (System.err), and the error message
+	// is stored in the errorMessage variable so that it can be retrieved and be used to generate an xhtml error page.
+	static public class ParseErrorHandler implements ErrorHandler
+	{
+		protected String errorMessage = null;
+
+		//  Receive notification of a recoverable error.
+		public void error(SAXParseException exception)
+		{
+			handleError("Error:\n", exception);
+		}
+
+		//   Receive notification of a non-recoverable error.
+		public void fatalError(SAXParseException exception)
+		{
+			handleError("Fatal Error:\n", exception);
+		}
+
+		// Receive notification of a warning.
+		public void warning(SAXParseException exception)
+		{
+			handleError("Warning:\n", exception);
+		}
+
+		public String toString(SAXParseException e)
+		{
+			String msg = e.getMessage();
+			msg += "\nOn line(column): " + e.getLineNumber() + "(" + e.getColumnNumber() + ")";
+			msg += (e.getPublicId() != null) ? ("\npublic ID: " + e.getPublicId()) : "\nNo public ID";
+			msg += (e.getSystemId() != null) ? ("\nsystem ID: " + e.getSystemId()) : "\nNo system ID";
+
+			return msg;
+		}
+
+		// clears the errorPage variable after first call to this method
+		public String getErrorMessage()
+		{
+			String errMsg = this.errorMessage;
+			if (this.errorMessage != null)
+			{
+				this.errorMessage = null;
+			}
+			return errMsg;
+		}
+
+		// sets the errorMessage member variable to the data stored in the exception
+		// and writes the errorMessage to the logger and tomcat's System.err
+		protected void handleError(String errorType, SAXParseException exception)
+		{
+			this.errorMessage = errorType + toString(exception);
+			System.err.println("\n****Error parsing xml:\n" + this.errorMessage + "\n****\n");
+			logger.error(this.errorMessage);
+		}
+	}
+
+  public static boolean writeDOM(Element elem, File file) {
+
+    BufferedWriter writer = null;
+    boolean success = false;
+    try {
+      String xml_string = getString(elem);
+      // need createNewFile???
+      writer = new BufferedWriter(new FileWriter(file));
+      writer.write(xml_string);
+      success = true;
+    }
+
+    catch (Exception e) {
+      logger.error(e.getMessage());
+      success = false;
+    }
+    finally {
+      try {
+	if (writer != null) {
+	  writer.close();
+	}
+      } catch(Exception e) {
+	logger.error("couldn't close the file"+e.getMessage());
+      }
+    }
+    return success;
+  }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLTransformer.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLTransformer.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XMLTransformer.java	(revision 31634)
@@ -0,0 +1,859 @@
+/*
+ *    XMLTransformer.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URI;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.ErrorListener;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.URIResolver;
+import javax.xml.transform.dom.DOMResult;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.apache.log4j.Logger;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+
+/**
+ * XMLTransformer - utility class for greenstone
+ * 
+ * transforms xml using xslt
+ * 
+ * @author Katherine Don
+ * @version $Revision$
+ */
+public class XMLTransformer
+{
+	private static int debugFileCount = 0; // for unique filenames when debugging XML transformations with physical files
+
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.XMLTransformer.class.getName());
+
+	/** The transformer factory we're using */
+	TransformerFactory t_factory = null;
+
+	/**
+	 * The no-arguments constructor.
+	 * 
+	 * Any exceptions thrown are caught internally
+	 * 
+	 * @see javax.xml.transform.TransformerFactory
+	 */
+	public XMLTransformer()
+	{
+		// http://download.oracle.com/docs/cd/E17476_01/javase/1.5.0/docs/api/index.html?javax/xml/transform/TransformerFactory.html states that
+		// TransformerFactory.newInstance() looks in jar files for a Factory specified in META-INF/services/javax.xml.transform.TransformerFactory, 
+		// else it will use the "platform default"
+		// In this case: xalan.jar's META-INF/services/javax.xml.transform.TransformerFactory contains org.apache.xalan.processor.TransformerFactoryImpl
+		// as required.
+
+		// This means we no longer have to do a System.setProperty("javax.xml.transform.TransformerFactory", "org.apache.xalan.processor.TransformerFactoryImpl");
+		// followed by a this.t_factory = org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+		// The System.setProperty step to force the TransformerFactory implementation that gets used, conflicts with
+		// Fedora (visiting the Greenstone server pages breaks the Greenstone-tomcat hosted Fedora pages) as Fedora 
+		// does not include the xalan.jar and therefore can't then find the xalan TransformerFactory explicitly set.
+
+		// Gone back to forcing use of xalan transformer, since other jars like crimson.jar, which may be on some
+		// classpaths, could be be chosen as the TransformerFactory implementation over xalan. This is what used to
+		// give problems before. Instead, have placed copies of the jars that Fedora needs (xalan.jar and serializer.jar 
+		// and the related xsltc.jar which it may need) into packages/tomcat/lib so that it's on the server's classpath
+		// and will be found by Fedora.
+
+		// make sure we are using the xalan transformer
+		System.setProperty("javax.xml.transform.TransformerFactory", "org.apache.xalan.processor.TransformerFactoryImpl");
+		try
+		{
+			this.t_factory = org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
+			//this.t_factory = TransformerFactory.newInstance();
+			this.t_factory.setErrorListener(new TransformErrorListener()); // handle errors in the xml Source used to instantiate transformers
+		}
+		catch (Exception e)
+		{
+			logger.error("exception creating t_factory " + e.getMessage());
+		}
+	}
+
+	/**
+	 * Transform an XML document using a XSLT stylesheet
+	 * 
+	 * @param stylesheet
+	 *            a filename for an XSLT stylesheet
+	 * @param xml_in
+	 *            the XML to be transformed
+	 * @return the transformed XML
+	 */
+	public String transform(String stylesheet, String xml_in)
+	{
+
+		try
+		{
+			TransformErrorListener transformerErrorListener = (TransformErrorListener) this.t_factory.getErrorListener();
+			transformerErrorListener.setStylesheet(stylesheet);
+			// Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+			Transformer transformer = this.t_factory.newTransformer(new StreamSource(stylesheet));
+
+			// Use the Transformer to transform an XML Source and send the output to a Result object.
+			StringWriter output = new StringWriter();
+			StreamSource streamSource = new StreamSource(new StringReader(xml_in));
+			transformer.setErrorListener(new TransformErrorListener(stylesheet, streamSource));
+			transformer.transform(streamSource, new StreamResult(output));
+			return output.toString();
+		}
+		catch (TransformerConfigurationException e)
+		{
+			logger.error("couldn't create transformer object: " + e.getMessageAndLocation());
+			logger.error(e.getLocationAsString());
+			return "";
+		}
+		catch (TransformerException e)
+		{
+			logger.error("couldn't transform the source: " + e.getMessageAndLocation());
+			return "";
+		}
+	}
+
+	public String transformToString(Document stylesheet, Document source)
+	{
+		return transformToString(stylesheet, source, null);
+	}
+
+	public String transformToString(Document stylesheet, Document source, HashMap parameters)
+	{
+
+		try
+		{
+			TransformErrorListener transformerErrorListener = (TransformErrorListener) this.t_factory.getErrorListener();
+			transformerErrorListener.setStylesheet(stylesheet);
+			// Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+			Transformer transformer = this.t_factory.newTransformer(new DOMSource(stylesheet));
+			if (parameters != null)
+			{
+				Set params = parameters.entrySet();
+				Iterator i = params.iterator();
+				while (i.hasNext())
+				{
+					Map.Entry m = (Map.Entry) i.next();
+					transformer.setParameter((String) m.getKey(), m.getValue());
+				}
+			}
+			//transformer.setParameter("page_lang", source.getDocumentElement().getAttribute(GSXML.LANG_ATT));
+
+			// Use the Transformer to transform an XML Source and send the output to a Result object.
+			StringWriter output = new StringWriter();
+			DOMSource domSource = new DOMSource(source);
+
+			transformer.setErrorListener(new TransformErrorListener(stylesheet, domSource));
+			transformer.transform(domSource, new StreamResult(output));
+			return output.toString();
+		}
+		catch (TransformerConfigurationException e)
+		{
+			logger.error("couldn't create transformer object: " + e.getMessageAndLocation());
+			logger.error(e.getLocationAsString());
+			return "";
+		}
+		catch (TransformerException e)
+		{
+			logger.error("couldn't transform the source: " + e.getMessageAndLocation());
+			return "";
+		}
+	}
+
+	/**
+	 * Transform an XML document using a XSLT stylesheet, but using a DOMResult
+	 * whose node should be set to the Document donated by resultNode
+	 */
+	public Node transform_withResultNode(Document stylesheet, Document source, Document resultNode)
+	{
+		return transform(stylesheet, source, null, null, resultNode);
+	}
+
+	public Node transform(Document stylesheet, Document source)
+	{
+		return transform(stylesheet, source, null, null, null);
+	}
+
+	public Node transform(Document stylesheet, Document source, HashMap<String, Object> parameters)
+	{
+		return transform(stylesheet, source, parameters, null, null);
+	}
+
+	public Node transform(Document stylesheet, Document source, HashMap<String, Object> parameters, Document docDocType)
+	{
+		return transform(stylesheet, source, parameters, docDocType, null);
+	}
+
+	// This method will now set the docType in the new document created and returned, if any are specified in the
+	// (merged) stylesheet being applied. The docDocType parameter is therefore no longer necessary nor used by default.
+	protected Node transform(Document stylesheet, Document source, HashMap<String, Object> parameters, Document docDocType, Document resultNode)
+	{
+		try
+		{
+			// Use the TransformerFactory to process the stylesheet Source and generate a Transformer.
+			TransformErrorListener transformerErrorListener = (TransformErrorListener) this.t_factory.getErrorListener();
+			transformerErrorListener.setStylesheet(stylesheet);
+			Transformer transformer = this.t_factory.newTransformer(new DOMSource(stylesheet));
+			//logger.info("XMLTransformer transformer is " + transformer); //done in ErrorListener
+
+			if (parameters != null)
+			{
+				Set params = parameters.entrySet();
+				Iterator i = params.iterator();
+				while (i.hasNext())
+				{
+					Map.Entry m = (Map.Entry) i.next();
+					transformer.setParameter((String) m.getKey(), m.getValue());
+				}
+			}
+
+			// When we transform the DOMResult, we need to make sure the result of
+			// the transformation has a DocType. For that to happen, we need to create
+			// the DOMResult using a Document with a predefined docType.
+
+			// When the DOCType is not explicitly specified (default case), the docDocType variable is null
+			// In such a case, the transformer will work out the docType and output method and the rest 
+			// from the stylesheet. Better to let the transformer work this out than GS manually aggregating 
+			// all xsls being applied and the GS code deciding on which output method and doctype to use.
+
+			//DOMResult result = docDocType == null ? new DOMResult() : new DOMResult(docDocType);
+			DOMResult result = null;
+
+			Properties props = transformer.getOutputProperties();
+			if(docDocType == null) { // default case
+			
+			    String outputMethod = props.getProperty(OutputKeys.METHOD);
+			    if(outputMethod.equals("html")) {				
+				String doctype_public = props.getProperty(OutputKeys.DOCTYPE_PUBLIC);
+				String doctype_system = props.getProperty(OutputKeys.DOCTYPE_SYSTEM);
+				
+				if(doctype_public == null) {
+				  //doctype_public = ""; // or default to PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"?
+				  doctype_public = "-//W3C//DTD HTML 4.01 Transitional//EN";
+				}
+				if(doctype_system == null) {
+				  //doctype_system = ""; // or default to "http://www.w3.org/TR/html4/loose.dtd"?
+				  doctype_system = "http://www.w3.org/TR/html4/loose.dtd";
+				}
+
+				Document docDocTypeFromTransformer = XMLConverter.newDOM(outputMethod, doctype_public, doctype_system);
+				result = new DOMResult(docDocTypeFromTransformer);
+			    } 
+			    // if output method=xml, the <?xml ?> processing method goes missing hereafter, although it
+			    // still exists in OutputKeys' VERSION, ENCODING and OMIT_XML_DECLARATION props at this point
+					    
+			} else { // if document with doctype was already specified (no longer the default case)
+			    result = new DOMResult(docDocType);
+			}
+			// At this point if we haven't initialised result yet, set it to an empty DOMResult
+			if(result == null) {
+			    result = new DOMResult();
+			}
+
+
+			if (resultNode != null)
+			{
+				result.setNode(resultNode);
+			}
+			DOMSource domSource = new DOMSource(source);
+			transformer.setErrorListener(new TransformErrorListener(stylesheet, domSource));
+			transformer.transform(domSource, result);
+			return result.getNode(); // pass the entire document
+		}
+		catch (TransformerConfigurationException e)
+		{
+			return transformError("XMLTransformer.transform(Doc, Doc, HashMap, Doc)" + "\ncouldn't create transformer object", e);
+		}
+		catch (TransformerException e)
+		{
+			return transformError("XMLTransformer.transform(Doc, Doc, HashMap, Doc)" + "\ncouldn't transform the source", e);
+		}
+	}
+
+    /**
+      When transforming an XML with an XSLT from the command-line, there's two problems if calling 
+      XMLTransformer.transform(File xslt, File xml):
+
+      1. Need to run the transformation process from the location of the stylesheet, else it can't find
+      the stylesheet in order to load it. This is resolved by setting the SystemID for the stylesheet.
+
+      2. The XSLT stylesheet has &lt;xsl:import&gt; statements importing further XSLTs which are furthermore 
+      specified by relative paths, requiring that the XSLT (and perhaps also XML) input file is located
+      in the folder from which the relative paths of the import statements are specified. This is resolved
+      by working out the absolute path of each imported XSLT file and setting their SystemIDs too.
+
+      Without solving these problems, things will only work if we 1. run the transformation from the 
+      web/interfaces/default/transform folder and 2. need to have the input XSLT file (and XML?)
+      placed in the web/interfacces/default/transform folder too
+
+      Both parts of the solution are described in:
+      http://stackoverflow.com/questions/3699860/resolving-relative-paths-when-loading-xslt-filse
+
+      After the systemID on the XSLT input file is set, we can run the command from the toplevel GS3 
+      folder.
+      After resolving the URIs of the XSLT files imported by the input XSLT, by setting their systemIDs,
+      the input XSLT (and XML) file need not be located in web/interfaces/default/transform anymore.
+     */
+    public Node transform(File stylesheet, File source, String interfaceName, Document docDocType)
+	{
+		try
+		{
+			TransformErrorListener transformerErrorListener = (TransformErrorListener) this.t_factory.getErrorListener();
+			transformerErrorListener.setStylesheet(stylesheet);
+
+			// http://stackoverflow.com/questions/3699860/resolving-relative-paths-when-loading-xslt-files
+			Source xsltSource = new StreamSource(new InputStreamReader(new FileInputStream(stylesheet), "UTF-8"));
+			URI systemID = stylesheet.toURI();
+			try {			    
+			    URL url = systemID.toURL();			
+			    xsltSource.setSystemId(url.toExternalForm());
+			} catch (MalformedURLException mue) {
+			    logger.error("Warning: Unable to set systemID for stylesheet to " + systemID);
+			    logger.error("Got exception: " + mue.getMessage(), mue);
+			}
+
+			this.t_factory.setURIResolver(new ClasspathResourceURIResolver(interfaceName));			
+			Transformer transformer = this.t_factory.newTransformer(xsltSource);
+			// Alternative way of instantiating a newTransformer():
+			//Templates cachedXSLT = this.t_factory.newTemplates(xsltSource);
+			//Transformer transformer = cachedXSLT.newTransformer();
+
+			DOMResult result = (docDocType == null) ? new DOMResult() : new DOMResult(docDocType);
+			StreamSource streamSource = new StreamSource(new InputStreamReader(new FileInputStream(source), "UTF-8"));
+
+			transformer.setErrorListener(new TransformErrorListener(stylesheet, streamSource));
+
+			transformer.transform(streamSource, result);
+			return result.getNode().getFirstChild();
+		}
+		catch (TransformerConfigurationException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't create transformer object for files\n" + stylesheet + "\n" + source, e);
+		}
+		catch (TransformerException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't transform the source for files\n" + stylesheet + "\n" + source, e);
+		}
+		catch (UnsupportedEncodingException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't read file due to an unsupported encoding\n" + stylesheet + "\n" + source, e);
+		}
+		catch (FileNotFoundException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't find the file specified\n" + stylesheet + "\n" + source, e);
+		}
+	}
+
+    /** 
+     * Class for resolving the relative paths used for &lt;xsl:import&gt;s 
+     * when transforming XML with an XSLT
+    */
+    class ClasspathResourceURIResolver implements URIResolver {
+	String interface_name;
+
+	public ClasspathResourceURIResolver(String interfaceName) {
+	    interface_name = interfaceName;	    
+	}
+
+	/** 
+	 * Override the URIResolver.resolve() method to turn relative paths of imported xslt files into
+	 * absolute paths and set these absolute paths as their SystemIDs when loading them into memory.
+	 * @see http://stackoverflow.com/questions/3699860/resolving-relative-paths-when-loading-xslt-files
+	 */
+	
+	@Override
+	    public Source resolve(String href, String base) throws TransformerException {
+	    
+	    //System.err.println("href: " + href); // e.g. href: layouts/main.xsl
+	    //System.err.println("base: " + base); // e.g for *toplevel* base: file:/path/to/gs3-svn/CL1.xslt
+
+
+	    // 1. for any xsl imported by the original stylesheet, try to work out its absolute path location
+	    // by concatenating the parent folder of the base stylesheet file with the href of the file it
+	    // imports (which is a path relative to the base file). This need not hold true for the very 1st
+	    // base stylesheet: it may be located somewhere entirely different from the greenstone XSLT
+	    // files it refers to. This is the case when running transforms on the commandline for testing
+
+	    File importXSLfile = new File(href); // assume the xsl imported has an absolute path
+
+	    if(!importXSLfile.isAbsolute()) { // imported xsl specifies a path relative to parent of base
+		try {
+		    URI baseURI = new URI(base);
+		    importXSLfile = new File(new File(baseURI).getParent(), href);
+
+		    if(!importXSLfile.exists()) { // if the imported file does not exist, it's not
+			// relative to the base (the stylesheet file importing it). So look in the
+			// interface's transform subfolder for the xsl file to be imported
+			importXSLfile = new File(GSFile.interfaceStylesheetFile(GlobalProperties.getGSDL3Home(), interface_name, href));
+		    }
+		} catch(URISyntaxException use) {
+		    importXSLfile = new File(href); // try
+		}
+	    }
+	    
+	    String importXSLfilepath = ""; // for printing the expected file path on error
+	    try {
+		// path of import XSL file after resolving any .. and . dir paths
+		importXSLfilepath = importXSLfile.getCanonicalPath();
+	    } catch(IOException ioe) { // resort to using the absolute path
+		importXSLfilepath = importXSLfile.getAbsolutePath();
+	    }
+
+	    // 2. now we know where the XSL file being imported lives, so set the systemID to its
+	    // absolute path when loading it into a Source object
+	    URI systemID = importXSLfile.toURI();
+	    Source importXSLsrc = null;
+	    try {
+		importXSLsrc = new StreamSource(new InputStreamReader(new FileInputStream(importXSLfile), "UTF-8"));		
+		URL url = systemID.toURL();			
+		importXSLsrc.setSystemId(url.toExternalForm());
+	    } catch (MalformedURLException mue) {
+		logger.error("Warning: Unable to set systemID for imported stylesheet to " + systemID);
+		logger.error("Got exception: " + mue.getMessage(), mue);
+	    } catch(FileNotFoundException fne) {
+		logger.error("ERROR: importXSLsrc file does not exist " + importXSLfilepath);
+		logger.error("\tError msg: " + fne.getMessage(), fne);
+	    } catch(UnsupportedEncodingException uee) {
+		logger.error("ERROR: could not resolve relative path of import XSL file " + importXSLfilepath
+			     + " because of encoding issues: " + uee.getMessage(), uee);
+	    }	    
+	    return importXSLsrc;
+	    //return new StreamSource(this.getClass().getClassLoader().getResourceAsStream(href)); // won't work
+	}	
+    }
+
+	public Node transform(File stylesheet, File source)
+	{
+		return transform(stylesheet, source, null);
+	}
+
+	// debugAsFile is only to be set to true when either the stylesheet or source parameters 
+	// are not objects of type File. The debugAsFile variable is passed into the 
+	// TransformErrorListener. When set to true, the TransformErrorListener will itself create
+	// two files containing the stylesheet and source XML, and try to transform the new source
+	// file with the stylesheet file for debugging purposes.
+	protected Node transform(File stylesheet, File source, Document docDocType)
+	{
+		try
+		{
+			TransformErrorListener transformerErrorListener = (TransformErrorListener) this.t_factory.getErrorListener();
+			transformerErrorListener.setStylesheet(stylesheet);
+			Transformer transformer = this.t_factory.newTransformer(new StreamSource(new InputStreamReader(new FileInputStream(stylesheet), "UTF-8")));
+			DOMResult result = (docDocType == null) ? new DOMResult() : new DOMResult(docDocType);
+			StreamSource streamSource = new StreamSource(new InputStreamReader(new FileInputStream(source), "UTF-8"));
+
+			transformer.setErrorListener(new TransformErrorListener(stylesheet, streamSource));
+
+			transformer.transform(streamSource, result);
+			return result.getNode().getFirstChild();
+		}
+		catch (TransformerConfigurationException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't create transformer object for files\n" + stylesheet + "\n" + source, e);
+		}
+		catch (TransformerException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't transform the source for files\n" + stylesheet + "\n" + source, e);
+		}
+		catch (UnsupportedEncodingException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't read file due to an unsupported encoding\n" + stylesheet + "\n" + source, e);
+		}
+		catch (FileNotFoundException e)
+		{
+			return transformError("XMLTransformer.transform(File, File)" + "\ncouldn't find the file specified\n" + stylesheet + "\n" + source, e);
+		}
+	}
+
+	// Given a heading string on the sort of transformation error that occurred and the exception object itself, 
+	// this method prints the exception to the tomcat window (system.err) and the greenstone log and then returns
+	// an xhtml error page that is constructed from it.
+	protected Node transformError(String heading, Exception e)
+	{
+		String message = heading + "\n" + e.getMessage();
+		logger.error(heading + ": " + e.getMessage());
+
+		if (e instanceof TransformerException)
+		{
+			String location = ((TransformerException) e).getLocationAsString();
+			if (location != null)
+			{
+				logger.error(location);
+				message = message + "\n" + location;
+			}
+		}
+		System.err.println("****\n" + message + "\n****");
+		return constructErrorXHTMLPage(message);
+	}
+
+	// Given an error message, splits it into separate lines based on any newlines present and generates an xhtml page
+	// (xml Element)  with paragraphs for each line. This is then returned so that it can be displayed in the browser.
+	public static Element constructErrorXHTMLPage(String message)
+	{
+		try
+		{
+			String[] lines = message.split("\n");
+
+			Document xhtmlDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
+			// <html></html>
+			Node htmlNode = xhtmlDoc.createElement("html");
+			xhtmlDoc.appendChild(htmlNode);
+			// <head></head>
+			Node headNode = xhtmlDoc.createElement("head");
+			htmlNode.appendChild(headNode);
+			// <title></title>
+			Node titleNode = xhtmlDoc.createElement("title");
+			headNode.appendChild(titleNode);
+			Node titleString = xhtmlDoc.createTextNode("Error occurred");
+			titleNode.appendChild(titleString);
+
+			// <body></body>
+			Node bodyNode = xhtmlDoc.createElement("body");
+			htmlNode.appendChild(bodyNode);
+
+			// finally put the message in the body
+			Node h1Node = xhtmlDoc.createElement("h1");
+			bodyNode.appendChild(h1Node);
+			Node headingString = xhtmlDoc.createTextNode("The following error occurred:");
+			h1Node.appendChild(headingString);
+
+			//Node textNode = xhtmlDoc.createTextNode(message);
+			//bodyNode.appendChild(textNode);
+
+			for (int i = 0; i < lines.length; i++)
+			{
+				Node pNode = xhtmlDoc.createElement("p");
+				Node textNode = xhtmlDoc.createTextNode(lines[i]);
+				pNode.appendChild(textNode);
+				bodyNode.appendChild(pNode);
+			}
+
+			return xhtmlDoc.getDocumentElement();
+
+		}
+		catch (Exception e)
+		{
+			String errmsg = "Exception trying to construct error xhtml page from message: " + message + "\n" + e.getMessage();
+			System.err.println(errmsg);
+			logger.error(errmsg);
+			return null;
+		}
+	}
+
+	// ErrorListener class for both Transformer objects and TransformerFactory objects.
+	// This class can be used to register a handler for any fatal errors, errors and warnings that
+	// may occur when either transforming an xml file with an xslt stylesheet using the XMLTransformer,
+	// or when instantiating a Transformer object using the XMLTransformer's TransformerFactory member var.
+	// The latter case occurs when the xml Source used to instantiate a Transformer from a TransformerFactory
+	// is invalid in some manner, which results in a null Transformer object. However, as no 
+	// TransformerConfigurationException or TransformerException are thrown in this case, the errors 
+	// would have not been noticed until things go wrong later when trying to use the (null) Transformer.
+	//
+	// The errors caught by this ErrorListener class are printed both to the greenstone.log and to the 
+	// tomcat console (System.err), and the error message is stored in the errorMessage variable so that 
+	// it can be retrieved and be used to generate an xhtml error page.
+	public class TransformErrorListener implements ErrorListener
+	{
+		protected String errorMessage = null;
+		protected String stylesheet = null;
+		protected Source source = null; // can be DOMSource or StreamSource
+		protected boolean debugAsFile = true; // true if xslt or source are not real physical files
+
+		// *********** METHODS TO BE CALLED WHEN SETTING AN ERROR LISTENER ON TRANSFORMERFACTORY OBJECTS
+		// The default constructor is only for when setting an ErrorListener on TransformerFactory objects
+		public TransformErrorListener()
+		{
+			this.stylesheet = null;
+			this.source = null;
+			XMLTransformer.debugFileCount++;
+		}
+
+		public void setStylesheet(Document xslt)
+		{
+			this.debugAsFile = true;
+			this.stylesheet = GSXML.elementToString(xslt.getDocumentElement(), true);
+			this.source = null;
+		}
+
+		public void setStylesheet(String xslt)
+		{
+			this.debugAsFile = true;
+			this.stylesheet = xslt;
+			this.source = null;
+		}
+
+		public void setStylesheet(File xslt)
+		{
+			this.debugAsFile = false; // if this constructor is called, we're dealing with physical files for both xslt and source			
+			this.stylesheet = xslt.getAbsolutePath();
+			this.source = null;
+		}
+
+		// *********** METHODS TO BE CALLED WHEN SETTING AN ERROR LISTENER ON TRANSFORMERFACTORY OBJECTS
+		// When setting an ErrorListener on Transformer object, the ErrorListener takes a Stylesheet xslt and a Source		
+		public TransformErrorListener(String xslt, Source source)
+		{
+			this.stylesheet = xslt;
+			this.source = source;
+			XMLTransformer.debugFileCount++;
+		}
+
+		public TransformErrorListener(Document xslt, Source source)
+		{
+			this.stylesheet = GSXML.elementToString(xslt.getDocumentElement(), true);
+			this.source = source;
+			XMLTransformer.debugFileCount++;
+		}
+
+		public TransformErrorListener(File xslt, Source source)
+		{
+			this.debugAsFile = false; // if this constructor is called, we're dealing with physical files for both xslt and source
+			this.source = source;
+			this.stylesheet = xslt.getAbsolutePath(); // not necessary to get the string from the file
+			// all we were going to do with it *on error* was write it out to a file anyway		
+		}
+
+		// *********** METHODS CALLED AUTOMATICALLY ON ERROR
+
+		//  Receive notification of a recoverable error.
+		public void error(TransformerException exception)
+		{
+			handleError("Error:\n", exception);
+		}
+
+		// Receive notification of a non-recoverable error.
+		public void fatalError(TransformerException exception)
+		{
+			handleError("Fatal Error:\n", exception);
+		}
+
+		// Receive notification of a warning.
+		public void warning(TransformerException exception)
+		{
+			handleError("Warning:\n", exception);
+		}
+
+		public String toString(TransformerException e)
+		{
+			String msg = "Exception encountered was:\n\t";
+			String location = e.getLocationAsString();
+			if (location != null)
+			{
+				msg = msg + "Location: " + location + "\n\t";
+			}
+
+			return msg + "Message: " + e.getMessage();
+		}
+
+		// clears the errorPage variable after the first call to this method
+		public String getErrorMessage()
+		{
+			String errMsg = this.errorMessage;
+			if (this.errorMessage != null)
+			{
+				this.errorMessage = null;
+			}
+			return errMsg;
+		}
+
+		// sets the errorMessage member variable to the data stored in the exception
+		// and writes the errorMessage to the logger and tomcat's System.err
+		protected void handleError(String errorType, TransformerException exception)
+		{
+
+			this.errorMessage = errorType + toString(exception);
+
+			// If either the stylesheet or the source to be transformed with it were not files,
+			// so that the transformation was performed in-memory, then the "location" information
+			// during the error handling (if any) wouldn't have been helpful.
+			// To allow proper debugging, we write both stylesheet and source out as physical files
+			// and perform the same transformation again, so that when a transformation error does 
+			// occur, the files are not in-memory but can be viewed, and any location information
+			// for the error given by the ErrorListener will be sensible (instead of the unhelpful
+			// "line#0 column#0 in file://somewhere/dummy.xsl").
+			// Note that if the stylesheet and the source it is to transform were both physical 
+			// files to start off with, we will not need to perform the same transformation again
+			// since the error reporting would have provided accurate locations for those.
+			if (debugAsFile)
+			{
+			    System.err.println("\n****Original error transforming xml:\n" + this.errorMessage + "\n****\n"); 
+			    System.err.println("***** About to perform the transform again with actual files\n******\n");
+			    logger.error("Original transformation error: " + this.errorMessage);
+			    logger.error("**** About to perform transform again with actual files");
+
+
+				performTransformWithPhysicalFiles(); // will give accurate line numbers
+
+				// No need to print out the current error message (seen in the Else statement below), 
+				// as the recursive call to XMLTransformer.transform(File, File, false) in method
+				// performTransformWithPhysicalFiles() will do this for us.
+			}
+			else
+			{
+				// printing out the error message
+				// since !debugAsFile, we are dealing with physical files, 
+				// variable stylesheet would have stored the filename instead of contents
+				this.errorMessage = this.errorMessage + "\nstylesheet filename: " + stylesheet;
+
+				this.errorMessage += "\nException CAUSE:\n" + exception.getCause();
+				System.err.println("\n****Error transforming xml:\n" + this.errorMessage + "\n****\n");
+				//System.err.println("Stylesheet was:\n + this.stylesheet + "************END STYLESHEET***********\n\n");		    
+
+				logger.error(this.errorMessage);
+
+				// now print out the source to a file, and run the stylesheet on it using a transform()
+				// then any error will be referring to one of these two input files.
+			}
+		}
+
+		// This method will redo the transformation that went wrong with *real* files: 
+		// it writes out the stylesheet and source XML to files first, then performs the transformation
+		// to get the actual line location of where things went wrong (instead of "line#0 column#0 in dummy.xsl")
+		protected void performTransformWithPhysicalFiles()
+		{
+			File webLogsTmpFolder = new File(GlobalProperties.getGSDL3Home() + File.separator + "logs" + File.separator + "tmp");
+			if (!webLogsTmpFolder.exists())
+			{
+				webLogsTmpFolder.mkdirs(); // create any necessary folders
+			}
+			File styleFile = new File(webLogsTmpFolder + File.separator + "stylesheet" + XMLTransformer.debugFileCount + ".xml");
+			File sourceFile = new File(webLogsTmpFolder + File.separator + "source" + XMLTransformer.debugFileCount + ".xml");
+			try
+			{
+				// write stylesheet to a file called stylesheet_systemID in tmp
+				FileWriter styleSheetWriter = new FileWriter(styleFile);
+				styleSheetWriter.write(stylesheet, 0, stylesheet.length());
+				styleSheetWriter.flush();
+				styleSheetWriter.close();
+			}
+			catch (Exception e)
+			{
+				System.err.println("*** Exception when trying to write out stylesheet to " + styleFile.getAbsolutePath());
+			}
+
+			if (this.source != null)
+			{ // ErrorListener was set on a Transformer object
+				try
+				{
+					FileWriter srcWriter = new FileWriter(sourceFile);
+					String contents = "";
+					if (source instanceof DOMSource)
+					{
+						DOMSource domSource = (DOMSource) source;
+						Document doc = (Document) domSource.getNode();
+						contents = GSXML.elementToString(doc.getDocumentElement(), true);
+						//contents = GSXML.xmlNodeToXMLString(domSource.getNode());
+					}
+					else if (source instanceof StreamSource)
+					{
+						StreamSource streamSource = (StreamSource) source;
+						BufferedReader reader = new BufferedReader(streamSource.getReader());
+						String line = "";
+						while ((line = reader.readLine()) != null)
+						{
+							contents = contents + line + "\n";
+						}
+					}
+					srcWriter.write(contents, 0, contents.length());
+					srcWriter.flush();
+					srcWriter.close();
+				}
+				catch (Exception e)
+				{
+					System.err.println("*** Exception when trying to write out stylesheet to " + sourceFile.getAbsolutePath());
+				}
+			}
+
+			System.err.println("*****************************************");
+			System.err.println("Look for stylesheet in: " + styleFile.getAbsolutePath());
+			if (this.source != null)
+			{ // ErrorListener was set on a Transformer object
+				System.err.println("Look for source XML in: " + sourceFile.getAbsolutePath());
+			}
+
+			// now perform the transform again, which will assign another TransformErrorListener
+			// but since debuggingAsFile is turned off, we won't recurse into this section of
+			// handling the error again
+			if (this.source != null)
+			{ // ErrorListener was set on a Transformer object
+				XMLTransformer.this.transform(styleFile, sourceFile); // calls the File, File version, so debugAsFile will be false		
+			}
+			else
+			{ // ErrorListener was set on a TransformerFactory object
+
+				// The recursive step in this case is to perform the instantiation 
+				// of the Transformer object again.
+				// Only one TransformerFactory object per XMLTransformer, 
+				// and only one TransformerHandler object set on any TransformerFactory
+				// But the stylesheet used to create a Transformer from that TransformerFactory
+				// object changes each time, by calls to setStylesheet(), 
+				// Therefore, the debugAsFile state for the single TransformerFactory's 
+				// TransformerHandler changes each time also.
+
+				try
+				{
+					debugAsFile = false;
+					this.stylesheet = styleFile.getAbsolutePath();
+					//TransformErrorListener transformerErrorListener = (TransformErrorListener)XMLTransformer.this.t_factory.getErrorListener();
+					//transformerErrorListener.setStylesheet(styleFile);
+					Transformer transformer = XMLTransformer.this.t_factory.newTransformer(new StreamSource(styleFile));
+					if (transformer == null)
+					{
+						String msg = "XMLTransformer transformer is " + transformer;
+						logger.info(msg);
+						System.out.println(msg + "\n****\n");
+					}
+				}
+				catch (TransformerConfigurationException e)
+				{
+					String message = "Couldn't create transformer object: " + e.getMessageAndLocation();
+					logger.error(message);
+					logger.error(e.getLocationAsString());
+					System.out.println(message);
+				}
+			}
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XSLTUtil.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XSLTUtil.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/XSLTUtil.java	(revision 31634)
@@ -0,0 +1,864 @@
+/*
+ *    XSLTUtil.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Locale;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import net.tanesha.recaptcha.ReCaptcha;
+import net.tanesha.recaptcha.ReCaptchaFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.greenstone.util.GlobalProperties;
+import org.w3c.dom.Node;
+
+/**
+ * a class to contain various static methods that are used by the xslt
+ * stylesheets
+ */
+public class XSLTUtil
+{
+	protected static HashMap<String, ArrayList<String>> _foundTableValues = new HashMap<String, ArrayList<String>>();
+	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.util.XSLTUtil.class.getName());
+	protected static HashMap<String, String> _stringVariables = new HashMap<String, String>();
+
+	public static void storeString(String name, String value)
+	{
+		_stringVariables.put(name, value);
+	}
+
+	public static String getString(String name)
+	{
+		return _stringVariables.get(name);
+	}
+
+	/* some tests */
+	public static boolean equals(String s1, String s2)
+	{
+		return s1.equals(s2);
+	}
+
+	public static boolean notEquals(String s1, String s2)
+	{
+		return !s1.equals(s2);
+	}
+
+	public static boolean exists(String s1, String s2)
+	{
+		return !s1.equals("");
+	}
+
+	public static boolean contains(String s1, String s2)
+	{
+		return (s1.indexOf(s2) != -1);
+	}
+
+	public static boolean startsWith(String s1, String s2)
+	{
+		return s1.startsWith(s2);
+	}
+
+	public static boolean endsWith(String s1, String s2)
+	{
+		return s1.endsWith(s2);
+	}
+
+	public static boolean lessThan(String s1, String s2)
+	{
+		return (s1.compareTo(s2) < 0);
+	}
+
+	public static boolean lessThanOrEquals(String s1, String s2)
+	{
+		return (s1.compareTo(s2) <= 0);
+	}
+
+	public static boolean greaterThan(String s1, String s2)
+	{
+		return (s1.compareTo(s2) > 0);
+	}
+
+	public static boolean greaterThanOrEquals(String s1, String s2)
+	{
+		return (s1.compareTo(s2) >= 0);
+	}
+
+	public static boolean oidIsMatchOrParent(String first, String second)
+	{
+		if (first.equals(second))
+		{
+			return true;
+		}
+
+		String[] firstParts = first.split(".");
+		String[] secondParts = second.split(".");
+
+		if (firstParts.length >= secondParts.length)
+		{
+			return false;
+		}
+
+		for (int i = 0; i < firstParts.length; i++)
+		{
+			if (!firstParts[i].equals(secondParts[i]))
+			{
+				return false;
+			}
+		}
+
+		return true;
+	}
+
+	public static String oidDocumentRoot(String oid)
+	{
+		String[] oidParts = oid.split("\\.");
+
+		return oidParts[0];
+	}
+
+	public static String replace(String orig, String match, String replacement)
+	{
+		return orig.replace(match, replacement);
+	}
+
+	public static String getNumberedItem(String list, int number)
+	{
+	    String[] items = list.split(",", -1); //String[] items = StringUtils.split(list, ",", -1); 
+	    // Using StringUtils.split() causes an off-by-one error for the boolean operators (fqk)
+	    // where boolean operators combining rows in multiforms are shifted up by 1 row.
+
+		if (items.length > number)
+		{
+			return items[number];
+		}
+		return ""; // index out of bounds
+	}
+
+	/**
+	 * Generates links to equivalent documents for a document with a default
+	 * document icon/type. Links are generated from the parameters: a list of
+	 * document icons which are each in turn embedded in the matching starting
+	 * link tag in the list of docStartLinks (these starting links link to the
+	 * equivalent documents in another format). Each link's start tag is closed
+	 * with the corresponding closing tag in the docEndLinks list. Parameter
+	 * token is the list separator. Parameter divider is the string that should
+	 * separate each final link generated from the next. Returns a string that
+	 * represents a sequence of links to equivalent documents, where the anchor
+	 * is a document icon.
+	 */
+	public static String getEquivDocLinks(String token, String docIconsString, String docStartLinksString, String docEndLinksString, String divider)
+	{
+		String[] docIcons = StringUtils.split(docIconsString, token, -1);
+		String[] startLinks = StringUtils.split(docStartLinksString, token, -1);
+		String[] endLinks = StringUtils.split(docEndLinksString, token, -1);
+
+		StringBuffer buffer = new StringBuffer();
+		for (int i = 0; i < docIcons.length; i++)
+		{
+			if (i > 0)
+			{
+				buffer.append(divider);
+			}
+			buffer.append(startLinks[i] + docIcons[i] + endLinks[i]);
+		}
+
+		return buffer.toString();
+	}
+
+	public static String getInterfaceText(String interface_name, String lang, String key)
+	{
+		return getInterfaceText(interface_name, lang, key, null);
+	}
+
+	public static String getInterfaceText(String interface_name, String lang, String key, String args_str)
+	{
+		key = key.replaceAll("__INTERFACE_NAME__", interface_name);
+
+		String[] args = null;
+		if (args_str != null && !args_str.equals(""))
+		{
+			args = StringUtils.split(args_str, ";");
+		}
+		Dictionary dict = new Dictionary("interface_" + interface_name, lang);
+		String result = dict.get(key, args);
+		if (result == null)
+		{ // not found
+			//if not found, search a separate subdirectory named by the interface name
+			String sep_interface_dir = interface_name + File.separatorChar + lang + File.separatorChar + "interface";
+			dict = new Dictionary(sep_interface_dir, lang);
+			result = dict.get(key, args);
+			if (result != null)
+			{
+				result = result.replaceAll("__INTERFACE_NAME__", interface_name);
+				return result;
+			}
+		}
+
+		if (result == null && !interface_name.equals("default"))
+		{ // not found, try the default interface
+			dict = new Dictionary("interface_default", lang);
+			result = dict.get(key, args);
+		}
+
+		if (result == null)
+		{ // not found
+			return "_" + key + "_";
+		}
+		result = result.replaceAll("__INTERFACE_NAME__", interface_name);
+		return result;
+	}
+
+	public static String getInterfaceText(String interfaceName, String dictionaryName, String lang, String key, String args_str)
+	{
+		key = key.replaceAll("__INTERFACE_NAME__", interfaceName);
+
+		String[] args = null;
+		if (args_str != null && !args_str.equals(""))
+		{
+			args = StringUtils.split(args_str, ";");
+		}
+		Dictionary dict = new Dictionary(dictionaryName, lang);
+		String result = dict.get(key, args);
+		if (result == null)
+		{ // not found
+			//if not found, search a separate subdirectory named by the interface name
+			String sep_interface_dir = interfaceName + File.separatorChar + lang + File.separatorChar + "interface";
+			dict = new Dictionary(sep_interface_dir, lang);
+			result = dict.get(key, args);
+			if (result != null)
+			{
+				result = result.replaceAll("__INTERFACE_NAME__", interfaceName);
+				return result;
+			}
+		}
+
+		if (result == null && !interfaceName.equals("default"))
+		{ // not found, try the default interface
+			dict = new Dictionary("interface_default", lang);
+			result = dict.get(key, args);
+		}
+
+		if (result == null)
+		{ // not found
+			return "_" + key + "_";
+		}
+		result = result.replaceAll("__INTERFACE_NAME__", interfaceName);
+		return result;
+	}
+
+	public static String getInterfaceTextWithDOM(String interface_name, String lang, String key, Node arg_node)
+	{
+		String[] args = new String[1];
+
+		String node_str = XMLConverter.getString(arg_node);
+		args[0] = node_str;
+		Dictionary dict = new Dictionary("interface_" + interface_name, lang);
+		String result = dict.get(key, args);
+		if (result == null)
+		{ // not found
+			//if not found, search a separate subdirectory named by the interface name
+			String sep_interface_dir = interface_name + File.separatorChar + lang + File.separatorChar + "interface";
+			dict = new Dictionary(sep_interface_dir, lang);
+			result = dict.get(key, args);
+			if (result != null)
+			{
+				return result;
+			}
+		}
+
+		if (result == null && !interface_name.equals("default"))
+		{ // not found, try the default interface
+			dict = new Dictionary("interface_default", lang);
+			result = dict.get(key, args);
+		}
+
+		if (result == null)
+		{ // not found
+			return "_" + key + "_";
+		}
+
+		return result;
+	}
+
+	public static String getInterfaceTextWithDOM(String interface_name, String lang, String key, Node arg1_node, Node arg2_node)
+	{
+		String[] args = new String[2];
+
+		String node_str = XMLConverter.getString(arg1_node);
+		args[0] = node_str;
+		node_str = XMLConverter.getString(arg2_node);
+		args[1] = node_str;
+		Dictionary dict = new Dictionary("interface_" + interface_name, lang);
+		String result = dict.get(key, args);
+		if (result == null)
+		{ // not found
+			//if not found, search a separate subdirectory named by the interface name
+			String sep_interface_dir = interface_name + File.separatorChar + lang + File.separatorChar + "interface";
+			dict = new Dictionary(sep_interface_dir, lang);
+			result = dict.get(key, args);
+			if (result != null)
+			{
+				return result;
+			}
+		}
+
+		if (result == null && !interface_name.equals("default"))
+		{ // not found, try the default interface
+			dict = new Dictionary("interface_default", lang);
+			result = dict.get(key, args);
+		}
+
+		if (result == null)
+		{ // not found
+			return "_" + key + "_";
+		}
+
+		return result;
+	}
+
+
+    	public static String getInterfaceTextSubstituteArgs(String value, String args_str)
+	{
+	    String[] args = null;
+	    if (args_str != null && !args_str.equals("")) {
+		args = StringUtils.split(args_str, ";");
+	    }
+
+	    return Dictionary.processArgs(value,args);
+	}
+	    
+	public static Node getCollectionText(String collection, String site_name, String lang, String key)
+	{
+		return getCollectionTextWithArgs(collection, site_name, lang, key, null);
+	}
+
+	public static Node getCollectionText(String collection, String site_name, String lang, String key, String args_str)
+	{
+
+		String[] args = null;
+		if (args_str != null && !args_str.equals(""))
+		{
+			args = StringUtils.split(args_str, ";");
+		}
+
+		return getCollectionTextWithArgs(collection, site_name, lang, key, args);
+	}
+
+	// xslt didn't like calling the function with Node varargs, so have this hack for now
+	public static Node getCollectionTextWithDOM(String collection, String site_name, String lang, String key, Node n1)
+	{
+		return getCollectionTextWithDOMMulti(collection, site_name, lang, key, n1);
+	}
+
+	public static Node getCollectionTextWithDOM(String collection, String site_name, String lang, String key, Node n1, Node n2)
+	{
+		return getCollectionTextWithDOMMulti(collection, site_name, lang, key, n1, n2);
+	}
+
+	public static Node getCollectionTextWithDOM(String collection, String site_name, String lang, String key, Node n1, Node n2, Node n3)
+	{
+		return getCollectionTextWithDOMMulti(collection, site_name, lang, key, n1, n2, n3);
+	}
+
+	public static Node getCollectionTextWithDOM(String collection, String site_name, String lang, String key, Node n1, Node n2, Node n3, Node n4)
+	{
+		return getCollectionTextWithDOMMulti(collection, site_name, lang, key, n1, n2, n3, n4);
+	}
+
+	public static Node getCollectionTextWithDOMMulti(String collection, String site_name, String lang, String key, Node... nodes)
+	{
+		int num_nodes = nodes.length;
+		String[] args = null;
+		if (num_nodes != 0)
+		{
+			args = new String[num_nodes];
+
+			for (int i = 0; i < num_nodes; i++)
+			{
+				String node_str = XMLConverter.getString(nodes[i]);
+				args[i] = node_str;
+			}
+		}
+		return getCollectionTextWithArgs(collection, site_name, lang, key, args);
+	}
+
+	public static Node getCollectionTextWithArgs(String collection, String site_name, String lang, String key, String[] args)
+	{
+		try
+		{
+			DocumentBuilder docBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+
+			CustomClassLoader class_loader = new CustomClassLoader(XSLTUtil.class.getClassLoader(), GSFile.collectionResourceDir(GSFile.siteHome(GlobalProperties.getGSDL3Home(), site_name), collection));
+			Dictionary dict = new Dictionary(collection, lang, class_loader);
+			String result = dict.get(key, args);
+			if (result != null)
+			{
+				return docBuilder.parse(new ByteArrayInputStream(("<fragment>" + result + "</fragment>").getBytes("UTF-8"))).getDocumentElement();
+			}
+			return docBuilder.parse(new ByteArrayInputStream(("<fragment>" + "text:" + collection + ":" + key + "</fragment>").getBytes())).getDocumentElement();
+		}
+		catch (Exception ex)
+		{
+			return null;
+		}
+	}
+
+	public static boolean isImage(String mimetype)
+	{
+		if (mimetype.startsWith("image/"))
+		{
+			return true;
+		}
+		return false;
+	}
+
+	// formatting /preprocessing functions
+	// some require a language, so we'll have a language param for all
+	public static String toLower(String orig, String lang)
+	{
+		return orig.toLowerCase();
+	}
+
+	public static String toUpper(String orig, String lang)
+	{
+		return orig.toUpperCase();
+	}
+
+	public static String tidyWhitespace(String original, String lang)
+	{
+
+		if (original == null || original.equals(""))
+		{
+			return original;
+		}
+		String new_s = original.replaceAll("\\s+", " ");
+		return new_s;
+	}
+
+	public static String stripWhitespace(String original, String lang)
+	{
+
+		if (original == null || original.equals(""))
+		{
+			return original;
+		}
+		String new_s = original.replaceAll("\\s+", "");
+		return new_s;
+	}
+
+	public static byte[] toUTF8(String orig, String lang)
+	{
+		try
+		{
+			byte[] utf8 = orig.getBytes("UTF-8");
+			return utf8;
+		}
+		catch (Exception e)
+		{
+			logger.error("unsupported encoding");
+			return orig.getBytes();
+		}
+	}
+
+	public static String formatDate(String date, String lang)
+	{
+		String in_pattern = "yyyyMMdd";
+		String out_pattern = "dd MMMM yyyy";
+		if (date.length() == 6)
+		{
+			in_pattern = "yyyyMM";
+		}
+
+		SimpleDateFormat formatter = new SimpleDateFormat(in_pattern, new Locale(lang));
+		try
+		{
+			Date d = formatter.parse(date);
+			formatter.applyPattern(out_pattern);
+			String new_date = formatter.format(d);
+			return new_date;
+		}
+		catch (Exception e)
+		{
+			return date;
+		}
+
+	}
+
+  public static final int TS_SECS = 0;
+  public static final int TS_MILLISECS = 1;
+  public static final int F_DATE = 0;
+  public static final int F_TIME = 1;
+  public static final int F_DATETIME = 2;
+  public static final int F_DAYSAGO = 3;
+
+  public static String formatTimeStamp(String timestamp, int ts_type, int format_type, String lang) {
+    try {
+      long ts = Long.parseLong(timestamp);
+      if (ts_type == TS_SECS) {
+	ts = ts * 1000;
+      }
+      if (format_type == F_DAYSAGO) {
+	long current_time = new Date().getTime();
+	long days = (current_time - ts)/86400000;
+	return String.valueOf(days);
+      }
+      Date d = new Date(ts);
+      DateFormat df;
+      switch (format_type) {
+      case F_DATE:
+	df = DateFormat.getDateInstance(DateFormat.DEFAULT, new Locale(lang));
+	break;
+      case F_TIME:
+	df = DateFormat.getTimeInstance(DateFormat.DEFAULT, new Locale(lang));
+	break;
+      case F_DATETIME:
+	df = DateFormat.getDateTimeInstance(DateFormat.DEFAULT, DateFormat.DEFAULT, new Locale(lang));
+	break;
+      default:
+	df = DateFormat.getDateInstance(DateFormat.DEFAULT, new Locale(lang));
+	break;
+      }
+      
+      return df.format(d);
+    } catch (Exception e) {
+      
+      return timestamp + e.getMessage();
+    }
+    
+  }
+	public static String getDetailFromDate(String date, String detail, String lang)
+	{
+		String in_pattern = "yyyyMMdd";
+		if (date.length() == 6)
+		{
+			in_pattern = "yyyyMM";
+		}
+
+		SimpleDateFormat formatter = new SimpleDateFormat(in_pattern, new Locale(lang));
+		try
+		{
+			Date d = formatter.parse(date);
+			if (detail.toLowerCase().equals("day"))
+			{
+				formatter.applyPattern("dd");
+			}
+			else if (detail.toLowerCase().equals("month"))
+			{
+				formatter.applyPattern("MMMM");
+			}
+			else if (detail.toLowerCase().equals("year"))
+			{
+				formatter.applyPattern("yyyy");
+			}
+			else
+			{
+				return "";
+			}
+			return formatter.format(d);
+		}
+		catch (Exception ex)
+		{
+			return "";
+		}
+	}
+
+	public static String formatLanguage(String display_lang, String lang)
+	{
+
+		return new Locale(display_lang).getDisplayLanguage(new Locale(lang));
+	}
+	
+	public static boolean checkFileExistence(String site_name, String filePath){
+
+		String gsdl3_home = GlobalProperties.getGSDL3Home();
+		//Remove leading file separator
+		filePath = filePath.replaceAll("^/+", "");
+		//Remove duplicates and replace by separator for current platform
+		filePath = filePath.replaceAll("/+", File.separator);
+		//Create full path to check
+		String fullPath = GSFile.siteHome(gsdl3_home, site_name) + File.separator + filePath;
+		File file = new File(fullPath);
+		if (file.exists() && file.isFile()) {
+			return true;
+		}
+		return false;
+	}
+	
+	public static String uriEncode(String input)
+	{
+		String result = "";
+		try {
+			result = URLEncoder.encode(input, "UTF-8");
+		} catch (UnsupportedEncodingException e) {
+			e.printStackTrace();
+		}
+		
+		return result;
+		
+	}
+
+	public static String cgiSafe(String original, String lang)
+	{
+
+		original = original.replace('&', ' ');
+		original = original.replaceAll(" ", "%20");
+		return original;
+	}
+
+	public static String formatBigNumber(String num, String lang)
+	{
+
+		String num_str = num;
+		char[] num_chars = num_str.toCharArray();
+		String zero_str = "";
+		String formatted_str = "";
+
+		for (int i = num_chars.length - 4; i >= 0; i--)
+		{
+			zero_str += '0';
+		}
+
+		String sig_str = "";
+		for (int i = 0; i < 3 && i < num_chars.length; i++)
+		{
+			sig_str = sig_str + num_chars[i];
+			if (i == 1 && i + 1 < num_chars.length)
+			{
+				sig_str = sig_str + ".";
+			}
+		}
+
+		int sig_int = Math.round(Float.parseFloat(sig_str));
+		String new_sig_str = sig_int + "";
+		if (sig_str.length() > 2)
+		{
+			new_sig_str = sig_int + "0";
+		}
+
+		char[] final_chars = (new_sig_str + zero_str).toCharArray();
+		int count = 1;
+		for (int i = final_chars.length - 1; i >= 0; i--)
+		{
+			formatted_str = final_chars[i] + formatted_str;
+			if (count == 3 && i != 0)
+			{
+				formatted_str = "," + formatted_str;
+				count = 1;
+			}
+			else
+			{
+				count++;
+			}
+		}
+		return formatted_str;
+	}
+
+	public static String hashToSectionId(String hashString)
+	{
+		if (hashString == null || hashString.length() == 0)
+		{
+			return "";
+		}
+
+		int firstDotIndex = hashString.indexOf(".");
+		if (firstDotIndex == -1)
+		{
+			return "";
+		}
+
+		String sectionString = hashString.substring(firstDotIndex + 1);
+
+		return sectionString;
+	}
+
+	public static String hashToDepthClass(String hashString)
+	{
+		if (hashString == null || hashString.length() == 0)
+		{
+			return "";
+		}
+
+		String sectionString = hashToSectionId(hashString);
+
+		int count = sectionString.split("\\.").length;
+
+		if (sectionString.equals(""))
+		{
+			return "sectionHeaderDepthTitle";
+		}
+		else
+		{
+			return "sectionHeaderDepth" + count;
+		}
+	}
+
+	public static String escapeNewLines(String str)
+	{
+		if (str == null || str.length() < 1)
+		{
+			return null;
+		}
+		return str.replace("\n", "\\\n");
+	}
+
+	public static String escapeQuotes(String str)
+	{
+		if (str == null || str.length() < 1)
+		{
+			return null;
+		}
+		return str.replace("\"", "\\\"");
+	}
+
+	public static String escapeNewLinesAndQuotes(String str)
+	{
+		if (str == null || str.length() < 1)
+		{
+			return null;
+		}
+		return escapeNewLines(escapeQuotes(str));
+	}
+
+	public static String getGlobalProperty(String name)
+	{
+		return GlobalProperties.getProperty(name);
+	}
+
+	public static void clearMetadataStorage()
+	{
+		_foundTableValues.clear();
+	}
+
+	public static boolean checkMetadataNotDuplicate(String name, String value)
+	{
+		if (_foundTableValues.containsKey(name))
+		{
+			for (String mapValue : _foundTableValues.get(name))
+			{
+				if (mapValue.equals(value))
+				{
+					return false;
+				}
+			}
+			_foundTableValues.get(name).add(value);
+			return true;
+		}
+
+		ArrayList<String> newList = new ArrayList<String>();
+		newList.add(value);
+
+		_foundTableValues.put(name, newList);
+
+		return true;
+	}
+
+	public static String reCAPTCHAimage(String publicKey, String privateKey)
+	{
+		ReCaptcha c = ReCaptchaFactory.newReCaptcha(publicKey, privateKey, false);
+		return c.createRecaptchaHtml(null, null);
+	}
+
+	public static String getInterfaceStringsAsJavascript(String interface_name, String lang, String prefix)
+	{
+		String prependToPrefix = "gs.text";
+		return XSLTUtil.getInterfaceStringsAsJavascript(interface_name, lang, prefix, prependToPrefix);
+	}
+
+	// generates javascript: 2 arrays are declared and populated with strings that declare variables and assign their values
+	// to be strings loaded from the interface_name.properties file for the language.    
+	public static String getInterfaceStringsAsJavascript(String interface_name, String lang, String prefix, String prependToPrefix)
+	{
+	  String prefixwithdot = prefix+".";
+		// 1. Generating Javascript of the form:
+		// if(!gs.text) { gs.text = new Array(); }
+		// if(!gs.text.dse) { gs.text.dse = new Array(); }
+		StringBuffer outputStr = new StringBuffer();
+		outputStr.append("if(!gs.text) { ");
+		outputStr.append(prependToPrefix + " = new Array(); ");
+		outputStr.append("}\n");
+		outputStr.append("if(!gs.text." + prefix + ") { ");
+		outputStr.append(prependToPrefix + "." + prefix + " = new Array(); ");
+		outputStr.append("}\n");
+
+		int foundCount = 0;
+
+		for (String dictName : new String[] { "interface_" + interface_name, "interface_default", "interface_default2" })
+		{
+		  // get all the keys from the english dictionary as this is a complete set
+			Dictionary dict = new Dictionary(dictName, "en");
+			Enumeration keys = dict.getKeys();
+			if (keys == null)
+			{
+				continue;
+			}
+
+			// Get all properties in the language-specific dictionary with the given key prefix
+			// Create Javascript strings of the form:
+			// prependToPrefix.key= "value";\n
+			while (keys.hasMoreElements())
+			{
+				String key = (String) keys.nextElement();
+				if (key.startsWith(prefixwithdot))
+				{
+				  // get the language dependent value for the key. This will return the english if no value found for the given lang
+					String value = getInterfaceText(interface_name, dictName, lang, key, null);
+
+					outputStr.append(prependToPrefix);
+					outputStr.append(".");
+					outputStr.append(key);
+					outputStr.append("=\"");
+					outputStr.append(value);
+					outputStr.append("\";\n");
+				}
+			}
+
+			if (foundCount > 0)
+			{
+				break;
+			}
+		}
+
+		return outputStr.toString();
+	}
+
+	public static String xmlNodeToString(Node node)
+	{
+		return GSXML.xmlNodeToString(node);
+	}
+
+	// Test from cmdline with:
+	// java -classpath /research/ak19/gs3-svn/web/WEB-INF/lib/gsdl3.jar:/research/ak19/gs3-svn/web/WEB-INF/lib/log4j-1.2.8.jar:/research/ak19/gs3-svn/web/WEB-INF/classes/ org.greenstone.gsdl3.util.XSLTUtil
+	public static void main(String args[])
+	{
+		System.out.println("\n@@@@@\n" + XSLTUtil.getInterfaceStringsAsJavascript("default", "en", "dse", "gs.text") + "@@@@@\n");
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/txt2usersDB.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/txt2usersDB.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/txt2usersDB.java	(revision 31634)
@@ -0,0 +1,235 @@
+/*
+ *    txt2usersDB.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.File;
+import java.io.FileReader;
+import java.sql.SQLException;
+
+import org.greenstone.gsdl3.service.Authentication;
+
+/** 
+    To run this from the command-line, first make sure that the derby networked server is running (ant start-derby),
+    then run:
+
+    java -Dgsdl3.writablehome=/full/path/to/GS3/web -cp web/WEB-INF/lib/gsdl3.jar:web/WEB-INF/lib/gutil.jar:web/WEB-INF/lib/derby.jar:./web/WEB-INF/lib/derbyclient.jar:./web/WEB-INF/lib/log4j-1.2.8.jar:./web/WEB-INF/classes org.greenstone.gsdl3.util.txt2usersDB <filename>.txt web/etc/usersDB/ [-append]
+
+    Don't forget to stop the networked derby server again at the end, if you had started it: ant stop-derby
+
+    Or if using embedded derby, ensure that tomcat is stopped, then run:
+    java -cp /full/path/to/GS3/web/WEB-INF/lib/gsdl3.jar:/full/path/to/GS3/web/WEB-INF/lib/derby.jar org.greenstone.gsdl3.util.txt2usersDB <filename>.txt web/etc/usersDB/ [-append]
+*/
+public class txt2usersDB
+{
+
+	public static void main(String[] args) throws SQLException
+	{
+		boolean appending = false;
+
+		String usage = "Usage: java org.greenstone.gsdl3.txt2usersDB full_path_of_the_text_file full_path_of_the_usersDB [-append]";
+		if (args.length < 2)
+		{
+			System.out.println(usage);
+			System.exit(0);
+		}
+		File txtfile = new File(args[0]);
+		if (!txtfile.exists())
+		{
+			System.out.println("File " + args[0] + " does not exist.");
+			System.out.println(usage);
+			System.exit(0);
+		}
+
+		try
+		{
+			BufferedReader in = new BufferedReader(new FileReader(args[0]));
+			String str;
+			DerbyWrapper dw = new DerbyWrapper(args[1]);
+
+			if (args.length > 2 && args[2].equals("-append"))
+			{
+				appending = true;
+			}
+			else
+			{
+				// no appending, replace existing database: the text file 
+				// represents the new database, so delete the existing DB first
+				boolean delete_rows = dw.deleteAllUser();
+				dw.closeDatabase();
+				if (!delete_rows)
+				{
+					System.out.println("Couldn't delete rows of the users table");
+					System.exit(0);
+				}
+			}
+
+			String username = null;
+			String password = null;
+			String groups = null;
+			String accountstatus = null;
+			String comment = null;
+			String email = null;
+
+			while ((str = in.readLine()) != null)
+			{
+				//ystem.out.println(str);
+
+				if (str.indexOf(" = ") != -1)
+				{ // works with DerbyWrapper.db2txt() and usersDB2txt.java. Fields listed as: USERNAME = admin
+					String field = str.substring(0, str.indexOf(" = "));
+					if (field.equalsIgnoreCase("email"))
+					{
+						email = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+					if (field.equalsIgnoreCase("comment"))
+					{
+						comment = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+					if (field.equalsIgnoreCase("status"))
+					{
+						accountstatus = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+					if (field.equalsIgnoreCase("groups"))
+					{
+						groups = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+					if (field.equalsIgnoreCase("password"))
+					{
+						//password=dw.rot13(str.substring(str.indexOf(">")+1,str.length()));
+						password = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+					if (field.equalsIgnoreCase("username"))
+					{
+						username = str.substring(str.indexOf(" = ") + 3, str.length());
+					}
+				}
+				else if (str.startsWith("<"))
+				{ // fields listed as: <username>admin
+					String field = str.substring(1, str.indexOf(">"));
+					if (field.equals("email"))
+					{
+						email = str.substring(str.indexOf(">") + 1, str.length());
+					}
+					if (field.equals("comment"))
+					{
+						comment = str.substring(str.indexOf(">") + 1, str.length());
+					}
+					if (field.equals("enabled") || field.equals("status"))
+					{
+						accountstatus = str.substring(str.indexOf(">") + 1, str.length());
+					}
+					if (field.equals("groups"))
+					{
+						groups = str.substring(str.indexOf(">") + 1, str.length());
+					}
+					if (field.equals("password"))
+					{
+						password = str.substring(str.indexOf(">") + 1, str.length());
+					}
+					if (field.equals("username"))
+					{
+						username = str.substring(str.indexOf(">") + 1, str.length());
+					}
+				}
+				else if (str.equals("----------------------------------------------------------------------") || str.equals("-------------------------------------"))
+				{
+
+					if ((username != null) && (password != null) && (groups != null) && (accountstatus != null) && (comment != null))
+					{
+						dw.connectDatabase(args[1], false);
+
+						// check if it's a new user or already exists in the database
+						UserQueryResult findUserResult = dw.findUser(username);
+
+						if (findUserResult == null)
+						{ // add new user
+							if (password.length() >= 3 && password.length() <= 8)
+							{ // if not yet encrypted, encrypt first
+								password = Authentication.hashPassword(password);
+							} // if > 8 chars, password for user being added was already encrypted (hashed-and-hexed)
+							dw.addUser(username, password, groups, accountstatus, comment, email);
+						}
+
+						else
+						{ // modify existing user
+							// if any of the other fields are not specified, get them from the database
+							UserTermInfo user = findUserResult.getUserTerms().get(0);
+
+							if (password.length() < 3 || password.length() > 8)
+							{ // includes empty string case
+								password = user.password;
+							}
+							else
+							{ // need to first encrypt (hash-and-hex) the user-entered password
+								// Use the same encryption technique used by the Admin Authentication page
+								// This ensures that the password generated for a string remains consistent
+								password = Authentication.hashPassword(password);
+							}
+							groups = groups.equals("") ? user.groups : groups;
+							accountstatus = accountstatus.equals("") ? user.accountstatus : accountstatus;
+							comment = comment.equals("") ? user.comment : comment;
+
+							if (email == null)
+							{ // special checking for backwards compatibility since old DB did not have email field
+								email = "";
+							}
+							if (user.email == null)
+							{
+								user.email = "";
+							}
+							if (email.equals(""))
+							{
+								email = user.email;
+							}
+
+							//System.err.println("**** Password: " + password);				
+							//System.err.println("**** " + username + " " + password + " " + groups + " " + accountstatus + " " + comment + " " + email);
+							dw.modifyUserInfo(username, password, groups, accountstatus, comment, email);
+						}
+
+						username = null;
+						password = null;
+						groups = null;
+						accountstatus = null;
+						comment = null;
+						email = null;
+						//dw.connectDatabase(args[1],false); // should this be closeDatabase()????
+						dw.closeDatabase();
+					}
+				}
+
+				// only true back when when hashed passwords weren't being converted to hex
+				//else { // encrypted passwords can span multiple lines for some reason
+				// assume that is the case here
+				//if(password != null) { 
+				//	password = password + "\n" + str;
+				//  }
+				//}
+
+			}
+			//dw.closeDatabase();
+			in.close();
+		}
+		catch (IOException e)
+		{
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/usersDB2txt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/usersDB2txt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/gsdl3/util/usersDB2txt.java	(revision 31634)
@@ -0,0 +1,55 @@
+/*
+ *    usersDB2txt.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.gsdl3.util;
+
+import javax.swing.*;
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.Statement;
+import java.util.Properties;
+
+/** 
+    To run this from the command-line, first make sure that the networked derby server is running (ant start-derby)
+    then run: 
+ 
+    java -Dgsdl3.writablehome=/full/path/to/GS3/web -cp ./web/WEB-INF/lib/gsdl3.jar:./web/WEB-INF/lib/gutil.jar:./web/WEB-INF/lib/derby.jar:./web/WEB-INF/lib/derbyclient.jar:./web/WEB-INF/lib/log4j-1.2.8.jar:./web/WEB-INF/classes org.greenstone.gsdl3.util.usersDB2txt web/etc/usersDB/
+
+    if redirecting to a file append ">& filename.txt" to the above command 
+    since the usersDB2txt program output goes to System.err and needs to be redirected to the file too
+
+    Don't forget to stop the networked derby server again at the end, if you had started it: ant stop-derby
+
+    Or if using embedded derby, ensure that tomcat is stopped, then run:
+    java -cp /full/path/to/GS3/web/WEB-INF/lib/gsdl3.jar:/full/path/to/GS3/web/WEB-INF/lib/derby.jar org.greenstone.gsdl3.util.usersDB2txt web/etc/usersDB/ [>& <output file>]
+*/
+public class usersDB2txt
+{
+    public static void main(String[] args) throws SQLException{
+	if (args.length!=1){
+	    System.out.println("The path of usersDB has to be given!");
+	    System.exit(0);
+	}
+	DerbyWrapper derbyWrapper=new DerbyWrapper(args[0]);
+	derbyWrapper.db2txt();
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseProperty.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseProperty.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseProperty.java	(revision 31634)
@@ -0,0 +1,44 @@
+package org.greenstone.server;
+
+public class BaseProperty {
+
+    public final String GSDL_HOME;
+    public final String GSDL_VERSION;
+    public final String AUTOSTART;
+    public final String KEEPPORT;
+    public final String START_BROWSER;
+    public final String ALLOW_EXTERNAL_ACCESS;
+
+    public final String DEFAULT_SERVLET = "server.default.servlet";
+
+    public final String WEB_PORT;
+
+    public static final String WIN_9X_OPEN_COMMAND = "command.com /c start %1";
+    public static final String WIN_OPEN_COMMAND = "cmd.exe /c start \"\" %1";
+    public static final String MAC_OPEN_COMMAND = "open %1";
+
+    public final String BROWSER_PATH = "browser.path";
+
+    public final String SERVER_CONTROL;
+    public final String SERVER_SETTINGS;
+    
+    protected BaseProperty(String version, 
+			   String web_port,
+			   String autostart, 
+			   String startbrowser, 
+			   String keepport,
+			   String allowExternalAccess) 
+    {
+	// property names
+	WEB_PORT = web_port;
+	GSDL_HOME = "gsdl"+version +".home";
+	GSDL_VERSION = "gsdl"+version+".version";
+	SERVER_CONTROL = "Server"+version+"Control";
+	SERVER_SETTINGS = "Server"+version+"Settings";
+	AUTOSTART = autostart;
+	START_BROWSER = startbrowser;
+	KEEPPORT = keepport;
+	ALLOW_EXTERNAL_ACCESS = allowExternalAccess;
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServer.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServer.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServer.java	(revision 31634)
@@ -0,0 +1,271 @@
+
+package org.greenstone.server;
+
+import java.awt.Dimension;
+import java.awt.Toolkit;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
+import java.util.Properties;
+import javax.swing.JOptionPane;
+
+import org.apache.log4j.*;
+
+import org.greenstone.util.RunTarget;
+import org.greenstone.util.BrowserLauncher;
+
+import org.greenstone.gsdl3.util.Dictionary;
+
+public abstract class BaseServer 
+{
+    static protected String START_CMD;
+    static protected String RESTART_CMD;
+    static protected String CONFIGURE_CMD;
+    static protected String STOP_CMD;
+
+    static protected final int SERVER_STARTED = 0;
+    static protected final int SERVER_START_FAILED = 1;
+    static protected final int BROWSER_LAUNCHED = 2;
+    static protected final int BROWSER_LAUNCH_FAILED = 3;
+    static protected final int START_SERVER = 4;
+ 
+    static protected Properties config_properties;
+    static protected Logger logger_;
+
+    static public File config_properties_file;
+    static public Dictionary dictionary;
+    static public BaseProperty Property;
+
+    protected int server_state_ = -1;    
+    protected boolean configure_required_ = true;
+    protected String gsdl_home;
+    protected String logs_folder;
+    protected boolean start_browser;
+
+    protected BaseServerControl server_control_;    
+    
+    protected BaseServer(String gsdl_home, String lang, String config_properties_path, String logs)
+    {
+	this.gsdl_home = gsdl_home;
+	// expand the relative location of the logs folder
+	this.logs_folder = this.gsdl_home+File.separator+logs;
+
+	// make sure we write to the correct logs
+	initLogger();
+	logger_ = Logger.getLogger(BaseServer.class.getName());
+
+	config_properties_file = new File(config_properties_path);
+	
+	if (!config_properties_file.exists()) {
+	    logger_.fatal("Can't find configuration file "+config_properties_path);
+	    System.exit(1);
+	}
+	
+	config_properties = new Properties();
+	reloadConfigProperties(true); // first time starting the server, work out port_number
+
+	dictionary = new Dictionary("server", lang, this.getClass().getClassLoader());	
+    }
+
+    public void autoStart()
+    {
+	String auto_start = config_properties.getProperty(BaseServer.Property.AUTOSTART, "true");
+	if (auto_start.equals("true") || auto_start.equals("1")) {
+	    String start_browser = config_properties.getProperty(BaseServer.Property.START_BROWSER, "true");
+
+	    if (start_browser.equals("true") || start_browser.equals("1")) {
+		restart();
+	    }
+	    else{
+		start();
+	    }
+	    
+	    server_control_.setState(java.awt.Frame.ICONIFIED); // minimise the server interface window
+	} else {
+	    if (configure_required_){
+		server_control_.displayMessage(dictionary.get("ServerControl.Configuring"));
+		int state = runTarget(CONFIGURE_CMD);
+		
+		if (state != RunTarget.SUCCESS){
+		    recordError(CONFIGURE_CMD);
+		}
+	    }
+	    reload(); // browser URL or other important properties may not yet be initialised
+	    configure_required_ = false;
+
+	    server_state_ = START_SERVER;
+	    server_control_.updateControl();
+	}
+    }
+
+    // package access methods
+    BaseServerControl getServerControl() { 
+	return server_control_; 
+    }
+
+    protected int getServerState()
+    {
+	return server_state_;
+    }
+
+    // override to write to the correct logs
+    protected void initLogger() {}
+
+    protected abstract int runTarget(String cmd);
+    public abstract String getBrowserURL();
+    public abstract void reload(); // reload properties, since they may have changed
+    protected void preStop() {}
+    protected void postStart() {}
+    
+    public void reconfigRequired()
+    {
+	configure_required_ = true;
+    }
+
+    public void start() 
+    {
+        int state = -1;
+        server_state_ = -1;
+	server_control_.updateControl();
+
+	server_control_.displayMessage(dictionary.get("ServerControl.Starting"));
+	stop(true); // silent, no messages displayed
+	
+  	// reconfigure if necessary
+        if (configure_required_){
+	    server_control_.displayMessage(dictionary.get("ServerControl.Configuring"));
+	    state = runTarget(CONFIGURE_CMD);
+	    
+	   if (state != RunTarget.SUCCESS){
+	       recordError(CONFIGURE_CMD);
+	   }
+	   reload(); // work out the browserURL again
+	   configure_required_ = false;
+	}
+	else{
+	    recordSuccess(CONFIGURE_CMD);
+	}
+	
+	try{
+	    Thread.sleep(5000);
+	} catch(Exception e) {
+	    logger_.error("Exception trying to sleep: " + e);
+	}
+        state = runTarget(START_CMD);
+	
+	if (state != RunTarget.SUCCESS){
+	    recordError(START_CMD);
+            server_state_ = SERVER_START_FAILED;            
+	}	            
+	else{
+	    recordSuccess(START_CMD); 
+            server_state_ = SERVER_STARTED; 
+	    postStart();
+       	} 
+
+       	server_control_.updateControl();
+    }
+    
+    protected void recordError(String message){
+        message = dictionary.get("ServerControl.Error",new String[]{message,logs_folder});
+	server_control_.displayMessage(message);
+	logger_.error(dictionary.get("ServerControl.Failed",new String[]{message}));
+    }
+
+
+    protected void recordError(String message, Exception e){
+        message = dictionary.get("ServerControl.Error",new String[]{message,logs_folder});
+	server_control_.displayMessage(message);
+	logger_.error(dictionary.get("ServerControl.Failed",new String[]{message}),e);
+    }
+
+    protected void recordSuccess(String message){
+        message = dictionary.get("ServerControl.Success",new String[]{message});
+	server_control_.displayMessage(message);
+	logger_.info(message);
+    }
+
+    public void launchBrowser() {
+	server_state_ = -1;
+        server_control_.updateControl();   
+	String url = getBrowserURL();
+        String message = dictionary.get("ServerControl.LaunchingBrowser");
+    	server_control_.displayMessage(message); 
+	//recordError("**** browserURL: " + url);
+	BrowserLauncher launcher = new BrowserLauncher(config_properties.getProperty(BaseServer.Property.BROWSER_PATH, ""), url);
+        logger_.info(message);
+
+	launcher.start();
+     
+        //wait for a while
+        while(launcher.getBrowserState() == -1){
+            try{
+		Thread.sleep(3000);
+	    }
+	    catch(Exception e){
+		logger_.error(e);
+	    }
+	}
+	
+	if (launcher.getBrowserState() != BrowserLauncher.LAUNCHSUCCESS ){
+	    recordError(dictionary.get("ServerControl.LaunchBrowser")); 
+	    server_state_ = BROWSER_LAUNCH_FAILED;
+	}
+	else{
+	    recordSuccess(dictionary.get("ServerControl.LaunchBrowser")); 
+	    server_state_ = BROWSER_LAUNCHED;
+	}    
+        
+	server_control_.updateControl();        
+    }
+    
+    public void restart(){
+	start();       
+        if (server_state_ == SERVER_STARTED){
+	    launchBrowser();
+	}  
+    }
+
+    // Preserving the current behaviour of stop() which is to
+    // display the message on stopping
+    public void stop() {
+	stop(false);
+    }
+
+    public void stop(boolean silent) {
+	preStop();
+	if(!silent) {
+	    server_control_.displayMessage(dictionary.get("ServerControl.Stopping"));
+	}
+	int state = runTarget(STOP_CMD);
+	
+        if (state != RunTarget.SUCCESS){
+	    recordError(STOP_CMD);
+	}
+	else{
+	    recordSuccess(STOP_CMD);
+	}
+
+    }
+    
+	// returns true on success
+    public boolean reloadConfigProperties(boolean port_has_changed) {
+	try {
+	    FileInputStream in = new FileInputStream(config_properties_file);
+
+	    if (in != null) {
+		logger_.info("loading configuration properties: " + config_properties_file);
+		config_properties.load(in);
+		in.close();
+	    } else {
+		logger_.error("Couldn't load configuration properties from " + config_properties_file + "!");
+	    }
+	} catch (Exception e) {
+	    logger_.error("Exception trying to reload configuration properties " +config_properties_file + ": " +e);
+	}
+	
+	return true;
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerControl.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerControl.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerControl.java	(revision 31634)
@@ -0,0 +1,286 @@
+package org.greenstone.server;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.border.*;
+
+import org.apache.log4j.*;
+/**
+ * Base class to help start or restart the library server
+ * 
+ */
+public abstract class BaseServerControl extends JFrame {
+
+    static Logger logger = Logger.getLogger(BaseServerControl.class.getName());
+ 
+    /** The dimension of the frame */
+    static final private Dimension FRAME_SIZE = new Dimension(430, 310);
+
+    Color bg_color = Color.white;    
+    
+    /** some components we need to refer to later */
+    protected JLabel info_label;
+    protected JButton enter_button;
+    protected JMenu fMenu;
+    protected BaseServer server;
+    protected JFrame thisframe;
+    
+    public BaseServerControl(BaseServer server,String frame_title) 
+    {
+	super(frame_title);
+ 
+	this.server = server;
+        thisframe = this;
+
+	setSize(FRAME_SIZE);
+	setDefaultCloseOperation(EXIT_ON_CLOSE);
+
+	// set the icon for the Greenstone Server Interface
+	try {
+	    ImageIcon image = new ImageIcon(getClass().getResource("/images/servericon.png"));
+	    if (image != null) {
+		this.setIconImage(image.getImage());
+	    }	
+	}
+	catch (Exception exception) {
+	    System.err.println("Error: Could not load servericon.png");
+	    logger.error("Error: Could not load servericon.png");
+	}	
+
+	Dimension screen_size = Toolkit.getDefaultToolkit().getScreenSize();
+	setLocation((screen_size.width - FRAME_SIZE.width) / 2,
+		    (screen_size.height - FRAME_SIZE.height) / 2);
+	setBackground(Color.white);
+        addWindowListener(new MyAdapter()); 
+   
+        JPanel title_panel = new JPanel();
+	title_panel.setLayout(new BorderLayout());
+
+        JLabel title_label = new JLabel();
+
+
+        String title = BaseServer.dictionary.get(BaseServer.Property.SERVER_CONTROL+".Title");           
+        title_label.setText(stringToHTML(title)); 
+        title_label.setOpaque(false);	
+	title_label.setHorizontalAlignment(SwingConstants.CENTER);
+        title_label.setFont(new Font("SansSerif",Font.PLAIN,18));
+       
+
+        JLabel version_label = new JLabel();
+        String version = BaseServer.dictionary.get(BaseServer.Property.SERVER_CONTROL+".Version").toLowerCase();
+         
+        version_label.setText(stringToHTML(version)); 
+        version_label.setOpaque(false);	
+	version_label.setHorizontalAlignment(SwingConstants.CENTER);
+        version_label.setFont(new Font("SansSerif",Font.PLAIN,14));
+        
+        title_panel.add(title_label,BorderLayout.CENTER);
+        title_panel.add(version_label,BorderLayout.SOUTH);
+	title_panel.setBackground(bg_color);	
+	title_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+
+   	info_label = new JLabel();
+        info_label.setOpaque(false);
+        info_label.setHorizontalAlignment(SwingConstants.LEFT);
+        info_label.setVerticalAlignment(SwingConstants.CENTER);
+        info_label.setFont(new Font("SansSerif",Font.PLAIN,14));
+	info_label.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+			
+        JPanel button_panel = new JPanel(new FlowLayout(FlowLayout.CENTER));
+	enter_button = new JButton();
+	enter_button.setEnabled(false);
+	enter_button.addActionListener(new EnterButtonListener());
+        enter_button.setText(BaseServer.dictionary.get("ServerControl.EnterLibrary"));
+	button_panel.add(enter_button);
+	button_panel.setBackground(bg_color);	
+	button_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+	getRootPane().setDefaultButton(enter_button); // button gets the focus for enterpress
+
+	getContentPane().setLayout(new BorderLayout());
+	getContentPane().add(title_panel,BorderLayout.NORTH);
+	getContentPane().add(info_label,BorderLayout.CENTER);
+	getContentPane().add(button_panel,BorderLayout.SOUTH);
+	getContentPane().setBackground(bg_color);
+    	setJMenuBar(createMenu());
+	setVisible(true);
+    }
+
+    protected abstract JMenuBar createMenu();
+
+    protected JMenuBar createMenu(JMenuItem iConf) {
+	JMenuBar menuBar = new JMenuBar();
+         fMenu = new JMenu(BaseServer.dictionary.get("ServerControl.Menu.File"));
+	JMenuItem iExit = new JMenuItem(BaseServer.dictionary.get("ServerControl.Menu.Exit"));
+	iExit.setBackground(Color.white);
+	iExit.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent ae) {
+		    /*Thread runInThread = new Thread(new Runnable(){
+		      public void run(){
+		      server.stop();
+		      
+		      }
+		      },"stop server");
+		      try{
+		      runInThread.start(); 
+		      
+		      }
+		    catch(Exception e){
+		    logger.error(e);
+		    }*/    
+		    thisframe.dispose();
+		    server.stop();
+		    System.exit(0);
+		}
+	    });
+
+	fMenu.add(iConf);
+	fMenu.add(iExit);
+	fMenu.setEnabled(false);
+	menuBar.add(fMenu);
+	menuBar.setBorder(new BevelBorder(BevelBorder.RAISED));
+	return menuBar;
+    }
+
+    public void updateControl(){
+      	switch (server.getServerState()){
+	case BaseServer.START_SERVER:
+	    {
+		info_label.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.Help_EnterLibrary")));
+		enter_button.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.EnterLibrary")));
+		enter_button.setEnabled(true);
+		fMenu.setEnabled(true);
+		break;
+	    }
+	case BaseServer.SERVER_STARTED:
+	    {
+		info_label.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.Help_RestartLibrary")));
+		enter_button.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.RestartLibrary")));
+		enter_button.setEnabled(true);
+		fMenu.setEnabled(true);
+		break;
+	    }
+	case BaseServer.SERVER_START_FAILED:
+	    {
+		enter_button.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.StartServer")));
+		enter_button.setEnabled(true);
+		fMenu.setEnabled(true);
+		break;	
+	    }
+	case BaseServer.BROWSER_LAUNCHED:
+	    {
+		info_label.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.BrowserLaunched",new String[]{server.getBrowserURL()})
+						+ BaseServer.dictionary.get("ServerControl.Help_RestartLibrary")));
+		enter_button.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.RestartLibrary")));
+		thisframe.setState(Frame.ICONIFIED);
+		enter_button.setEnabled(true);
+                fMenu.setEnabled(true);
+		break;
+	    }
+	case BaseServer.BROWSER_LAUNCH_FAILED:
+	    {
+		info_label.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.BrowserLaunchFailed",new String[]{server.getBrowserURL()})));
+		enter_button.setText(stringToHTML(BaseServer.dictionary.get("ServerControl.EnterLibrary")));
+		enter_button.setEnabled(true);
+		fMenu.setEnabled(true);
+		break;
+	    }
+        default:
+	    {
+		enter_button.setText(BaseServer.dictionary.get("ServerControl.EnterLibrary"));
+		enter_button.setEnabled(false);
+		fMenu.setEnabled(false);
+	    }
+	}
+    }
+           
+   
+    public void displayMessage(String message){
+	info_label.setText(stringToHTML("<br>"+message));           
+    }
+
+    private class MyAdapter extends WindowAdapter{   
+	public void windowClosing(WindowEvent env){
+	    /*          Thread runInThread = new Thread(new Runnable(){
+			public void run(){
+			server.stop();
+			
+			}
+			},"stop server");
+			try{
+			runInThread.start(); 
+			}
+			catch(Exception e){
+			logger.error(e);
+			}  
+			thisframe.dispose();
+	    */
+	    thisframe.dispose();
+	    server.stop();
+	    System.exit(0);
+	}
+    } 
+
+    private String stringToHTML(String s){
+	return "<html><body>"+s+"</body></html>";	
+    }
+    
+    private class EnterButtonListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent ev) {
+	    switch (server.getServerState()){
+	    case BaseServer.START_SERVER:
+		{
+		 Thread runInThread = new Thread(new Runnable(){
+			    public void run(){
+				server.start();
+				server.launchBrowser();
+			    }
+			},"start server and launch browser");
+		    
+		    runInThread.start(); 
+                   break;     
+		}
+	    case BaseServer.SERVER_STARTED:
+		{
+		 Thread runInThread = new Thread(new Runnable(){
+			    public void run(){
+				server.launchBrowser();
+			    }
+			},"launch browser");
+		    
+		    runInThread.start(); 
+                   break;     
+		}
+	    case BaseServer.SERVER_START_FAILED:
+		{
+		    Thread runInThread = new Thread(new Runnable(){
+			    public void run(){
+				server.start();
+			    }
+			},"start server");
+		    runInThread.start();
+		    break;
+		}
+	    case BaseServer.BROWSER_LAUNCHED: case BaseServer.BROWSER_LAUNCH_FAILED:
+		{
+		    Thread runInThread = new Thread(new Runnable(){
+			    public void run(){
+				server.restart();
+				  
+			    }
+			},"restart server");
+		    runInThread.start();
+		}  
+	    }   
+	  
+	}
+    }
+	
+	public void errorMessage(String message) {
+		JOptionPane.showMessageDialog(null,message,"Error", JOptionPane.ERROR_MESSAGE);
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerSettings.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerSettings.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/BaseServerSettings.java	(revision 31634)
@@ -0,0 +1,339 @@
+package org.greenstone.server;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import javax.swing.*;
+
+import org.w3c.dom.*;
+
+import org.apache.log4j.*;
+
+import org.greenstone.util.ScriptReadWrite;
+
+public abstract class BaseServerSettings extends JDialog implements ActionListener
+{
+    static Logger logger = Logger.getLogger(BaseServerSettings.class.getName());
+    static final int DEFPORT = 8080;
+    static final Color bg_color = Color.white;
+
+    protected JCheckBox autoEnter;
+    protected JCheckBox keepPortToggle;
+    protected JCheckBox allowConnections;
+
+    protected JSpinner portNumber_spinner = null;
+    protected JTextField program_path_field = null;
+    protected JButton browse_button = null; 
+    protected JRadioButton default_browser_button = null ;
+    protected JRadioButton other_browser_button = null;
+    protected Dimension screen = Toolkit.getDefaultToolkit().getScreenSize();
+
+    protected int portNum = DEFPORT;
+    protected boolean autoStart = false;
+    protected boolean keepPort = false;
+    protected String browserPath = "";
+    protected boolean useDefaultBrowser = true;
+    protected boolean externalaccess = false;
+
+    protected JDialog self;
+    protected BaseServer server;
+     
+    public BaseServerSettings(BaseServer server) 
+    {
+	super(server.getServerControl(), "", true);
+	this.self = this;
+        this.server = server;
+
+        try {
+	    this.portNum = Integer.parseInt(server.config_properties.getProperty(BaseServer.Property.WEB_PORT));
+	}
+        catch(Exception e){
+	    logger.error(e);
+	} 
+
+        this.browserPath = server.config_properties.getProperty(BaseServer.Property.BROWSER_PATH);
+        
+        if (this.browserPath == null || this.browserPath.equals("")){
+	    useDefaultBrowser = true;
+            this.browserPath = "";
+	}
+	else{
+	    useDefaultBrowser = false;
+	}
+
+	String auto_start_str = server.config_properties.getProperty(BaseServer.Property.AUTOSTART, "1").trim();
+	if (auto_start_str.equals("true") || auto_start_str.equals("1")) {
+	    this.autoStart = true;
+	} else {
+	    this.autoStart = false;
+	}
+
+	String keep_port_str = server.config_properties.getProperty(BaseServer.Property.KEEPPORT, "false").trim();
+	if (keep_port_str.equals("true") || keep_port_str.equals("1")) {
+	    this.keepPort = true;
+	} else {
+	    this.keepPort = false;
+	}
+
+
+	setTitle(server.dictionary.get("ServerSettings.Title"));
+	setDefaultCloseOperation(DO_NOTHING_ON_CLOSE);
+
+	JLabel port_label = new JLabel(BaseServer.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".Port"));
+
+	portNumber_spinner = new JSpinner(new SpinnerNumberModel(portNum,1,65535,1));
+	portNumber_spinner.setEditor(new JSpinner.NumberEditor(portNumber_spinner, "#####"));
+
+	autoEnter = new JCheckBox(server.dictionary.get("ServerSettings.Auto_Start"));
+	keepPortToggle = new JCheckBox(server.dictionary.get("ServerSettings.Keep_Port"));
+	
+	if (autoStart) {
+	    autoEnter.setSelected(true);
+	} else {
+	    autoEnter.setSelected(false);
+	}
+	autoEnter.setBackground(bg_color);
+
+	if(keepPort) {
+	    keepPortToggle.setSelected(true);
+	} else {
+	    keepPortToggle.setSelected(false);
+	}
+	keepPortToggle.setBackground(bg_color);
+
+	boolean allowCons = false;
+	String externalAccessStr = server.config_properties.getProperty(BaseServer.Property.ALLOW_EXTERNAL_ACCESS, "1").trim();
+	allowConnections = new JCheckBox(server.dictionary.get("ServerSettings.ExternalAccess"), allowCons);
+	if(externalAccessStr.equals("1") || externalAccessStr.equals("true")) {
+	    this.externalaccess = true;
+	    allowCons = true;
+	    allowConnections.setSelected(true);
+	} else {
+	    allowConnections.setSelected(false);
+	}
+	
+	allowConnections.setBackground(bg_color);
+
+	JButton save_button = new JButton(BaseServer.dictionary.get("ServerSettings.OK"));
+	save_button.addActionListener(new SaveActionListener());
+
+	JButton exit_button = new JButton(BaseServer.dictionary.get("ServerSettings.Cancel"));
+	exit_button.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent ev) {
+		    self.dispose();
+		}
+	    });
+
+	addWindowListener(new WindowAdapter() {
+		public void windowClosing(WindowEvent we) { 
+		    self.dispose();
+		}
+	    });
+
+        JPanel port_panel = new JPanel();    
+        port_panel.setLayout(new FlowLayout(FlowLayout.LEFT));
+	port_panel.add(port_label);
+	port_panel.add(portNumber_spinner);
+	port_panel.setBackground(bg_color);
+
+	JPanel top_panel = new JPanel(new GridLayout(4,1));
+	top_panel.add(port_panel);
+	top_panel.add(keepPortToggle);
+	top_panel.add(autoEnter);
+	top_panel.add(allowConnections);
+
+	JPanel comb_panel = createServletPanel();
+	comb_panel.setBackground(bg_color);
+
+	JPanel mid_panel = new JPanel();
+	mid_panel.setLayout(new BorderLayout());
+	mid_panel.add(top_panel, BorderLayout.NORTH);
+	mid_panel.add(comb_panel, BorderLayout.CENTER);
+	mid_panel.setBackground(bg_color);
+	mid_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+
+
+	JPanel browser_panel = new JPanel();
+        browser_panel.setLayout(new GridLayout(4,1));
+        default_browser_button = new JRadioButton(BaseServer.dictionary.get("ServerSettings.DefaultBrowser"));
+        other_browser_button = new JRadioButton(BaseServer.dictionary.get("ServerSettings.OtherBrowser"));
+	default_browser_button.addActionListener(this);
+	other_browser_button.addActionListener(this);	
+
+	ButtonGroup bg = new ButtonGroup();
+        bg.add(default_browser_button);
+        bg.add(other_browser_button);
+        default_browser_button.setBorder(BorderFactory.createEmptyBorder(5,10,5,0));
+        default_browser_button.setBackground(bg_color);     
+        other_browser_button.setBorder(BorderFactory.createEmptyBorder(5,10,5,0)); 
+        other_browser_button.setBackground(bg_color);
+        JPanel browse_program_panel = new JPanel();     
+        browse_program_panel.setLayout(new BorderLayout());
+        program_path_field = new JTextField(this.browserPath);
+        browse_button = new JButton(BaseServer.dictionary.get("ServerSettings.Browse"));
+	browse_program_panel.add(program_path_field,BorderLayout.CENTER);
+        browse_program_panel.add(browse_button,BorderLayout.EAST);
+        browse_program_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,0));
+	browse_program_panel.setBackground(bg_color);
+        browser_panel.add(new JLabel(BaseServer.dictionary.get("ServerSettings.ChooseBrowser")));
+        browser_panel.add(default_browser_button);
+	browser_panel.add(other_browser_button);
+        browser_panel.add(browse_program_panel);
+	browser_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+        browser_panel.setBackground(bg_color);
+	if (useDefaultBrowser){
+	    default_browser_button.setSelected(true);
+	    program_path_field.setEnabled(false);
+	    browse_button.setEnabled(false); 
+	}
+	else{
+	    other_browser_button.setSelected(true);
+	}
+	browse_button.addActionListener(this);
+        
+        JPanel down_panel = new JPanel();
+	down_panel.setLayout(new FlowLayout(FlowLayout.CENTER, 5, 5));
+	down_panel.add(save_button);
+	down_panel.add(exit_button);
+	down_panel.setBackground(bg_color);
+	down_panel.setBorder(BorderFactory.createEmptyBorder(5,10,5,10));
+
+        getContentPane().add(mid_panel, BorderLayout.NORTH);
+        getContentPane().add(browser_panel, BorderLayout.CENTER);
+	getContentPane().add(down_panel, BorderLayout.SOUTH);
+	getContentPane().setBackground(bg_color);
+
+	pack();
+	setLocation((screen.width - getWidth()) / 2,
+		    (screen.height - getHeight()) / 2);
+	setVisible(true);
+    }
+
+
+    protected boolean[] onSave() {
+	boolean[] returnValues = { false, false }; // hasChanged, requireRestart
+	return returnValues;
+    }
+    protected void save(ScriptReadWrite scriptReadWrite, ArrayList<String> newFileLines) {}
+    protected abstract JPanel createServletPanel();
+
+
+    public void actionPerformed(ActionEvent ev) {
+	if(ev.getSource() == browse_button) {
+	    JFileChooser chooser = new JFileChooser();
+	    int returnVal = chooser.showOpenDialog(self);        
+	    
+	    if(returnVal == JFileChooser.APPROVE_OPTION) {
+		File program_path = chooser.getSelectedFile();
+		program_path_field.setText(program_path.getPath());
+	   } 
+	} else { // one of the two Choose Browser radiobuttons
+	    if (default_browser_button.isSelected()){
+		program_path_field.setEnabled(false);
+		browse_button.setEnabled(false);
+	    }
+	    else{
+		program_path_field.setEnabled(true);
+		browse_button.setEnabled(true);
+	    }
+	}       
+    }
+    
+
+    private class SaveActionListener
+	implements ActionListener {
+	
+	public void actionPerformed(ActionEvent ev) {
+	    // save everything to config_properties if things have changed
+	    boolean has_changed = false;
+		boolean require_restart = false;
+		boolean port_has_changed = false;
+
+	    if (portNum != ((Integer)portNumber_spinner.getValue()).intValue()) {
+			port_has_changed = true;
+			has_changed = true;
+			require_restart = true;
+                server.reconfigRequired();
+                portNum = ((Integer)portNumber_spinner.getValue()).intValue();
+             	logger.info("port changed, new port is "+portNumber_spinner.getValue());
+	    }
+	    if (autoStart != autoEnter.isSelected()) {
+            	has_changed = true;
+	    }
+	    if (keepPort != keepPortToggle.isSelected()) {
+            	has_changed = true;
+	    }
+
+	    boolean oldExternalAccess = externalaccess;
+	    externalaccess = allowConnections.isSelected() ? true : false;
+	    if (oldExternalAccess != externalaccess) {
+		has_changed = true;
+		require_restart = true;
+		server.reconfigRequired();
+	    } // else require_restart remains what it had been
+
+	    // call subclass' onSave method, which may indicate (further) changes,
+	    // and which may or may not require a restart
+	    boolean[] returnValues = onSave();
+	    has_changed = has_changed || returnValues[0];
+	    require_restart = require_restart || returnValues[1];
+
+            //changed to use other browser
+            if (useDefaultBrowser && other_browser_button.isSelected()){
+		browserPath = program_path_field.getText();
+		has_changed = true;
+	    } 
+	    //the browser path has been changed 
+	    if (!useDefaultBrowser && !browserPath.equals(program_path_field.getText())){
+		browserPath = program_path_field.getText();
+		has_changed = true;
+	    }
+	    
+            //changed to use the default browser
+	    if (default_browser_button.isSelected() && !useDefaultBrowser){
+		browserPath = "";
+		has_changed = true;
+	    }
+             
+
+	    if (has_changed) {
+		ArrayList<String> oldFileLines = null;
+		ArrayList<String> newFileLines = null;
+		
+		ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+		oldFileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+		
+		newFileLines = scriptReadWrite.queryReplace(oldFileLines, BaseServer.Property.WEB_PORT, portNum+"");
+		
+		// call the subclass' save() method to save custom elements
+		// and to save externalAccess in a custom manner for both GS2 and GS3
+		save(scriptReadWrite, newFileLines);
+		
+		String osName = System.getProperty("os.name");
+		if (osName.startsWith("Windows")){
+		    browserPath = browserPath.replaceAll("\\\\","/");
+		}
+		newFileLines = scriptReadWrite.replaceOrAddLine(newFileLines, BaseServer.Property.BROWSER_PATH, browserPath, true);
+
+		scriptReadWrite.writeOutFile(BaseServer.config_properties_file, newFileLines);
+
+		boolean success = server.reloadConfigProperties(port_has_changed); // if failure, it would have displayed an error message
+		server.reload(); // work out the URL again in case it has changed
+		if(success) { // successful/valid configuration changes
+			if (require_restart){
+				JOptionPane.showMessageDialog(null,server.dictionary.get("ServerSettings.SettingChanged"),"Info", JOptionPane.INFORMATION_MESSAGE);
+				if(autoStart) {
+					server.autoStart();
+					server.getServerControl().updateControl();
+				}
+			}
+		}
+	    } 
+
+	    self.dispose();
+	}
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2.java	(revision 31634)
@@ -0,0 +1,546 @@
+
+package org.greenstone.server;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.net.URL;
+//import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.Properties;
+import java.util.StringTokenizer;
+
+import org.apache.log4j.PropertyConfigurator;
+
+import org.greenstone.util.PortFinder;
+import org.greenstone.util.ScriptReadWrite;
+import org.greenstone.util.RunMake;
+import org.greenstone.util.RunAnt;
+
+import org.greenstone.server.BaseServer;
+import org.greenstone.server.BaseProperty;
+
+
+public class Server2 extends BaseServer 
+{
+    private static final int WAITING_TIME = 10; // time to wait and check for whether the server is running
+    private static final String URL_PENDING="URL_pending";	
+	
+    protected String libraryURL;
+    protected String property_prefix;
+    protected String gsdlos_gsdlarch;
+	
+    private class QuitListener extends Thread 
+    {
+	int quitPort = -1;
+	ServerSocket serverSocket = null;
+
+	public QuitListener(int quitport) throws Exception {
+	    ///Server2.this.recordSuccess("In QuitListener constructor");
+	    this.quitPort = quitport;
+	    serverSocket = new ServerSocket(quitPort);
+	}
+
+	public void run() {
+	    Socket connection = null;
+
+	    try {
+		// wait for a connection
+		connection = serverSocket.accept();
+		boolean stop = false;
+
+		// read input
+		try {
+		    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+		    String line = null;
+		    while((line = reader.readLine()) != null) {
+			if(line.equals("QUIT")) {
+			    stop = true;			    
+			    // Server2.this.recordSuccess("In QuitListener - line is QUIT");
+			    reader.close();
+			    reader = null;
+			    serverSocket.close();
+			    serverSocket = null;
+			    break;
+			} else if(line.equals("RECONFIGURE")) {
+			    server_control_.displayMessage(dictionary.get("ServerControl.Configuring"));
+			    reconfigRequired();
+			} else if(line.equals("RESTART")) {
+
+			    // If the GSI is set to NOT autoenter/autostart the server, then write url=URL_PENDING out to the file.
+			    // When the user finally presses the Enter Library button and so has started up the server, the correct
+			    // url will be written out to the configfile.
+				String url = property_prefix+"url";
+			    if(config_properties.getProperty(BaseServer.Property.AUTOSTART, "").equals("0")) {
+				if(config_properties.getProperty(url) == null) {
+				    config_properties.setProperty(url, URL_PENDING);
+				    ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+				    ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+				    scriptReadWrite.replaceOrAddLine(fileLines, url, URL_PENDING, true);
+				    scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+				}
+			    }
+
+			    autoStart();
+			}
+		    }
+		} catch(Exception e) {
+		    Server2.this.recordError("Exception in QuitListener thread.");
+		} finally {
+		    if(stop) {
+			Server2.this.stop();
+			System.exit(0);
+		    }
+		}
+	    } catch(IOException ioe) {
+		Server2.this.recordError("Server2.QuitListener: Unable to make the connection with the client socket." + ioe);
+	    }
+	}
+    }
+
+
+    public Server2(String gsdl2_home, String gsdlos_arch, String lang, String configfile, int quitPort, String mode)
+    {
+	super(gsdl2_home, lang, configfile, "etc"+File.separator+"logs-gsi");	
+	               // configfile can be either glisite.cfg or llssite.cfg	
+
+	//logger_.error("gsdlhome: " + gsdl2_home + " | lang: " + lang + " | configfile: " 
+		//+ configfile + " | gsdlos_gsdlarch: " + gsdlos_arch + " | mode: " + property_prefix + " | quitport: " + quitPort);
+	
+	// to get to the OS specific subdirectory of cgi-bin
+	gsdlos_gsdlarch = gsdlos_arch;
+
+	// property_prefix is the mode we're running in (gli or empty) and contains the prefix
+	// string to look for in config file for auto_enter, start_browser and url properties
+	property_prefix = mode;
+	if(!property_prefix.equals("") && !property_prefix.endsWith(".")) { // ensure a '.' is suffixed if non-empty
+		property_prefix += ".";
+	}
+	Property = new Server2Property(property_prefix);
+	
+	
+	String frame_title = dictionary.get("ServerControl.Frame_Title");
+	server_control_ = new Server2Control(this,frame_title);
+
+	/* Make command targets for managing Web server */
+	START_CMD     = "web-start";
+	RESTART_CMD   = "web-restart";
+	CONFIGURE_CMD = "configure-web \"" + configfile + "\"";
+	STOP_CMD      = "web-stop";
+
+	// now we can monitor for the quit command if applicable
+	if(quitPort != -1) { 
+	    // First check if given port is within the range of allowed ports
+	    if(PortFinder.isAssignablePortNumber(quitPort)) {
+		try {
+		    new QuitListener(quitPort).start();
+		} catch(Exception e) {
+		    Server2.this.recordError("Exception constructing the QuitListener thread.");
+		}
+	    }
+	    else {
+		recordError("QuitPort provided is not within acceptable range: ("
+			    + PortFinder.PORTS_RESERVED + " - " + PortFinder.MAX_PORT + "]" );
+		quitPort = -1;
+	    }
+	}
+	
+	// For some machines, localhost is not sufficient, 
+	// need hostname defined as well (e.g. Ubuntu 10.10)
+	InetAddress inetAddress = null;
+	try {
+	   	inetAddress = InetAddress.getLocalHost();
+		String hosts = inetAddress.getHostName();
+		ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+		ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+		scriptReadWrite.replaceOrAddLine(fileLines, "hosts", hosts, true);
+		scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+	} catch(UnknownHostException e) {
+	   	// Unable to get hostname, need to try for the default localhost without it
+	}		
+
+	// If the GSI is set to NOT autoenter/autostart the server, then write url=URL_PENDING out to the file.
+	// When the user finally presses the Enter Library button and so has started up the server, the correct
+	// url will be written out to the configfile.
+	if(config_properties.getProperty(BaseServer.Property.AUTOSTART, "").equals("0")) {//if(configfile.endsWith("llssite.cfg")) {
+		String url = property_prefix+"url";
+	    if(config_properties.getProperty(url) == null) {
+		config_properties.setProperty(url, URL_PENDING);
+		ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+		ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+		scriptReadWrite.replaceOrAddLine(fileLines, url, URL_PENDING, true);
+		scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+	    }
+	}
+	
+	autoStart();
+    }
+
+    // Prepare the log4j.properties for GS2
+    protected void initLogger() {
+
+	String libjavaFolder = gsdl_home+File.separator+"lib"+File.separator+"java"+File.separator;
+	File propsFile = new File(libjavaFolder+"log4j.properties");
+
+	// create it from the template file log4j.properties.in
+	if(!propsFile.exists()) {
+	    try {
+		// need to set gsdl2.home property's value to be gsdl_home,
+		// so that the location  of the log files gets resolved correctly
+		
+		// load the template log4j.properties.in file into logProps
+		FileInputStream infile = new FileInputStream(new File(libjavaFolder+"log4j.properties.in"));
+		if(infile != null) {
+		    Properties logProps = new Properties();
+		    logProps.load(infile);
+		    infile.close();
+
+		    // set gsdl3.home to gsdl_home
+		    logProps.setProperty("gsdl2.home", gsdl_home);
+		    
+		    // write the customised properties out to a custom log4j.properties file
+		    FileOutputStream outfile = new FileOutputStream(propsFile);
+		    if(outfile != null) {
+			logProps.store(outfile, "Customised log4j.properties file");
+			outfile.close();
+		    } else {
+			System.err.println("Could not store properties file " + propsFile + " for Server2.");
+		    }
+		}
+	    } catch(Exception e) {
+		System.err.println("Exception occurred when custom-configuring the logger for Server2.\n" + e);
+	    }
+	}
+
+	// now configure the logger with the custom log4j.properties file
+	if(propsFile.exists()) {
+	    PropertyConfigurator.configure(propsFile.getAbsolutePath());
+	} else {
+	    System.err.println("Could not create properties file " + propsFile + " for Server2.");
+	}
+    }
+    
+
+    protected int runTarget(String cmd)
+    {
+	RunMake runMake = new RunMake();
+	runMake.setTargetCmd(cmd);
+	runMake.run();
+   	return runMake.getTargetState();
+    }
+
+    public String getBrowserURL() {
+	return libraryURL;
+    }
+
+    // works out the library URL again
+    public void reload() {
+	// default values, to be replaced with what's in gsdlsite.cfg
+	String host = "localhost";
+	String port = "80";
+	String gwcgi;
+	String httpprefix = "/greenstone";
+	String suffix = "/cgi-bin/library.cgi";
+
+	// get the prefix from the gsdlsite.cfg and build.properties files (port number and servername)
+	try{
+	    File gsdlsite_cfg = new File(gsdl_home + File.separator + "cgi-bin" + File.separator 
+					 + gsdlos_gsdlarch + File.separator + "gsdlsite.cfg");
+	    FileInputStream fin = new FileInputStream(gsdlsite_cfg); 
+	    Properties gsdlProperties = new Properties();
+	    if(fin != null) {
+		gsdlProperties.load(fin);
+	    
+		gwcgi = gsdlProperties.getProperty("gwcgi");
+		if(gwcgi != null) {
+		    suffix = gwcgi;
+		} else {
+		    httpprefix = gsdlProperties.getProperty("httpprefix", httpprefix);
+		    suffix = httpprefix + suffix;
+		}
+		fin.close();
+	    } else {
+		recordError("Could not open gsdlsite_cfg for reading, using default library prefix.");
+	    }
+	    //reloadConfigProperties();
+	    port = config_properties.getProperty("portnumber", port);
+	    
+	    // The "hosts" property in the config file contains more than one allowed host
+	    // Need to work out the particular host chosen from the address_resolution_method
+	    // Default is address_resolution_method 2: localhost
+	    String addressResolutionMethod = config_properties.getProperty("address_resolution_method");
+	    int address_resolution_method = (addressResolutionMethod == null) ? 2 : Integer.parseInt(addressResolutionMethod);
+	    InetAddress inetAddress = null;
+	    try {
+		inetAddress = InetAddress.getLocalHost();
+	    } catch(UnknownHostException e) {
+		logger_.error(e);
+		logger_.info("Defaulting host IP to "+ host); // use the default		
+		address_resolution_method = 2;
+		inetAddress = null;
+	    }
+	    switch(address_resolution_method) {
+	    case 0:
+		host = inetAddress.getHostName();
+		break;
+	    case 1:
+		host = inetAddress.getHostAddress();
+		break;
+	    case 2:
+		host = "localhost";
+		break;
+	    case 3:
+		host = "127.0.0.1";
+		break;
+	    default:
+		host = "localhost";		    
+	    }
+	} catch(Exception e) {
+	    recordError("Exception trying to load properties from gsdlsite_cfg. Using default library prefix.", e);
+	    suffix = httpprefix + suffix;
+	}
+
+	libraryURL = "http://" + host + ":" + port + suffix;
+    }
+
+    public boolean reloadConfigProperties(boolean port_has_changed) {
+	super.reloadConfigProperties(port_has_changed);
+
+	// make sure the port is okay, otherwise find another port
+	// first choice is port 80, second choice starts at 8282
+	String port = config_properties.getProperty("portnumber", "80");
+	String keepport = config_properties.getProperty("keepport", "0"); // default is to not try to force the same port if in use by other servers
+
+	int portDefault = 8282;
+	try {
+	  int portNum = Integer.parseInt(port);
+	  boolean verbose = true;
+	  if(port_has_changed) { // this is the test that prevents the server from arbitrarily shifting the port. 
+							// only check at configured port if it's not the current port (the port we
+							// are still running on), because that will always be in use and unavailable.
+		if(!PortFinder.isPortAvailable(portNum, verbose)) { // first time, print any Port Unavailable messages
+		if(keepport.equals("1")) {
+			server_control_.errorMessage(dictionary.get("ServerSettings.SettingsUnchangedPortOccupied", new String[]{port}));
+		    String errorMsg = "Unable to run the Greenstone server on port " + port + ". It appears to already be in use.";
+		    System.err.println("\n******************");
+			logger_.error(errorMsg);
+		    System.err.println("If you wish to try another port, go to File > Settings of the Greenstone Server interface and either change the port number or untick the \"Do Not Modify Port\" option there. Then press the \"Enter Library\" button.");
+		    System.err.println("******************\n");
+			
+			return false; // property change is unsuccessful
+			
+		} else { // can modify port, try to find a new port
+		
+		    PortFinder portFinder = new PortFinder(portDefault, 101);
+		    // Search for a free port silently from now on--don't want more
+		    // messages saying that a port could not be found...
+		    portNum = portFinder.findPortInRange(!verbose);
+		    
+		    if (portNum == -1) {
+			// If we've still not found a free port, do we try the default port again?
+			System.err.println("No free port found. Going to try on " + portDefault + " anyway.");
+			port = Integer.toString(portDefault);
+		    } else {
+			port = Integer.toString(portNum);
+		    }
+		    config_properties.setProperty("portnumber", port); // store the correct port
+		    
+		    // write this updated port to the config file, since the configure target uses the file to run
+		    ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+		    ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+		    scriptReadWrite.replaceOrAddLine(fileLines, "portnumber", port, false); // write the correct port
+		    scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+		    
+		    configure_required_ = true;
+		    System.err.println("Running server on port " + port + ".");
+		}
+	    }
+	  }	
+	} catch (Exception e) {
+	    recordError("Exception in Server2.reload(): " + e.getMessage());
+	    port = Integer.toString(portDefault);
+	}
+	
+	return true;
+    }
+
+    
+    // About to stop the webserver
+    // Custom GS2 action: remove the url property from the config file
+    protected void preStop() {
+	ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+	ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+
+	// Remove the url=... line, start searching from the end
+	boolean done = false;
+	for (int i = fileLines.size()-1; i >= 0 && !done; i--) {
+	    String line = fileLines.get(i).trim();
+	    if(line.startsWith(property_prefix+"url=")) {
+		fileLines.remove(i);
+		done = true;
+	    }
+	}
+	scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+    }
+
+    // Called when the URL has been changed (called after a reload() and starting the server).
+    // By the time we get here, reload() would already have been called and have set
+    // both the port and worked out libraryURL
+    // This method needs to write the URL to the configfile since things should work
+    // like GS2's Local Lib Server for Windows
+    protected void postStart() {
+
+	URL libURL = null;
+	try {
+	    libURL = new URL(libraryURL);
+	} catch (Exception e) {
+	    recordError("Unable to convert library URL string into a valid URL, Server2.java." + e);
+	}
+	
+	// 1. Test that the server is running at the libraryURL before writing it out to glisite.cfg/configfile 
+	// A quick test involves opening a connection to get the home page for this collection
+	if(libURL != null && !libraryURL.equals(URL_PENDING)) {
+
+	    boolean ready = false; 
+	    for(int i = 0; i < WAITING_TIME && !ready; i++) {
+		try {
+		    libURL.openConnection();
+		    //URLConnection connection = new URL(libraryURL).openConnection();
+		    //connection.getContent();
+		    ready = true;
+		    recordSuccess("Try connecting to server on url: '" + libraryURL + "'");
+		} catch (IOException bad_url_connection) {
+		    // keep looping
+		    recordSuccess("NOT YET CONNECTED. Waiting to try again...");
+		    try {
+			Thread.sleep(1000);
+		    } catch (InterruptedException ie) {
+			ready = true;
+			recordError("Unexpected: got an InterruptedException in sleeping thread, Server2.java." + ie);
+		    }
+		} catch (Exception e) {
+		    ready = true;
+		    recordError("Got an Exception while waiting for the connection to become live, Server2.java." + e);
+		}
+	    }
+	}
+
+	// 2. Now write the URL to the config file
+	String port = config_properties.getProperty("portnumber");
+
+	ScriptReadWrite scriptReadWrite = new ScriptReadWrite();
+	ArrayList<String> fileLines = scriptReadWrite.readInFile(BaseServer.config_properties_file);
+	scriptReadWrite.replaceOrAddLine(fileLines, property_prefix+"url", libraryURL, true);
+	scriptReadWrite.replaceOrAddLine(fileLines, "portnumber", port, false); // write the correct port
+	scriptReadWrite.writeOutFile(config_properties_file, fileLines);
+    }
+
+
+    public static void main (String[] args)
+    {
+    	if ((args.length < 2) || (args.length > 6)) {
+	    System.err.println(
+	       "Usage: java org.greenstone.server.Server2 <gsdl2-home-dir> <GSDLOS_GSDLARCH> [lang] [--mode=\"gli\"] [--config=configfile] [--quitport=portNum]");
+	    System.exit(1);
+		}
+	
+	String gsdl2_home = args[0];
+	File gsdl2_dir = new File(gsdl2_home);
+	if (!gsdl2_dir.isDirectory()) {
+	    System.err.println("gsdl-home-dir directory does not exist!");
+	    System.exit(1);
+	}
+	
+	String gsdlos_arch = args[1];
+
+	//for(int i = 0; i < args.length; i++) { System.err.println("Arg[" + i + "]: |" + args[i] + "|");	}
+	
+	// for every subsequent argument, check whether we're dealing with amalgamated
+	// parameters in that argument. If so, split on whitespace and reconstruct arguments
+	for(int i = 2; i < args.length; i++) {
+	
+		// if the *last* occurrence of a flag parameter is not at the start of 
+		// this argument then we're dealing with amalgamated parameters in this 
+		// argument. Need to split on whitespace and reconstitute the args list
+		int lastIndex = args[i].lastIndexOf("--");
+		if(lastIndex > 0) { 
+		
+			StringTokenizer tokenizer = new StringTokenizer(args[i]); // splits on whitespace
+			String[] tokens = new String[tokenizer.countTokens()+i];
+			int j = 0;
+			for(; j < i; j++) { // copy over previous arguments
+				tokens[j] = args[j];
+			}
+			
+			// the remainder comes from the tokens
+			for(; tokenizer.hasMoreTokens(); j++) {
+				tokens[j] = tokenizer.nextToken();
+			}
+			args = null;
+			args = tokens;
+		}
+		
+	}	
+	
+	// Defaults for optional arguments
+	
+	// if no config file is given, then it defaults to llssite.cfg 
+	// (embedded in quotes to preserve spaces in the filepath)
+	File defaultConfigFile = new File(gsdl2_dir, "llssite.cfg");	
+	String configfile = defaultConfigFile.getAbsolutePath();
+	int port = -1;
+	String mode = "";
+	String lang = "en";
+	
+	int index = 2; // move onto any subsequent arguments (past 1st GSDLHOME arg and 2nd GSDLOS_GSDLARCH arg)
+	if (args.length > index && !args[index].startsWith("--")) {
+		lang = args[index]; // 2nd arg is not a flag option, so must be language
+		index++;
+	}	
+	
+	// Cycle through arguments, parsing and storing them
+	// note that the batch file could have split arguments in the config-
+	// filepath over several parameters. Here we join them back up again.
+	while(args.length > index) {
+			
+		if(args[index].startsWith("--config=")) {
+			configfile = args[index].substring(args[index].indexOf('=')+1); // get value after '=' sign			
+			gsdl2_dir = null;
+			defaultConfigFile = null;
+		}
+		
+		else if(args[index].startsWith("--quitport=")) {
+			String quitport = args[index].substring(args[index].indexOf('=')+1);			
+			try {				
+				port = Integer.parseInt(quitport);
+			} catch(Exception e) { // parse fails
+				System.err.println("Port must be numeric. Continuing without it.");
+			}		
+		}
+		
+		// mode can be: "gli" if launched by GLI or unspecified. If unspecified, then 
+		// the gs2-server was launched independently.
+		else if(args[index].startsWith("--mode=")) {
+			mode = args[index].substring(args[index].indexOf('=')+1);
+		}
+		
+		else if(!args[index].startsWith("--")) { // assume it's part of the config file name (that this path had spaces in it)
+			configfile = configfile + " " + args[index]; // reinstate space in path
+		}
+		
+		index++;
+	}
+
+	configfile = configfile.trim(); // remove any trailing whitespace
+	//System.err.println("\n\n*******\n\ngsdlhome: " + gsdl2_home + " | lang: " + lang + " |configfile:"
+	//	+ configfile + "| mode: " + mode + " | quitport: " + port + "\n\n*************\n");
+	new Server2(gsdl2_home, gsdlos_arch, lang, configfile, port, mode);
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Control.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Control.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Control.java	(revision 31634)
@@ -0,0 +1,35 @@
+package org.greenstone.server;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+
+import org.greenstone.server.BaseServerControl;
+
+/**
+ * This class is to start or restart the Greenstone library server
+ * 
+ */
+
+public class Server2Control extends BaseServerControl 
+{
+    
+    public Server2Control(BaseServer server,String frame_title) 
+    {
+	super(server,frame_title);
+    }
+
+    protected JMenuBar createMenu() 
+    {
+	JMenuItem iConf = new JMenuItem(BaseServer.dictionary.get("ServerControl.Menu.Settings"));
+	iConf.setBackground(Color.white);
+	iConf.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent ae) {
+		    BaseServerSettings serverSetting = new Server2Settings(server);
+		}
+	    });
+
+	return createMenu(iConf);
+    }
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Property.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Property.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Property.java	(revision 31634)
@@ -0,0 +1,15 @@
+package org.greenstone.server;
+
+import org.greenstone.server.BaseProperty;
+
+public class Server2Property extends BaseProperty 
+{
+    Server2Property(String property_prefix)
+    {
+	// Initialising customised final variables 
+	// Version number, WEB_PORT
+	super("2", "portnumber", property_prefix+"autoenter", property_prefix+"start_browser",
+	      "keepport", "externalaccess");
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Settings.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Settings.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server2Settings.java	(revision 31634)
@@ -0,0 +1,125 @@
+package org.greenstone.server;
+
+import java.awt.BorderLayout;
+import java.awt.FlowLayout;
+import java.awt.GridLayout;
+import java.awt.event.*;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import javax.swing.*;
+import javax.swing.event.*;
+
+import org.greenstone.server.BaseServerSettings;
+import org.greenstone.util.ScriptReadWrite;
+
+public class Server2Settings extends BaseServerSettings
+{
+    protected JComboBox prefix_combobox;
+    protected JRadioButton[] hostRadioButtons = new JRadioButton[4];
+
+    // 0 to 3: 0 is resolved (hostname) from local IP, 1 is local IP address, 2 is localhost, 3 is 127.0.0.1
+    protected int address_resolution_method = 2;
+
+    public Server2Settings(BaseServer server) 
+    {
+	super(server);	
+    }
+
+    protected JPanel createServletPanel()
+    {
+	JPanel server2panel = new JPanel();
+	server2panel.setLayout(new BorderLayout());
+	
+	JPanel connect_panel = new JPanel(new GridLayout(4, 1));
+	connect_panel.setBackground(bg_color);
+	connect_panel.setBorder(BorderFactory.createTitledBorder(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".AddressResolutionMethod")));
+
+	hostRadioButtons = new JRadioButton[4];
+	hostRadioButtons[0] = new JRadioButton(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".ResolveIP"));
+	hostRadioButtons[1] = new JRadioButton(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".LocalIP"));
+	hostRadioButtons[2] = new JRadioButton(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".AlwaysUse")+" localhost");
+	hostRadioButtons[3] = new JRadioButton(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".AlwaysUse")+" 127.0.0.1");
+
+	ButtonGroup hostGroup = new ButtonGroup();
+	for(int i = 0; i < hostRadioButtons.length; i ++) {
+	    connect_panel.add(hostRadioButtons[i]);
+	    hostGroup.add(hostRadioButtons[i]);
+	    hostRadioButtons[i].setBackground(bg_color);
+	}	
+
+	String addressResolutionMethod = server.config_properties.getProperty("address_resolution_method", "").trim();
+	if(!addressResolutionMethod.equals("")) {
+	    this.address_resolution_method = Integer.parseInt(addressResolutionMethod);
+	}
+	hostRadioButtons[address_resolution_method].setSelected(true);
+
+	JPanel comb_panel = new JPanel(new BorderLayout());	
+	comb_panel.add(connect_panel, BorderLayout.CENTER);
+	return comb_panel;
+    }
+
+    public boolean[] onSave()
+    {
+	// superclass detects changes to port and autoenter
+	// handle changes to address_resolution_method here
+
+	boolean hasChanged = false;
+	boolean requireRestart = false;
+	
+	for(int i = 0; i < hostRadioButtons.length; i++) {
+	    if(hostRadioButtons[i].isSelected() && address_resolution_method != i) {
+		address_resolution_method = i;
+		hasChanged = true;
+		requireRestart = true;
+		server.reconfigRequired();
+	    }
+	}
+
+	boolean[] returnValues = { hasChanged, requireRestart };
+	return returnValues;
+    }
+
+    public void save(ScriptReadWrite scriptReadWrite, ArrayList<String> newFileLines) 
+    {
+	// write only 1 or 0 (and not true or false) for Server2
+	boolean auto_enter = autoEnter.isSelected();
+	if(autoStart != auto_enter) {
+	    String newAutoEnter = auto_enter ? "1" : "0";
+	    newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.AUTOSTART, newAutoEnter);
+	}
+	boolean keep_port = keepPortToggle.isSelected();
+	if(keepPort != keep_port) {
+	    String newKeepPort = keep_port ? "1" : "0";
+	    newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.KEEPPORT, newKeepPort);
+	}
+
+	// external access - onSave() would have updated this value
+	// Its possible values are specific to the version of Greenstone: 0 or 1 for GS2 (true or false for GS3)
+	newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.ALLOW_EXTERNAL_ACCESS, externalaccess ? "1" : "0");
+	
+	// work out the host (default is address_resolution_method 2: localhost)
+	String hostIP = "127.0.0.1";
+	String hosts = "";
+	InetAddress inetAddress = null;
+	try {
+	    inetAddress = InetAddress.getLocalHost();
+		hosts = inetAddress.getHostName();
+	    hostIP = inetAddress.getHostAddress(); // used for all cases unless an Exception is thrown here
+	} catch(UnknownHostException e) {
+	    logger.error(e);
+	    logger.info("Server2.java reload(): Defaulting host URL to localhost");
+	    hostIP = "127.0.0.1";
+	    hosts = "";
+	    address_resolution_method = 2;	    
+	}
+
+	newFileLines = scriptReadWrite.replaceOrAddLine(newFileLines, "hostIP", hostIP, true);
+	newFileLines = scriptReadWrite.replaceOrAddLine(newFileLines, "hosts", hosts, true);
+
+	// address resolution method - onSave() would have updated
+	// this value (or the UnknownHostException above might have)
+	newFileLines = scriptReadWrite.queryReplace(newFileLines, "address_resolution_method", Integer.toString(address_resolution_method));
+	
+    }    
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3.java	(revision 31634)
@@ -0,0 +1,142 @@
+package org.greenstone.server;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.Properties;
+
+import org.apache.log4j.PropertyConfigurator;
+
+import org.greenstone.server.BaseServer;
+import org.greenstone.server.BaseProperty;
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.util.RunAnt;
+
+public class Server3 extends BaseServer
+{
+        String opt_ant_properties = null;
+
+	public Server3(String gsdl3_src_home, String lang)
+	{
+		super(gsdl3_src_home, lang, gsdl3_src_home + File.separatorChar + "build.properties", "web"+File.separator+"logs");
+
+		Property = new Server3Property();
+
+		String frame_title = dictionary.get("ServerControl.Frame_Title");
+		server_control_ = new Server3Control(this, frame_title);
+
+		/** Ant command targets for managing Web server */
+		START_CMD = "start";
+		RESTART_CMD = "restart";
+		CONFIGURE_CMD = "configure";
+		STOP_CMD = "force-stop-tomcat force-stop-derby";
+
+		String is_read_only = System.getProperty("gsdl3home.isreadonly","false");
+		if (is_read_only.equals("true")) {
+		    String default_gsdl3_home = gsdl3_src_home + File.separatorChar + "web";
+		    String gsdl3_writablehome = System.getProperty("gsdl3.writablehome",default_gsdl3_home);
+		    opt_ant_properties = "-Dgsdl3home.isreadonly=true -Dgsdl3.writablehome="+gsdl3_writablehome;
+		}
+		
+		autoStart();
+	}
+
+	protected int runTarget(String cmd)
+	{
+		RunAnt runAnt = new RunAnt(opt_ant_properties);
+		runAnt.setTargetCmd(cmd);
+		runAnt.run();
+		return runAnt.getTargetState();
+	}
+
+	public String getBrowserURL()
+	{
+		return GlobalProperties.getFullGSDL3WebAddress() + config_properties.getProperty(BaseServer.Property.DEFAULT_SERVLET);
+	}
+
+	public String fallbackGSDL3Home() 
+	{ 
+		return gsdl_home + File.separator + "web"; //System.getenv("GSDL3SRCHOME") + File.separator + "web"; 
+	} 
+
+	public void reload()
+	{
+	        String gsdl3_writablehome = System.getProperty("gsdl3.writablehome",fallbackGSDL3Home());
+
+		GlobalProperties.loadGlobalProperties(gsdl3_writablehome); // properties file may have changed, so reload it
+	}
+
+	public static void main(String[] args)
+	{
+		if ((args.length < 1) || (args.length > 2))
+		{
+			System.err.println("Usage: java org.greenstone.server.Server3 <gsdl3-src-home> [lang]");
+			System.exit(1);
+		}
+
+		String gsdl3_src_home = args[0];
+		File gsdl3_src_dir = new File(gsdl3_src_home);
+		if (!gsdl3_src_dir.isDirectory())
+		{
+			System.err.println("src directory does not exist!");
+			System.exit(1);
+		}
+
+		String lang = (args.length == 2) ? args[1] : "en";
+		new Server3(gsdl3_src_home, lang);
+	}
+
+    // Prepare the log4j.properties for GS3
+    protected void initLogger() {
+
+	String gsdl3_home = GlobalProperties.getGSDL3Home(); // may not be initialised at this stage	
+	if(gsdl3_home == null) { // use gsdl_home/web
+	    gsdl3_home = fallbackGSDL3Home();
+	}
+
+	String gsdl3_writablehome = System.getProperty("gsdl3.writablehome",gsdl3_home);
+
+	String propsFolder = gsdl3_writablehome + File.separator + "WEB-INF" + File.separator + "classes" + File.separator;
+	File propsFile = new File(propsFolder+"log4j.properties");
+
+	// create it from the template file GS3/resources/web/log4j.properties
+	// Always do this => helps make Greenstone3 portable
+
+
+	    try {
+		// need to set gsdl3.home property's value to be gsdl_home/web,
+		// so that the location  of the log files gets resolved correctly
+		
+		// load the template log4j.properties.in file into logProps
+		FileInputStream infile = new FileInputStream(new File(gsdl_home+File.separator+"resources"+File.separator+"web"+File.separator+"log4j.properties"));
+		if(infile != null) {
+		    Properties logProps = new Properties();
+		    logProps.load(infile);
+		    infile.close();
+
+		    // set gsdl3.home to web home
+		    logProps.setProperty("gsdl3.home", gsdl3_home);
+		    logProps.setProperty("gsdl3.writablehome", gsdl3_writablehome);
+		    
+		    // write the customised properties out to a custom log4j.properties file
+		    FileOutputStream outfile = new FileOutputStream(propsFile);
+		    if(outfile != null) {
+			logProps.store(outfile, "Customised log4j.properties file");
+			outfile.close();
+		    } else {
+			System.err.println("Could not store properties file " + propsFile + " for Server3.");
+		    }
+		}
+	    } catch(Exception e) {
+		System.err.println("Exception occurred when custom-configuring the logger for Server3.\n" + e);
+		e.printStackTrace();
+	    }
+
+	// now configure the logger with the custom log4j.properties file
+	if(propsFile.exists()) {
+	    PropertyConfigurator.configure(propsFile.getAbsolutePath());
+	} else {
+	    System.err.println("Could not create properties file " + propsFile + " for Server3.");
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Control.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Control.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Control.java	(revision 31634)
@@ -0,0 +1,33 @@
+package org.greenstone.server;
+
+import java.awt.*;
+import java.awt.event.*;
+import javax.swing.*;
+
+import org.greenstone.server.BaseServerControl;
+
+/**
+ * This class is to start or restart the Greenstone 3 library server
+ * 
+ */
+public class Server3Control extends BaseServerControl 
+{
+    
+    public Server3Control(BaseServer server,String frame_title) 
+    {
+	super(server,frame_title);
+    }
+
+    protected JMenuBar createMenu() 
+    {
+	JMenuItem iConf = new JMenuItem(BaseServer.dictionary.get("ServerControl.Menu.Settings"));
+	iConf.setBackground(Color.white);
+	iConf.addActionListener(new ActionListener() {
+		public void actionPerformed(ActionEvent ae) {
+		    BaseServerSettings serverSetting = new Server3Settings(server);
+		}
+	    });
+
+	return createMenu(iConf);
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Property.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Property.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Property.java	(revision 31634)
@@ -0,0 +1,16 @@
+package org.greenstone.server;
+
+import org.greenstone.server.BaseProperty;
+
+public class Server3Property extends BaseProperty 
+{
+    Server3Property()
+    {
+	// Initialising customised final variables 
+	// Version number, WEB_PORT, autoenter, startbrowser
+	// For GS3, the last two are controlled by the same property
+	super("3", "tomcat.port", "server.auto.start", "server.auto.start", 
+	      "server.keep.port", "server.external.access");
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Settings.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Settings.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/server/Server3Settings.java	(revision 31634)
@@ -0,0 +1,120 @@
+package org.greenstone.server;
+
+import java.awt.*;
+import java.awt.event.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import javax.swing.*;
+
+import org.w3c.dom.*;
+import org.xml.sax.InputSource;
+
+import org.greenstone.util.ScriptReadWrite;
+
+import org.greenstone.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSEntityResolver;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.XMLConverter;
+
+import org.greenstone.server.BaseServerSettings;
+
+public class Server3Settings extends BaseServerSettings
+{
+    protected String servletDefault = null;
+    protected JComboBox servlet_combobox;
+    protected HashMap<String, String> url_mappings = null;
+
+    public Server3Settings(BaseServer server) 
+    {
+	super(server);
+    }
+
+    protected JPanel createServletPanel()
+    {
+	JLabel servlet_label = new JLabel(server.dictionary.get(BaseServer.Property.SERVER_SETTINGS+".URL"));
+
+	this.servletDefault = server.config_properties.getProperty(BaseServer.Property.DEFAULT_SERVLET);
+
+	servlet_combobox = new JComboBox();
+	servlet_combobox.setMaximumRowCount(5);
+	servlet_combobox.setBackground(bg_color);
+
+	File web_xml = new File(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "web.xml");
+	XMLConverter converter = new XMLConverter();
+	// web.xml now includes the new file servlets.xml which defines entities used therein
+	// So when loading web.xml's DOM, need to resolve entities in web.xml. And for this, 
+	// need to instruct the entity resolver to load included files relative to web.xml's location
+	Document web_config = converter.getDOM(web_xml, new GSEntityResolver(web_xml.getParentFile()));	
+	if (web_config == null) {
+	    logger.error("web.xml is null! "+web_xml.getAbsolutePath());
+	    return null;
+	}
+
+	NodeList servlet_mappings = web_config.getElementsByTagName("servlet-mapping");
+	// make a little map class
+	url_mappings = new HashMap<String, String>();
+	for (int i = 0; i < servlet_mappings.getLength(); i++) {
+	    Element map = (Element) servlet_mappings.item(i);
+	    Element servlet_name_elem = (Element) GSXML.getChildByTagName(map, "servlet-name");
+	    String name = GSXML.getNodeText(servlet_name_elem);
+	    Element url_pattern_elem = (Element) GSXML.getChildByTagName(map, "url-pattern");
+	    String pattern = GSXML.getNodeText(url_pattern_elem);
+	    // Ignore the Axis servlets
+	    if (!(name.equals("AxisServlet"))) {
+		servlet_combobox.addItem(name.trim());
+		url_mappings.put(name, pattern);
+	    }
+
+	    if (pattern.replaceAll("/\\*","").equals(servletDefault)) { // urlmapping maybe something like "/library/*", want "/library"
+		servlet_combobox.setSelectedItem(name);
+	    }
+	}
+
+
+        JPanel comb_panel = new JPanel();
+	comb_panel.setLayout(new FlowLayout(FlowLayout.LEFT));
+	comb_panel.add(servlet_label);
+	comb_panel.add(servlet_combobox);
+
+	return comb_panel;
+    }
+
+    public boolean[] onSave()
+    {
+	boolean hasChanged = false;
+	boolean requireRestart = false;
+	String urlMapping = url_mappings.get(servlet_combobox.getSelectedItem());
+	if(urlMapping.endsWith("/*")) {	// urlmapping maybe something like "/library/*"
+	    urlMapping = urlMapping.substring(0, urlMapping.length()-2);
+	}
+	if (!servletDefault.equals(urlMapping)) { 
+	    hasChanged = true;
+	    requireRestart = true;
+	}
+	boolean[] returnValues = { hasChanged, requireRestart };
+	return returnValues;
+    }
+
+    public void save(ScriptReadWrite scriptReadWrite, ArrayList<String> newFileLines) 
+    {
+	String newAutoEnter = (new Boolean(autoEnter.isSelected())).toString();
+	newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.AUTOSTART, newAutoEnter);
+
+	String newKeepPort = (new Boolean(keepPortToggle.isSelected())).toString();
+	newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.KEEPPORT, newKeepPort);
+	
+	// external access - BaseServerSettings.actionPerformed() would have updated this value
+	// Its possible values are specific to the version of Greenstone: 0 or 1 for GS2 (true or false for GS3)
+	newFileLines = scriptReadWrite.queryReplace(newFileLines, BaseServer.Property.ALLOW_EXTERNAL_ACCESS, externalaccess ? "true" : "false");
+
+	String newServletDef = (String) servlet_combobox.getSelectedItem();
+	String servletDefName = url_mappings.get(newServletDef);
+	if(servletDefName.endsWith("/*")) { // urlmapping maybe something like "/library/*"	    
+	  servletDefName = servletDefName.substring(0, servletDefName.length()-2);
+	}
+	newFileLines = scriptReadWrite.queryReplace(newFileLines,BaseServer.Property.DEFAULT_SERVLET, servletDefName);
+    }
+
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/testing/ClassFinder.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/testing/ClassFinder.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/testing/ClassFinder.java	(revision 31634)
@@ -0,0 +1,83 @@
+/** this file originally downloaded from 
+ *http://www.javaworld.com/javaworld/jw-12-2000/junit/jw-1221-junit.zip
+ *
+ */
+
+package org.greenstone.testing;
+
+import java.util.*;
+import java.io.*;
+/*
+ * This used to use JCF. However, JCF seems to have dissapeared from the web.
+ * import lti.java.jcf.*;
+ */
+
+
+/**
+ * This class is responsible for searching a directory for class files. It builds
+ * a list of fully qualified class names from the class files in the directory tree.
+ */
+public class ClassFinder {
+    final private Vector<String> classNameList = new Vector<String> ();
+    final private int startPackageName;
+
+    /**
+     * Construct the class finder and locate all the classes in the directory structured
+     * pointed to by <code>classPathRoot</code>. Only classes in the package <code>packageRoot</code>
+     * are considered.
+     */
+    public ClassFinder(final File classPathRoot, final String packageRoot) throws IOException {
+        startPackageName = classPathRoot.getAbsolutePath().length() + 1;
+        String directoryOffset = packageRoot.replace ('.', File.separatorChar);
+        findAndStoreTestClasses (new File (classPathRoot, directoryOffset));
+    }
+    
+    /**
+     * Given a file name, guess the fully qualified class name.
+     */
+    private String computeClassName (final File file) {
+        String absPath = file.getAbsolutePath();
+        String packageBase = absPath.substring (startPackageName, absPath.length () - 6);
+        String className;
+        className = packageBase.replace(File.separatorChar, '.');
+        return className;
+    }
+
+    /**
+     * This method does all the work. It runs down the directory structure looking
+     * for java classes.
+     */
+    private void findAndStoreTestClasses (final File currentDirectory) throws IOException {
+        String files[] = currentDirectory.list();
+        for(int i = 0;i < files.length;i++) {
+            File file = new File(currentDirectory, files[i]);
+            String fileBase = file.getName ();
+            int idx = fileBase.indexOf(".class");
+            final int CLASS_EXTENSION_LENGTH = 6;
+            
+            if(idx != -1 && (fileBase.length() - idx) == CLASS_EXTENSION_LENGTH) {
+/* 
+ * This used to use JCF. However, JCF seems to have dissapeared from the web so we fallback
+ * to a less elegant method. We compute the class name from the file name :-(
+ *               JcfClassInputStream inputStream = new JcfClassInputStream(new FileInputStream (file));
+ *               JcfClassFile classFile = new JcfClassFile (inputStream);
+ *               System.out.println ("Processing: " + classFile.getFullName ().replace ('/','.'));
+ *               classNameList.add (classFile.getFullName ().replace ('/','.'));
+ */
+                String className = computeClassName (file);
+                classNameList.add (className);
+            } else {
+                if(file.isDirectory()) {
+                    findAndStoreTestClasses (file);
+                }
+            }
+        }
+    }
+
+    /**
+     * Return the found classes.
+     */
+    public Iterator<String> getClasses () {
+        return classNameList.iterator ();
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/testing/TestCaseLoader.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/testing/TestCaseLoader.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/testing/TestCaseLoader.java	(revision 31634)
@@ -0,0 +1,83 @@
+package org.greenstone.testing;
+import java.util.*;
+import java.lang.reflect.*;
+import junit.framework.*;
+/**
+ * Responsible for loading classes representing valid test cases.
+ */
+public class TestCaseLoader {
+    final private Vector<Class> classList = new Vector<Class> ();
+    final private String requiredType;
+    
+    /** 
+     * Adds <code>testCaseClass</code> to the list of classdes
+     * if the class is a test case we wish to load. Calls
+     * <code>shouldLoadTestCase ()</code> to determine that.
+     */
+    private void addClassIfTestCase (final Class testCaseClass) {
+        if (shouldAddTestCase (testCaseClass)) {
+            classList.add (testCaseClass);
+        }
+    }
+
+    /**
+     * Determine if we should load this test case. 
+     */
+    private boolean shouldAddTestCase (final Class testCaseClass) {
+        boolean isOfTheCorrectType = false;
+        if (TestCase.class.isAssignableFrom(testCaseClass)) {
+            try {
+                Field testAllIgnoreThisField = testCaseClass.getDeclaredField("TEST_ALL_TEST_TYPE");
+                final int EXPECTED_MODIFIERS = Modifier.STATIC | Modifier.PUBLIC | Modifier.FINAL;
+                if (((testAllIgnoreThisField.getModifiers() & EXPECTED_MODIFIERS) != EXPECTED_MODIFIERS) ||
+                    (testAllIgnoreThisField.getType() != String.class)) {
+                    throw new IllegalArgumentException ("TEST_ALL_TEST_TYPE should be static public final String");
+                }
+                String testType = (String)testAllIgnoreThisField.get(testCaseClass);
+                isOfTheCorrectType = requiredType.equals (testType);
+            } catch (NoSuchFieldException e) {
+            } catch (IllegalAccessException e) {
+                throw new IllegalArgumentException ("The field " + testCaseClass.getName () + ".TEST_ALL_TEST_TYPE is not accessible.");
+            }
+        }
+        return isOfTheCorrectType;
+    }
+    
+       
+    
+    /**
+     * Load the classes that represent test cases we are interested. 
+     * @param classNamesIterator An iterator over a collection of fully qualified class names
+     */
+    public void loadTestCases (final Iterator classNamesIterator) {
+        while (classNamesIterator.hasNext ()) {
+            String className = (String)classNamesIterator.next ();
+	    System.out.println("trying class "+className);
+            try {
+                Class candidateClass = Class.forName (className);
+                addClassIfTestCase (candidateClass);
+            } catch (ClassNotFoundException e) {
+                System.err.println ("Cannot load class: " + className + " " + e.getMessage ());
+            } catch (NoClassDefFoundError e) {
+                System.err.println ("Cannot load class that " + className + " is dependant on");
+            }
+        }
+    }
+        
+    /**
+     * Construct this instance. Load all the test cases possible that derive
+     * from <code>baseClass</code> and cannot be ignored. 
+     * @param classNamesIterator An iterator over a collection of fully qualified class names
+     */
+    public TestCaseLoader(final String requiredType) {
+        if (requiredType == null) throw new IllegalArgumentException ("requiredType is null");
+        this.requiredType = requiredType;
+    }
+    
+    /**
+     * Obtain an iterator over the collection of test case classes loaded by <code>loadTestCases</code>
+     */
+    public Iterator<Class> getClasses () {
+        return classList.iterator ();
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/BrowserLauncher.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/BrowserLauncher.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/BrowserLauncher.java	(revision 31634)
@@ -0,0 +1,136 @@
+package org.greenstone.util;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.greenstone.server.BaseProperty;
+import org.greenstone.util.Misc;
+import org.apache.log4j.*;
+
+public class BrowserLauncher 
+    extends Thread {
+    private String url = "about:blank" ;
+    static Logger logger = Logger.getLogger(org.greenstone.util.BrowserLauncher.class.getName());
+    private String[] default_browsers = new String[]{"firefox","mozilla"};
+    private String command = "";
+    private int state = -1; //0: launch success, 1: launch failed 
+    public final  static int LAUNCHSUCCESS = 0;
+    public final static int LAUNCHFAILED = 1;
+    private String browserPath = ""; 
+
+    public BrowserLauncher(String browserPath, String url) {
+	this.url = url;
+        this.browserPath = browserPath;
+        //use the default browser
+	if (this.browserPath.equals("")){
+	    setBrowserCommand();
+	}
+	else{
+	    this.command = this.browserPath + " " + this.url;
+	}
+    }
+
+    public BrowserLauncher(){
+
+    }
+
+    // we should try and use settings from the settings panel??
+    protected void setBrowserCommand() {
+	if(Misc.isWindows()) {
+	    // we use cmd and start
+	    if (Misc.isWindows9x()) {
+		this.command = BaseProperty.WIN_9X_OPEN_COMMAND;//"command.com /c start \""+url+"\"";
+	    } else {
+		this.command = BaseProperty.WIN_OPEN_COMMAND;//"cmd.exe /c start \"\" \""+url+"\"";
+	    }
+	} else if (Misc.isMac()) {
+	    this.command = BaseProperty.MAC_OPEN_COMMAND; // "open %1"
+	} else {
+	     // we try to look for a browser
+ 	    for (int i=0; i<default_browsers.length; i++) {
+ 		if (isAvailable(default_browsers[i])) {
+ 		    this.command = default_browsers[i] + " %1";
+ 		    break;
+ 		}
+ 	    }
+	}
+
+	this.command = this.command.replaceAll("%1",this.url);
+    }
+
+    protected boolean isAvailable(String program) {
+	try {
+             Runtime run = Runtime.getRuntime();
+             Process process = run.exec("which "+ program);
+             BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
+	     String line = null;
+	     while ((line = br.readLine()) != null){
+		 if (line.indexOf("no "+program) !=-1){
+		     return false;
+		 }
+	     }    	
+	     
+	    return true;
+	} catch (Exception e) {
+	    return false;
+	}
+    }
+
+    public  int getBrowserState(){
+	return state;
+    }
+    
+    public void run() {
+	// Call an external process using the args.
+	if(command.equals("")) {
+	    state = LAUNCHFAILED;
+            logger.error("launching command is empty: no browser found.");
+	    return;
+	}
+	
+        try {
+	    int exitCode = 0;
+	    String prog_name = this.command.substring(0,this.command.indexOf(" "));
+	    String lower_name = prog_name.toLowerCase();
+	    if (lower_name.indexOf("mozilla") != -1 || lower_name.indexOf("firefox") != -1) { 
+		logger.info("found mozilla or firefox, trying to remotely launch it");
+                // mozilla and netscape, try using a remote command to get things in the same window
+		String new_command = prog_name +" -raise -remote openURL("+url+",new-tab)";
+		logger.info(new_command);
+		Runtime rt = Runtime.getRuntime();
+		Process process = rt.exec(new_command);
+               	state = LAUNCHSUCCESS;
+		exitCode = process.waitFor();
+		logger.info("ExitCode:" + exitCode);              
+		if (exitCode != 0) { // if Netscape or mozilla was not open
+		    logger.info("couldn't do remote, trying original command");
+                    logger.info(this.command);
+		    process = rt.exec(this.command); // try the original command
+                    state = LAUNCHSUCCESS;
+		    //for some reason the following part is not executed sometimes.
+                    exitCode = process.waitFor();
+		}
+	    } else {
+		logger.info(this.command);
+                Runtime rt = Runtime.getRuntime();
+		Process process = rt.exec(this.command);
+                state = LAUNCHSUCCESS;
+              	//for some reason the following part is not executed sometimes.
+              	exitCode = process.waitFor();
+	    }
+
+	    logger.info("ExitCode:" + exitCode);
+	    if (exitCode != 0) { // if trying to launch the browser above failed
+	       state = LAUNCHFAILED;
+	       logger.error("Failed to launch web browser when running command:");
+	       logger.error("\t" + this.command);		
+	    }
+	}
+	catch (Exception e) {
+	    logger.error(e);
+            state = LAUNCHFAILED;
+	}
+	
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/Configuration.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/Configuration.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/Configuration.java	(revision 31634)
@@ -0,0 +1,132 @@
+/**
+ *#########################################################################
+ * Configuration
+ * A component of the GAI application, part of the Greenstone digital
+ * library suite from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * <BR><BR>
+ *
+ * Copyright (C) 1999 New Zealand Digital Library Project
+ *
+ * <BR><BR>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * <BR><BR>
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * <BR><BR>
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *########################################################################
+ */
+package org.greenstone.util;
+
+import java.awt.Color;
+import java.util.Locale;
+import java.util.Hashtable;
+import java.util.StringTokenizer;
+
+// eventually this class will hold configuration settings from a config file. For now, lets use static info
+// loosely copied from Gatherer Configuration
+public class Configuration {
+    
+    static private Hashtable<String, String> hash = null;
+
+    /** The first of three patterns used during tokenization, this pattern handles a comma separated list. */
+    static final private String TOKENIZER_PATTERN1 = " ,\n\t";
+    /** The second of three patterns used during tokenization, this pattern handles an underscore separated list. */
+    static final private String TOKENIZER_PATTERN2 = "_\n\t";
+    /** The last of three patterns used during tokenization, this pattern handles an comma separated list containing spaces. */
+    static final private String TOKENIZER_PATTERN3 = ",\n\t";
+
+    public Configuration() {
+	
+	hash = new Hashtable<String, String>();
+	
+	
+	// load up all the initial stuff
+	hash.put("admin.log", "true");
+	hash.put("admin.conf", "true");
+	hash.put("admin.ext", "true");
+	hash.put("admin.monitor", "true");
+	hash.put("coloring.workspace_selection_foreground", "0, 0, 0");
+	hash.put("coloring.workspace_selection_background", "176, 208, 176");
+	hash.put("coloring.table_noneditable_background", "255, 255, 255");
+	hash.put("coloring.table_editable_background", "255, 255, 255");
+	hash.put("coloring.collection_tree_background", "224, 240, 224");
+	hash.put("coloring.collection_tree_foreground", "0, 0, 0");
+	hash.put("coloring.workspace_tree_foreground", "0, 0, 0");
+	hash.put("coloring.workspace_tree_background", "218, 237, 252");
+	hash.put("coloring.workspace_heading_background", "128, 180, 216");
+	hash.put("coloring.workspace_heading_foreground", "0, 0, 0");
+	
+    }
+
+    /** Retrieve whether the named property is set or not */
+    static public boolean get(String property) {
+	String value = hash.get(property);
+	return (value != null && value.equalsIgnoreCase("true"));
+    }
+    
+    /** Retrieve the value of the named property as a String */
+    static public String getString(String property) {
+	return hash.get(property);
+    }
+
+    /** Retrieve the value of the named property as a Locale. */
+    static public Locale getLocale(String property) {
+	Locale result = Locale.getDefault();
+	try {
+	    String raw = hash.get(property);
+	    if (raw==null) {
+		return result;
+	    }
+	    // Locale is a underscore separated code.
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN2);
+	    String language = tokenizer.nextToken();
+	    if(tokenizer.hasMoreTokens()) {
+		String country = tokenizer.nextToken();
+		result = new Locale(language, country);
+	    }
+	    else {
+		result = new Locale(language);
+	    }
+	}
+	catch(Exception error) {
+	    error.printStackTrace();
+	}
+	return result;
+    }
+    
+    /** Retrieve the value of the named property as a Color. */
+    static public Color getColor(String property) {
+	Color result = Color.white; // Default
+	String raw = hash.get(property);
+	if (raw == null || raw.equals("")) {
+	    return result;
+	}
+	try {
+	    // Color is a RGB triplet list, comma separated (also remove whitespace)
+	    StringTokenizer tokenizer = new StringTokenizer(raw, TOKENIZER_PATTERN1);
+	    int red = Integer.parseInt(tokenizer.nextToken());
+	    int green = Integer.parseInt(tokenizer.nextToken());
+	    int blue = Integer.parseInt(tokenizer.nextToken());
+	    result = new Color(red, green, blue);
+	}
+	catch(Exception error) {
+	    error.printStackTrace();
+	}
+	return result;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/GlobalProperties.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/GlobalProperties.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/GlobalProperties.java	(revision 31634)
@@ -0,0 +1,219 @@
+/*
+ *    GlobalProperties.java
+ *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+package org.greenstone.util;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.FileInputStream;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * holds some global properties for the application. Read from a properties file
+ */
+public class GlobalProperties
+{
+
+	static Logger logger = Logger.getLogger(org.greenstone.util.GlobalProperties.class.getName());
+	private static Properties properties = new Properties();
+	private static String properties_filename = "global.properties";
+	private static String gsdl3_home = null;
+	private static String gsdl3_writablehome = null;
+	private static String gsdl3_web_address = null;
+	private static String full_gsdl3_web_address = null;
+
+	/** get the value of the property 'key'. returns null if not found */
+	public static String getProperty(String key)
+	{
+		return properties.getProperty(key);
+	}
+
+	/**
+	 * get the value of the property 'key'. returns default_value if not found
+	 */
+	public static String getProperty(String key, String default_value)
+	{
+		return properties.getProperty(key, default_value);
+	}
+
+	/** some special ones */
+	public static String getGSDL3Home()
+	{
+		return gsdl3_home;
+	}
+
+	public static String getGSDL3WritableHome()
+	{
+		return gsdl3_writablehome;
+	}
+
+	public static String getGS2Build()
+	{
+		return gsdl3_home + File.separator + ".." + File.separator + "gs2build";
+	}
+
+	public static String getGSDL3WebAddress()
+	{
+		return gsdl3_web_address;
+	}
+
+	public static String getFullGSDL3WebAddress()
+	{
+		return full_gsdl3_web_address;
+	}
+
+	public static void loadGlobalProperties(String optionalGS3Home)
+	{
+		try
+		{
+			InputStream in = Class.forName("org.greenstone.util.GlobalProperties").getClassLoader().getResourceAsStream(properties_filename);
+
+			if (in==null)
+			{
+			        // Try to find it through gsdl3.writablehome
+			        logger.info("Couldn't find '" + properties_filename + "' through ClassLoader");
+			        logger.info("Trying alternative path through System property gsdl3.writablehome");
+				gsdl3_writablehome = System.getProperty("gsdl3.writablehome");
+				if (gsdl3_writablehome!=null) {
+
+				    String gp_full_filename = gsdl3_writablehome + File.separator + "WEB-INF"
+					+ File.separator + "classes" 
+					+ File.separator + properties_filename; 
+
+				    try {
+					in = new FileInputStream(gp_full_filename);
+					logger.info("Found '" + properties_filename + "'");
+				    }
+				    catch (Exception e) {
+					logger.info("Unable to locate '" + gp_full_filename + "'");
+				    }
+				}
+				else {
+				    logger.info("Java property gsdl3.writablehome was not set");
+				}
+			}
+
+			if (in != null)
+			{
+				logger.debug("Loading global properties");
+				properties.load(in);
+				in.close();
+			}
+			else {
+			        logger.error("Failed to find '" + properties_filename + "'");
+			}    
+
+			gsdl3_home = properties.getProperty("gsdl3.home");
+			if ((gsdl3_home == null || gsdl3_home.length() == 0) 
+			    && optionalGS3Home != null && optionalGS3Home.length() > 0)
+			{
+				gsdl3_home = optionalGS3Home;
+			}
+
+			// if gsdl3_home is still null, fall back to default: gsdl3srchome/web
+			if (gsdl3_home == null) { 
+			    gsdl3_home = System.getenv("GSDL3SRCHOME") + File.separator + "web";
+			    logger.warn("** Note: falling back to using GSDL3SRCHOME to set gsdl3.home to: " + gsdl3_home);
+			}
+
+			// make sure the path separators are correct
+			// gsdl3_home may be null, e.g., when we are loading properties from Server3
+			if (gsdl3_home != null)
+			{
+				File gs3_file = new File(gsdl3_home);
+				gsdl3_home = gs3_file.getPath();
+			}
+
+			gsdl3_writablehome = properties.getProperty("gsdl3.writablehome");
+			// if gsdl3_writablehome is null, then defaults to gsdl3_home
+			if (gsdl3_writablehome == null) { 
+			    gsdl3_writablehome = gsdl3_home;
+			}
+
+			//build the gsdl3 web address, in a way resilient to errors and ommisions in global.properties, simplifying where possible
+			//aiming for a string with no trailing slash, eg "http://localhost:8080/greenstone3" or "http://www.mygreenstonelibrary.com"
+			String protocolSpecifier = null, hostSpecifier = null, portSpecifier = null, contextSpecifier = null;
+
+			//protocol
+			if (properties.getProperty("tomcat.protocol") == null || properties.getProperty("tomcat.protocol").equals(""))
+			{
+				protocolSpecifier = "http://";
+			}
+			else
+			{
+				if (properties.getProperty("tomcat.protocol").endsWith("://"))
+				{
+					protocolSpecifier = properties.getProperty("tomcat.protocol");
+				}
+				else
+				{
+					protocolSpecifier = properties.getProperty("tomcat.protocol") + "://";
+				}
+			}
+
+			//hostname
+			if (properties.getProperty("tomcat.server") == null)
+			{
+				hostSpecifier = "localhost";
+			}
+			else
+			{
+				hostSpecifier = properties.getProperty("tomcat.server");
+				while (hostSpecifier.endsWith("/"))
+				{
+					hostSpecifier = hostSpecifier.substring(0, hostSpecifier.length() - 1);
+				}
+			}
+
+			//port
+			if (properties.getProperty("tomcat.port") == null || properties.getProperty("tomcat.port").equals("") || (protocolSpecifier.equals("http://") && properties.getProperty("tomcat.port").equals("80")) || (protocolSpecifier.equals("https://") && properties.getProperty("tomcat.port").equals("443")))
+			{
+				portSpecifier = "";
+			}
+			else
+			{
+				portSpecifier = ":" + properties.getProperty("tomcat.port");
+			}
+
+			//context path
+			if (properties.getProperty("tomcat.context") == null || properties.getProperty("tomcat.context").equals("") || properties.getProperty("tomcat.context").equals("/"))
+			{
+				contextSpecifier = "";
+			}
+			else
+			{
+				contextSpecifier = properties.getProperty("tomcat.context");
+				if (!contextSpecifier.startsWith("/"))
+				{
+					contextSpecifier = "/" + contextSpecifier;
+				}
+			}
+
+			//string it all together
+			full_gsdl3_web_address = protocolSpecifier + hostSpecifier + portSpecifier + contextSpecifier;
+			gsdl3_web_address = contextSpecifier;
+		}
+		catch (Exception e)
+		{
+			logger.error("Exception trying to reload global.properties: " + e);
+			e.printStackTrace();
+		}
+	}
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/LogStreamGobbler.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/LogStreamGobbler.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/LogStreamGobbler.java	(revision 31634)
@@ -0,0 +1,41 @@
+package org.greenstone.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.apache.log4j.*;
+
+// The LogStreamGobbler class, despite the name, is not connected to Input- and OutputStreamGobbler
+// classes. This class has to do with logging and not with thread based input/output streams. 
+class LogStreamGobbler{
+    static Logger logger = Logger.getLogger(org.greenstone.util.LogStreamGobbler.class.getName());
+    
+    public static void logError(InputStream in)
+      {
+	  try {
+	      BufferedReader br = new BufferedReader(new InputStreamReader(in));
+	      String line = null;
+              //using null as a guard doesn't work on windows 
+  	      while ((line = br.readLine()) != null){
+		  logger.error(line);
+	      }
+	  } catch (IOException ioe) {
+	      logger.error(ioe);
+	  }
+      }
+    
+    
+    public void logInfo(InputStream in){
+	try {
+	    BufferedReader br = new BufferedReader(new InputStreamReader(in));
+	    String line = null;
+	    while ((line = br.readLine()) != null){
+		  logger.info(line);
+	      }
+	  } catch (IOException ioe) {
+	      logger.error(ioe);
+	  }
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/Misc.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/Misc.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/Misc.java	(revision 31634)
@@ -0,0 +1,137 @@
+ /*
+ *    Misc.java
+ *    Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+package org.greenstone.util;
+
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Map;
+import java.util.Iterator;
+import java.util.Properties;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+/** contains miscellaneous functions */
+public class Misc {
+
+    // Can initialise static final vars on declaration or in static initialisation code block
+    // http://stackoverflow.com/questions/2339932/java-can-final-variables-be-initialized-in-static-initialization-block
+    // Initialise object member final vars on declaration or in constructors
+    public static final String NEWLINE;
+    
+    // Before Java 7, no System.lineSeparator() or System.getProperty("line.separator")
+    // And on local linux, am compiling with JDK 6, so need this.
+    // http://stackoverflow.com/questions/207947/how-do-i-get-a-platform-dependent-new-line-character
+
+    static {
+	// http://stackoverflow.com/questions/2591083/getting-java-version-at-runtime
+	// https://www.tutorialspoint.com/java/lang/package_getspecificationversion.htm
+	
+	double java_version = Double.parseDouble(System.getProperty("java.specification.version"));
+	if(java_version >= 1.7) { // https://docs.oracle.com/javase/tutorial/essential/environment/sysprop.html
+	    NEWLINE = System.getProperty("line.separator");
+	} else {
+	    NEWLINE = isWindows() ? "\r\n" : "\n";
+	}	
+    }
+
+    public static void printHash(HashMap map) {
+	Set entries = map.entrySet();
+	Iterator i = entries.iterator();
+	while (i.hasNext()) {
+	    Map.Entry m = (Map.Entry)i.next();
+	    String name = (String)m.getKey();
+	    String value = (String)m.getValue();
+	}  
+    }
+    
+    /** Method to determine if the host system is Windows based.
+     * @return a boolean which is true if the platform is Windows, 
+     * false otherwise
+     */
+    public static boolean isWindows() {
+	Properties props = System.getProperties();
+	String os_name = props.getProperty("os.name","");
+	if(os_name.startsWith("Windows")) {
+	    return true;
+	}
+	return false;
+    }
+    
+    public static boolean isWindows9x() {
+        Properties props = System.getProperties();
+        String os_name = props.getProperty("os.name","");
+        if(os_name.startsWith("Windows") && os_name.indexOf("9") != -1) {
+            return true;
+        }
+        return false;
+    }
+
+    /** Method to determine if the host system is MacOS based.
+     * @return a boolean which is true if the platform is MacOS, false otherwise
+     */
+    public static boolean isMac() {
+        Properties props = System.getProperties();
+        String os_name = props.getProperty("os.name","");
+        if(os_name.startsWith("Mac OS")) {
+            return true;
+        }
+        return false;
+    }
+
+    public static String getGsdlOS() {
+		if (isWindows()) {
+			return "windows";
+		}
+        if (isMac()) {
+			return "darwin";
+		}
+		return "linux";
+    }
+    
+    public static boolean isBigEndian() {
+
+	if (System.getProperty("sun.cpu.endian").equals("big")) {
+	    return true;
+	}
+	return false;
+
+    }
+
+    public static BufferedReader makeHttpConnection(String url_string)
+	throws java.net.MalformedURLException, java.io.IOException {
+	BufferedReader reader = null;
+	URL url = new URL(url_string);
+	HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+	InputStream input = connection.getInputStream();
+	reader = new BufferedReader(new InputStreamReader(input));  
+	return reader;
+    }
+
+    public static void main(String [] args) {
+	isBigEndian();
+    }
+
+}
+
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/PortFinder.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/PortFinder.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/PortFinder.java	(revision 31634)
@@ -0,0 +1,135 @@
+package org.greenstone.util;
+
+/** Verbatim copy of what's in GLI's org.greenstone.gatherer.util */
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.BindException;
+
+// When needing to use sockets, this class can be used to find an available port
+public class PortFinder {
+
+    // Port numbers range from 0 to 65,535. RESERVED PORTS are below 1025
+    // see http://www.gnu.org/software/hello/manual/libc/Ports.html
+    public static final int MAX_PORT = 65535;
+    public static final int PORTS_RESERVED = 1024;
+
+    // Considering a limited range of ports that will be reused (circular buffer)
+    public final int PORT_BASE;
+    public final int PORT_BLOCK_SIZE;
+
+    // Keep track of what port numbers we have checked for availability
+    private int nextFreePort;
+
+    // The socket port number that we will use
+    private int port = -1;
+
+    public PortFinder() {
+	this( PORTS_RESERVED+1, (MAX_PORT - PORTS_RESERVED) );
+    }
+
+    public PortFinder(int base, int blocksize) {
+	PORT_BASE = base;
+	PORT_BLOCK_SIZE = blocksize;
+
+	nextFreePort = PORT_BASE;
+    }
+
+    /** Circular buffer. Modifies the value of nextFreePort (the buffer index). */
+    private void incrementNextFreePort() {
+	int offset = nextFreePort - PORT_BASE;
+	offset = (offset + 1) % PORT_BLOCK_SIZE;
+	nextFreePort = PORT_BASE + offset;
+    } 
+
+    /** Finds the first available port in the range specified during Constructor call.
+     *  @return the number of an available port.	
+     */
+    public int findPortInRange(boolean verbose) throws Exception {	
+	try {
+	    boolean foundFreePort = false;
+	    for(int i = 0; i < PORT_BLOCK_SIZE; i++) {
+		
+		if(isPortAvailable(nextFreePort, verbose)) {
+		    foundFreePort = true;
+		    break;
+		    
+		} else {
+		    incrementNextFreePort();
+		}
+	    }
+	    
+	    if(foundFreePort) {
+		// Free port number currently found becomes the port number of the socket
+		// that will be used
+		this.port = nextFreePort;
+		incrementNextFreePort();
+		
+	    } else {
+		throw new Exception("Cannot find an available port in the range " 
+				    + PORT_BASE + "-" + (PORT_BASE+PORT_BLOCK_SIZE));
+	    }
+	    
+	} catch(IOException e) {
+	    System.err.println("Error when trying to find an available port. In PortFinder.findPort() " + e);
+	}
+	return port;
+    }
+
+    /** @return true if the portnumber is in the valid range. */
+    public static boolean isValidPortNumber(int portNum) {
+	return (portNum >= 0 && portNum <= MAX_PORT);
+    }
+
+    /** @return true if the portnumber is in the useable range. */
+    public static boolean isAssignablePortNumber(int portNum) {
+	return (portNum > PORTS_RESERVED && portNum <= MAX_PORT);
+    }
+
+    /** Finds an available port.
+     * @return the number of an available port.
+     */
+    public static int findAnyFreePort() throws Exception {	
+	ServerSocket tmpSocket = null;
+	int portNumber = -1;
+	try {
+	    // Creates a server socket, bound to the specified port.
+	    // A port of 0 creates a socket on any free port. 
+	    tmpSocket = new ServerSocket(0);
+	    portNumber = tmpSocket.getLocalPort();
+	    tmpSocket.close();
+	} catch(Exception e) {
+	    System.err.println("Unable to find a free port or close it. Got Exception: " + e);
+	    tmpSocket = null;
+	}
+	return portNumber;
+    }
+
+    /** @return true if the portnum is available for use */
+    public static boolean isPortAvailable(int portnum, boolean verbose) {
+	ServerSocket tmpSocket = null;
+	try {
+	    tmpSocket = new ServerSocket(portnum);
+	    tmpSocket.close();
+	    //if(verbose) {
+	    //System.err.println("Port " + portnum + " not yet in use.");
+	    //}
+	    return true;
+	    
+	} catch(BindException ex){
+	    // "Signals that an error occurred while attempting to bind a 
+	    // socket to a local address and port. Typically, the port is 
+	    // in use, or the requested local address could not be assigned."
+	    if(verbose) {
+		System.err.println("Port " + portnum + " already in use or can't be assigned.");
+	    }
+	    tmpSocket = null;
+	    return false;	    
+	} catch(Exception ex) { 
+	    // Some other problem creating or closing the server socket
+	    System.err.println("Problem creating or closing server socket at port " + portnum);
+	    tmpSocket = null;
+	    return false;
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/RunAnt.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/RunAnt.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/RunAnt.java	(revision 31634)
@@ -0,0 +1,37 @@
+package org.greenstone.util;
+
+import org.greenstone.util.RunTarget;
+
+public class RunAnt extends RunTarget 
+{
+    String opt_cmd_args = "";
+
+    public RunAnt(String opt_args)
+    {
+	super();
+
+	if (opt_args!=null) {
+	    opt_cmd_args = opt_args + " ";
+	}
+
+	targetSuccess  = "BUILD SUCCESSFUL";
+	targetFailed   = "BUILD FAILED";
+	targetFinished = "Total time";
+    }
+
+    public RunAnt()
+    {
+	this(null);
+    }
+
+    
+    public void setTargetCmd(String target) 
+    {
+	String osName = System.getProperty("os.name");
+      	if (osName.startsWith("Windows")) {
+	    targetCmd = "ant.bat " + opt_cmd_args + target;
+	} else {
+	    targetCmd = "ant " + opt_cmd_args + target;
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/RunMake.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/RunMake.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/RunMake.java	(revision 31634)
@@ -0,0 +1,28 @@
+package org.greenstone.util;
+
+import org.greenstone.util.RunTarget;
+
+public class RunMake extends RunTarget 
+{
+
+    public RunMake()
+    {
+	super();
+
+	targetSuccess  = "MAKE SUCCESSFUL";
+	targetFailed   = "MAKE FAILED";
+	targetFinished = "MAKE DONE";
+    }
+
+
+    public void setTargetCmd(String target) 
+    {
+	String osName = System.getProperty("os.name");
+      	if (osName.startsWith("Windows")) {
+	    //targetCmd = "fakemake.bat " + target;
+	    targetCmd = "gsicontrol.bat " + target;
+	} else {
+	    targetCmd = "./gsicontrol.sh " + target;
+	}
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/RunTarget.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/RunTarget.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/RunTarget.java	(revision 31634)
@@ -0,0 +1,79 @@
+package org.greenstone.util;
+
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.greenstone.util.LogStreamGobbler;
+
+import org.apache.log4j.*;
+
+public abstract class RunTarget extends Thread 
+{
+    protected String targetCmd = "";
+    protected static Logger logger = Logger.getLogger(RunTarget.class.getName());
+
+    protected int state = -1; //success: 0 error: 1    
+    public static int SUCCESS = 0;
+    public static int FAILED  = 1; 
+
+    protected String targetSuccess;
+    protected String targetFailed;
+    protected String targetFinished;
+
+    public void run() 
+    {
+	 
+	 try {
+	     state = -1;
+	     Runtime run = Runtime.getRuntime();
+
+	     String targetCmd = getTargetCmd();
+	     logger.info("Target: " + targetCmd);
+
+             Process process = run.exec(targetCmd);
+	     BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
+	     String line = null;
+             
+	     while ((line = br.readLine()) != null) {
+
+		 logger.info(line);
+
+		 if (line.equals(targetSuccess)){
+		     state = 0;
+		 }
+		 
+		 if (line.equals(targetFailed)){
+		     state = 1; 
+		 }
+
+		 if(line.startsWith(targetFinished)){
+		     break;		 
+		 }
+	     }
+	     
+	     br.close();
+
+	     if(state < 0) {
+		 logger.info("Unexpected end of input when running target: " + targetCmd);
+	     }
+	 } catch (Exception e) {
+	     e.printStackTrace();
+	     logger.error(e);
+	     state = 1;
+	 } 
+    }
+  
+    public int getTargetState()
+    {
+	return state;     
+    }
+    
+    public abstract void setTargetCmd(String cmd);
+
+    public String getTargetCmd() 
+    {
+	return this.targetCmd;
+    }
+}
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/SafeProcess.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/SafeProcess.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/SafeProcess.java	(revision 31634)
@@ -0,0 +1,674 @@
+package org.greenstone.util;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import java.util.Arrays;
+
+import org.apache.log4j.*;
+
+// Use this class to run a Java Process. It follows the good and safe practices at
+// http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+// to avoid blocking problems that can arise from a Process' input and output streams.
+
+public class SafeProcess {
+    
+    public static final int STDERR = 0;
+    public static final int STDOUT = 1;
+    public static final int STDIN = 2;    
+
+    static Logger logger = Logger.getLogger(org.greenstone.util.SafeProcess.class.getName());
+
+    // input to SafeProcess and initialising it
+    private String command = null;
+    private String[] command_args = null;
+    private String[] envp = null;
+    private File dir = null;
+    private String inputStr = null;
+
+    // output from running SafeProcess.runProcess()
+    private String outputStr = ""; 
+    private String errorStr = "";
+    private int exitValue = -1;
+
+    // allow callers to process exceptions of the main process thread if they want
+    private ExceptionHandler exceptionHandler = null;
+
+    // whether std/err output should be split at new lines
+    private boolean splitStdOutputNewLines = false;
+    private boolean splitStdErrorNewLines = false;
+
+    // call one of these constructors
+
+    // cmd args version
+    public SafeProcess(String[] cmd_args)
+    {
+	command_args = cmd_args;
+    }
+
+    // cmd string version
+    public SafeProcess(String cmdStr)
+    {
+	command = cmdStr;
+    }
+
+    // cmd args with env version, launchDir can be null.
+    public SafeProcess(String[] cmd_args, String[] envparams, File launchDir)
+    {
+	command_args = cmd_args;
+	envp = envparams;
+	dir = launchDir;
+    }
+
+    // The important methods:
+    // to get the output from std err and std out streams after the process has been run
+    public String getStdOutput() { return outputStr; }
+    public String getStdError() { return errorStr; }
+    public int getExitValue() { return exitValue; }
+
+    
+    // set any string to send as input to the process spawned by SafeProcess
+    public void setInputString(String sendStr) {
+	inputStr = sendStr;
+    }
+
+    // register a SafeProcess ExceptionHandler whose gotException() method will
+    // get called for each exception encountered
+    public void setExceptionHandler(ExceptionHandler exception_handler) {
+	exceptionHandler = exception_handler;	
+    }
+
+    // set if you want the std output or err output to have \n at each newline read from the stream
+    public void setSplitStdOutputNewLines(boolean split) {
+	splitStdOutputNewLines = split;
+    }
+    public void setSplitStdErrorNewLines(boolean split) {
+	splitStdErrorNewLines = split;
+    }
+
+    private Process doRuntimeExec() throws IOException {
+	Process prcs = null;
+	Runtime rt = Runtime.getRuntime();
+	
+	if(this.command != null) {
+	    prcs = rt.exec(this.command);
+	}
+	else { // at least command_args must be set now
+	    
+	    // http://stackoverflow.com/questions/5283444/convert-array-of-strings-into-a-string-in-java
+	    logger.info("SafeProcess running: " + Arrays.toString(command_args));
+	    //System.err.println("SafeProcess running: " + Arrays.toString(command_args));
+	    
+	    if(this.envp == null) { 
+		prcs = rt.exec(this.command_args);
+	    } else { // launch process using cmd str with env params		
+		
+		if(this.dir == null) {
+		    ///logger.info("\twith: " + Arrays.toString(this.envp));
+		    ///System.err.println("\twith: " + Arrays.toString(this.envp));
+		    prcs = rt.exec(this.command_args, this.envp);
+		} else {
+		    ///logger.info("\tfrom directory: " + this.dir);
+		    ///logger.info("\twith: " + Arrays.toString(this.envp));
+		    ///System.err.println("\tfrom directory: " + this.dir);
+		    ///System.err.println("\twith: " + Arrays.toString(this.envp));
+			prcs = rt.exec(this.command_args, this.envp, this.dir);
+		}
+	    }
+	}
+
+	return prcs;
+    }
+
+    // Copied from gli's gui/FormatConversionDialog.java
+    private int waitForWithStreams(Process prcs,
+				   SafeProcess.OutputStreamGobbler inputGobbler,
+				   SafeProcess.InputStreamGobbler outputGobbler,
+				   SafeProcess.InputStreamGobbler errorGobbler)
+	throws IOException, InterruptedException
+    {
+
+            // kick off the stream gobblers
+            inputGobbler.start();
+            errorGobbler.start();
+            outputGobbler.start();
+
+            // any error???
+            this.exitValue = prcs.waitFor(); // can throw an InterruptedException if process did not terminate
+
+            ///logger.info("Process exitValue: " + exitValue); 
+	    ///System.err.println("Process exitValue: " + exitValue); 
+
+	    // From the comments of 
+	    // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+	    // To avoid running into nondeterministic failures to get the process output
+	    // if there's no waiting for the threads, call join() on each Thread (StreamGobbler) object.
+	    // From Thread API: join() "Waits for this thread (the thread join() is invoked on) to die."
+	    outputGobbler.join(); 
+	    errorGobbler.join();
+	    inputGobbler.join(); 
+	    
+
+	    // set the variables the code that created a SafeProcess object may want to inspect
+	    this.outputStr = outputGobbler.getOutput();
+	    this.errorStr = errorGobbler.getOutput();
+	    
+	    // Since we didn't have an exception, process should have terminated now (waitFor blocks until then)
+	    // Set process to null so we don't forcibly terminate it below with process.destroy()
+	    prcs = null;
+
+	    return this.exitValue;
+    }
+
+
+    // Run a very basic process: with no reading from or writing to the Process' iostreams,
+    // this just execs the process and waits for it to return
+    public int runBasicProcess() {
+	Process prcs = null;
+	try {
+	    // 1. create the process
+	    prcs = doRuntimeExec();
+	    // 2. basic waitFor the process to finish
+	    this.exitValue = prcs.waitFor();
+
+	    
+	} catch(IOException ioe) {
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ioe);
+	    } else {
+		logger.error("IOException: " + ioe.getMessage(), ioe);
+		//System.err.println("IOException " + ioe.getMessage());
+		//ioe.printStackTrace();
+	    }
+	} catch(InterruptedException ie) {
+
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ie);
+	    } else {
+		logger.error("Process InterruptedException: " + ie.getMessage(), ie);
+		//System.err.println("Process InterruptedException " + ie.getMessage());
+		///ie.printStackTrace(); // an interrupt here is not an error, it can be a cancel action
+	    }
+	    
+	    Thread.currentThread().interrupt();
+	} finally { 
+
+	    if( prcs != null ) {
+		prcs.destroy(); // see runProcess() below
+	    }
+	}
+	return this.exitValue;
+    }
+
+    // Runs a process with default stream processing. Returns the exitValue
+    public int runProcess() {
+	return runProcess(null, null, null); // use default processing of all 3 of the process' iostreams
+    }
+
+    // Run a process with custom stream processing (any custom handlers passed in that are null
+    // will use the default stream processing). 
+    // Returns the exitValue from running the Process
+    public int runProcess(CustomProcessHandler procInHandler,
+			   CustomProcessHandler procOutHandler,
+			   CustomProcessHandler procErrHandler)
+    {
+	Process prcs = null;
+	SafeProcess.OutputStreamGobbler inputGobbler = null;
+	SafeProcess.InputStreamGobbler errorGobbler = null;
+	SafeProcess.InputStreamGobbler outputGobbler = null;
+
+	try {
+	    // 1. get the Process object
+	    prcs = doRuntimeExec();
+	    
+
+	    // 2. create the streamgobblers and set any specified handlers on them
+
+	    // PROC INPUT STREAM
+	    if(procInHandler == null) {
+		// send inputStr to process. The following constructor can handle inputStr being null
+		inputGobbler = // WriterToProcessInputStream
+		    new SafeProcess.OutputStreamGobbler(prcs.getOutputStream(), this.inputStr);
+	    } else { // user will do custom handling of process' InputStream 
+		inputGobbler = new SafeProcess.OutputStreamGobbler(prcs.getOutputStream(), procInHandler);
+	    }
+
+	    // PROC ERR STREAM to monitor for any error messages or expected output in the process' stderr
+	    if(procErrHandler == null) {
+		errorGobbler // ReaderFromProcessOutputStream
+		    = new SafeProcess.InputStreamGobbler(prcs.getErrorStream(), splitStdErrorNewLines);
+	    } else {
+		errorGobbler
+		    = new SafeProcess.InputStreamGobbler(prcs.getErrorStream(), procErrHandler);
+	    }
+
+            // PROC OUT STREAM to monitor for the expected std output line(s)
+	    if(procOutHandler == null) {
+		outputGobbler
+		    = new SafeProcess.InputStreamGobbler(prcs.getInputStream(), splitStdOutputNewLines);
+	    } else {
+		outputGobbler
+		    = new SafeProcess.InputStreamGobbler(prcs.getInputStream(), procOutHandler);
+	    }
+
+
+            // 3. kick off the stream gobblers
+	    this.exitValue = waitForWithStreams(prcs, inputGobbler, outputGobbler, errorGobbler);
+       
+	} catch(IOException ioe) {
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ioe);
+	    } else {
+		logger.error("IOexception: " + ioe.getMessage(), ioe);
+		//System.err.println("IOexception " + ioe.getMessage());
+		//ioe.printStackTrace();
+	    }
+	} catch(InterruptedException ie) {
+
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ie);
+	    } else {
+		logger.error("Process InterruptedException: " + ie.getMessage(), ie);
+		//System.err.println("Process InterruptedException " + ie.getMessage());
+		///ie.printStackTrace(); // an interrupt here is not an error, it can be a cancel action
+	    }
+
+	    // propagate interrupts to worker threads here?
+	    // unless the interrupt emanated from any of them in any join()...
+	    // Only if the thread SafeProcess runs in was interrupted
+	    // do we propagate the interrupt to the worker threads.
+	    // http://stackoverflow.com/questions/2126997/who-is-calling-the-java-thread-interrupt-method-if-im-not
+	    // "I know that in JCiP it is mentioned that you should never interrupt threads you do not own"
+	    // But SafeProcess owns the worker threads, so it have every right to interrupt them
+	    // Also read http://stackoverflow.com/questions/13623445/future-cancel-method-is-not-working?noredirect=1&lq=1
+	    if(Thread.currentThread().isInterrupted()) {
+		inputGobbler.interrupt();
+		errorGobbler.interrupt();
+		outputGobbler.interrupt();	    
+	    }
+
+	    // On catchingInterruptedException, re-interrupt the thread.
+	    // This is just how InterruptedExceptions tend to be handled
+	    // See also http://stackoverflow.com/questions/4906799/why-invoke-thread-currentthread-interrupt-when-catch-any-interruptexception
+	    // and https://praveer09.github.io/technology/2015/12/06/understanding-thread-interruption-in-java/
+
+	    // http://stackoverflow.com/questions/3976344/handling-interruptedexception-in-java
+	    // http://stackoverflow.com/questions/4906799/why-invoke-thread-currentthread-interrupt-when-catch-any-interruptexception
+	    // "Only code that implements a thread's interruption policy may swallow an interruption request. General-purpose task and library code should never swallow interruption requests."
+	    // Does that mean that since this code implements this thread's interruption policy, it's ok
+	    // to swallow the interrupt this time and not let it propagate by commenting out the next line?
+	    Thread.currentThread().interrupt(); // re-interrupt the thread - which thread? Infinite loop?
+	} finally { 
+
+	    // Moved into here from GS2PerlConstructor which said
+	    // "I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine."
+	    // http://steveliles.github.io/invoking_processes_from_java.html
+	    // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+	    // http://mark.koli.ch/leaky-pipes-remember-to-close-your-streams-when-using-javas-runtimegetruntimeexec	    
+	    if( prcs != null ) {		
+		prcs.destroy();
+	    }
+	}
+	
+	return this.exitValue;
+    }
+    
+    public int runProcess(LineByLineHandler outLineByLineHandler, LineByLineHandler errLineByLineHandler)
+    {
+	Process prcs = null;
+	SafeProcess.OutputStreamGobbler inputGobbler = null;
+	SafeProcess.InputStreamGobbler errorGobbler = null;
+	SafeProcess.InputStreamGobbler outputGobbler = null;
+
+	try {
+	    // 1. get the Process object
+	    prcs = doRuntimeExec();
+	    
+
+	    // 2. create the streamgobblers and set any specified handlers on them
+
+	    // PROC INPUT STREAM
+	    // send inputStr to process. The following constructor can handle inputStr being null
+	    inputGobbler = // WriterToProcessInputStream
+		new SafeProcess.OutputStreamGobbler(prcs.getOutputStream(), this.inputStr);
+
+	    // PROC ERR STREAM to monitor for any error messages or expected output in the process' stderr    
+	    errorGobbler // ReaderFromProcessOutputStream
+		    = new SafeProcess.InputStreamGobbler(prcs.getErrorStream(), splitStdErrorNewLines);		
+            // PROC OUT STREAM to monitor for the expected std output line(s)
+	    outputGobbler
+		= new SafeProcess.InputStreamGobbler(prcs.getInputStream(), splitStdOutputNewLines);
+
+
+	    // 3. register line by line handlers, if any were set, for the process stderr and stdout streams
+	    if(outLineByLineHandler != null) {
+		outputGobbler.setLineByLineHandler(outLineByLineHandler);
+	    }
+	    if(errLineByLineHandler != null) {
+		errorGobbler.setLineByLineHandler(errLineByLineHandler);
+	    }	    
+
+
+            // 4. kick off the stream gobblers
+	    this.exitValue = waitForWithStreams(prcs, inputGobbler, outputGobbler, errorGobbler);
+       
+	} catch(IOException ioe) {
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ioe);
+	    } else {
+		logger.error("IOexception: " + ioe.getMessage(), ioe);
+		//System.err.println("IOexception " + ioe.getMessage());
+		//ioe.printStackTrace();
+	    }
+	} catch(InterruptedException ie) {
+
+	    if(exceptionHandler != null) {
+		exceptionHandler.gotException(ie);
+	    } else {
+		logger.error("Process InterruptedException: " + ie.getMessage(), ie);
+		//System.err.println("Process InterruptedException " + ie.getMessage());
+		///ie.printStackTrace(); // an interrupt here is not an error, it can be a cancel action
+	    }
+
+	    // propagate interrupts to worker threads here?
+	    // unless the interrupt emanated from any of them in any join()...
+	    // Only if the thread SafeProcess runs in was interrupted
+	    // do we propagate the interrupt to the worker threads.
+	    // http://stackoverflow.com/questions/2126997/who-is-calling-the-java-thread-interrupt-method-if-im-not
+	    // "I know that in JCiP it is mentioned that you should never interrupt threads you do not own"
+	    // But SafeProcess owns the worker threads, so it have every right to interrupt them
+	    // Also read http://stackoverflow.com/questions/13623445/future-cancel-method-is-not-working?noredirect=1&lq=1
+	    if(Thread.currentThread().isInterrupted()) {
+		inputGobbler.interrupt();
+		errorGobbler.interrupt();
+		outputGobbler.interrupt();	    
+	    }
+
+	    // On catchingInterruptedException, re-interrupt the thread.
+	    // This is just how InterruptedExceptions tend to be handled
+	    // See also http://stackoverflow.com/questions/4906799/why-invoke-thread-currentthread-interrupt-when-catch-any-interruptexception
+	    // and https://praveer09.github.io/technology/2015/12/06/understanding-thread-interruption-in-java/
+
+	    // http://stackoverflow.com/questions/3976344/handling-interruptedexception-in-java
+	    // http://stackoverflow.com/questions/4906799/why-invoke-thread-currentthread-interrupt-when-catch-any-interruptexception
+	    // "Only code that implements a thread's interruption policy may swallow an interruption request. General-purpose task and library code should never swallow interruption requests."
+	    // Does that mean that since this code implements this thread's interruption policy, it's ok
+	    // to swallow the interrupt this time and not let it propagate by commenting out the next line?
+	    Thread.currentThread().interrupt(); // re-interrupt the thread - which thread? Infinite loop?
+	} finally { 
+
+	    // Moved into here from GS2PerlConstructor which said
+	    // "I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine."
+	    // http://steveliles.github.io/invoking_processes_from_java.html
+	    // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+	    // http://mark.koli.ch/leaky-pipes-remember-to-close-your-streams-when-using-javas-runtimegetruntimeexec	    
+	    if( prcs != null ) {		
+		prcs.destroy();
+	    }
+	}
+	
+	return this.exitValue;
+    }
+
+
+//******************** Inner class and interface definitions ********************//
+// Static inner classes can be instantiated without having to instantiate an object of the outer class first
+
+// Can have public static interfaces too,
+// see http://stackoverflow.com/questions/71625/why-would-a-static-nested-interface-be-used-in-java
+// Implementors need to take care that the implementations are thread safe
+// http://stackoverflow.com/questions/14520814/why-synchronized-method-is-not-included-in-interface
+public static interface ExceptionHandler {
+
+    // when implementing ExceptionHandler.gotException(), if it manipulates anything that's
+    // not threadsafe, declare gotException() as a synchronized method to ensure thread safety
+    public void gotException(Exception e); // can't declare as synchronized in interface method declaration
+}
+
+// Write your own run() body for any StreamGobbler. You need to create an instance of a class
+// extending CustomProcessHandler for EACH IOSTREAM of the process that you want to handle.
+// Do not create a single CustomProcessHandler instance and reuse it for all three streams,
+// i.e. don't call SafeProcess' runProcess(x, x, x); It should be runProcess(x, y, z).
+// Make sure your implementation is threadsafe if you're sharing immutable objects between the threaded streams
+// example implementation is in the GS2PerlConstructor.SynchronizedProcessHandler class.
+// CustomProcessHandler is made an abstract class instead of an interface to force classes that want
+// to use a CustomProcessHandler to create a separate class that extends CustomProcessHandler, rather than
+// that the classes that wish to use it "implementing" the CustomProcessHandler interface itself: the
+// CustomProcessHandler.run() method may then be called in the major thread from which the Process is being
+// executed, rather than from the individual threads that deal with each iostream of the Process.
+public static abstract class CustomProcessHandler {
+
+    protected final int source;
+
+    protected CustomProcessHandler(int src) {
+	this.source = src; // STDERR or STDOUT or STDIN
+    }
+    
+    public abstract void run(Closeable stream); //InputStream or OutputStream
+}
+
+// When using the default stream processing to read from a process' stdout or stderr stream, you can
+// create a class extending LineByLineHandler for the process' err stream and one for its output stream
+// to do something on a line by line basis, such as sending the line to a log
+public static abstract class LineByLineHandler {
+    protected final int source;
+
+    protected LineByLineHandler(int src) {
+	this.source = src; // STDERR or STDOUT
+    }
+
+    public abstract void gotLine(String line); // first non-null line
+    public abstract void gotException(Exception e); // for when an exception occurs instead of getting a line
+}
+
+//**************** StreamGobbler Inner class definitions (stream gobblers copied from GLI) **********//
+
+// http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+// This class is used in FormatConversionDialog to properly read from the stdout and stderr
+// streams of a Process, Process.getInputStream() and Process.getErrorSream()
+public static class InputStreamGobbler extends Thread
+{
+    private InputStream is = null;
+    private StringBuffer outputstr = new StringBuffer();
+    private boolean split_newlines = false;
+    private CustomProcessHandler customHandler = null;
+    private LineByLineHandler lineByLineHandler = null;
+
+    public InputStreamGobbler(InputStream is)
+    {
+	this.is = is;
+	this.split_newlines = false;
+    }
+    
+    public InputStreamGobbler(InputStream is, boolean split_newlines)
+    {
+	this.is = is;
+	this.split_newlines = split_newlines;
+    }
+    
+    public InputStreamGobbler(InputStream is, CustomProcessHandler customHandler)
+    {
+	this.is = is;
+	this.customHandler = customHandler;
+    }
+
+    public void setLineByLineHandler(LineByLineHandler lblHandler) {
+	this.lineByLineHandler = lblHandler;
+    }
+
+    // default run() behaviour
+    public void runDefault()
+    {
+	BufferedReader br = null;
+	try {
+	    br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+	    String line=null;
+	    while ( (line = br.readLine()) != null ) {
+
+		if(this.isInterrupted()) { // should we not instead check if SafeProcess thread was interrupted?
+		    logger.info("Got interrupted when reading lines from process err/out stream.");
+		    //System.err.println("InputStreamGobbler.runDefault() Got interrupted when reading lines from process err/out stream.");
+		    break; // will go to finally block
+		}
+
+		//System.out.println("@@@ GOT LINE: " + line);
+		outputstr.append(line);
+		if(split_newlines) {
+		    outputstr.append(Misc.NEWLINE); // "\n" is system dependent (Win must be "\r\n")
+		}
+
+		if(lineByLineHandler != null) { // let handler deal with newlines
+		    lineByLineHandler.gotLine(line);
+		}
+	    }
+	} catch (IOException ioe) {
+	    if(lineByLineHandler != null) {
+		lineByLineHandler.gotException(ioe);
+	    } else {
+		logger.error("Exception when reading from a process' stdout/stderr stream: ", ioe);
+		//System.err.println("Exception when reading from a process' stdout/stderr stream: ");
+		//ioe.printStackTrace();  
+	    }
+	} finally {
+	    SafeProcess.closeResource(br);
+	}
+    }
+    
+    public void runCustom() {
+	this.customHandler.run(is);
+    }
+    
+    public void run() {
+	if(this.customHandler == null) {
+	    runDefault();
+	} else {
+	    runCustom();
+	}
+    }
+
+    public String getOutput() { 
+	return outputstr.toString(); // implicit toString() call anyway. //return outputstr; 
+    }
+} // end static inner class InnerStreamGobbler
+
+
+// http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+// This class is used in FormatConversionDialog to properly write to the inputstream of a Process
+// Process.getOutputStream()
+public static class OutputStreamGobbler extends Thread
+{
+    private OutputStream os = null;
+    private String inputstr = "";
+    private CustomProcessHandler customHandler = null;
+
+    public OutputStreamGobbler(OutputStream os) {
+	this.os = os;
+    }
+
+    public OutputStreamGobbler(OutputStream os, String inputstr)
+    {
+	this.os = os;
+	this.inputstr = inputstr;
+    }
+
+    public OutputStreamGobbler(OutputStream os, CustomProcessHandler customHandler) {
+	this.os = os;
+	this.customHandler = customHandler;
+    }
+
+    // default run() behaviour
+    public void runDefault() {
+	
+	if (inputstr == null) {
+		return;
+	}
+	
+	BufferedWriter osw = null;
+	try {
+	    osw = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+	    //System.out.println("@@@ SENDING LINE: " + inputstr);
+	    osw.write(inputstr, 0, inputstr.length());
+	    osw.newLine();//osw.write("\n");
+	    osw.flush();
+	    
+	    // Don't explicitly send EOF when using StreamGobblers as below, 
+	    // as the EOF char is echoed to output.
+	    // Flushing the write handle and/or closing the resource seems 
+	    // to already send EOF silently.
+	    
+	    /*if(Utility.isWindows()) {
+	      osw.write("\032"); // octal for Ctrl-Z, EOF on Windows
+	      } else { // EOF on Linux/Mac is Ctrl-D
+	      osw.write("\004"); // octal for Ctrl-D, see http://www.unix-manuals.com/refs/misc/ascii-table.html
+	      }
+	      osw.flush();
+	    */
+	} catch (IOException ioe) {
+	    logger.error("Exception writing to SafeProcess' inputstream: ", ioe);
+	    //System.err.println("Exception writing to SafeProcess' inputstream: ");
+	    //ioe.printStackTrace();	
+	} finally {
+	    SafeProcess.closeResource(osw);
+	}
+    }
+
+    // call the user's custom handler for the run() method
+    public void runCustom() {
+	this.customHandler.run(os);	
+    }
+
+    public void run()
+    {
+	if(this.customHandler == null) {
+	    runDefault();
+	} else {
+	    runCustom();
+	}
+
+    }	
+} // end static inner class OutputStreamGobbler
+
+
+//**************** Useful static methods. Copied from GLI's Utility.java ******************
+    // For safely closing streams/handles/resources. 
+    // For examples of use look in the Input- and OutputStreamGobbler classes.
+    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html
+    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks
+    public static boolean closeResource(Closeable resourceHandle) {
+	boolean success = false;
+	try {
+	    if(resourceHandle != null) {
+		resourceHandle.close();
+		resourceHandle = null;
+		success = true;
+	    }
+	} catch(Exception e) {
+	    logger.error("Exception closing resource: ", e);
+	    //System.err.println("Exception closing resource: " + e.getMessage());
+	    //e.printStackTrace();
+	    resourceHandle = null;
+	    success = false;
+	} finally {
+	    return success;
+	}
+    }
+
+    public static boolean closeProcess(Process prcs) {
+	boolean success = true;
+	if( prcs != null ) { 
+	    success = success && closeResource(prcs.getErrorStream()); 
+	    success = success && closeResource(prcs.getOutputStream()); 
+	    success = success && closeResource(prcs.getInputStream()); 
+	    prcs.destroy(); 
+	}
+	return success;
+    }
+
+} // end class SafeProcess
Index: main/tags/GS3-src-SafeProcess/org/greenstone/util/ScriptReadWrite.java
===================================================================
--- main/tags/GS3-src-SafeProcess/org/greenstone/util/ScriptReadWrite.java	(revision 31634)
+++ main/tags/GS3-src-SafeProcess/org/greenstone/util/ScriptReadWrite.java	(revision 31634)
@@ -0,0 +1,115 @@
+package org.greenstone.util;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+public class ScriptReadWrite {
+
+    public ScriptReadWrite() {
+    }
+
+    public ArrayList<String> readInFile(File file) {
+	try {
+	    ArrayList<String> fileLines = new ArrayList<String>();
+	    String oneLine = null;
+	    BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
+	    while ((oneLine = bufferedReader.readLine()) != null) {
+		fileLines.add(oneLine);
+	    }
+	    bufferedReader.close();
+	    return fileLines;
+	} catch (Exception e) {
+	    System.err.println("exception:" + e);
+	    return null;
+	}
+    }
+
+    public void writeOutFile(File file, ArrayList<String> fileLines_in) {
+	try {
+	    PrintWriter printWriter = new PrintWriter(new BufferedWriter(new FileWriter(file)));
+	    for (int i = 0; i < fileLines_in.size(); i++) {
+		printWriter.println(fileLines_in.get(i));
+	    }
+	    printWriter.flush();
+	    printWriter.close();
+	} catch (Exception e) {
+	    System.err.println("exception:" + e);
+	}
+    }
+
+    public ArrayList<String> queryReplace(ArrayList<String> fileLines_ex, String param,
+				  String newValue) {
+	// only replace existing, don't append if param does not exist
+	return replaceOrAddLine(fileLines_ex, param, newValue, false); 
+    }
+    
+    // if the parameter exists, then a replace is performed, else the parameter-value
+    // is appended
+    public ArrayList<String> replaceOrAddLine(ArrayList<String> fileLines_ex, String param, String newValue, 
+				      boolean replaceElseAdd) 
+    {	
+	String oneLine = null;
+	String newLine = null;
+	String name = null;
+	// int oneLine_length = 0;
+	param = param.trim();
+	// int param_length = param.length();
+
+	for (int i = 0; i < fileLines_ex.size(); i++) {
+	    oneLine = fileLines_ex.get(i).trim();
+	    // oneLine_length = oneLine.length();
+	    StringTokenizer st = new StringTokenizer(oneLine, "=");
+	    if (st.hasMoreTokens()) {
+		name = st.nextToken();
+		if (param.equals(name)) {
+		    newLine = param + "=" + newValue;
+		    fileLines_ex.set(i, newLine);
+		    replaceElseAdd = false; // replaced, no longer need to add in any case
+		    break;
+		}
+	    }
+	}
+	// If we've made no replacement and need to append the new item
+	if(replaceElseAdd) {
+	    fileLines_ex.add(param+"="+newValue);
+	}
+	return fileLines_ex;
+    }
+
+    public String existValue(ArrayList fileLines, String querytext) {
+	String retValue = null;
+	String oneline = null;
+	String name = null;
+	// String value=null;
+	// int oneline_length = 0;
+	querytext = querytext.trim();
+	// int querytext_length = querytext.length();
+	for (int i = 0; i < fileLines.size(); i++) {
+	    oneline = ((String) fileLines.get(i)).trim();
+	    // oneline_length = oneline.length();
+	    StringTokenizer st = new StringTokenizer(oneline, "=");
+	    if (st.hasMoreTokens()) {
+		name = st.nextToken();
+		if (querytext.equals(name)) {
+		    if (st.hasMoreTokens()) {
+			retValue = st.nextToken();
+			retValue = retValue.trim();
+			if (retValue.equals("")) {
+			    retValue = null;
+			}
+		    } else {
+			retValue = null;
+		    }
+		    break;
+		}
+	    }
+	}
+	return retValue;
+    }
+}
