Index: /other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java
===================================================================
--- /other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java	(revision 18167)
+++ /other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java	(revision 18167)
@@ -0,0 +1,897 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+import javax.xml.parsers.*;
+
+import org.xml.sax.*;
+import org.w3c.dom.*;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSPath;
+
+public class MatServlet extends HttpServlet {
+    
+    private int port_number = 0;
+    private String oaiPrefix ="";
+    private String titleString;
+    private String h1String;
+    private String maxRecord;
+    private GlobalProperties globalProperty;
+    private String fileSeparator;
+    private String cssString;
+    private String headerString;
+    private String javaScript;
+    private String headerString2;
+    private String gsdl3Home;
+    private String logoURL;
+    private String description;
+    private String hostName;
+    private String proxyHost="http.proxyHost";
+    private String proxyPort="http.proxyPort";
+    private String proxyHostContent = "";
+    private String proxyPortContent = "";
+    private String collection_creator = "";
+    
+    
+    public void doGet(HttpServletRequest request,HttpServletResponse response)throws ServletException, IOException {
+	
+	port_number = request.getLocalPort();
+	loadRuntimeSettings();
+	PrintWriter out = response.getWriter();   
+	
+	response.setContentType("text/html");
+	response.setHeader("pragma", "no-cache");
+	
+	out.println("<html>");
+	out.println(headerString);
+	out.println("<body>");
+	out.println("<img src="+logoURL+">");
+	out.println(h1String);
+	out.println("<p>"+description+"</p>");
+	out.println("<div><form method=\"post\" action=\"http://"+hostName+":"+port_number+"/greenstone3/mat\">");
+	out.println("<p>OAI URL: <input type=\"text\" name=\"oaiurl\" size=\"140\"></p>"); 
+	out.println("<p><input type=\"submit\" value=\"Analyse repository\"></p>");
+	out.println("</form>");
+	
+	out.println ("</body>");
+	out.println ("</html>");
+	out.close();
+    }
+
+    protected void doPost(HttpServletRequest req, HttpServletResponse res)throws ServletException, IOException {
+
+    port_number = req.getLocalPort();
+	loadRuntimeSettings();
+	
+   	res.setContentType("text/html");
+	res.setHeader("pragma", "no-cache");
+	PrintWriter out = res.getWriter();
+
+	out.println("<html>");
+	
+	if (req.getParameter("metadataPrefix") != null) {
+	    
+	    oaiPrefix = req.getParameter("metadataPrefix");
+	    String maxRecords = req.getParameter("maxrecords");
+	    Pattern pa = Pattern.compile("[0-9]{1,5}");
+	    Matcher ma = pa.matcher(maxRecords);
+		
+	    out.println(headerString2);
+	    out.println("<body onLoad=\"autoSubmit()\">");
+	    out.println(h1String);
+	    out.println("<p>Please wait .... <br> It's downloading OAI records </p>");
+	    out.println("<input type=\"button\" onClick=\"showdiv()\" value=\"show debug infomation\">");	
+	    
+	    if(ma.matches()){    
+		int num = Integer.parseInt(maxRecords);
+		if(num>0){
+		    downloadCollection(out, req, res, oaiPrefix, maxRecords);
+		}
+		else{
+		    downloadCollection(out, req, res, oaiPrefix, maxRecord);
+		}
+	    }
+	    else{
+		downloadCollection(out, req, res, oaiPrefix, maxRecord);
+	    }
+	    
+	}
+	else if(req.getParameter("matShell") != null){
+	    out.println(headerString2);
+	    out.println("<body onLoad=\"autoSubmit2()\">");
+	    out.println(h1String);
+	    out.println("<p>Please wait ...</p><p>It's building collection now.</p>");
+	    out.println("<input type=\"button\" onClick=\"showdiv()\" value=\"show debug infomation\">");	   
+	    buildCollection(out,req.getParameter("matShell"),req.getParameter("collectionName"),req.getParameter("collectionURL"),req.getParameter("oaiPrefix"));
+	}
+	
+	else if (req.getParameter("collName") != null){
+	    out.println(headerString);
+	    //out.write("<script type=\"text/javascript\" src=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/getInfomation.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/yahoo/yahoo-min.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/event/event-min.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/connection/connection-min.js\"></script>\r\n");
+	    out.println("<body>");
+	    out.println(h1String);  
+	    analyzeCollection(out, req.getParameter("collName"),req.getParameter("collURL"),req.getParameter("collHost"),req.getParameter("oaiPrefix"));
+	}
+	
+	else {
+	    out.println(headerString);
+	    out.println("<body>");
+	    out.println(h1String);
+	    
+	    java.net.URL oaiURL;
+	    String oaiURLString = req.getParameter("oaiurl");
+	    
+	    if (! (oaiURLString.startsWith("http://"))) {
+	    	oaiURLString = "http://" + oaiURLString; // add on protocol if missing 
+	    }
+	    
+	    // URL checks
+	    // check if Java can make a URL from the string
+	    
+	    try {
+	    	oaiURL = new URL(oaiURLString);
+	    }
+	    catch (MalformedURLException e) {
+	    	out.println("<p>Malformed URL Exception caught: " + e.getMessage() + "</p>");
+	    	out.println("<p>The system cannot recognise the URL you have entered.</p>");
+		return;  // go no further
+	    }
+	    
+	    //out.println("<p>host: " + oaiURL.getHost() + "</p>" );
+	    //out.println("<p>protocol: " + oaiURL.getProtocol() + "</p>" );
+	    
+	    // disallow anything with waikato as part of the host
+	    // note: this blocks things like waikato.uiuc.edu - 
+	    // but guess this is highly unlikely to occur
+	    // (also turns off the researchcommons)
+	    // could use either of these approaches, waikato or waikato.ac.nz
+	    
+	    if  (oaiURL.getHost().indexOf("researchcommons.waikato.ac.nz") == -1 ) { // not the RC
+		/*
+		  if((oaiURL.getHost().indexOf("waikato") != -1 ) || 
+		  (oaiURL.getHost().indexOf("waikato.ac.nz") != -1 ) ) {
+		  out.println("<p>This service cannot be used to access Waikato URLs</p>");
+		  return;
+		  }*/
+	    }
+	    
+	    // need to prevent machine names on their own, e.g. smith 
+	    // being allowed through
+	    
+	    // approach 1: explicit blacklisting
+	    
+	    if (oaiURL.getHost().equals("smith") ||
+		oaiURL.getHost().equals("wesson") ) {
+	    	out.println("<p>This service cannot be used to access these URLs</p>");
+		return;
+	    }
+	    
+	    // approach 2: require at least one . in the host URL
+	    if (oaiURL.getHost().indexOf(".") == -1 ) { // i.e. no . in URL host
+	    	out.println("<p>This service cannot be used to access URLs of this form.</p>");  
+		return;
+	    }
+	    
+	    // jones.cs would get through to this point, might this mean something 
+	    // from the perspective of the host machine?? Does it matter?
+	    // will automatic domain completion be applied? by java? by 'the network'?
+	    // require 2 dots in the host? does that help at all ?
+	    // domain suffix whitelisting is impractical here, as we'd have to 
+	    // list all countries in the world
+	    
+	    // do an OAI verb=identify check to make sure it is an OAI server
+	    // trim the URL back and then add on verb=Identify
+	    
+	    
+	    //System.setProperty("http.proxyHost", "wwwcache.cs.waikato.ac.nz");
+	    //System.setProperty("http.proxyPort","80");
+	    //System.setProperty("http.nonProxyHosts", "localhost|*.waikato.ac.nz");
+	    
+	    Properties systemSettings = System.getProperties();
+	    
+	    if( proxyHostContent.equals("") || proxyPortContent.equals("")){}
+	    else{
+	    	systemSettings.put(proxyHost, proxyHostContent);
+	    	systemSettings.put(proxyPort, proxyPortContent);
+	    }
+	    
+	    URL url = oaiURL;
+	    String identifyVerb = "";
+	    
+	    if (! url.getPath().endsWith("?")) {
+	    	identifyVerb += "?";
+	    }
+	    
+	    identifyVerb += "verb=Identify";
+	    
+	    URL identifyURL = new URL (url.toString() + identifyVerb);
+	    HttpURLConnection connection = (HttpURLConnection)identifyURL.openConnection();
+	    //URLConnection connection = identifyURL.openConnection();          
+	    connection.connect(); 
+	    
+	    //BufferedReader inIdentify = new BufferedReader(new InputStreamReader(
+	    //connection.getInputStream()));
+	        
+	    Document identifyDocument;
+	    
+	    try {
+	    	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	    	DocumentBuilder builder = factory.newDocumentBuilder();
+	    	identifyDocument = builder.parse( connection.getInputStream() );
+		
+	    	Element oaiElement = identifyDocument.getDocumentElement();
+		
+		if (oaiElement.getTagName() == "OAI-PMH" ) {
+		    //out.println("<p>OAI-PMH element found...</p>");
+		}
+		else {
+		    out.println("<p>Error: OAI-PMH element not found..exiting</p>");
+		    return;
+		}
+		
+		NodeList identifyNodeList = identifyDocument.getElementsByTagName("Identify");
+		Node identifyNode = null;
+	
+		if (identifyNodeList.getLength() == 1 ) {
+		    identifyNode = identifyNodeList.item(0);
+		    System.out.println("<Identify>\n");
+		}
+		else {
+		    out.println("<p>Error: Identify node not found... exiting</p>");
+		    return;
+		}
+		
+		NodeList identifyChildList = identifyNode.getChildNodes();
+		out.println("<table rules =\"none\">");
+		
+		for (int i=0; i < identifyChildList.getLength(); i++) {
+		    if (identifyChildList.item(i).getNodeName() == "repositoryName" ) {
+			out.println("<tr><td><strong>Repository Name:</strong></td><td> " + identifyChildList.item(i).getNodeValue() + "</td></tr>");
+		    }
+		    if (identifyChildList.item(i).getNodeName() == "baseURL" ) {
+		    	
+		    	Node n = identifyChildList.item(i);
+		    	String baseURL = getTextContent(n);
+
+
+		    	
+			if(baseURL.startsWith("http://")){
+
+			    out.println("<tr><td><strong>Base URL:</strong></td><td><code><a href=\"" + baseURL + "\">" + baseURL + "</a></code></td></tr>");
+		    	}
+		    	else{
+
+			    out.println("<tr><td><strong>Base URL:</strong></td><td><code><a href=\"" + oaiURLString + "\">" + oaiURLString + "</a></code></td></tr>");
+			}
+		    }
+		}
+		out.println("</table>");
+	    }
+	    catch (SAXParseException spe) {
+		out.println(spe.getMessage());
+	    }
+	    catch (SAXException sxe) {
+		out.println(sxe.toString());
+	    }
+	    catch (ParserConfigurationException pce) {
+		out.println(pce.toString());
+	    }
+	    catch (IOException ioe) {
+		out.println(ioe.toString());
+	    }
+	    
+	    // now get the metadata prefixes
+	    
+	    String metadataFormats = "";
+	    
+	    if (! url.getPath().endsWith("?")) {
+		metadataFormats  += "?";
+	    }
+	    metadataFormats += "verb=ListMetadataFormats";
+	    
+	    URL metadataFormatsURL = new URL (url.toString() + metadataFormats);
+	    URLConnection connection2 = metadataFormatsURL.openConnection();   
+	    connection.connect(); 
+	    Document document2;
+	    
+	    try {
+		DocumentBuilderFactory factory =  DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		document2 = builder.parse( connection2.getInputStream() ); 
+		
+		NodeList prefixList = document2.getElementsByTagName("metadataPrefix");
+			
+		// create UI based on response
+
+		if ( prefixList.getLength() > 0 ) {
+		    
+		    out.println("<p>Choose one metadata prefix to use:</p>");
+		    out.println("<form method=\"post\" action=\"http://"+hostName+":"+port_number+"/greenstone3/mat\">");
+		    out.println("<table rules =\"none\">");
+		    
+		    for (int i=0; i < prefixList.getLength(); i++) {
+			out.println("<tr><td>");
+			String id = "radioID" + i;
+			
+	    	Node n = prefixList.item(i);
+	    	String prefix = getTextContent(n);
+	    	
+			//String prefix = prefixList.item(i).getNodeValue();
+			
+			if (prefix.equals("oai_dc")) {
+			    out.println("<label for=\"" + id + "\">" + prefix + " (Dublin Core)</label>"); 
+			    //System.out.println("<MetadataElement>Dublin Core<\\MetadataElement>\n");
+			}
+			else {
+			    out.println("<label for=\"" + id + "\">" + prefix + "</label>"); 
+			    //System.out.println("<MetadataElement>"+prefix+"<\\MetadataElement>\n");
+			}
+			
+			out.print("</td><td><input type=\"radio\" name=\"metadataPrefix\" value=\"" + prefix + "\" ");
+			
+			if (prefix.equals("oai_dc")) {
+			    out.println("checked=\"checked\" ");
+			}
+			out.println(" id=\"" + id + "\"><br>");
+			out.println("</td></tr>");
+		    }
+		    
+		    //System.out.println("</Identify>");
+		    out.println("</table>");
+		    
+		    // need to pass oaiurl through as well (again)
+		    
+		    out.println("<input type=\"hidden\" name=\"oaiURL\" value=\"" + url.toString() + "\"><br>");
+		    out.println("Max records: <input type=\"text\" name=\"maxrecords\" value=\"500\"><br>");
+		    
+		    // submit button
+		    
+		    out.println("<p><input type=\"submit\" value=\"Continue\"></p>");
+		    out.println("</form>");
+		    
+		    out.println("<p> Warning: Generating the statistics and visualization will take some time: </p> ");
+		    
+		    /*
+		    out.println("<table border = \"1\">");
+		    out.println("<tr align=\"right\"><td> No.of Records <td> Estimated Time");
+		    out.println("<tr align=\"right\"><td>100<td> 5 minutes");
+		    out.println("<tr align=\"right\"><td>500 <td> 10 minutes");
+		    out.println("<tr align=\"right\"><td>1000 <td> 18 minutes");
+		    out.println("<tr align=\"right\"><td>2000<td> 30 minutes");
+		    out.println("</table>");
+		    */
+		    
+		    out.println("<p> This tool is designed to work with Dublin Core metadata: note that the mapping of qualified Dublin Core to simple Dublin Core (as in <code>oai_dc</code>) may affect the results.");
+		}
+		else {
+		    out.println("<p>Error: no metadata prefixes found... exiting</p>");
+		    return;               
+		}
+	    }
+	    catch (SAXParseException spe) {
+		out.println(spe.getMessage());
+	    }
+	    catch (SAXException sxe) { 
+		out.println(sxe.toString());
+	    }
+	    catch (ParserConfigurationException pce) {
+		out.println(pce.toString());
+	    }
+	    catch (IOException ioe) {           
+		out.println(ioe.toString());
+	    }    
+	    
+	    // check we get a valid XML document back
+	    // check OAI-PMH element
+	    // check we have a <repositoryName> and <baseURL> elements
+	    // maybe check the baseURL against the query we issued?
+	    
+	    
+	    // if we get here we have a valid non-waikato non-local OAI server
+	    // /* process stuff here */
+	    // test with University home page - as it doesn't require
+	    // dealing with the proxy server
+	    // URL url = new URL("http://waikato.ac.nz");
+	    // URL url = new URL(oaiURL);
+	    // need to go through the proxy here
+	    // http://dn.codegear.com/article/29783
+	    // http://java.ittoolbox.com/groups/technical-functional/java-l/response-to-proxy-authentication-exception-71438
+	    // http://www.jguru.com/faq/view.jsp?EID=13186
+	    // http://www.developer.com/java/other/article.php/1551421
+	    // System.setProperty("http.proxyHost","http://proxy.scms.waikato.ac.nz");
+	    // System.setProperty("http.proxyPort","80");
+	    // System.setProperty("http.nonProxyHosts", "localhost|*.waikato.ac.nz");
+	    // build a GS 3 collection from the OAI URL 
+	    // http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html
+	    // http://codon.kribb.re.kr/wiki/display/Java/Using+Runtime.exec+to+invoke+child+process
+	    
+	    out.print("</body></html>");
+	    out.close();   
+	} // endif
+    } // end doPost
+    
+    
+    private static String getTextContent(Node node) {
+        Node child;
+        String sContent = node.getNodeValue() != null ? node.getNodeValue() : "";
+
+        NodeList nodes = node.getChildNodes();
+        for(int i = 0; i < nodes.getLength(); i++) {
+            child = nodes.item(i);
+            sContent += child.getNodeValue() != null ? child.getNodeValue() : "";
+            if(nodes.item(i).getChildNodes().getLength() > 0) {
+                sContent += getTextContent(nodes.item(i));
+            }
+        }
+
+        return sContent;
+    
+    }
+    protected void buildCollection( PrintWriter out, String matShell2, String collName, String oaiURLString, String oaiPrefix)  throws ServletException, IOException{
+	
+	out.println("<br><br><div id=\"hideshow\" style=\"visibility:hidden\">");
+	out.println("<input type=\"button\" onClick=\"hidediv()\" value=\"hide debug infomation\"/>");
+	out.println("<p>Building collection...</p>");
+	
+	String host = "http://"+hostName+":"+port_number+"/greenstone3/mat/";
+	
+	out.println("<form method=\"post\" name=\"aForm\">");
+	out.println("<input type=\"hidden\" name=\"collName\" value=\"" + collName + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"collURL\" value=\"" + oaiURLString + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"collHost\" value=\"" + host + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"oaiPrefix\" value=\"" + oaiPrefix + "\"><br>");
+	out.println("</form>");
+	out.println("<p><pre>" + matShell2 + "</pre></p>");
+	out.flush();
+	
+	
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+	    gsdl3Home = gsdl3Home.replaceAll("\\\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replaceAll("/","\\\\");
+	    gsdl3Home = gsdl3Home.replaceAll("/", "\\\\");
+	}
+
+	// downloading
+	//String gs3Root = "/research/cc108/greenstone3Project";
+	String gs3Root = gsdl3Root;
+	String collectDir = gsdl3Home + fileSeparator +"sites"+fileSeparator+"localsite"+ fileSeparator +"collect";
+	String logFile = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + "log.txt";
+	String cacheDir = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + collName;
+		
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+		File wd = new File(gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script");
+
+		String[] arrays = new String[12];
+		arrays[0] = "cmd";
+		arrays[1] = "/c";
+		arrays[2] = "start";	
+		arrays[3] = "mat-colbuild.bat";
+		arrays[4] = collName;
+		arrays[5] = oaiURLString;
+		arrays[6] = cacheDir;
+		arrays[7] = "100";
+		arrays[8] = oaiPrefix;
+		arrays[9] = gs3Root;
+		arrays[10] = collectDir;
+		arrays[11] = logFile;
+		
+		Process p2 = processBatch(arrays,out,wd);
+		out.print("</div></body></html>");
+		p2 = null;
+		p2.destroy();
+	}
+	else{
+	Process p2 = processShell(matShell2, out);
+	
+	if (p2.exitValue() == 0){
+		out.println("<p>Collection built.</p>");
+	}
+	else{
+	    out.println("<p><b>Collection not built.</b></p>");
+	}
+	
+	out.print("</div></body></html>");
+	p2 = null;
+	p2.destroy();
+	}
+	out.close();
+    }
+    
+    private void downloadCollection( PrintWriter out, HttpServletRequest req, HttpServletResponse res, String oaiPrefix, String Records)  throws ServletException, IOException
+    {
+	String oaiURLString = req.getParameter("oaiURL");
+	String metadataprefix = req.getParameter("metadataPrefix");
+	String collName = generateCollName();
+	String host = "http://"+hostName+":"+port_number+"/";
+
+	out.println("<br><br><div id=\"hideshow\" style=\"visibility:hidden\">");
+	out.println("<input type=\"button\" onClick=\"hidediv()\" value=\"hide debug infomation\"/>");
+	out.println("<p>Downloading OAI documents...</p>");
+	out.flush();
+	// move to correct directory
+
+
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+	    gsdl3Home = gsdl3Home.replaceAll("\\\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replaceAll("/","\\\\");
+	    gsdl3Home = gsdl3Home.replaceAll("/", "\\\\");
+	}
+	String maxRecords = Records;
+	String cacheDir = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + collName;
+
+	// downloading
+	//String gs3Root = "/research/cc108/greenstone3Project";
+	String gs3Root = gsdl3Root;
+	String collectDir = gsdl3Home + fileSeparator +"sites"+fileSeparator+"localsite"+ fileSeparator +"collect";
+	String logFile = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + "log.txt";
+	//source /research/cc108/greenstone3Project/gs2build/bin/script/mat-colbuild-download.bash
+	//source /research/cc108/greenstone3Project/ext/mat/bin/script/mat-colbuild-download.bash
+	
+	String matShell  = "";
+	
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+		
+		File wd = new File(gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script");
+		ArrayList alist = new ArrayList();
+		alist.add("cmd");
+		alist.add("/c");
+		alist.add("start");
+		alist.add("mat-colbuild-download.bat");
+		alist.add(collName);
+		alist.add(oaiURLString);
+		alist.add(cacheDir);
+		alist.add(maxRecords);
+		alist.add(metadataprefix);
+		alist.add(gs3Root);
+		alist.add(collectDir);
+		alist.add(logFile);
+		alist.add(collection_creator);
+		
+		String[] arrays = new String[alist.size()];
+		for(int i = 0; i< arrays.length; i++){
+			arrays[i] = (String)alist.get(i);
+			System.out.println(arrays[i]);
+		}
+		System.out.println(wd.getAbsolutePath());
+		Process p = processBatch(arrays,out,wd);
+		out.flush();
+		
+		
+		p.destroy();
+		
+	}
+	else{
+	matShell = "source "+ gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script" + fileSeparator + "mat-colbuild-download.bash "
+	    + collName
+	    + " "
+	    + oaiURLString 
+	    + " "
+	    + cacheDir
+	    + " "
+	    + maxRecords
+	    + " "
+	    + metadataprefix
+	    + " " 
+	    + gs3Root
+	    + " "
+	    + collectDir
+	    + " " 
+	    + logFile
+	    + " "
+	    + collection_creator;
+	
+	out.println("<p><pre>" + matShell + "</pre></p>");
+	out.flush();
+	Process p = processShell(matShell, out);
+	
+	if (p.exitValue() == 0){
+	    out.println("<p>Collection downloaded.</p>");
+	}
+	else {
+	    out.println("<p><b>Collection not downloaded properly.</b></p>");
+	}
+	out.flush();
+	//p = null;
+	p.destroy();
+	
+	}
+
+	String matShell2 = "";
+
+		matShell2 = "source "+ gsdl3Root + fileSeparator + "ext" + fileSeparator +"mat"+ fileSeparator +"bin" + fileSeparator + "script" + fileSeparator + "mat-colbuild.bash "
+	    + collName
+	    + " "
+	    + oaiURLString 
+	    + " "
+	    + cacheDir
+	    + " "
+	    + maxRecords
+	    + " "
+	    + metadataprefix
+	    + " " 
+	    + gs3Root
+	    + " "
+	    + collectDir
+	    + " " 
+	    + logFile;
+
+		
+	out.println("<form method=\"post\" name=\"collectionForm\">");
+	out.println("<input type=\"hidden\" name=\"matShell\" value=\"" + matShell2 + "\">");
+	out.println("<input type=\"hidden\" name=\"collectionName\" value=\"" + collName + "\">");
+	out.println("<input type=\"hidden\" name=\"collectionURL\" value=\"" + oaiURLString + "\">");
+	out.println("<input type=\"hidden\" name=\"oaiPrefix\" value=\"" + oaiPrefix + "\">");	
+	out.println("<input type=\"hidden\" name=\"host\" value=\"" + host + "\">");
+	//out.println("<input type=\"submit\" value=\"start\"><br>");
+	out.println("</form>");
+	out.println("</div>");
+	out.print("</body></html>");
+	out.flush();
+	out.close();
+
+}
+
+    /* produce a random 7 letter collection name */
+    
+    private String generateCollName () {
+	Random random  = new Random();
+	StringBuffer message = new StringBuffer();
+	int offset = 97; // = "a"
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	return message.toString();
+    }
+	
+    private Process processBatch(String[] command, PrintWriter out, File wd){
+    	
+    	Process proc;
+    	String s;
+    	try{
+    		proc = Runtime.getRuntime().exec(command,null,wd);
+    	    BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+    	    BufferedReader stdError = new BufferedReader(new InputStreamReader(proc.getErrorStream()));	
+    	    StringBuffer stdInputBuffer = new StringBuffer();
+    	    while ((s = stdInput.readLine()) != null) {
+        		stdInputBuffer.append(s+"\n");
+        		if(s.indexOf("-->")!=-1){
+    		    s = s.replaceAll("-->", "");
+    		    out.println("<!-- " + s + " -->");
+        		}
+        		else{
+    		    out.println("<!-- " + s + " -->");
+        		}
+    		out.flush();
+    	    }
+    	    
+    	    StringBuffer stdErrorBuffer = new StringBuffer();
+    	    
+            while ((s = stdError.readLine()) != null) {
+	    stdErrorBuffer.append(s+"\n");
+		
+	    if(s.indexOf("-->")!=-1){
+		s = s.replaceAll("-->", "");
+		out.println("<!-- " + s + " -->");
+	    }
+	    else{
+		out.println("<!-- " + s + " -->");
+	    }
+	    out.flush();
+            }
+    	    
+            
+    		out.println("<p>Here is the standard output:</p>\n");
+    		out.println("<p><pre>" + stdInputBuffer + "</pre></p>");   
+    		out.println("<p>Here is the standard error (if any):</p>\n");
+    		out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+    		out.flush();
+    		//if (p.exitValue() != 0)
+    		if (false){
+    		    out.println("<p>An error occurred while building the collection.</p>");
+    		    out.println("<p>Here is the standard output:</p>\n");
+    		    out.println("<p><pre>" + stdInputBuffer + "</pre></p>");
+    		    out.println("<p>Here is the standard error (if any):</p>\n");
+    		    out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+    		}
+    		
+    		InputStream is = proc.getInputStream();
+    		is.close();
+    		
+    		OutputStream os = proc.getOutputStream();
+    		os.close();
+    		
+    		InputStream es = proc.getErrorStream();
+    		es.close();
+    		
+    		stdInput.close();
+    		stdError.close();
+    		proc.waitFor();
+            return proc;
+            
+    	}catch(Exception ex){
+    		ex.printStackTrace();
+    	}
+    	return null;
+    }
+    private Process processShell( String command, PrintWriter out) { 
+    	
+    	String s= "";
+    	try {
+    	    String[] args = new String[]{"sh", "-c", command};
+	    Process p = Runtime.getRuntime().exec(args);
+	    BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
+	    BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));	    
+	    StringBuffer stdInputBuffer = new StringBuffer();
+    	    
+	    while ((s = stdInput.readLine()) != null) {
+    		stdInputBuffer.append(s+"\n");
+    		if(s.indexOf("-->")!=-1){
+		    s = s.replaceAll("-->", "");
+		    out.println("<!-- " + s + " -->");
+    		}
+    		else{
+		    out.println("<!-- " + s + " -->");
+    		}
+		out.flush();
+	    }
+
+    	    StringBuffer stdErrorBuffer = new StringBuffer();
+    	    
+                while ((s = stdError.readLine()) != null) {
+		    stdErrorBuffer.append(s+"\n");
+    		
+		    if(s.indexOf("-->")!=-1){
+			s = s.replaceAll("-->", "");
+			out.println("<!-- " + s + " -->");
+		    }
+		    else{
+			out.println("<!-- " + s + " -->");
+		    }
+		    out.flush();
+                }
+		
+		out.println("<p>Here is the standard output:</p>\n");
+		out.println("<p><pre>" + stdInputBuffer + "</pre></p>");   
+		out.println("<p>Here is the standard error (if any):</p>\n");
+		out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+		out.flush();
+		//if (p.exitValue() != 0)
+		if (false){
+		    out.println("<p>An error occurred while building the collection.</p>");
+		    out.println("<p>Here is the standard output:</p>\n");
+		    out.println("<p><pre>" + stdInputBuffer + "</pre></p>");
+		    out.println("<p>Here is the standard error (if any):</p>\n");
+		    out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+		}
+		
+		InputStream is = p.getInputStream();
+		is.close();
+		
+		OutputStream os = p.getOutputStream();
+		os.close();
+		
+		InputStream es = p.getErrorStream();
+		es.close();
+		
+		stdInput.close();
+		stdError.close();
+		
+		return p;
+	}
+	catch (IOException e) {
+	    out.println("exception happened - here's what I know: ");
+    	    out.println(e.toString());        
+    	    out.flush();
+	}
+	return null;
+    }
+
+    private void analyzeCollection(PrintWriter out, String collectionName,String collectionURL,String collectionHost, String Prefix){
+	
+	String collName = collectionName;
+	String oaiURLString = collectionURL;
+	String host = collectionHost;
+	
+	try{
+	    DescribeMessenger dm = new DescribeMessenger(collName,oaiURLString);
+	    out.println("<p>Generating statistics and visualisations...</p>");
+	    out.flush();
+	    out.println("<p>please wait.</p>");
+	    out.flush();
+	    boolean status = dm.describeMatadata(out,collName,oaiURLString,Prefix,port_number);
+	    
+	    if(status){
+		    
+	    	out.println("<script type=\"text/javascript\" language=\"JavaScript\">");
+	    	out.println("reconfig();");
+	    	out.println("</script>");
+	    	
+	    	out.println("<a href=\""+host+collName+"/Overall.html\">View the report</a>");
+	    	out.flush();
+	    }
+	    out.println("</body></html>");
+	    out.flush();
+	}catch(Exception e){e.printStackTrace(out);
+	    out.println("<p><pre>"+e.toString()+"</pre></p>");
+	}
+	out.close();
+    }
+    
+    private void loadRuntimeSettings(){
+    
+    	
+	fileSeparator = File.separator;	
+	gsdl3Home = globalProperty.getGSDL3Home();
+	globalProperty = new GlobalProperties();
+	maxRecord ="10";
+	
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().indexOf("windows")!=-1){
+		gsdl3Home = gsdl3Home.replaceAll("\\\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replaceAll("/","\\\\");
+	    gsdl3Home = gsdl3Home.replaceAll("/", "\\\\");
+	}
+    	
+	try{
+	    Properties prop = new Properties();
+	    FileInputStream fis = new FileInputStream(gsdl3Root+fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"properties.xml");
+	    prop.load(fis);
+	    
+	    titleString ="<title>"+ prop.getProperty("Servlet.Title")+"</title>";
+	    h1String ="<h2>"+ prop.getProperty("Servlet.Head")+"</h2>";
+	    cssString = "<link rel=\"stylesheet\" href=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/mat.css\" type=\"text/css\" >";
+	    javaScript = "<script type=\"text/javascript\" src=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/status3.js\"></script>";;
+	    headerString2 = "<head>" + titleString + "\n" +  javaScript + cssString+"</head>\n";
+	    headerString ="<head>" + titleString + "\n" + cssString + "</head>\n";
+	    logoURL =prop.getProperty("Servlet.Logo");
+	    description = prop.getProperty("Servlet.Description");
+	    
+	    proxyHostContent = prop.getProperty("Servlet.proxyHost");
+	    proxyPortContent = prop.getProperty("Servlet.proxyPort");
+	    
+	    collection_creator = prop.getProperty("Servlet.Collection.Creator");
+	    
+	}catch(Exception ex){
+	    ex.printStackTrace();
+	}
+	
+	
+	try {
+		java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost();
+		hostName = localMachine.getHostName();
+	}catch (java.net.UnknownHostException uhe) { 
+			uhe.printStackTrace();
+	}
+		
+    }
+}
