Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DataMaker.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DataMaker.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DataMaker.java	(revision 18104)
@@ -0,0 +1,982 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.math.BigDecimal;
+
+import org.w3c.dom.Node;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.core.MessageRouter;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+public class DataMaker {
+    
+    MetadataStats  ms;
+    ArrayList nameList;
+    PrintWriter out;
+
+    private int Mode;
+    private int TotalDoc; 
+    private String path;
+    
+    protected Document doc=null;
+    protected MessageRouter mr = null;
+    protected XMLConverter converter=null;
+    
+    private ArrayList removedID = new ArrayList();
+    private static final int DEF_DIV_SCALE = 10;
+    
+    public DataMaker(MetadataStats arg){
+    	
+	ms = arg;
+    	nameList = arg.metadataNameList;
+    	path = arg.StatsDirectory;
+    	setTotalDocNumber();
+    }
+    
+    private Element getRootNode(String core_element){
+	
+	try{
+	    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+	    Document doc = docBuilder.newDocument();
+	    doc = docBuilder.parse (new File(path+ms.fileSeparator+core_element+".xml"));
+	    Element rootNode = doc.getDocumentElement();
+	    
+	    return rootNode;  
+	}catch (Exception e) {
+    	    e.printStackTrace();
+    	    return null;
+	}
+    }
+    
+    public void setTotalDocNumber(){
+    	
+	Element ex = getRootNode(ms.RootDirectory);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	TotalDoc = listOfFrequency.getLength();
+    }
+    
+    public int getTotalElementUsed(){
+	
+	int totalNumber = 0;
+	
+	for(int i=0 ; i<nameList.size(); i++){
+	    
+	    Element ex = getRootNode((String)nameList.get(i));
+	    NodeList listOfFrequency = ex.getElementsByTagName(ms.FrequencyNode);
+	
+	    for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    	Node FrequencyNode = listOfFrequency.item(s);
+	    	NodeList textFNList = FrequencyNode.getChildNodes();
+	    	String TextNode = textFNList.item(0).getNodeValue();
+	    	totalNumber = totalNumber + Integer.parseInt(TextNode);
+	    }
+	}
+	
+	return totalNumber;
+    }
+      
+    public int getFrequency(String name){
+	
+        int totalNumber = 0;
+      	Element ex = getRootNode(name);
+      	NodeList listOfFrequency = ex.getElementsByTagName(ms.FrequencyNode);
+      	
+	if(listOfFrequency.getLength()==0){
+	    return 0;
+	}
+	
+      	for(int s=0; s<listOfFrequency.getLength(); s++){
+	    Node FrequencyNode = listOfFrequency.item(s);
+	    NodeList textFNList = FrequencyNode.getChildNodes();
+	    String TextNode = textFNList.item(0).getNodeValue();
+	    int x = Integer.parseInt(TextNode);
+	    totalNumber = totalNumber + x;
+        }
+	
+	return totalNumber;
+    }
+    
+    public int getDistinctNumber(String name){
+	
+        ArrayList alist = new ArrayList();
+      	Element ex = getRootNode(name);
+      	NodeList listOfFrequency = ex.getElementsByTagName(ms.ActualValueNode);
+      	if(listOfFrequency.getLength()==0){return 0;}
+        for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node ActualValueNode = listOfFrequency.item(s);
+	    NodeList textFNList = ActualValueNode.getChildNodes();
+	    String TextNode = textFNList.item(0).getNodeValue();
+	    if(!alist.contains(TextNode) && !TextNode.equals(" ")){alist.add(TextNode);}
+        }
+	return alist.size();
+    }
+    
+    public int getDocumentUsedElement(String core_element){
+	
+        int totalNumber = 0;
+      	Element ex = getRootNode(core_element);
+      	NodeList listOfFrequency = ex.getElementsByTagName(ms.FrequencyNode);
+	
+      	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    totalNumber++;
+        }
+	return totalNumber;
+    }
+       
+    public double Mean(String core_element){
+    	
+	int due = getDocumentUsedElement(core_element);
+    	
+	if(due==0 || TotalDoc==0){
+	    return 0;
+	}
+	
+    	Double d1 = new Double(due);
+    	Double d2 = new Double(TotalDoc);
+    	Double result = div(d1,d2);
+    	result = mul(result, new Double(100));
+    	
+	return round(result.doubleValue(),1);
+    }
+    
+    public int getMinRange(String core_element){
+	
+      	Element ex = getRootNode(core_element);
+      	NodeList listOfFrequency = ex.getElementsByTagName(ms.FrequencyNode);
+      	
+        if(listOfFrequency.getLength()==0){
+	    return 0;
+	}
+        
+    	Node FrequencyNode = listOfFrequency.item(0);
+    	NodeList textFNList = FrequencyNode.getChildNodes();
+    	String TextNode = textFNList.item(0).getNodeValue();
+        int minNumber = 0;
+        
+        if(listOfFrequency.getLength()==TotalDoc){
+	    minNumber = Integer.parseInt(TextNode);
+        }
+	else {
+	    minNumber = 0;
+        }
+        
+        for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    FrequencyNode = listOfFrequency.item(s);
+	    textFNList = FrequencyNode.getChildNodes();
+	    TextNode = textFNList.item(0).getNodeValue();
+	    int x = Integer.parseInt(TextNode);
+	    
+	    if(x<minNumber){
+		minNumber=x;
+	    }
+        }
+	
+	return minNumber;
+    }
+    
+    public int getMaxRange(String core_element){
+	
+      	Element ex = getRootNode(core_element);
+      	NodeList listOfFrequency = ex.getElementsByTagName(ms.FrequencyNode);
+       
+	if(listOfFrequency.getLength()==0){
+	    return 0;
+	}
+	
+    	Node FrequencyNode = listOfFrequency.item(0);
+    	NodeList textFNList = FrequencyNode.getChildNodes();
+    	String TextNode = textFNList.item(0).getNodeValue();
+        int maxNumber = 0;
+        
+        for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    FrequencyNode = listOfFrequency.item(s);
+	    textFNList = FrequencyNode.getChildNodes();
+	    TextNode = textFNList.item(0).getNodeValue();
+	    int x = Integer.parseInt(TextNode);
+	    
+	    if(x>maxNumber){
+		maxNumber=x;
+	    }
+        }
+	
+	return maxNumber;
+    }
+    
+    public int getMode(String core_element){
+	
+    	Element ex = getRootNode(core_element);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	
+	if(listOfFrequency.getLength()==0){
+	    Mode = 0; 
+	    return 0;
+	}
+	
+    	ArrayList alist = new ArrayList();
+    	String[] idsx = getDocumentIDs(core_element);
+    	
+	for(int i = 0; i<idsx.length; i++){
+	    alist.add(idsx[i]);
+    	}
+	
+    	int[] list = new int[TotalDoc];
+    	
+	for(int i = 0; i<list.length; i++){
+	    list[i] = 0;
+    	}
+	
+    	for(int s=0; s< listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    NamedNodeMap NodeIDMap = docNode.getAttributes();
+	    Node DocNodeID = NodeIDMap.item(0);
+	    String DocID = DocNodeID.getNodeValue();
+	    Element xNode = (Element)docNode;
+	    int location = alist.indexOf(DocID);
+	    NodeList xList = xNode.getElementsByTagName(ms.FrequencyNode);
+	    int fre = Integer.parseInt(xList.item(0).getChildNodes().item(0).getNodeValue());
+	    list[location] = fre;
+    	}
+    	
+        Arrays.sort(list);
+	
+	int max_idx = 0; // Index of the maximum count
+	int max_cnt = 0;
+	int count = 0;
+	
+	for ( int i = 0; i <list.length; i++) {
+	    count = 0;
+	    for ( int j = 0; j < list.length; j++) {
+		if (list[i] == list[j]) {
+		    count++;
+		}
+	    }
+	    if (count > max_cnt) {
+		max_cnt = count;
+		max_idx = i;
+	    }	
+	}
+	Mode = list [max_idx];
+	
+	return list [max_idx];
+    }
+    
+    public double ModeFrequency(String core_element){
+	
+    	Element ex = getRootNode(core_element);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	
+	if(listOfFrequency.getLength()==0){ 
+	    return 100;
+	}
+	
+    	ArrayList alist = new ArrayList();
+    	String[] idsx = getDocumentIDs(core_element);
+    	
+	for(int i = 0; i<idsx.length; i++){
+	    alist.add(idsx[i]);
+    	}
+	
+    	int[] list = new int[TotalDoc];
+    	
+	for(int i = 0; i<list.length; i++){
+	    list[i] = 0;
+    	}
+    	
+    	int length = alist.size();
+    	int counter = 0;
+    	
+    	for(int s=0; s< listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    NamedNodeMap NodeIDMap = docNode.getAttributes();
+	    Node DocNodeID = NodeIDMap.item(0);
+	    String DocID = DocNodeID.getNodeValue();
+	    Element xNode = (Element)docNode;
+	    int location = alist.indexOf(DocID);
+	    NodeList xList = xNode.getElementsByTagName(ms.FrequencyNode);
+	    int fre = Integer.parseInt(xList.item(0).getChildNodes().item(0).getNodeValue());
+	    list[location] = fre;
+    	}
+	
+    	for(int i =0; i<list.length; i++){
+	    if(list[i]==Mode){counter++;}
+    	}
+
+        Double result = div(new Double(counter), new Double(length));
+        result = mul(result ,new Double (100));
+        
+	return round(result.doubleValue(),1);
+    }
+    
+    public double Median(String core_element){
+	   	
+    	Element ex = getRootNode(core_element);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+  
+    	ArrayList alist = new ArrayList();
+    	String[] idsx = getDocumentIDs(core_element);
+
+    	for(int i = 0; i<idsx.length; i++){
+	    alist.add(idsx[i]);
+    	}
+
+    	int[] list = new int[TotalDoc];
+
+    	for(int i = 0; i<list.length; i++){
+	    list[i] = 0;
+    	}
+    	
+    	for(int s=0; s< listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    NamedNodeMap NodeIDMap = docNode.getAttributes();
+	    Node DocNodeID = NodeIDMap.item(0);
+	    String DocID = DocNodeID.getNodeValue();
+	    Element xNode = (Element)docNode;
+	    int location = alist.indexOf(DocID);
+	    NodeList xList = xNode.getElementsByTagName(ms.FrequencyNode);
+	    int fre = Integer.parseInt(xList.item(0).getChildNodes().item(0).getNodeValue());
+	    list[location] = fre;
+    	}
+    	      	
+        int length = alist.size();
+      	int middle = length/2 -1;
+      	
+        Arrays.sort(list);
+	
+	if(length % 2 == 1){
+	    middle = middle + 1;
+	    return list[middle];
+	}
+	
+	else{
+	    return  round((double)(list[middle]+list[middle+1])/2,1);
+	}
+    }
+        
+    public double Average(String core_element){
+    	
+    	int t1 = getFrequency(core_element);
+    	int t2 = getDocumentUsedElement(core_element);
+    	
+	if(t1==0 || t2==0){
+	    return 0;
+	}
+    	
+    	Double result = div(new Double(t1),new Double(t2));
+    	
+	return	round(result.doubleValue(),1);
+    }
+    
+    public Object[][] AllInformation(){
+    	
+	int rows = nameList.size();
+    	int cols = 11;
+    	int y = 0;
+
+    	Object[][] info = new Object[rows][cols];
+    	String[] list = new String[rows];
+    	
+	for(int i = 0 ; i < list.length; i++){
+	    list[i] = nameList.get(i).toString();
+    	}
+	
+    	Arrays.sort(list);
+    	
+    	for(int iu = 0; iu<list.length; iu++){ 
+    	    String xi = list[iu];
+	    info[y][0] = xi ;
+	    info[y][1] = new Integer(getFrequency(xi));
+	    info[y][2] = new Integer(getDocumentUsedElement(xi));
+	    info[y][3] = new Double(Mean(xi));
+	    info[y][4] = new Double(Median(xi));
+	    info[y][5] = new Integer(getDistinctNumber(xi));
+	    info[y][6] = new Integer(getMinRange(xi));
+	    info[y][7] = new Integer(getMaxRange(xi));
+	    info[y][8] = new Double(Average(xi));
+	    info[y][9] = new Integer(getMode(xi));
+	    info[y][10] = ModeFrequency(xi)+"%";
+	    y++;
+	}
+	
+    	return info;
+    }
+    
+    public String[] getSortList(String core_element,String sort){
+	
+    	if(sort.equals("ASCII")){
+	    HashMap hp = getDistinctValueMap (core_element);
+	    String[] temp = new String[hp.size()];
+	    int counter = 0;
+	    Set s = hp.keySet();
+	    Iterator i = s.iterator();
+
+	    while(i.hasNext()){
+		temp[counter] = (String)i.next();
+		counter++;
+	    }
+	    
+	    Arrays.sort(temp);
+	    
+	    return temp;
+    	}
+	
+    	else{
+	    Map m = getDistinctValueMap (core_element);
+	    ArrayList outputList = sortMap(m);
+	    String[] temp = new String[outputList.size()];
+	    
+	    for(int i = 0; i< outputList.size(); i++){
+    		Map.Entry entry = (Map.Entry) outputList.get(i);
+    		temp[i] = (String) entry.getKey();
+	    }
+	    
+	    return temp;
+    	}
+    }
+    
+    public HashMap getDistinctValueMap(String core_element){
+    	
+	Element ex = getRootNode(core_element);
+    	HashMap hp = new HashMap();
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	
+	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    Element docElement = (Element)docNode;
+	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+	    
+	    for(int y = 0; y<valueList.getLength(); y++){
+		Element valueElement = (Element)valueList.item(y);
+		NodeList textFNList = valueElement.getChildNodes();
+		String text = ((Node)textFNList.item(0)).getNodeValue();
+		
+		if(!text.equals(" ")){
+		    if(hp.containsKey(text)){
+			Integer i = (Integer)hp.get(text);
+			int number = i.intValue();
+			number++;
+			hp.put(text,new Integer(number));
+		    }
+		    else{
+			Integer i = new Integer(1);
+			hp.put(text, i);
+		    }
+		}
+	    }
+    	}   
+            
+    	return hp;
+    }
+    
+    
+    public HashMap getDocFrequencyMap(String core_element){
+    	
+    	Element ex = getRootNode(core_element);
+    	HashMap hp = new HashMap();
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	
+	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    
+	    Node docNode = listOfFrequency.item(s);  
+	    NamedNodeMap NodeMap = docNode.getAttributes();
+	    Node AttributeNode = NodeMap.item(0);
+	    String att_name = AttributeNode.getNodeValue();
+	    
+	    Element docElement = (Element)docNode;
+	    NodeList valueList = docElement.getElementsByTagName(ms.FrequencyNode);
+	    Element frequencyElement = (Element)valueList.item(0);
+	    NodeList textFNList = frequencyElement.getChildNodes();
+	    String text = ((Node)textFNList.item(0)).getNodeValue();
+	    Integer i = new Integer(Integer.parseInt(text));
+	    hp.put(att_name, i);
+    	}
+	
+    	return hp;
+    }
+    
+
+    public String[] getDocumentIDs(String core_element){
+	
+    	Element ex = getRootNode(ms.RootDirectory);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	String[] ids = new String[listOfFrequency.getLength()];
+    	
+	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    NamedNodeMap NodeMap = docNode.getAttributes();
+	    Node AttributeNode = NodeMap.item(0);
+	    String att_name = AttributeNode.getNodeValue();
+	    ids[s] = att_name;
+    	}
+	
+    	return (String[])ids.clone();
+    }
+    
+    public int[] getMetadataRows(String core_element){
+    	
+	Element ex = getRootNode(core_element);
+	ArrayList alist = new ArrayList();
+    	String[] idsx = getDocumentIDs(core_element);
+	
+    	for(int i = 0; i<idsx.length; i++){
+    		alist.add(idsx[i]);
+    	}
+
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	int[] row = new int[TotalDoc];
+	
+    	for(int i = 0; i<row.length; i++){
+    		row [i] = 0;
+    	}
+    	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){   
+	    Node docNode = listOfFrequency.item(s);
+	    int location = alist.indexOf(docNode.getAttributes().item(0).getNodeValue());
+	    row[location] =1;
+       	}
+	
+    	return row;
+    }
+     
+    public ArrayList sortMap(Map map) {
+	
+	ArrayList outputList = null;
+	int count = 0;
+	Set set = null;
+	Map.Entry[] entries = null;
+	//     Logic:
+	//     get a set from Map
+	//     Build a Map.Entry[] from set
+	//     Sort the list using Arrays.sort
+	//     Add the sorted Map.Entries into arrayList and return
+	
+	set = (Set) map.entrySet();
+	Iterator iterator = set.iterator();
+	entries = new Map.Entry[set.size()];
+	while(iterator.hasNext()) {
+	    entries[count++] = (Map.Entry) iterator.next();
+	}
+	
+	//     Sort the entries with your own comparator for the values:
+	Arrays.sort(entries, new Comparator() {
+		public int compareTo(Object lhs, Object rhs) {
+		    Map.Entry le = (Map.Entry)lhs;
+		    Map.Entry re = (Map.Entry)rhs;
+		    return ((Comparable)le.getValue()).compareTo((Comparable)re.getValue());
+		}
+		
+		public int compare(Object lhs, Object rhs) {
+		    Map.Entry le = (Map.Entry)lhs;
+		    Map.Entry re = (Map.Entry)rhs;
+		    return ((Comparable)le.getValue()).compareTo((Comparable)re.getValue());
+		}
+	    });
+
+	outputList = new ArrayList();
+	
+	for(int i = 0; i < entries.length; i++) {
+	    outputList.add(entries[i]);
+	}
+	return outputList;
+    }
+    
+    private Double div(Double d1, Double d2){
+    	
+	BigDecimal b1 = new BigDecimal(d1.toString());
+        BigDecimal b2 = new BigDecimal(d2.toString());
+        
+	return new Double(b1.divide(b2,DEF_DIV_SCALE,BigDecimal.ROUND_HALF_UP).doubleValue());
+    }
+    
+    private Double mul(Double d1,Double d2){
+        
+	BigDecimal b1 = new BigDecimal(d1.toString());
+        BigDecimal b2 = new BigDecimal(d2.toString());
+        
+	return new Double(b1.multiply(b2).doubleValue());
+    }
+    
+    public double round(double v,int scale){
+        
+	if(scale<0){
+            throw new IllegalArgumentException("The scale must be a positive integer or zero");
+        }
+	
+        BigDecimal b = new BigDecimal(Double.toString(v));
+        BigDecimal one = new BigDecimal("1");
+        
+	return b.divide(one,scale,BigDecimal.ROUND_HALF_UP).doubleValue();
+    }
+    
+    public double getSingleMetadataSetCompleteness(ArrayList mds_list){
+    	
+    	int totalElement = 0;
+    	int totalElementUsed = 0;
+    	
+	for(int a = 0; a<mds_list.size(); a++){
+	    MetadataSet mds = (MetadataSet)mds_list.get(a);
+	    ArrayList alist = mds.getIndexsList();
+	    int length = alist.size();
+	    totalElement = totalElement + length * ms.getDocNum();
+	    
+	    for(int i = 0; i<alist.size(); i++){
+		String name = (String)alist.get(i);
+		totalElementUsed = totalElementUsed + getDocumentUsedElement(name);
+	    }
+	}
+
+    	double x = (double)totalElementUsed/totalElement;
+    	Double d1 = new Double(x);
+    	Double d2 = new Double(100);
+    	Double result = mul(d1,d2);
+    	
+	return round(result.doubleValue(),1);
+    }
+    
+    public Object[][] getMetadataInfo(MetadataSet mds){
+    	
+    	ArrayList alist = mds.getIndexsList();
+    	int rows = alist.size();
+    	int cols = 11;
+    	int y = 0;
+    	Object[][] dataset = new Object[rows][cols];
+    	
+       	String[] list = new String[rows];
+    	
+	for(int i = 0 ; i < list.length; i++){
+	    list[i] = alist.get(i).toString();
+    	}
+	
+    	Arrays.sort(list);
+    	
+    	for(int iu = 0; iu<list.length; iu++){
+    	    String xi = list[iu];
+    	    dataset[y][0] = xi ;
+    	    dataset[y][1] = new Integer(getFrequency(xi));
+    	    dataset[y][2] = new Integer(getDocumentUsedElement(xi));
+    	    dataset[y][3] = new Double (Mean(xi));
+    	    dataset[y][4] = new Double (Median(xi));
+    	    dataset[y][5] = new Integer(getDistinctNumber(xi));
+    	    dataset[y][6] = new Integer(getMinRange(xi));
+    	    dataset[y][7] = new Integer(getMaxRange(xi));
+    	    dataset[y][8] = new Double (Average(xi));
+    	    dataset[y][9] = new Integer(getMode(xi));
+    	    dataset[y][10] = ModeFrequency(xi)+"%";
+	    y++;
+	}
+	
+    	return dataset;
+    }
+    
+    public boolean IsElementEmpty(String core_element){
+    	
+    	int[] list = getMetadataRows(core_element);
+    	boolean status = true;
+    	
+	for(int i = 0; i< list.length; i++){
+    		if(list[i]==1){status = false;}
+    	}
+	
+    	return status;
+    }
+    
+    public boolean IsElementFull(String core_element){
+    	
+    	int[] list = getMetadataRows(core_element);
+    	boolean status = true;
+    	
+	for(int i = 0; i< list.length; i++){
+    		if(list[i]==0){status = false;}
+    	}
+    	return status;
+    }
+    
+    public ArrayList removeDocument(ArrayList dataset, String[] ids, int number){
+    	
+	removedID = new ArrayList();
+    	int[] metadataLevel =(int[])dataset.get(0);
+    	int docIDslength = metadataLevel.length;
+    	int[][] valueMap = new int[dataset.size()][docIDslength];
+    	
+    	for(int i = 0; i< docIDslength; i++){
+    		
+    		boolean status = true;
+    		
+    		for(int j = 0; j<dataset.size(); j++){
+    		 int[] metadataLevelArray = (int[])dataset.get(j);
+    		 valueMap[j][i] =  metadataLevelArray[i];
+    		 if(metadataLevelArray[i]!=number){status = false;}
+
+    		}
+    		if(status == true){
+
+		    for(int j = 0; j<dataset.size(); j++){
+
+			valueMap[j][i]=-1;
+
+		    }
+		    removedID.add(ids[i]);
+    		}
+    	}
+
+    	ArrayList wholeList = new ArrayList();
+
+    	for(int i = 0; i<valueMap.length; i++){
+	    ArrayList numberList = new ArrayList();
+	    
+	    for(int j = 0; j<valueMap[i].length; j++){
+		numberList.add(new Integer(valueMap[i][j]));
+	    }
+	    wholeList.add(numberList);
+    	}
+    	
+    	for(int i =0; i< wholeList.size(); i++){
+	    ArrayList numberList = (ArrayList)wholeList.get(i);
+	    Integer value = new Integer(-1);
+	    while(numberList.contains(value)){
+		numberList.remove(value);
+	    }
+	    int[] valueList = new int [numberList.size()];
+	    
+	    for(int j = 0; j< valueList.length; j++){
+		valueList[j] = ((Integer)numberList.get(j)).intValue();
+	    }
+	    wholeList.remove(i);
+	    wholeList.add(i,valueList);
+    	}
+
+    	return wholeList;
+    }
+    
+    public ArrayList getRemovedID(){
+    	return removedID;
+    }
+    
+    
+    public HashMap getLinks(String[] args, String core_element){
+	
+	Element ex = getRootNode(core_element);
+	HashMap hp = new HashMap();
+	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	ArrayList tempList = new ArrayList();
+    	String urlFile = "dc.Identifier";
+    	
+    	for(int i = 0; i<args.length; i++){
+    		tempList.add(args[i]);		
+    	}
+    	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    Element docElement = (Element)docNode;
+	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+	    
+	    if(valueList.getLength()==1){
+		String id = docNode.getAttributes().item(0).getNodeValue();
+		Element valueElement = (Element)valueList.item(0);
+		NodeList textFNList = valueElement.getChildNodes();
+		String text = ((Node)textFNList.item(0)).getNodeValue();
+    		
+		if(tempList.contains(text)){
+		    hp.put(id, text);
+		}
+	    }	
+    	}
+    	
+    	ex = getRootNode(urlFile);
+    	listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	HashMap newHp = new HashMap();
+    	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    Element docElement = (Element)docNode;
+	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+	    
+	    if(valueList.getLength()==1){
+		String id = docNode.getAttributes().item(0).getNodeValue();
+		Element valueElement = (Element)valueList.item(0);
+		NodeList textFNList = valueElement.getChildNodes();
+		String text = ((Node)textFNList.item(0)).getNodeValue();
+		newHp.put(text, id);  			
+	    }	
+    	}
+    	
+    	HashMap tempMap = new HashMap();
+    	Collection c = hp.values();
+    	Iterator i = c.iterator();
+    	
+	while(i.hasNext()){
+	    String id = (String)i.next();
+	    if(newHp.containsKey(id)){
+		String text = (String)newHp.get(id);
+		if(text.contains("http")){
+		    tempMap.put((String)tempMap.get(id),text);
+		}
+	    }
+    	}
+    	
+    	return tempMap;
+    }
+    
+    public ArrayList getURLMap(String elementName){
+    
+    	String core_element =elementName;
+    	Element ex = getRootNode(core_element);
+    	ArrayList alist = new ArrayList();
+    	if(ex.equals(null)){
+	    return new ArrayList();
+	}
+	
+        NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+        	
+        if(listOfFrequency.getLength()==0){
+	    return new ArrayList();
+	}
+	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){
+    	    Node docNode = listOfFrequency.item(s);
+    	    Element docElement = (Element)docNode;
+    	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+    	    
+    	    for(int y = 0; y<valueList.getLength(); y++){
+    	    	Element valueElement = (Element)valueList.item(y);
+    	    	NodeList textFNList = valueElement.getChildNodes();
+    	    	String text = ((Node)textFNList.item(0)).getNodeValue();
+    		
+    	    	if(!text.equals(" ")){
+		    NamedNodeMap NodeIDMap = docNode .getAttributes();
+		    Node DocNodeID = NodeIDMap.item(0);
+		    String DocID = DocNodeID.getNodeValue();
+		    if(alist.contains(DocID)){}
+		    else{
+			alist.add(DocID);
+		    }
+    	    	}
+    	    }
+       } 
+	
+	return alist;
+}
+
+    public HashMap getIdentifierLink(String core_element){
+    	
+    	Element ex = getRootNode(core_element);
+        HashMap hp = new HashMap();
+        NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+        	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){
+    	    Node docNode = listOfFrequency.item(s);
+    	    String HashID = docNode.getAttributes().item(0).getNodeValue();
+	    Element docElement = (Element)docNode;
+    	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+    	    
+    	    for(int y = 0; y<valueList.getLength(); y++){
+    		Element valueElement = (Element)valueList.item(y);
+    		NodeList textFNList = valueElement.getChildNodes();
+    		String text = ((Node)textFNList.item(0)).getNodeValue();
+    		
+    		if(!text.equals(" ") && text.startsWith("http:")){
+    		    if(hp.containsKey(HashID)){
+    			InternalLink il = (InternalLink)hp.get(HashID);
+    			il.increaseElement(text);
+    			hp.put(HashID,il);
+    		    }
+    		    else{
+    		    InternalLink il = new InternalLink();
+    			il.setValue(HashID);
+    			il.increaseElement(text);
+    			hp.put(HashID, il);
+    		    }
+    		}
+    	    }
+	}
+	
+	return hp;
+    }
+
+    public String[] getDocumentIDList(String core_element){
+    	
+    	Element ex = getRootNode(core_element);
+    	NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+    	String[] ids = new String[listOfFrequency.getLength()];
+    	
+	for(int s=0; s<listOfFrequency.getLength() ; s++){
+	    Node docNode = listOfFrequency.item(s);
+	    NamedNodeMap NodeMap = docNode.getAttributes();
+	    Node AttributeNode = NodeMap.item(0);
+	    String att_name = AttributeNode.getNodeValue();
+	    ids[s] = att_name;
+    	}
+	
+    	return (String[])ids.clone();
+    }
+  
+    public HashMap getInternalIdentifier(String core_element){
+    	
+    	Element ex = getRootNode(core_element);
+        HashMap hp = new HashMap();
+        NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+        	
+    	for(int s=0; s<listOfFrequency.getLength() ; s++){
+    	    Node docNode = listOfFrequency.item(s);
+    	    String HashID = docNode.getAttributes().item(0).getNodeValue();
+	    Element docElement = (Element)docNode;
+    	    NodeList valueList = docElement.getElementsByTagName(ms.ActualValueNode);
+    	    
+    	    for(int y = 0; y<valueList.getLength(); y++){
+    		Element valueElement = (Element)valueList.item(y);
+    		NodeList textFNList = valueElement.getChildNodes();
+    		String text = ((Node)textFNList.item(0)).getNodeValue();
+    		
+    		if(!text.equals(" ")){
+    		    if(hp.containsKey(text)){
+    			InternalLink il = (InternalLink)hp.get(text);
+    			il.increaseElement(HashID);
+    			hp.put(text,il);
+    		    }
+    		    else{
+    		    InternalLink il = new InternalLink();
+		    il.setValue(text);
+		    il.increaseElement(HashID);
+		    hp.put(text, il);
+    		    }
+    		}
+    	    }
+	}
+	
+	return hp;
+    }
+    
+    public HashMap getIdentifierLinkNoIdentifier(){
+    	
+    	Element ex = getRootNode(ms.RootDirectory);
+        HashMap hp = new HashMap();
+        NodeList listOfFrequency = ex.getElementsByTagName(ms.DocumentNode);
+        String url ="No Source Available";
+    	
+	for(int s=0; s<listOfFrequency.getLength() ; s++){
+    	    Node docNode = listOfFrequency.item(s);
+    	    String HashID = docNode.getAttributes().item(0).getNodeValue();
+    	    InternalLink il = new InternalLink();
+	    il.increaseElement(HashID);
+	    hp.put(HashID, il);
+	}               
+	return hp;
+    }
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DescribeMessager.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DescribeMessager.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DescribeMessager.java	(revision 18104)
@@ -0,0 +1,141 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.io.*;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.w3c.dom.Element;
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GDBMWrapper;
+
+public class DescribeMessager {
+    
+    private final String site_name = "localsite";
+    private MessageRouter mr = null;
+    private String collection_Name = null;
+    private MetadataStats ms;
+    private String site_home = null;
+    private String gsdl3_home = null;
+    
+    public DescribeMessager(String collectionName, String URL) {
+	
+    	collection_Name = collectionName; 	
+    	gsdl3_home = GlobalProperties.getGSDL3Home();
+    
+    	if (gsdl3_home == null || gsdl3_home.equals("")) {
+    	    System.err.println("Couldn't access GSDL3Home from GlobalProperties.getGSDL3HOME, can't initialize the SOAP Server.");
+    	    return;
+    	}
+    	
+    	site_home = GSFile.siteHome(gsdl3_home, this.site_name);
+    	
+    	File site_file = new File(site_home);
+	
+    	if (!site_file.isDirectory()) {
+    	    System.err.println("The site directory "+site_file.getPath()+" doesn't exist. Can't initialize the SOAP Server.");
+    	    return;
+    	} 
+    	
+    	mr = new MessageRouter();
+    	mr.setSiteName(this.site_name);
+    	mr.configure();
+    }
+    
+    public void setCollectionName(String arg){
+    	collection_Name = arg;
+    }
+    
+    public String getCollectionName(){
+    	return collection_Name;
+    }
+
+    public boolean describeMatadata(PrintWriter out, String collectionName,String OAIURL, String oaiPrefix){
+	
+	ms = new MetadataStats(this.site_home,collectionName,OAIURL,out, oaiPrefix);
+    	
+	if(ms.getStatus()){
+	    printWebPage(ms,out);
+	    return true;
+	}
+     	else{
+	    out.print("<p> This collection could not be analysed.</p>");
+	    return false;
+	}
+    }
+
+    public void printWebPage(MetadataStats ms,PrintWriter out){
+	
+    	DataMaker dm = new DataMaker(ms);
+    	String[] elements =  new String [ms.metadataNameList.size()];
+        
+    	for(int i = 0; i<elements.length; i++){
+    		elements[i] = (String)ms.metadataNameList.get(i);
+        }
+	
+        Arrays.sort(elements);
+	
+        /*
+    	String[] metaDataElementName = (String[])elements.clone();	
+    	String[] ids = dm.getDocumentIDs("archivedir");
+		ArrayList alist = new ArrayList();
+    	
+		for(int a = 0; a<elements.length; a++){
+	    	alist.add(dm.getMetadataRows(elements[a]));
+    	}
+		*/
+        
+    	PrintHTML phtml = new PrintHTML(ms,out);
+    	HashMap hp = ms.getMetadataSetMap();
+    	Collection c = hp.values();
+    	Iterator i = c.iterator();
+
+    	while(i.hasNext()){
+	    MetadataSet mds  = (MetadataSet)i.next();
+	    ArrayList elementList = mds.getIndexsList();
+	    String[] mdsElement = new String[elementList.size()];
+	    
+	    for(int x = 0; x<mdsElement.length; x++){
+		mdsElement[x]=(String)elementList.get(x);
+	    }
+	    
+	    Arrays.sort(mdsElement);
+	    
+	    String[] metaDataElement = (String[])mdsElement.clone();	
+	    String[] idList = dm.getDocumentIDs("archivedir");
+	    ArrayList datalist = new ArrayList();
+        	
+	    for(int a = 0; a<mdsElement.length; a++){
+	    	datalist.add(dm.getMetadataRows(mdsElement[a]));
+	    }
+	    phtml.generateAllPossibleGraph(datalist,idList,metaDataElement,mds.getName());
+	}
+    	
+	phtml.generateOverallStatisticsPage(hp);
+
+    	Collection cx = ms.getMetadataSetMap().values();
+    	Iterator ix = cx.iterator();
+
+    	while(ix.hasNext()){
+	    MetadataSet temp = (MetadataSet)ix.next();
+	    phtml.WriteMetadataSetDetailHTML(temp);
+	    phtml.generateMetadataElementDetailPage(temp,out);
+	}
+    }
+ 
+    public MetadataStats getElementClass(){
+    	return ms;
+    }
+    
+    /*
+      public static void main(String[] args) {
+      DescribeMessager dm = new DescribeMessager("testCOLLECTION","cs.waikato.ac.nz");
+      }
+    */
+}
+
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DocumentInfo.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DocumentInfo.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/DocumentInfo.java	(revision 18104)
@@ -0,0 +1,42 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.util.ArrayList;
+
+public class DocumentInfo {
+    
+    private String DocumentID;
+    private ArrayList ActualValue;
+    private int Frequence;
+    
+    public DocumentInfo(){
+    	DocumentID = null;
+    	ActualValue = new ArrayList();
+    	Frequence = 0;
+    }
+    
+    public void IncreaseFrequence(){
+    	Frequence++;
+    }
+     
+    public void addActualValue(String v){
+    	ActualValue.add(v);
+    }
+    
+    public void setDocumentID(String ID){
+    	DocumentID = ID;
+    }
+    
+    public ArrayList getActualValue(){
+    	return ActualValue;
+    }
+
+    public int getFrequence(){
+    	return Frequence;
+    }
+    
+    public String getDocumentID(){
+    	return DocumentID;
+    }
+
+}
+
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/InternalLink.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/InternalLink.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/InternalLink.java	(revision 18104)
@@ -0,0 +1,29 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.util.ArrayList;
+
+public class InternalLink {
+	private ArrayList alist = new ArrayList();
+	private ArrayList nodeList = new ArrayList();
+	private String value = "";
+	
+	public void setValue(String arg){
+		value = arg;
+	}
+	public void increaseElement(String arg){
+		alist.add(arg);
+	}
+	public String getValue(){
+		return value;
+	}
+	public ArrayList retrieveList(){
+		return (ArrayList)alist.clone();
+	}
+	public ArrayList retrieveNodeList(){
+		return (ArrayList)nodeList.clone();
+	}
+	public void increaseNode(InternalLink il){
+		nodeList.add(il);
+	}
+}
+
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MatServlet.java	(revision 18104)
@@ -0,0 +1,870 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+import javax.xml.parsers.*;
+
+import org.xml.sax.*;
+import org.w3c.dom.*;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.GSPath;
+
+public class MatServlet extends HttpServlet {
+    
+    private int port_number = 0;
+    private String oaiPrefix ="";
+    private String titleString;
+    private String h1String;
+    private String maxRecord;
+    private GlobalProperties globalProperty;
+    private String fileSeparator;
+    private String cssString;
+    private String headerString;
+    private String javaScript;
+    private String headerString2;
+    private String gsdl3Home;
+    private String logoURL;
+    private String description;
+    private String hostName;
+    private String proxyHost="http.proxyHost";
+    private String proxyPort="http.proxyPort";
+    private String proxyHostContent = "";
+    private String proxyPortContent = "";
+    private String collection_creator = "";
+    
+    
+    public void doGet(HttpServletRequest request,HttpServletResponse response)throws ServletException, IOException {
+	
+	port_number = request.getLocalPort();
+	loadRuntimeSettings();
+	PrintWriter out = response.getWriter();   
+	
+	response.setContentType("text/html");
+	response.setHeader("pragma", "no-cache");
+	
+	out.println("<html>");
+	out.println(headerString);
+	out.println("<body>");
+	out.println("<img src="+logoURL+">");
+	out.println(h1String);
+	out.println("<p>"+description+"</p>");
+	out.println("<div><form method=\"post\" action=\"http://"+hostName+":"+port_number+"/greenstone3/mat\">");
+	out.println("<p>OAI URL: <input type=\"text\" name=\"oaiurl\" size=\"140\"></p>"); 
+	out.println("<p><input type=\"submit\" value=\"Analyse repository\"></p>");
+	out.println("</form>");
+	
+	out.println ("</body>");
+	out.println ("</html>");
+	out.close();
+    }
+
+    protected void doPost(HttpServletRequest req, HttpServletResponse res)throws ServletException, IOException {
+
+    port_number = req.getLocalPort();
+	loadRuntimeSettings();
+	
+   	res.setContentType("text/html");
+	res.setHeader("pragma", "no-cache");
+	PrintWriter out = res.getWriter();
+
+	out.println("<html>");
+	
+	if (req.getParameter("metadataPrefix") != null) {
+	    
+	    oaiPrefix = req.getParameter("metadataPrefix");
+	    String maxRecords = req.getParameter("maxrecords");
+	    Pattern pa = Pattern.compile("[0-9]{1,5}");
+	    Matcher ma = pa.matcher(maxRecords);
+		
+	    out.println(headerString2);
+	    out.println("<body onLoad=\"autoSubmit()\">");
+	    out.println(h1String);
+	    out.println("<p>Please wait .... <br> It's downloading OAI records </p>");
+	    out.println("<input type=\"button\" onClick=\"showdiv()\" value=\"show debug infomation\">");	
+	    
+	    if(ma.matches()){    
+		int num = Integer.parseInt(maxRecords);
+		if(num>0){
+		    downloadCollection(out, req, res, oaiPrefix, maxRecords);
+		}
+		else{
+		    downloadCollection(out, req, res, oaiPrefix, maxRecord);
+		}
+	    }
+	    else{
+		downloadCollection(out, req, res, oaiPrefix, maxRecord);
+	    }
+	    
+	}
+	else if(req.getParameter("matShell") != null){
+	    out.println(headerString2);
+	    out.println("<body onLoad=\"autoSubmit2()\">");
+	    out.println(h1String);
+	    out.println("<p>Please wait ...</p><p>It's building collection now.</p>");
+	    out.println("<input type=\"button\" onClick=\"showdiv()\" value=\"show debug infomation\">");	   
+	    buildCollection(out,req.getParameter("matShell"),req.getParameter("collectionName"),req.getParameter("collectionURL"),req.getParameter("oaiPrefix"));
+	}
+	
+	else if (req.getParameter("collName") != null){
+	    out.println(headerString);
+	    //out.write("<script type=\"text/javascript\" src=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/getInfomation.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/yahoo/yahoo-min.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/event/event-min.js\"></script>\r\n");
+	    //out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/connection/connection-min.js\"></script>\r\n");
+	    out.println("<body>");
+	    out.println(h1String);  
+	    analyzeCollection(out, req.getParameter("collName"),req.getParameter("collURL"),req.getParameter("collHost"),req.getParameter("oaiPrefix"));
+	}
+	
+	else {
+	    out.println(headerString);
+	    out.println("<body>");
+	    out.println(h1String);
+	    
+	    java.net.URL oaiURL;
+	    String oaiURLString = req.getParameter("oaiurl");
+	    
+	    if (! (oaiURLString.startsWith("http://"))) {
+	    	oaiURLString = "http://" + oaiURLString; // add on protocol if missing 
+	    }
+	    
+	    // URL checks
+	    // check if Java can make a URL from the string
+	    
+	    try {
+	    	oaiURL = new URL(oaiURLString);
+	    }
+	    catch (MalformedURLException e) {
+	    	out.println("<p>Malformed URL Exception caught: " + e.getMessage() + "</p>");
+	    	out.println("<p>The system cannot recognise the URL you have entered.</p>");
+		return;  // go no further
+	    }
+	    
+	    //out.println("<p>host: " + oaiURL.getHost() + "</p>" );
+	    //out.println("<p>protocol: " + oaiURL.getProtocol() + "</p>" );
+	    
+	    // disallow anything with waikato as part of the host
+	    // note: this blocks things like waikato.uiuc.edu - 
+	    // but guess this is highly unlikely to occur
+	    // (also turns off the researchcommons)
+	    // could use either of these approaches, waikato or waikato.ac.nz
+	    
+	    if  (oaiURL.getHost().indexOf("researchcommons.waikato.ac.nz") == -1 ) { // not the RC
+		/*
+		  if((oaiURL.getHost().indexOf("waikato") != -1 ) || 
+		  (oaiURL.getHost().indexOf("waikato.ac.nz") != -1 ) ) {
+		  out.println("<p>This service cannot be used to access Waikato URLs</p>");
+		  return;
+		  }*/
+	    }
+	    
+	    // need to prevent machine names on their own, e.g. smith 
+	    // being allowed through
+	    
+	    // approach 1: explicit blacklisting
+	    
+	    if (oaiURL.getHost().equals("smith") ||
+		oaiURL.getHost().equals("wesson") ) {
+	    	out.println("<p>This service cannot be used to access these URLs</p>");
+		return;
+	    }
+	    
+	    // approach 2: require at least one . in the host URL
+	    if (oaiURL.getHost().indexOf(".") == -1 ) { // i.e. no . in URL host
+	    	out.println("<p>This service cannot be used to access URLs of this form.</p>");  
+		return;
+	    }
+	    
+	    // jones.cs would get through to this point, might this mean something 
+	    // from the perspective of the host machine?? Does it matter?
+	    // will automatic domain completion be applied? by java? by 'the network'?
+	    // require 2 dots in the host? does that help at all ?
+	    // domain suffix whitelisting is impractical here, as we'd have to 
+	    // list all countries in the world
+	    
+	    // do an OAI verb=identify check to make sure it is an OAI server
+	    // trim the URL back and then add on verb=Identify
+	    
+	    
+	    //System.setProperty("http.proxyHost", "wwwcache.cs.waikato.ac.nz");
+	    //System.setProperty("http.proxyPort","80");
+	    //System.setProperty("http.nonProxyHosts", "localhost|*.waikato.ac.nz");
+	    
+	    Properties systemSettings = System.getProperties();
+	    
+	    if( proxyHostContent.equals("") || proxyPortContent.equals("")){}
+	    else{
+	    	systemSettings.put(proxyHost, proxyHostContent);
+	    	systemSettings.put(proxyPort, proxyPortContent);
+	    }
+	    
+	    URL url = oaiURL;
+	    String identifyVerb = "";
+	    
+	    if (! url.getPath().endsWith("?")) {
+	    	identifyVerb += "?";
+	    }
+	    
+	    identifyVerb += "verb=Identify";
+	    
+	    URL identifyURL = new URL (url.toString() + identifyVerb);
+	    HttpURLConnection connection = (HttpURLConnection)identifyURL.openConnection();
+	    //URLConnection connection = identifyURL.openConnection();          
+	    connection.connect(); 
+	    
+	    //BufferedReader inIdentify = new BufferedReader(new InputStreamReader(
+	    //connection.getInputStream()));
+	        
+	    Document identifyDocument;
+	    
+	    try {
+	    	DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+	    	DocumentBuilder builder = factory.newDocumentBuilder();
+	    	identifyDocument = builder.parse( connection.getInputStream() );
+		
+	    	Element oaiElement = identifyDocument.getDocumentElement();
+		
+		if (oaiElement.getTagName() == "OAI-PMH" ) {
+		    //out.println("<p>OAI-PMH element found...</p>");
+		}
+		else {
+		    out.println("<p>Error: OAI-PMH element not found..exiting</p>");
+		    return;
+		}
+		
+		NodeList identifyNodeList = identifyDocument.getElementsByTagName("Identify");
+		Node identifyNode = null;
+	
+		if (identifyNodeList.getLength() == 1 ) {
+		    identifyNode = identifyNodeList.item(0);
+		    System.out.println("<Identify>\n");
+		}
+		else {
+		    out.println("<p>Error: Identify node not found... exiting</p>");
+		    return;
+		}
+		
+		NodeList identifyChildList = identifyNode.getChildNodes();
+		out.println("<table rules =\"none\">");
+		
+		for (int i=0; i < identifyChildList.getLength(); i++) {
+		    if (identifyChildList.item(i).getNodeName() == "repositoryName" ) {
+			out.println("<tr><td><strong>Repository Name:</strong></td><td> " + identifyChildList.item(i).getTextContent() + "</td></tr>");
+		    }
+		    if (identifyChildList.item(i).getNodeName() == "baseURL" ) {
+		    	String baseURL = identifyChildList.item(i).getTextContent();
+		    	
+			if(baseURL.startsWith("http://")){
+			    out.println("<tr><td><strong>Base URL:</strong></td><td><code><a href=\"" + baseURL + "\">" + baseURL + "</a></code></td></tr>");
+		    	}
+		    	else{
+			    out.println("<tr><td><strong>Base URL:</strong></td><td><code><a href=\"" + oaiURLString + "\">" + oaiURLString + "</a></code></td></tr>");
+			}
+		    }
+		}
+		out.println("</table>");
+	    }
+	    catch (SAXParseException spe) {
+		out.println(spe.getMessage());
+	    }
+	    catch (SAXException sxe) {
+		out.println(sxe.toString());
+	    }
+	    catch (ParserConfigurationException pce) {
+		out.println(pce.toString());
+	    }
+	    catch (IOException ioe) {
+		out.println(ioe.toString());
+	    }
+	    
+	    // now get the metadata prefixes
+	    
+	    String metadataFormats = "";
+	    
+	    if (! url.getPath().endsWith("?")) {
+		metadataFormats  += "?";
+	    }
+	    metadataFormats += "verb=ListMetadataFormats";
+	    
+	    URL metadataFormatsURL = new URL (url.toString() + metadataFormats);
+	    URLConnection connection2 = metadataFormatsURL.openConnection();   
+	    connection.connect(); 
+	    Document document2;
+	    
+	    try {
+		DocumentBuilderFactory factory =  DocumentBuilderFactory.newInstance();
+		DocumentBuilder builder = factory.newDocumentBuilder();
+		document2 = builder.parse( connection2.getInputStream() ); 
+		
+		NodeList prefixList = document2.getElementsByTagName("metadataPrefix");
+			
+		// create UI based on response
+
+		if ( prefixList.getLength() > 0 ) {
+		    
+		    out.println("<p>Choose one metadata prefix to use:</p>");
+		    out.println("<form method=\"post\" action=\"http://"+hostName+":"+port_number+"/greenstone3/mat\">");
+		    out.println("<table rules =\"none\">");
+		    
+		    for (int i=0; i < prefixList.getLength(); i++) {
+			out.println("<tr><td>");
+			String id = "radioID" + i;
+			String prefix = prefixList.item(i).getTextContent();
+			
+			if (prefix.equals("oai_dc")) {
+			    out.println("<label for=\"" + id + "\">" + prefix + " (Dublin Core)</label>"); 
+			    //System.out.println("<MetadataElement>Dublin Core<\\MetadataElement>\n");
+			}
+			else {
+			    out.println("<label for=\"" + id + "\">" + prefix + "</label>"); 
+			    //System.out.println("<MetadataElement>"+prefix+"<\\MetadataElement>\n");
+			}
+			
+			out.print("</td><td><input type=\"radio\" name=\"metadataPrefix\" value=\"" + prefix + "\" ");
+			
+			if (prefix.equals("oai_dc")) {
+			    out.println("checked=\"checked\" ");
+			}
+			out.println(" id=\"" + id + "\"><br>");
+			out.println("</td></tr>");
+		    }
+		    
+		    //System.out.println("</Identify>");
+		    out.println("</table>");
+		    
+		    // need to pass oaiurl through as well (again)
+		    
+		    out.println("<input type=\"hidden\" name=\"oaiURL\" value=\"" + url.toString() + "\"><br>");
+		    out.println("Max records: <input type=\"text\" name=\"maxrecords\" value=\"500\"><br>");
+		    
+		    // submit button
+		    
+		    out.println("<p><input type=\"submit\" value=\"Continue\"></p>");
+		    out.println("</form>");
+		    
+		    out.println("<p> Warning: Generating the statistics and visualization will take some time: </p> ");
+		    
+		    /*
+		    out.println("<table border = \"1\">");
+		    out.println("<tr align=\"right\"><td> No.of Records <td> Estimated Time");
+		    out.println("<tr align=\"right\"><td>100<td> 5 minutes");
+		    out.println("<tr align=\"right\"><td>500 <td> 10 minutes");
+		    out.println("<tr align=\"right\"><td>1000 <td> 18 minutes");
+		    out.println("<tr align=\"right\"><td>2000<td> 30 minutes");
+		    out.println("</table>");
+		    */
+		    
+		    out.println("<p> This tool is designed to work with Dublin Core metadata: note that the mapping of qualified Dublin Core to simple Dublin Core (as in <code>oai_dc</code>) may affect the results.");
+		}
+		else {
+		    out.println("<p>Error: no metadata prefixes found... exiting</p>");
+		    return;               
+		}
+	    }
+	    catch (SAXParseException spe) {
+		out.println(spe.getMessage());
+	    }
+	    catch (SAXException sxe) { 
+		out.println(sxe.toString());
+	    }
+	    catch (ParserConfigurationException pce) {
+		out.println(pce.toString());
+	    }
+	    catch (IOException ioe) {           
+		out.println(ioe.toString());
+	    }    
+	    
+	    // check we get a valid XML document back
+	    // check OAI-PMH element
+	    // check we have a <repositoryName> and <baseURL> elements
+	    // maybe check the baseURL against the query we issued?
+	    
+	    
+	    // if we get here we have a valid non-waikato non-local OAI server
+	    // /* process stuff here */
+	    // test with University home page - as it doesn't require
+	    // dealing with the proxy server
+	    // URL url = new URL("http://waikato.ac.nz");
+	    // URL url = new URL(oaiURL);
+	    // need to go through the proxy here
+	    // http://dn.codegear.com/article/29783
+	    // http://java.ittoolbox.com/groups/technical-functional/java-l/response-to-proxy-authentication-exception-71438
+	    // http://www.jguru.com/faq/view.jsp?EID=13186
+	    // http://www.developer.com/java/other/article.php/1551421
+	    // System.setProperty("http.proxyHost","http://proxy.scms.waikato.ac.nz");
+	    // System.setProperty("http.proxyPort","80");
+	    // System.setProperty("http.nonProxyHosts", "localhost|*.waikato.ac.nz");
+	    // build a GS 3 collection from the OAI URL 
+	    // http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html
+	    // http://codon.kribb.re.kr/wiki/display/Java/Using+Runtime.exec+to+invoke+child+process
+	    
+	    out.print("</body></html>");
+	    out.close();   
+	} // endif
+    } // end doPost
+    
+    protected void buildCollection( PrintWriter out, String matShell2, String collName, String oaiURLString, String oaiPrefix)  throws ServletException, IOException{
+	
+	out.println("<br><br><div id=\"hideshow\" style=\"visibility:hidden\">");
+	out.println("<input type=\"button\" onClick=\"hidediv()\" value=\"hide debug infomation\"/>");
+	out.println("<p>Building collection...</p>");
+	
+	String host = "http://"+hostName+":"+port_number+"/greenstone3/mat/";
+	
+	out.println("<form method=\"post\" name=\"aForm\">");
+	out.println("<input type=\"hidden\" name=\"collName\" value=\"" + collName + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"collURL\" value=\"" + oaiURLString + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"collHost\" value=\"" + host + "\"><br>");
+	out.println("<input type=\"hidden\" name=\"oaiPrefix\" value=\"" + oaiPrefix + "\"><br>");
+	out.println("</form>");
+	out.println("<p><pre>" + matShell2 + "</pre></p>");
+	out.flush();
+	
+	
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().contains("windows")){
+	    gsdl3Home = gsdl3Home.replace("\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replace("/","\\");
+	    gsdl3Home = gsdl3Home.replace("/", "\\");
+	}
+
+	// downloading
+	//String gs3Root = "/research/cc108/greenstone3Project";
+	String gs3Root = gsdl3Root;
+	String collectDir = gsdl3Home + fileSeparator +"sites"+fileSeparator+"localsite"+ fileSeparator +"collect";
+	String logFile = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + "log.txt";
+	String cacheDir = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + collName;
+		
+	if(System.getProperty("os.name").toLowerCase().contains("windows")){
+		File wd = new File(gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script");
+
+		String[] arrays = new String[12];
+		arrays[0] = "cmd";
+		arrays[1] = "/c";
+		arrays[2] = "start";	
+		arrays[3] = "mat-colbuild.bat";
+		arrays[4] = collName;
+		arrays[5] = oaiURLString;
+		arrays[6] = cacheDir;
+		arrays[7] = "100";
+		arrays[8] = oaiPrefix;
+		arrays[9] = gs3Root;
+		arrays[10] = collectDir;
+		arrays[11] = logFile;
+		
+		Process p2 = processBatch(arrays,out,wd);
+		out.print("</div></body></html>");
+		p2 = null;
+		p2.destroy();
+	}
+	else{
+	Process p2 = processShell(matShell2, out);
+	
+	if (p2.exitValue() == 0){
+		out.println("<p>Collection built.</p>");
+	}
+	else{
+	    out.println("<p><b>Collection not built.</b></p>");
+	}
+	
+	out.print("</div></body></html>");
+	p2 = null;
+	p2.destroy();
+	}
+	out.close();
+    }
+    
+    private void downloadCollection( PrintWriter out, HttpServletRequest req, HttpServletResponse res, String oaiPrefix, String Records)  throws ServletException, IOException
+    {
+	String oaiURLString = req.getParameter("oaiURL");
+	String metadataprefix = req.getParameter("metadataPrefix");
+	String collName = generateCollName();
+	String host = "http://"+hostName+":"+port_number+"/";
+
+	out.println("<br><br><div id=\"hideshow\" style=\"visibility:hidden\">");
+	out.println("<input type=\"button\" onClick=\"hidediv()\" value=\"hide debug infomation\"/>");
+	out.println("<p>Downloading OAI documents...</p>");
+	out.flush();
+	// move to correct directory
+
+
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().contains("windows")){
+	    gsdl3Home = gsdl3Home.replace("\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replace("/","\\");
+	    gsdl3Home = gsdl3Home.replace("/", "\\");
+	}
+	String maxRecords = Records;
+	String cacheDir = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + collName;
+
+	// downloading
+	//String gs3Root = "/research/cc108/greenstone3Project";
+	String gs3Root = gsdl3Root;
+	String collectDir = gsdl3Home + fileSeparator +"sites"+fileSeparator+"localsite"+ fileSeparator +"collect";
+	String logFile = gsdl3Root +fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"tmp"+fileSeparator + "log.txt";
+	//source /research/cc108/greenstone3Project/gs2build/bin/script/mat-colbuild-download.bash
+	//source /research/cc108/greenstone3Project/ext/mat/bin/script/mat-colbuild-download.bash
+	
+	String matShell  = "";
+	
+	if(System.getProperty("os.name").toLowerCase().contains("windows")){
+		
+		File wd = new File(gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script");
+		ArrayList alist = new ArrayList();
+		alist.add("cmd");
+		alist.add("/c");
+		alist.add("start");
+		alist.add("mat-colbuild-download.bat");
+		alist.add(collName);
+		alist.add(oaiURLString);
+		alist.add(cacheDir);
+		alist.add(maxRecords);
+		alist.add(metadataprefix);
+		alist.add(gs3Root);
+		alist.add(collectDir);
+		alist.add(logFile);
+		alist.add(collection_creator);
+		
+		String[] arrays = new String[alist.size()];
+		for(int i = 0; i< arrays.length; i++){
+			arrays[i] = (String)alist.get(i);
+			System.out.println(arrays[i]);
+		}
+		System.out.println(wd.getAbsolutePath());
+		Process p = processBatch(arrays,out,wd);
+		out.flush();
+		
+		
+		p.destroy();
+		
+	}
+	else{
+	matShell = "source "+ gsdl3Root + fileSeparator + "ext" + fileSeparator + "mat" + fileSeparator + "bin" + fileSeparator +"script" + fileSeparator + "mat-colbuild-download.bash "
+	    + collName
+	    + " "
+	    + oaiURLString 
+	    + " "
+	    + cacheDir
+	    + " "
+	    + maxRecords
+	    + " "
+	    + metadataprefix
+	    + " " 
+	    + gs3Root
+	    + " "
+	    + collectDir
+	    + " " 
+	    + logFile
+	    + " "
+	    + collection_creator;
+	
+	out.println("<p><pre>" + matShell + "</pre></p>");
+	out.flush();
+	Process p = processShell(matShell, out);
+	
+	if (p.exitValue() == 0){
+	    out.println("<p>Collection downloaded.</p>");
+	}
+	else {
+	    out.println("<p><b>Collection not downloaded properly.</b></p>");
+	}
+	out.flush();
+	//p = null;
+	p.destroy();
+	
+	}
+
+	String matShell2 = "";
+
+		matShell2 = "source "+ gsdl3Root + fileSeparator + "ext" + fileSeparator +"mat"+ fileSeparator +"bin" + fileSeparator + "script" + fileSeparator + "mat-colbuild.bash "
+	    + collName
+	    + " "
+	    + oaiURLString 
+	    + " "
+	    + cacheDir
+	    + " "
+	    + maxRecords
+	    + " "
+	    + metadataprefix
+	    + " " 
+	    + gs3Root
+	    + " "
+	    + collectDir
+	    + " " 
+	    + logFile;
+
+		
+	out.println("<form method=\"post\" name=\"collectionForm\">");
+	out.println("<input type=\"hidden\" name=\"matShell\" value=\"" + matShell2 + "\">");
+	out.println("<input type=\"hidden\" name=\"collectionName\" value=\"" + collName + "\">");
+	out.println("<input type=\"hidden\" name=\"collectionURL\" value=\"" + oaiURLString + "\">");
+	out.println("<input type=\"hidden\" name=\"oaiPrefix\" value=\"" + oaiPrefix + "\">");	
+	out.println("<input type=\"hidden\" name=\"host\" value=\"" + host + "\">");
+	//out.println("<input type=\"submit\" value=\"start\"><br>");
+	out.println("</form>");
+	out.println("</div>");
+	out.print("</body></html>");
+	out.flush();
+	out.close();
+
+}
+
+    /* produce a random 7 letter collection name */
+    
+    private String generateCollName () {
+	Random random  = new Random();
+	StringBuffer message = new StringBuffer();
+	int offset = 97; // = "a"
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	message.append( (char) ( random.nextInt( 26 ) + offset ) );
+	return message.toString();
+    }
+	
+    private Process processBatch(String[] command, PrintWriter out, File wd){
+    	
+    	Process proc;
+    	String s;
+    	try{
+    		proc = Runtime.getRuntime().exec(command,null,wd);
+    	    BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+    	    BufferedReader stdError = new BufferedReader(new InputStreamReader(proc.getErrorStream()));	
+    	    StringBuffer stdInputBuffer = new StringBuffer();
+    	    while ((s = stdInput.readLine()) != null) {
+        		stdInputBuffer.append(s+"\n");
+        		if(s.contains("-->")){
+    		    s = s.replace("-->", "");
+    		    out.println("<!-- " + s + " -->");
+        		}
+        		else{
+    		    out.println("<!-- " + s + " -->");
+        		}
+    		out.flush();
+    	    }
+    	    
+    	    StringBuffer stdErrorBuffer = new StringBuffer();
+    	    
+            while ((s = stdError.readLine()) != null) {
+	    stdErrorBuffer.append(s+"\n");
+		
+	    if(s.contains("-->")){
+		s = s.replace("-->", "");
+		out.println("<!-- " + s + " -->");
+	    }
+	    else{
+		out.println("<!-- " + s + " -->");
+	    }
+	    out.flush();
+            }
+    	    
+            
+    		out.println("<p>Here is the standard output:</p>\n");
+    		out.println("<p><pre>" + stdInputBuffer + "</pre></p>");   
+    		out.println("<p>Here is the standard error (if any):</p>\n");
+    		out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+    		out.flush();
+    		//if (p.exitValue() != 0)
+    		if (false){
+    		    out.println("<p>An error occurred while building the collection.</p>");
+    		    out.println("<p>Here is the standard output:</p>\n");
+    		    out.println("<p><pre>" + stdInputBuffer + "</pre></p>");
+    		    out.println("<p>Here is the standard error (if any):</p>\n");
+    		    out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+    		}
+    		
+    		InputStream is = proc.getInputStream();
+    		is.close();
+    		
+    		OutputStream os = proc.getOutputStream();
+    		os.close();
+    		
+    		InputStream es = proc.getErrorStream();
+    		es.close();
+    		
+    		stdInput.close();
+    		stdError.close();
+    		proc.waitFor();
+            return proc;
+            
+    	}catch(Exception ex){
+    		ex.printStackTrace();
+    	}
+    	return null;
+    }
+    private Process processShell( String command, PrintWriter out) { 
+    	
+    	String s= "";
+    	try {
+    	    String[] args = new String[]{"sh", "-c", command};
+	    Process p = Runtime.getRuntime().exec(args);
+	    BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream()));
+	    BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream()));	    
+	    StringBuffer stdInputBuffer = new StringBuffer();
+    	    
+	    while ((s = stdInput.readLine()) != null) {
+    		stdInputBuffer.append(s+"\n");
+    		if(s.contains("-->")){
+		    s = s.replace("-->", "");
+		    out.println("<!-- " + s + " -->");
+    		}
+    		else{
+		    out.println("<!-- " + s + " -->");
+    		}
+		out.flush();
+	    }
+
+    	    StringBuffer stdErrorBuffer = new StringBuffer();
+    	    
+                while ((s = stdError.readLine()) != null) {
+		    stdErrorBuffer.append(s+"\n");
+    		
+		    if(s.contains("-->")){
+			s = s.replace("-->", "");
+			out.println("<!-- " + s + " -->");
+		    }
+		    else{
+			out.println("<!-- " + s + " -->");
+		    }
+		    out.flush();
+                }
+		
+		out.println("<p>Here is the standard output:</p>\n");
+		out.println("<p><pre>" + stdInputBuffer + "</pre></p>");   
+		out.println("<p>Here is the standard error (if any):</p>\n");
+		out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+		out.flush();
+		//if (p.exitValue() != 0)
+		if (false){
+		    out.println("<p>An error occurred while building the collection.</p>");
+		    out.println("<p>Here is the standard output:</p>\n");
+		    out.println("<p><pre>" + stdInputBuffer + "</pre></p>");
+		    out.println("<p>Here is the standard error (if any):</p>\n");
+		    out.println("<p><pre>" + stdErrorBuffer + "</pre></p>");
+		}
+		
+		InputStream is = p.getInputStream();
+		is.close();
+		
+		OutputStream os = p.getOutputStream();
+		os.close();
+		
+		InputStream es = p.getErrorStream();
+		es.close();
+		
+		stdInput.close();
+		stdError.close();
+		
+		return p;
+	}
+	catch (IOException e) {
+	    out.println("exception happened - here's what I know: ");
+    	    out.println(e.toString());        
+    	    out.flush();
+	}
+	return null;
+    }
+
+    private void analyzeCollection(PrintWriter out, String collectionName,String collectionURL,String collectionHost, String Prefix){
+	
+	String collName = collectionName;
+	String oaiURLString = collectionURL;
+	String host = collectionHost;
+	
+	try{
+	    DescribeMessager dm = new DescribeMessager(collName,oaiURLString);
+	    out.println("<p>Generating statistics and visualisations...</p>");
+	    out.flush();
+	    out.println("<p>please wait.</p>");
+	    out.flush();
+	    boolean status = dm.describeMatadata(out,collName,oaiURLString,Prefix);
+	    
+	    if(status){
+		    
+	    	out.println("<script type=\"text/javascript\" language=\"JavaScript\">");
+	    	out.println("reconfig();");
+	    	out.println("</script>");
+	    	
+	    	out.println("<a href=\""+host+collName+"/Overall.html\">View the report</a>");
+	    	out.flush();
+	    }
+	    out.println("</body></html>");
+	    out.flush();
+	}catch(Exception e){e.printStackTrace(out);
+	    out.println("<p><pre>"+e.toString()+"</pre></p>");
+	}
+	out.close();
+    }
+    
+    private void loadRuntimeSettings(){
+    
+    	
+	fileSeparator = File.separator;	
+	gsdl3Home = globalProperty.getGSDL3Home();
+	globalProperty = new GlobalProperties();
+	maxRecord ="10";
+	
+	String os = "linux";
+	if(System.getProperty("os.name").toLowerCase().contains("windows")){
+		gsdl3Home = gsdl3Home.replace("\\", "/");
+	    os = "windows";
+	}
+	String gsdl3Root = GSPath.removeLastLink(gsdl3Home);
+	
+	if(os.equals("windows")){
+	    gsdl3Root = gsdl3Root.replace("/","\\");
+	    gsdl3Home = gsdl3Home.replace("/", "\\");
+	}
+    	
+	try{
+	    Properties prop = new Properties();
+	    FileInputStream fis = new FileInputStream(gsdl3Root+fileSeparator+"ext"+fileSeparator+"mat"+fileSeparator+"properties.xml");
+	    prop.load(fis);
+	    
+	    titleString ="<title>"+ prop.getProperty("Servlet.Title")+"</title>";
+	    h1String ="<h2>"+ prop.getProperty("Servlet.Head")+"</h2>";
+	    cssString = "<link rel=\"stylesheet\" href=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/mat.css\" type=\"text/css\" >";
+	    javaScript = "<script type=\"text/javascript\" src=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/status3.js\"></script>";;
+	    headerString2 = "<head>" + titleString + "\n" +  javaScript + cssString+"</head>\n";
+	    headerString ="<head>" + titleString + "\n" + cssString + "</head>\n";
+	    logoURL =prop.getProperty("Servlet.Logo");
+	    description = prop.getProperty("Servlet.Description");
+	    
+	    proxyHostContent = prop.getProperty("Servlet.proxyHost");
+	    proxyPortContent = prop.getProperty("Servlet.proxyPort");
+	    
+	    collection_creator = prop.getProperty("Servlet.Collection.Creator");
+	    
+	}catch(Exception ex){
+	    ex.printStackTrace();
+	}
+	
+	
+	try {
+		java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost();
+		hostName = localMachine.getHostName();
+	}catch (java.net.UnknownHostException uhe) { 
+			uhe.printStackTrace();
+	}
+		
+    }
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataElement.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataElement.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataElement.java	(revision 18104)
@@ -0,0 +1,86 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+
+
+public class MetadataElement {
+    
+    protected String metadataName = null;;
+    protected int frequency = 0;
+    protected ArrayList length = new ArrayList();
+    protected HashMap metadataList= new HashMap();
+    protected boolean used = false;
+    protected HashMap valueList = new HashMap();
+    
+    public String getMetadataName(){
+	return this.metadataName;
+    }
+    
+    public HashMap getValueList(){
+	return valueList;
+    }
+    
+    public int getFrequency(){
+	return frequency;
+    }
+    
+    public ArrayList getLength(){
+	return length;
+    }
+    
+    public HashMap getMetadataList(){
+	return metadataList;
+    }
+        
+    public void setMetadataName(String Meta_Name){
+	metadataName =  Meta_Name;
+    }
+    
+    public void IncreaseFrequency(){
+	frequency++;
+    }
+    
+    public void DecreaseFrequency(){
+	frequency--;
+    }
+    
+    public void SetLengthList(int len){
+	Integer in = new Integer (len);
+	length.add(in);
+    }
+    
+    public void SetmetadataList(String ID, DocumentInfo doc){
+	//Integer in = new Integer (num);
+	//metadataList.add(in);
+	//metadataList = alist;
+	metadataList.put(ID, doc);
+    }
+    
+    public void SetStatus(boolean status){
+	used = status;
+    }
+    
+    public int[] sortArrayList(){
+	
+	int[] sortList = new int [metadataList.size()];
+	Collection c = metadataList.values();
+	Iterator i = c.iterator();
+	int x = 0;
+	
+	while(i.hasNext()){
+	    sortList[x] = ((DocumentInfo) i.next()).getFrequence();
+	    x++;
+	}
+	/*
+	  for(int i = 0; i < metadataList.size(); i++){
+	  sortList[i] = ((Integer)metadataList.get(i)).intValue();
+	  }
+	*/
+	Arrays.sort(sortList);
+	return sortList;
+    }
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataSet.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataSet.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataSet.java	(revision 18104)
@@ -0,0 +1,44 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.util.ArrayList;
+
+public class MetadataSet {
+
+	private String Name;
+	private ArrayList indexs = new ArrayList();
+	private String abb;
+	
+	public void setName(String str){
+		Name = str;
+	}
+	
+	public void addIndex(String str){
+		if(!indexs.contains(str)){
+			indexs.add(str);
+		}
+	}
+	
+	public void setAbb(String str){
+		abb = str;
+	}
+	
+	public String getName(){
+		return Name;
+	}
+	
+	public ArrayList getIndexsList(){
+		return (ArrayList)indexs.clone();
+	}
+	
+	public String getAbb(){
+		return abb;
+	}
+	
+	public boolean IsContain(String str){
+		return indexs.contains(str);
+	}
+	
+	public void setIndexList(ArrayList alist){
+		indexs = (ArrayList)alist.clone();
+	}
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataStats.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataStats.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/MetadataStats.java	(revision 18104)
@@ -0,0 +1,718 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+
+import org.w3c.dom.*;
+import javax.xml.parsers.*;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.*;
+import javax.xml.transform.stream.*;
+
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.greenstone.gsdl3.core.MessageRouter;
+import org.greenstone.gsdl3.util.GDBMWrapper;
+import org.greenstone.gsdl3.util.GSFile;
+import org.greenstone.gsdl3.util.GSXML;
+import org.greenstone.gsdl3.util.GlobalProperties;
+import org.greenstone.gsdl3.util.XMLConverter;
+import org.greenstone.gsdl3.util.GSPath;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+
+public class MetadataStats {
+    
+    private Document doc=null;
+    private MessageRouter mr = null;
+    private XMLConverter converter=null;
+    private GDBMWrapper gdbm_src = null;
+    private GSPath gspath = null;
+    private GlobalProperties globalProperty = null;
+    private String site_name = "localsite";
+    //private String node_id = "browselist";
+    private String DLS_SET = "dls";
+    private String DC_SET = "dublin";
+    private String OAI_URL = "";
+    
+    private boolean status = false;
+    
+    private String oai_Prefix ="";
+    private PrintWriter pw;
+    public String StatsDirectory;
+    public String HTMLDirectory;
+
+    private String destination = "";
+    private int TotalDoc = 0;
+    String linkIdentifier ="";
+    private HashMap MDS_list = new HashMap();
+    private String collection = null;
+    public ArrayList metadataNameList = new ArrayList();
+    private String collection_Name = null;
+    
+    private String DB_TYPE = "gdbm";
+    protected final String FrequencyNode = "Frequency";
+    protected final String DocumentNode = "Document";
+    protected final String ActualValueNode = "ActualValue";
+    protected final String RootDirectory = "archivedir";
+    protected final String fileSeparator = File.separator;
+    
+    MetadataElement me;
+    
+    private static boolean deleteDir(File dir) {
+        if (dir.isDirectory()) {
+            String[] children = dir.list();
+            for (int i=0; i<children.length; i++) {
+                boolean success = deleteDir(new File(dir, children[i]));
+                if (!success) {
+                    return false;
+                }
+            }
+        }
+    
+        return dir.delete();
+    } 
+
+    public MetadataStats(String site_home, String collection,String url, PrintWriter out, String oaiPrefix){
+    	OAI_URL = url;
+    	collection_Name = collection; 
+    	oai_Prefix = oaiPrefix;
+    	pw = out;
+
+   	    destination = globalProperty.getGSDL3Home()+fileSeparator+"mat"+fileSeparator+collection+fileSeparator;
+   	    HTMLDirectory = destination;
+        StatsDirectory = HTMLDirectory+"metadataStats"+fileSeparator;
+
+        	if(new File(StatsDirectory).exists()){
+    	    deleteDir(new File(StatsDirectory));
+    	}
+        	new File(StatsDirectory).mkdirs();
+    	String gdbm = GSFile.collectionDatabaseFile(site_home,collection,collection,DB_TYPE);
+
+	mr = new MessageRouter();
+	mr.setSiteName(this.site_name);
+	mr.configure();
+	
+	this.collection = collection;
+	this.gdbm_src = new GDBMWrapper();
+	this.converter = new XMLConverter();
+	this.doc = this.converter.newDOM();
+	Setup(out, gdbm);
+    }
+    
+    public void getAvailableMetadataSets(String SetName,HashMap UsedMap, String SetAbbr){
+	
+    	
+    	   
+        String gsdl3Home = globalProperty.getGSDL3Home();
+        String os = "linux";
+        
+        if(System.getProperty("os.name").toLowerCase().contains("windows")){
+        	gsdl3Home = gsdl3Home.replace("\\", "/");
+        	os = "windows";
+        }
+    	String metadataSetHome = gspath.removeLastLink(gsdl3Home)+fileSeparator+"gli"+fileSeparator+"metadata";
+    	if(os.equals("windows")){
+    		metadataSetHome = metadataSetHome.replace("/", "\\");
+    	}
+    	
+        File metadata_directory = new File(metadataSetHome);
+
+	
+	if (metadata_directory.exists()) {
+	    // Load just those .mds files in this directory, and return them
+	    File[] directory_files = metadata_directory.listFiles();
+
+	    for (int i = 0; i < directory_files.length; i++) {
+		File child_file = directory_files[i];
+		
+		if (!child_file.isDirectory() && child_file.getName().endsWith("mds")) {
+                    String fileName = child_file.getName();
+		    
+		    /////////////////////////
+		    //LOADING DLS CORE
+		    //---if(fileName.equals(SetName)&& fileName.equals("dls.mds") && DLS_STATUS){
+		    if(!MDS_list.containsKey(SetName)&& fileName.equals(SetName+".mds")){
+            		    
+			MetadataSet ms = new MetadataSet();
+			ms.setName(SetName);
+			ms.setAbb(SetAbbr);
+			
+			converter.newDOM();
+			Document d = converter.getDOM(child_file);
+			     
+			NodeList e = d.getElementsByTagName("Element");
+			     
+			int length = e.getLength();
+			
+			for(int y = 0; y<length; y++){
+			    Node temp = e.item(y);
+			    NamedNodeMap mmp = temp.getAttributes();
+			    
+			    if(!mmp.item(0).getNodeValue().equals("dc.Description")){
+			    MetadataElement me = new MetadataElement();
+			    //me.setMetadataName((SetAbbr+"."+mmp.item(0).getNodeValue().toLowerCase()));
+			    me.setMetadataName((SetAbbr+"."+mmp.item(0).getNodeValue()));
+			    //System.out.println(me.getMetadataName());
+			    //.child_file............child_file
+			    if(!metadataNameList.contains(me.getMetadataName())){
+			    	metadataNameList.add(me.getMetadataName());
+			    	ms.addIndex(me.getMetadataName());
+			    }
+			    UsedMap.put(me.getMetadataName(),me);
+			    }
+			}
+			//System.out.println(ms.getName()+"loaded");
+			MDS_list.put(ms.getName(), ms);
+			//pw.write("load metadata set"+ms.getName());
+			//pw.flush();
+			//System.gc();
+		    }// DLS FINISH
+		}
+    	    }
+	}   	
+    }
+    
+    // doc_list is the list of document ids
+    //MetadataElement represents which metadata element we want to explore
+    private void parseMetadata(ArrayList doc_list, String MetadataElement){
+    
+	//System.out.println("parse metadata starts");
+	int counter = 0;
+	//int x = 0;
+	int docSize = doc_list.size();
+    	
+	pw.write("<!-- prepare request -->");
+	while(true){
+    		
+	    Node message = this.doc.createElement(GSXML.MESSAGE_ELEM);
+	
+	    Node request = GSXML.createBasicRequest(
+						       this.doc, GSXML.REQUEST_TYPE_PROCESS,collection+"/DocumentMetadataRetrieve","en", "");
+	    
+	    Node param_list = this.doc.createElement(
+							GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
+	    
+	    param_list.appendChild(GSXML.createParameter(
+							 this.doc, "metadata", MetadataElement));
+	
+	    Node documentNode_list = this.doc.createElement(
+							       GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
+	
+	    if(docSize>=500){
+		docSize = docSize - 500;
+		for(int i = 0; i<500; i++){
+		    Element documentNode = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		    documentNode.setAttribute("nodeID",(String)doc_list.get(counter));
+		    documentNode_list.appendChild(documentNode);
+		    counter++;
+		}
+		request.appendChild(param_list);
+		request.appendChild(documentNode_list);
+		message.appendChild(request);
+		//System.out.println(this.converter.getPrettyString(message));
+		//pw.write("send request");
+		//pw.flush();
+		Node response = mr.process(message);
+		//pw.write("receive request");
+		//pw.flush();
+		//System.out.println(this.converter.getPrettyString(response));
+		
+		getAttributes(response);
+		
+		response  = null;
+		request = null;
+		param_list = null;
+		documentNode_list = null;
+		//System.gc();	
+		}
+	
+	    else if(docSize<500){
+		for(int i = 0; i<docSize; i++){
+		    Element documentNode = this.doc.createElement(GSXML.DOC_NODE_ELEM);
+		    documentNode.setAttribute("nodeID",(String)doc_list.get(counter));
+		    documentNode_list.appendChild(documentNode);
+		    counter++;
+		}
+		request.appendChild(param_list);
+		request.appendChild(documentNode_list);
+		message.appendChild(request);
+		
+		//pw.write("send request");
+		Node response = mr.process(message);
+		//pw.write("receive request\n");
+		//pw.write(this.converter.getPrettyString(response));
+		//System.out.println(this.converter.getPrettyString(response));
+		//System.out.println(this.converter.getPrettyString(response));
+		getAttributes(response);
+		response  = null;
+		request = null;
+		param_list = null;
+		documentNode_list = null;
+		//System.gc();	
+		break;
+	    }
+	}
+    }
+    
+    public boolean getStatus(){
+    	return this.status;
+    }
+    
+
+    private void Setup(PrintWriter out, String gdbm){
+	
+	if (!this.gdbm_src.openDatabase(gdbm, GDBMWrapper.READ)) {
+	    out.println("Could not open GDBM database!");
+	    
+	}
+	
+	else{
+
+	    String info = this.gdbm_src.getValue("browselist");
+	    
+	    if (info == null) {
+		System.out.println("cannot locate the list");
+	    }
+	    else{
+		if (info == null) {
+		    System.out.println("the db does not contain any info");
+		}
+		
+		ArrayList children = new ArrayList();
+		StringTokenizer st = new StringTokenizer(info, ";");
+		
+		while (st.hasMoreTokens()) {
+		    String part = st.nextToken(";");
+
+		    if(part.contains("<contains>")){
+			part = part.replace("<contains>", "");
+		    }
+		    else if(part.contains("<thistype>")){
+			int location = part.indexOf("<thistype>");
+			part = part.substring(0,location-1);
+		    }
+		    children.add(part);
+		}
+		
+		gdbm_src.closeDatabase();
+		parseMetadata(children,"all");
+		TotalDoc = children.size();
+		status = true;
+	    }
+	}
+    }
+    
+    public void getAttributes(Node f){
+	
+	HashMap UsedMap = new HashMap();
+	Element e = (Element) f;
+	NodeList metadataNode =  e.getElementsByTagName("metadata");
+	int length = metadataNode.getLength();
+	
+	for (int j = 0; j < length; j++){
+	    Node aNode = metadataNode.item(j);
+	    NamedNodeMap NodeMap = aNode.getAttributes();
+	    Node AttributeNode = NodeMap.item(0);
+	    String att_name = AttributeNode.getNodeValue();
+	    
+	    if(att_name.indexOf("dls.")!=-1){getAvailableMetadataSets(DLS_SET,UsedMap,"dls");}
+	    else if(att_name.indexOf("dc.")!=-1){getAvailableMetadataSets(DC_SET,UsedMap,"dc");}
+	    
+	}
+	//UsedMap = combineList(UsedMap);
+	adjust(UsedMap,e);
+	UsedMap = null;
+	e = null;
+	System.gc();
+    }
+    
+
+    /* 
+     * MATCH UP THE ELEMENTS FROM BOTH DOCUMENTS AND PRE-DEFINED METADATA SET
+     * MOVE THE ELEMENT FROM NOUSEDMAP TO USEDMAP IF THE ELEMENT HAS NOT BEEN DISCOVERED BEFORE
+     * INCREASE THE COUNTER ONCE THE ELEMENT IS RECONGNIZED
+     * CALCULATE THE TIMES OF ELEMENTS USED AND STORE IT INTO THE LIST
+     */
+    
+    private void adjust(HashMap UsedMap, Element response){
+
+	pw.write("<!-- start parsing -->");
+	pw.flush();
+	//try{
+	String DocID;;
+	NodeList nList = response.getElementsByTagName("documentNode");
+	int length = nList.getLength();
+	for (int j = 0; j <length;  j++){
+	    
+	    Node aNode = nList.item(j);
+	    NamedNodeMap NodeMap = aNode.getAttributes();
+	    Node AttributeNode = NodeMap.item(0);
+	    String att_name = AttributeNode.getNodeValue();
+	    DocID = att_name;
+
+	    NodeList childList = aNode.getChildNodes();
+	    Node nNode = childList.item(0);
+	    NodeList grandChildList = nNode.getChildNodes();
+	    
+	    int length1 = grandChildList.getLength();
+	    
+	    for(int e = 0; e<length1 ; e++){
+
+		Node xNode = grandChildList.item(e);
+		NamedNodeMap xNodeMap = xNode.getAttributes();
+		Node xAttributeNode = xNodeMap.item(0);
+		String xatt_name = xAttributeNode.getNodeValue();
+
+		NodeList cList = xNode.getChildNodes();
+		
+		String ActValue = cList.item(0).getNodeValue();
+
+		//////////////
+		char firstchart = xatt_name.charAt(0);
+		//if(Character.isLetter(firstchart)
+		////////////////
+		pw.write("<!--"+xatt_name+"-->");
+		pw.flush();
+		//System.out.println(xatt_name+" "+UsedMap.containsKey(xatt_name));
+		
+		if(xatt_name.indexOf("dc.")!=-1){
+			int position = xatt_name.indexOf(".");
+			char  x = xatt_name.charAt(position+1);
+			x = Character.toUpperCase(x);
+			xatt_name = "dc."+x+xatt_name.substring(position+2);
+		}
+		
+		if(xatt_name.indexOf('^')!=-1){
+			xatt_name = xatt_name.replace('^', '.');
+		}
+		
+		if(UsedMap.containsKey(xatt_name)){
+			//pw.write("add element to list");
+		    MetadataElement met = (MetadataElement) UsedMap.get(xatt_name);
+		    met.IncreaseFrequency();
+		    HashMap metadataMap = met.getMetadataList();
+		    
+		    if(metadataMap.containsKey(DocID)){
+			//pw.write("element contains ID");
+			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
+			dc.IncreaseFrequence();
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+		    else{
+			//pw.write("element DOES NOT contain ID");
+			DocumentInfo dc = new DocumentInfo();
+			dc.IncreaseFrequence();
+			dc.setDocumentID(DocID);
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+		    
+		    HashMap valueMap = met.getValueList();
+		    if(valueMap.containsKey(ActValue)){
+			Integer f = (Integer)valueMap.get(ActValue);
+			int fx = f.intValue();
+			fx++;
+			valueMap.put(ActValue,new Integer(fx));
+		    }
+		    else{
+			valueMap.put(ActValue, new Integer(1));
+		    }
+		    
+		}
+		//else if ((!UsedMap.containsKey(xatt_name)) && (xatt_name.indexOf(".")==-1) && (!xatt_name.equals("prettymd")) && (!xatt_name.equals("Description")) && (Character.isLetter(firstchart)) && (xatt_name.indexOf("-")==-1)){
+		else if( (!UsedMap.containsKey(xatt_name)) && xatt_name.equals(RootDirectory)){   
+		    //pw.write("NEW EX element");
+		    String SetAbbr = "ex";
+		    String SetName = "extracted";
+		    
+		    MetadataSet ms = new MetadataSet();
+		    ms.setName(SetName);
+		    ms.setAbb(SetAbbr);
+		    
+		    MetadataElement me = new MetadataElement();
+		    me.setMetadataName(xatt_name);
+		    me.IncreaseFrequency();
+		    HashMap metadataMap = me.getMetadataList();
+	    	
+		    if(metadataMap.containsKey(DocID)){
+			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
+			dc.IncreaseFrequence();
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+		    else{
+			DocumentInfo dc = new DocumentInfo();
+			dc.IncreaseFrequence();
+			dc.setDocumentID(DocID);
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+
+		    HashMap valueMap = me.getValueList();
+		    if(valueMap.containsKey(ActValue)){
+			Integer f = (Integer)valueMap.get(ActValue);
+			int fx = f.intValue();
+			fx++;
+			valueMap.put(ActValue,new Integer(fx));
+		    }
+		    else{
+			//System.out.println(ActValue+" doesn't already exist");
+			valueMap.put(ActValue, new Integer(1));
+		    }
+		    if(!metadataNameList.contains(me.getMetadataName())){
+		    	metadataNameList.add(me.getMetadataName());
+		    	ms.addIndex(me.getMetadataName());
+		    }
+		    //pw.write("adding element");
+		    UsedMap.put(me.getMetadataName(), me);
+		    if(!MDS_list.containsKey(ms.getName())){MDS_list.put(ms.getName(), ms);}
+		    else{
+			MetadataSet msx = (MetadataSet)MDS_list.get(ms.getName());
+			msx.addIndex(me.getMetadataName());
+			MDS_list.put(ms.getName(), msx);
+		    }
+		}
+		
+		
+		else if((!UsedMap.containsKey(xatt_name)) && xatt_name.indexOf(".")!=-1 && xatt_name.indexOf("dc.")==-1){   
+		    //pw.write("NEW EX element");
+			int dotLocation = xatt_name.indexOf(".");
+			
+			String SetAbbr = new String();
+			String SetName = new String();
+			if(dotLocation==0){
+				SetAbbr = "Other";
+				SetName = "Other";
+			}
+			else{
+				SetAbbr = xatt_name.substring(0,dotLocation);
+				SetName = SetAbbr;
+			}
+		    MetadataSet ms = new MetadataSet();
+		    ms.setName(SetName);
+		    ms.setAbb(SetAbbr);
+		    
+		    MetadataElement me = new MetadataElement();
+		    me.setMetadataName(xatt_name);
+		    me.IncreaseFrequency();
+		    HashMap metadataMap = me.getMetadataList();
+	    	
+		    if(metadataMap.containsKey(DocID)){
+			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
+			dc.IncreaseFrequence();
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+		    else{
+			DocumentInfo dc = new DocumentInfo();
+			dc.IncreaseFrequence();
+			dc.setDocumentID(DocID);
+			dc.addActualValue(ActValue);
+			metadataMap.put(DocID, dc);
+		    }
+
+		    HashMap valueMap = me.getValueList();
+		    if(valueMap.containsKey(ActValue)){
+			Integer f = (Integer)valueMap.get(ActValue);
+			int fx = f.intValue();
+			fx++;
+			valueMap.put(ActValue,new Integer(fx));
+		    }
+		    else{
+			//System.out.println(ActValue+" doesn't already exist");
+			valueMap.put(ActValue, new Integer(1));
+		    }
+		    if(!metadataNameList.contains(me.getMetadataName())){
+		    	metadataNameList.add(me.getMetadataName());
+		    	ms.addIndex(me.getMetadataName());
+		    }
+		    //pw.write("adding element");
+		    UsedMap.put(me.getMetadataName(), me);
+		    if(!MDS_list.containsKey(ms.getName())){MDS_list.put(ms.getName(), ms);}
+		    else{
+			MetadataSet msx = (MetadataSet)MDS_list.get(ms.getName());
+			msx.addIndex(me.getMetadataName());
+			MDS_list.put(ms.getName(), msx);
+		    }
+		}
+		
+		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+		else if ((!UsedMap.containsKey(xatt_name)) && (xatt_name.indexOf("dc.")==0) && xatt_name.indexOf("dc.Description")!=0){
+		    
+		    //pw.write("QUALIFIED DC element");
+			
+		    String SetName = "dublin";
+		    xatt_name = xatt_name.replace('^', '.');
+		    
+		    MetadataElement me = new MetadataElement();
+		    me.setMetadataName(xatt_name);
+		    me.IncreaseFrequency();
+		    HashMap metadataMap = me.getMetadataList();
+		    
+		    DocumentInfo dc = new DocumentInfo();
+		    dc.IncreaseFrequence();
+		    dc.setDocumentID(DocID);
+		    dc.addActualValue(ActValue);
+		    metadataMap.put(DocID, dc);
+		    
+		    HashMap valueMap = me.getValueList();
+		    valueMap.put(ActValue, new Integer(1));
+			
+		    if(!metadataNameList.contains(me.getMetadataName())){
+		    	metadataNameList.add(me.getMetadataName());
+		    }
+		    
+		    //pw.write("adding element");
+		    UsedMap.put(me.getMetadataName(), me);
+		    
+		    MetadataSet msx = (MetadataSet)MDS_list.get(SetName);
+		    msx.addIndex(me.getMetadataName());
+		    MDS_list.put(SetName, msx);
+		}
+		/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	    }// element
+	    
+	}//doc id
+	////////////////////
+	//write file
+	int counter = 0;
+	//System.out.println("write");
+	//pw.write("write files" + UsedMap.size()+ metadataNameList.size() );
+	//pw.flush();
+	
+	Set s = UsedMap.keySet();
+	Iterator is = s.iterator();
+	//System.out.println(UsedMap.size());
+	while(is.hasNext()){
+
+	    String fileName = (String)is.next();
+	    MetadataElement me = (MetadataElement)UsedMap.get(fileName);
+	    HashMap hp = me.getMetadataList();
+	    Collection ks = hp.values();
+	    Iterator iks = ks.iterator();
+		
+	    try{
+		
+		DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+		DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+		Document doc = docBuilder.newDocument();
+		Element root = doc.createElement("root");
+		boolean fileExist = (new File(StatsDirectory+fileName+".xml")).exists();
+		
+		if(!fileExist){
+		    root = doc.createElement("root");
+		}
+		else{
+		    doc = docBuilder.parse (new File(StatsDirectory+fileName+".xml"));
+		    root = doc.getDocumentElement();
+		}
+		
+		while(iks.hasNext()){
+		    
+		    DocumentInfo dc = (DocumentInfo)iks.next();
+		    Element docID = doc.createElement(DocumentNode);
+		    docID.setAttribute("id", dc.getDocumentID());
+		    //pw.write("doc id"+dc.getDocumentID());
+            	
+		    Element freq = doc.createElement(FrequencyNode);
+		    Text text = doc.createTextNode(dc.getFrequence()+"");
+		    freq.appendChild(text);
+		    docID.appendChild(freq);
+            	
+		    Element actValue = doc.createElement(ActualValueNode);
+		    ArrayList alist = dc.getActualValue();
+		    if(alist.size()==0){
+            		text = doc.createTextNode(" ");
+            		actValue = doc.createElement(ActualValueNode);
+            		actValue.appendChild(text);
+            		docID.appendChild(actValue);
+            		}
+            
+		    for(int i = 0; i<alist.size(); i++){
+            		actValue = doc.createElement(ActualValueNode);
+            		String utf8String = new String(((String)alist.get(i)).getBytes(),"UTF-8");
+            		Text text1 = doc.createTextNode(utf8String);
+            		actValue.appendChild(text1);
+            		docID.appendChild(actValue);
+		    }	
+            	
+		    root.appendChild(docID);
+		    docID = null;
+		    counter++;
+            }
+ 
+		TransformerFactory tf=TransformerFactory.newInstance();     
+		Transformer transformer=tf.newTransformer();     
+		DOMSource source= new DOMSource(root);     
+		transformer.setOutputProperty(OutputKeys.INDENT,"yes");     
+		
+		Writer pwx= new  BufferedWriter(new OutputStreamWriter(new FileOutputStream(StatsDirectory+fileName+".xml"),"UTF-8"));     
+		StreamResult result= new StreamResult(pwx);     
+		transformer.transform(source,result);  
+		pwx.close();
+  			
+		root = null;
+		docBuilderFactory = null;
+		docBuilder = null;
+		doc = null;
+		
+	    }catch (Exception e) {
+		e.printStackTrace(pw);
+	    }
+	}
+    }
+    
+    public int getDocNum(){
+	return TotalDoc;
+    }
+        
+    public HashMap getMetadataSetMap(){
+    	MDS_list.remove("extracted");
+    	return (HashMap)MDS_list.clone();
+    }
+    
+    public String getCollectionName(){
+    	return collection_Name;
+    }
+    
+    public void setOAIURL(String url){
+    	OAI_URL = url;
+    }
+    
+    public String getOAIURL(){
+    	return OAI_URL;
+    }
+    
+    public String getOaiPrefix(){
+    	return oai_Prefix;
+    }
+    
+    public ArrayList getMetadataNameList(){
+    	return (ArrayList)metadataNameList.clone();
+    }
+    private String constructUpdateMessgae (){
+    	String message = "<message><request type='system' to=''><system type='configure' subset=''/></request></message>";
+    	return message;
+    }
+    public String getIdendifier(){
+    	return linkIdentifier;
+    }
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/PrintHTML.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/PrintHTML.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/PrintHTML.java	(revision 18104)
@@ -0,0 +1,3117 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.io.*;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.greenstone.gsdl3.*;
+import org.greenstone.gsdl3.util.GlobalProperties;
+
+public class PrintHTML {
+    
+    private final String NORMAL = "normal";
+    private final String WORST = "worst";
+    private final String BEST = "best";
+    private final String title1 ="Hide Empty Metadata ,";
+    private final String title2 ="Hide Completed Metadata  ,";
+    private final String title3 ="Hide Documents with Empty Metadata  ,";
+    private final String title4 ="Hide Documents with Completed Metadata  ,";
+    private final String title5 ="No Available Graph";
+    private final String metadataSet1 = "Dublin Core";
+    private final String metadataSet2 = "Extracted";
+    private final String spaceLeft = "<I>&laquo;";
+    private final String spaceRight = "&raquo;</I>";
+    private final String htmlSpace = "&nbsp;";
+    private final String oddChar ="odd Character";
+    private final String destination;
+    private final String space ="space";
+    private String hostName;
+    private String cssString;
+    private int port_number = 9090;
+    private GlobalProperties globalProperty = null;
+    private String fileSeparator = File.separator;
+    
+    String linkIdentifier = "";
+    
+    
+    
+    private HashMap url;
+
+    MetadataStats mds;
+    DataMaker dm;
+    Calendar cl;
+    SimpleDateFormat sdf;
+    PrintWriter printWriter;
+    
+    public PrintHTML(MetadataStats ms, PrintWriter arg){
+	
+	mds = ms;
+	printWriter = arg;
+	dm = new DataMaker(mds);
+	cl=Calendar.getInstance(); 
+	linkIdentifier = "dc.Identifier";
+	destination = mds.HTMLDirectory;
+	sdf = new SimpleDateFormat(" dd MMM yyyy 'at' HH:mm:ss z 'GMT'Z");
+	setupMetadataLink("metadataElementURL");
+	
+	
+	try {
+		java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost();
+		hostName = localMachine.getHostName();
+	}catch (java.net.UnknownHostException uhe) { 
+			uhe.printStackTrace();
+	}
+	
+	cssString = "<link rel=\"stylesheet\" href=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/mat.css\" type=\"text/css\" >";
+    }
+	
+    private ArrayList getHideEmptyMetadataDetail(ArrayList dataset, String[] id, String[] metadataName){
+    	
+	printWriter.write("<!-- generating all possible page -->");
+	printWriter.flush();
+    	
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	ArrayList detailList = new ArrayList();
+	ArrayList alist = new ArrayList();
+	ArrayList nameList = new ArrayList();
+	
+	for(int i = 0; i< metadataName.length; i++){
+	    if(!dm.IsElementEmpty(metadataName[i])){
+		alist.add(dm.getMetadataRows(metadataName[i]));
+		nameList.add(metadataName[i]);
+	    }
+	}
+	
+	String[] metaDataElementName = new String[nameList.size()];
+	
+	for(int i = 0; i< nameList.size(); i++){
+	    metaDataElementName[i] = nameList.get(i).toString();
+	}
+	
+	if(id.length==0 || metaDataElementName.length==0){
+	    return new ArrayList();
+	}
+	
+	detailList.add(alist);
+	detailList.add(id);
+	detailList.add(metaDataElementName);
+    	printWriter.write("<!-- generating all possible page-->");
+    	printWriter.flush();
+	
+	return detailList;
+    }
+	
+    private ArrayList getHideFullMetadataDetail(ArrayList dataset, String[] id, String[] metadataName){
+    	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    	
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	ArrayList detailList = new ArrayList();
+	ArrayList alist = new ArrayList();
+	ArrayList nameList = new ArrayList();
+	
+	for(int i = 0; i< metadataName.length; i++){
+	    if(!dm.IsElementFull(metadataName[i])){
+		alist.add(dm.getMetadataRows(metadataName[i]));
+		nameList.add(metadataName[i]);
+	    }
+	}
+	
+	String[] metaDataElementName = new String[nameList.size()];
+	
+	for(int i = 0; i< nameList.size(); i++){
+	    metaDataElementName[i] = nameList.get(i).toString();
+	}
+	
+	if(id.length==0 || metaDataElementName.length==0){
+	    return new ArrayList();
+	}
+	
+	detailList.add(alist);
+	detailList.add(id);
+	detailList.add(metaDataElementName);
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+	
+	return detailList;
+    }
+	
+    private ArrayList getHideEmptyDocumentDetail(ArrayList dataset, String[] id, String[] metadataName){
+    	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	ArrayList detailList = new ArrayList();
+	ArrayList alist = dm.removeDocument(dataset, id, 0);
+	ArrayList idList = dm.getRemovedID();
+	ArrayList newIDList = new ArrayList();
+	
+	for(int i = 0; i< id.length; i++){
+	    if(!idList.contains(id[i])){
+		newIDList.add(id[i]);
+	    }
+	}
+	
+	id = new String[newIDList.size()];
+	for(int i = 0; i<newIDList.size(); i++){
+	    id[i] = newIDList.get(i).toString();
+	}
+	
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	detailList.add(alist);
+	detailList.add(id);
+	detailList.add(metadataName);
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+	
+	return detailList;
+    }
+    
+    private ArrayList getHideFullDocumentDetail(ArrayList dataset, String[] id, String[] metadataName){
+    	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	ArrayList detailList = new ArrayList();
+	ArrayList alist = dm.removeDocument(dataset, id ,1);
+	ArrayList idList = dm.getRemovedID();
+	ArrayList newIDList = new ArrayList();
+	
+	for(int i = 0; i< id.length; i++){
+	    if(!idList.contains(id[i])){
+		newIDList.add(id[i]);
+	    }
+	}
+	
+	id = new String[newIDList.size()];
+	for(int i = 0; i<newIDList.size(); i++){
+	    id[i] = newIDList.get(i).toString();
+	}
+	
+	if(id.length==0 || metadataName.length==0){
+	    return new ArrayList();
+	}
+	
+	detailList.add(alist);
+	detailList.add(id);
+	detailList.add(metadataName);
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+	
+	return detailList;
+    }
+   
+    public void generateAllPossibleGraph(ArrayList Namelist,String[]ids,String[] names,String condition){
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    
+	ArrayList dataList= Namelist;
+	String[] id = ids;
+	String[] metadataName = names;
+    
+	ArrayList alist = new ArrayList();
+	ArrayList detailList = new ArrayList();
+	ArrayList new_dataList = new ArrayList();
+	String[] idList;
+	String[] metadataNameList;
+	
+	//---------------------------------------------------------------------------------------------------------SS
+	//-----------------------------------------------------------------------SSSS
+	if(id.length==0 || metadataName.length==0){
+	    generateEmptyGraph("SSSS",WORST,condition);
+	    generateEmptyGraph("SSSS",BEST,condition);
+	}
+	
+	else{
+	    new_dataList = transformDataList(dataList,id);
+	    WriteHTML("SSSS",new_dataList,id,metadataName,WORST,"Show completed graph",condition);
+	    WriteHTML("SSSS",new_dataList,id,metadataName,BEST,"Show completed graph",condition);
+	}
+	//-----------------------------------------------------------------------SSSH
+	detailList = getHideFullDocumentDetail(dataList,id,metadataName);
+	if(detailList.size()==0){
+	    generateEmptyGraph("SSSH",WORST,condition);
+	    generateEmptyGraph("SSSH",BEST,condition);
+	}
+	else{
+	    alist = (ArrayList)detailList.get(0);
+	    idList = (String[])detailList.get(1);
+	    metadataNameList = (String[])detailList.get(2);
+	    new_dataList = transformDataList(alist,idList);
+	    WriteHTML("SSSH",new_dataList,idList,metadataNameList,WORST,title4,condition);
+	    WriteHTML("SSSH",new_dataList,idList,metadataNameList,BEST,title4,condition);
+	}
+	//-----------------------------------------------------------------------SSHS
+	detailList = getHideEmptyDocumentDetail(dataList,id,metadataName);
+	if(detailList.size()==0){
+	    generateEmptyGraph("SSHS",WORST,condition);
+	    generateEmptyGraph("SSHS",BEST,condition);
+	    generateEmptyGraph("SSHH",WORST,condition);
+	    generateEmptyGraph("SSHH",BEST,condition);
+	}
+	else{
+	    alist = (ArrayList)detailList.get(0);
+	    idList = (String[])detailList.get(1);
+	    metadataNameList = (String[])detailList.get(2);
+	    new_dataList = transformDataList(alist,idList);
+	    WriteHTML("SSHS",new_dataList,idList,metadataNameList,WORST,title3,condition);
+	    WriteHTML("SSHS",new_dataList,idList,metadataNameList,BEST,title3,condition);
+	    //-----------------------------------------------------------------------SSHH
+	    detailList = getHideFullDocumentDetail(alist,idList,metadataNameList);
+	    if(detailList.size()==0){
+		generateEmptyGraph("SSHH",WORST,condition);
+		generateEmptyGraph("SSHH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("SSHH",new_dataList,idList,metadataNameList,WORST,title3+title4,condition);
+		WriteHTML("SSHH",new_dataList,idList,metadataNameList,BEST,title3+title4,condition);
+	    }
+	}
+	
+	//---------------------------------------------------------------------------------------------------------SH
+	ArrayList xList = new ArrayList(); 
+	String[] idListCopy;
+	String[] metadataNameListCopy;
+	
+	//-----------------------------------------------------------------------SHSS
+	detailList = getHideFullMetadataDetail(dataList,id,metadataName);
+	
+	if(detailList.size()==0){
+	    generateEmptyGraph("SHSS",WORST,condition);
+	    generateEmptyGraph("SHSS",BEST,condition);
+		
+	    generateEmptyGraph("SHSH",WORST,condition);
+	    generateEmptyGraph("SHSH",BEST,condition);
+		
+	    generateEmptyGraph("SHHS",WORST,condition);
+	    generateEmptyGraph("SHHS",BEST,condition);
+		
+	    generateEmptyGraph("SHHH",WORST,condition);
+	    generateEmptyGraph("SHHH",BEST,condition);
+	}
+	else{
+	    alist = (ArrayList)detailList.get(0);
+	    idList = (String[])detailList.get(1);
+	    metadataNameList = (String[])detailList.get(2);
+	    
+	    xList = (ArrayList)alist.clone();
+	    idListCopy = (String[]) idList.clone();
+	    metadataNameListCopy = (String[])metadataNameList.clone();
+	    new_dataList = transformDataList(alist,idList);
+	    WriteHTML("SHSS",new_dataList,idList,metadataNameList,WORST,title2,condition);
+	    WriteHTML("SHSS",new_dataList,idList,metadataNameList,BEST,title2,condition);
+	
+	    //-----------------------------------------------------------------------SHHS
+	    detailList = getHideEmptyDocumentDetail((ArrayList)xList.clone(),(String[])idListCopy.clone(),(String[])metadataNameListCopy.clone());
+	    if(detailList.size()==0){		
+		generateEmptyGraph("SHHS",WORST,condition);
+		generateEmptyGraph("SHHS",BEST,condition);
+		generateEmptyGraph("SHHH",WORST,condition);
+		generateEmptyGraph("SHHH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("SHHS",new_dataList,idList,metadataNameList,WORST,title2+title3,condition);
+		WriteHTML("SHHS",new_dataList,idList,metadataNameList,BEST,title2+title3,condition);
+		
+		//-----------------------------------------------------------------------SHHH
+		detailList = getHideFullDocumentDetail(alist,idList,metadataNameList);
+		if(detailList.size()==0){		
+		    generateEmptyGraph("SHHH",WORST,condition);
+		    generateEmptyGraph("SHHH",BEST,condition);
+		}
+		else{
+		    alist = (ArrayList)detailList.get(0);
+		    idList = (String[])detailList.get(1);
+		    metadataNameList = (String[])detailList.get(2);
+		    new_dataList = transformDataList(alist,idList);
+		    WriteHTML("SHHH",new_dataList,idList,metadataNameList,WORST,title2+title3+title4,condition);
+		    WriteHTML("SHHH",new_dataList,idList,metadataNameList,BEST,title2+title3+title4,condition);
+		}
+	    }
+	    
+	    //-----------------------------------------------------------------------SHSH	
+	    detailList = getHideFullDocumentDetail((ArrayList)xList.clone(),(String[])idListCopy.clone(),(String[])metadataNameListCopy.clone());
+	    if(detailList.size()==0){		
+		generateEmptyGraph("SHSH",WORST,condition);
+		generateEmptyGraph("SHSH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("SHSH",new_dataList,idList,metadataNameList,WORST,title2+title4,condition);
+		WriteHTML("SHSH",new_dataList,idList,metadataNameList,BEST,title2+title4,condition);
+	    }
+	}
+	
+	//---------------------------------------------------------------------------------------------------------HS
+	ArrayList xList1 = new ArrayList(); 
+	String[] idListCopy1;
+	String[] metadataNameListCopy1;
+	
+	//-----------------------------------------------------------------------HSSS
+	detailList = getHideEmptyMetadataDetail(dataList,id,metadataName);
+	
+	if(detailList.size()==0){
+	    generateEmptyGraph("HSSS",WORST,condition);
+	    generateEmptyGraph("HSSS",BEST,condition);
+	    
+	    generateEmptyGraph("HSHS",WORST,condition);
+	    generateEmptyGraph("HSHS",BEST,condition);
+		
+	    generateEmptyGraph("HSHH",WORST,condition);
+	    generateEmptyGraph("HSHH",BEST,condition);
+		
+	    generateEmptyGraph("HSSH",WORST,condition);
+	    generateEmptyGraph("HSSH",BEST,condition);
+	}
+	else{
+	    alist = (ArrayList)detailList.get(0);
+	    idList = (String[])detailList.get(1);
+	    metadataNameList = (String[])detailList.get(2);
+	    
+	    xList1 = (ArrayList)alist.clone();
+	    idListCopy1 = (String[]) idList.clone();
+	    metadataNameListCopy1 = (String[])metadataNameList.clone();
+	
+	    new_dataList = transformDataList(alist,idList);
+	    WriteHTML("HSSS",new_dataList,idList,metadataNameList,WORST,title1,condition);
+	    WriteHTML("HSSS",new_dataList,idList,metadataNameList,BEST,title1,condition);
+	
+	    //-----------------------------------------------------------------------HSHS
+	    detailList = getHideEmptyDocumentDetail((ArrayList)xList1.clone(),(String[])idListCopy1.clone(),(String[])metadataNameListCopy1.clone());
+	    if(detailList.size()==0){
+		generateEmptyGraph("HSHS",WORST,condition);
+		generateEmptyGraph("HSHS",BEST,condition);
+		
+		generateEmptyGraph("HSHH",WORST,condition);
+		generateEmptyGraph("HSHH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+			
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("HSHS",new_dataList,idList,metadataNameList,WORST,title1+title3,condition);
+		WriteHTML("HSHS",new_dataList,idList,metadataNameList,BEST,title1+title3,condition);
+	
+		//-----------------------------------------------------------------------HSHH
+		detailList = getHideFullDocumentDetail(alist,idList,metadataNameList);
+		if(detailList.size()==0){		
+		    generateEmptyGraph("HSHH",WORST,condition);
+		    generateEmptyGraph("HSHH",BEST,condition);
+		}
+		else{
+		    alist = (ArrayList)detailList.get(0);
+		    idList = (String[])detailList.get(1);
+		    metadataNameList = (String[])detailList.get(2);
+		    
+		    new_dataList = transformDataList(alist,idList);
+		    WriteHTML("HSHH",new_dataList,idList,metadataNameList,WORST,title1+title3+title4,condition); 
+		    WriteHTML("HSHH",new_dataList,idList,metadataNameList,BEST,title1+title3+title4,condition); 
+		}
+	    }
+	    
+	    //-----------------------------------------------------------------------HSSH
+	    detailList = getHideFullDocumentDetail((ArrayList)xList1.clone(),(String[])idListCopy1.clone(),(String[])metadataNameListCopy1.clone());
+	    if(detailList.size()==0){
+		generateEmptyGraph("HSSH",WORST,condition);
+		generateEmptyGraph("HSSH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+		
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("HSSH",new_dataList,idList,metadataNameList,WORST,title1+title4,condition);
+		WriteHTML("HSSH",new_dataList,idList,metadataNameList,BEST,title1+title4,condition);
+	    }
+	}
+	
+	//---------------------------------------------------------------------------------------------------------HH
+	ArrayList xList2 = new ArrayList(); 
+	String[] idListCopy2;
+	String[] metadataNameListCopy2;
+	
+	//-----------------------------------------------------------------------HHSS
+	detailList = getHideEmptyMetadataDetail(dataList,id,metadataName);
+	if(detailList.size()==0){
+	    generateEmptyGraph("HHSS",WORST,condition);
+	    generateEmptyGraph("HHSS",BEST,condition);
+	    
+	    generateEmptyGraph("HHHS",WORST,condition);
+	    generateEmptyGraph("HHHS",BEST,condition);
+		
+	    generateEmptyGraph("HHHH",WORST,condition);
+	    generateEmptyGraph("HHHH",BEST,condition);
+		
+	    generateEmptyGraph("HHSH",WORST,condition);
+	    generateEmptyGraph("HHSH",BEST,condition);
+	}
+	else{
+	    alist = (ArrayList)detailList.get(0);
+	    idList = (String[])detailList.get(1);
+	    metadataNameList = (String[])detailList.get(2);
+	    
+	    detailList = getHideFullMetadataDetail(alist,idList,metadataNameList);
+	    
+	    if(detailList.size()==0){
+		generateEmptyGraph("HHSS",WORST,condition);
+		generateEmptyGraph("HHSS",BEST,condition);
+			
+		generateEmptyGraph("HHHS",WORST,condition);
+		generateEmptyGraph("HHHS",BEST,condition);
+		
+		generateEmptyGraph("HHHH",WORST,condition);
+		generateEmptyGraph("HHHH",BEST,condition);
+			
+		generateEmptyGraph("HHSH",WORST,condition);
+		generateEmptyGraph("HHSH",BEST,condition);
+	    }
+	    else{
+		alist = (ArrayList)detailList.get(0);
+		idList = (String[])detailList.get(1);
+		metadataNameList = (String[])detailList.get(2);
+		
+		xList2 = (ArrayList)alist.clone();
+		idListCopy2 = (String[]) idList.clone();
+		metadataNameListCopy2 = (String[])metadataNameList.clone();
+		
+		new_dataList = transformDataList(alist,idList);
+		WriteHTML("HHSS",new_dataList,idList,metadataNameList,WORST,title1+title2,condition);
+		WriteHTML("HHSS",new_dataList,idList,metadataNameList,BEST,title1+title2,condition);
+		
+		//---------------------------------------------------------------------- HHSH	
+		detailList = getHideFullDocumentDetail((ArrayList)xList2.clone(),(String[])idListCopy2.clone(),(String[])metadataNameListCopy2.clone());
+		if(detailList.size()==0){
+		    generateEmptyGraph("HHSH",WORST,condition);
+		    generateEmptyGraph("HHSH",BEST,condition);
+		}
+		else{
+		    alist = (ArrayList)detailList.get(0);
+		    idList = (String[])detailList.get(1);
+		    metadataNameList = (String[])detailList.get(2);
+		    
+		    new_dataList = transformDataList(alist,idList);
+		    
+		    WriteHTML("HHSH",new_dataList,idList,metadataNameList,WORST,title1+title2+title4,condition);
+		    WriteHTML("HHSH",new_dataList,idList,metadataNameList,BEST,title1+title2+title4,condition);
+		}
+		//----------------------------------------------------------------------HHHS	
+		detailList = getHideEmptyDocumentDetail((ArrayList)xList2.clone(),(String[])idListCopy2.clone(),(String[])metadataNameListCopy2.clone());
+		if(detailList.size()==0){
+		    generateEmptyGraph("HHHS",WORST,condition);
+		    generateEmptyGraph("HHHS",BEST,condition);
+		    
+		    generateEmptyGraph("HHHH",WORST,condition);
+		    generateEmptyGraph("HHHH",BEST,condition);
+		}
+		else{
+		    alist = (ArrayList)detailList.get(0);
+		    idList = (String[])detailList.get(1);
+		    metadataNameList = (String[])detailList.get(2);
+	
+		    new_dataList = transformDataList(alist,idList);
+		    
+		    WriteHTML("HHHS",new_dataList,idList,metadataNameList,WORST,title1+title2+title3,condition);
+		    WriteHTML("HHHS",new_dataList,idList,metadataNameList,BEST,title1+title2+title3,condition);
+		    
+		    //----------------------------------------------------------------------HHHH
+		    detailList = getHideFullDocumentDetail((ArrayList)alist.clone(),(String[])idList.clone(),(String[])metadataNameList.clone());
+		    if(detailList.size()==0){
+			generateEmptyGraph("HHHH",WORST,condition);
+			generateEmptyGraph("HHHH",BEST,condition);
+		    }
+		    else{
+			alist = (ArrayList)detailList.get(0);
+			idList = (String[])detailList.get(1);
+			metadataNameList = (String[])detailList.get(2);
+			
+			new_dataList = transformDataList(alist,idList);
+			
+			WriteHTML("HHHH",new_dataList,idList,metadataNameList,WORST,title1+title2+title3+title4,condition);
+			WriteHTML("HHHH",new_dataList,idList,metadataNameList,BEST,title1+title2+title3+title4,condition);
+		    }
+		}
+	    }
+	}
+    }
+    
+    private ArrayList transformDataList(ArrayList list, String[] ids){
+    	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+	
+	ArrayList wholeList = new ArrayList();
+	
+	for(int i = 0; i< ids.length; i++){
+	    ArrayList idList = new ArrayList();
+	    for(int j = 0; j<list.size();j++){
+		int[] datarows = (int[]) list.get(j);
+		idList.add(new Integer (datarows[i]));
+	    }
+	    wholeList.add(idList);
+	}
+	
+	for(int i = 0; i< wholeList.size(); i++){
+	    ArrayList alist = (ArrayList)wholeList.get(i);
+	    int[] rows = new int[alist.size()];
+	    for(int j = 0; j< alist.size(); j++){
+		rows[j] = ((Integer)alist.get(j)).intValue();
+	    }
+	    wholeList.remove(i);
+	    wholeList.add(i,rows);
+	}
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    
+	return wholeList;
+	
+    }
+    
+    private void generateEmptyGraph(String fileName,String condition,String suffix){
+        
+	printWriter.write("<!-- generating all possible page-->");
+        printWriter.flush();
+        
+	String suf = suffix;
+        if(!suffix.equalsIgnoreCase("dublin")){
+        	suf = "other";
+        }
+	
+    	String cases = condition; 
+    	String collectionFolder = mds.getCollectionName();
+	
+	try{
+	    FileWriter fstream = new FileWriter(destination+suf+"_"+fileName+"_"+cases+".html");
+	    BufferedWriter out = new BufferedWriter(fstream);
+	    out.write("<!-- This comment keeps IE6/7 in the reliable quirks mode -->\r\n");
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\r\n");
+	    out.write("<html>\r\n");
+	    out.write("<head>\r\n<title> No Available Chart</title>\r\n");   
+	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<link rel=\"stylesheet\" href=\"../script/doby2.css\" type=\"text/css\"/>\r\n");
+	    out.write("</head>\r\n");
+	    out.write("<p><a href=\" Overall.html \">Summary</a></p>");
+	    out.write("<body>\r\n");
+	    out.write("<p>No data available to render chart.</p>\r\n");
+	    out.write("<p>Reason: Document number is zero or Metadata element number is zero </p>\r\n");
+	    out.write("</body></html>\r\n");
+	    out.close();
+	    fstream.close();
+	    
+    	}catch(Exception e){}
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    }
+    
+    private void WriteHTML(String fileName, ArrayList dataset, String[] ids, String[] metadataName, String condition, String title, String suffix){
+ 
+    	int blueDot = 0;
+    	String cases = condition; 
+        String suf = suffix;
+        if(!suffix.equalsIgnoreCase("dublin")){
+        	suf = "other";
+        }
+	try{
+	    printWriter.write("<!-- generating all possible page-->");
+	    printWriter.flush();
+
+	    ArrayList tempList = mds.getMetadataNameList();
+	    ArrayList urlIDList;
+	    if(tempList.contains("dc.Identifier") && linkIdentifier.equals("dc.Identifier")){
+	    	urlIDList = dm.getURLMap("dc.Identifier");
+	    }
+	    else if(tempList.contains("nzir_internal.Link") && linkIdentifier.equals("nzir_internal.Link")){
+	    	urlIDList = dm.getURLMap("nzir_internal.Link");
+	    }
+	    else{
+	    	urlIDList = new ArrayList();
+	    }
+
+	    FileWriter fstream = new FileWriter(destination+suf+"_"+fileName+"_"+cases+".html");
+	    BufferedWriter out = new BufferedWriter(fstream);
+
+	    out.write("<!-- This comment keeps IE6/7 in the reliable quirks mode -->\r\n");
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\r\n");
+	    out.write("<html>\r\n");
+	    out.write("<head>\r\n<title>"+title+"</title>\r\n");
+	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<link rel=\"stylesheet\" href=\"../script/doby2.css\" type=\"text/css\"/>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"../script/getInfomation.js\"></script>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/yahoo/yahoo-min.js\"></script>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/event/event-min.js\"></script>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/connection/connection-min.js\"></script>\r\n");
+	    out.write("</head>\r\n");
+	    out.write ("<p><a href=\" Overall.html \">Summary</a></p>");
+	    out.write("<body id=\""+mds.getCollectionName()+"\" onLoad=\"reconfig()\">\r\n");
+	    out.write("<div id=\"container\">\r\n");
+	    out.write("<div class=\"tableContainer\">\r\n");
+	    out.write ("<table cellspacing=\"0\">\r\n");   
+	    out.write ("<thead>\r\n");
+	    out.write ("<tr>\r\n");
+	    out.write("<td class=\"qh\">Info\r\n");
+	    out.write("<td class=\"qh\">URL\r\n");
+	    
+	    for(int a = 0; a< metadataName.length; a++){
+	    	out.write("<td>"+ metadataName[a]+"\r\n");
+	    }
+
+	    out.write ("</thead>\r\n<tfoot>\r\n<tr>\r\n");
+	    out.write("<td class=\"qh\">&nbsp;\r\n");
+	    out.write("<td class=\"qh\">&nbsp;\r\n");
+	    
+	    for(int a = 0; a< metadataName.length; a++){
+	    	out.write("<td>"+ dm.Mean(metadataName[a])+"%\r\n");
+	    }
+	    
+	    out.write ("</tfoot>\r\n<tbody>\r\n");
+	    
+	    if(cases.equals("normal")){
+	    	for(int i = ids.length; i>0; i--){
+		    out.write ("<tr>");
+		    int[] datarows = (int[])dataset.get(i-1);
+		    
+		    for(int j = 0; j<datarows.length; j++){
+			
+			if(datarows[j]==1){
+			    out.write ("<td class=\"b\">");
+			    blueDot++;
+			}
+			else {
+			    out.write ("<td>&nbsp;");
+			}
+		    }
+	    	}
+	    }
+	    else{
+	    	ArrayList idList = new ArrayList();
+	    	for(int i = 0; i<ids.length;i++){
+	    		idList.add(ids[i]);
+	    	}
+	    	
+	    	
+	    	HashMap hp = new HashMap();
+	    	
+	    	for(int i = ids.length; i>0; i--){
+		    String idValue = ids[i-1];
+		    int dots = 0;
+		    int[] datarows = (int[])dataset.get(i-1);
+		    for(int j = 0; j<datarows.length; j++){
+			if(datarows[j]==1){
+			    dots++;
+			}	
+		    }
+		    hp.put(idValue,new Integer(dots));
+		    
+	    	}
+	    	ArrayList alist = dm.sortMap(hp);
+	    	String[] idIntValue = new String[alist.size()]; 
+		for(int i = 0; i< alist.size(); i++){
+		    Map.Entry entry = (Map.Entry) alist.get(i);
+		    String idElement = ((String) entry.getKey());
+		    idIntValue[i] = idElement;
+		}
+		
+		if(cases.equals("worst")){
+
+		    for(int i = 0; i<idIntValue.length; i++){
+			
+		        printWriter.write("<!-- generating all possible page -->");
+		        printWriter.flush(); 
+		        
+	    		int value = idList.indexOf(idIntValue[i]);
+	    		int[] datarows = (int[])dataset.get(value);
+
+	    		out.write ("<tr id=\""+idIntValue[i].substring(4)+"\">\r\n");
+	    		out.write("<td class=\"E\" onclick=\"GD(this)\">&nbsp;\r\n");
+
+	    		if(urlIDList.contains(idIntValue[i])){
+			    out.write("<td class=\"qh\" onclick=\"GX(this)\"><span title=\"open URL in new window\">open</span>\r\n");
+	    		}
+	    		else{
+	    			out.write("<td>\r\n");
+	    		}
+			
+	    		for(int j = 0; j<datarows.length; j++){
+		    
+			    if(datarows[j]==1){
+				out.write ("<td class=\"b\">\r\n");
+				blueDot++;
+			    }
+			    else {
+				out.write ("<td class=\"w\">\r\n");
+			    }
+	    		}
+		    }
+		}
+		
+		if(cases.equals("best")){
+		    
+		    for(int i = idIntValue.length; i>0; i--){
+			printWriter.write("<!-- generating all possible page -->");
+		        printWriter.flush();
+		        
+	    		int value = idList.indexOf(idIntValue[i-1]);
+	    		int[] datarows = (int[])dataset.get(value);
+
+	    		out.write ("<tr id=\""+idIntValue[i-1].substring(4)+"\">\r\n");
+	    		out.write("<td class=\"E\" onclick=\"GD(this)\">&nbsp;\r\n");
+	    		
+	    		if(urlIDList.contains(idIntValue[i-1])){
+			    out.write("<td class=\"qh\" onclick=\"GX(this)\"><span title=\"open URL in new window\">open</span>\r\n");
+	    		}
+	    		else{
+			    out.write("<td>\r\n");
+	    		}
+	    		
+	    		for(int j = 0; j<datarows.length; j++){
+			    
+			    if(datarows[j]==1){
+				out.write ("<td class=\"b\">\r\n");
+				blueDot++;
+			    }
+			    else {
+				out.write ("<td class=\"w\">\r\n");
+			    }
+	    		}
+		    }
+		}
+	    }
+	    
+	    out.write ("</tbody>\r\n");
+	    out.write ("</table>\r\n</div></div>\r\n");
+
+	    int t1 = ids.length;
+	    int t3 = metadataName.length;
+	    int t4 = t1*t3;
+	    
+	    out.write("<table>\r\n<tbody class=\"table1\">");
+	    out.write("<tr>\r\n");
+	    
+	    HashMap tempMap = mds.getMetadataSetMap();
+	    MetadataSet ms = (MetadataSet)tempMap.get(suffix);
+	    
+	    out.write("<td class=\"bfont\">This subset shows "+t1+" out of "+mds.getDocNum()+" documents");
+	    out.write("<td class=\"bfont\">"+blueDot+" out of "+(mds.getDocNum()*t3)+" metadata items  are defined");
+	    out.write("<tr>\r\n");
+	    out.write("<td class=\"bfont\">This subset shows "+t3+" out of "+(ms.getIndexsList().size())+" metadata elements");
+	    out.write("<td class=\"bfont\">Subset completeness: "+dm.round((double)(blueDot*100/t4),5)+"%");
+	    out.write("</table>\r\n");
+	    out.write ("<p> <a href=\"Overall.html \">Summary</a></p>");
+	    out.write ("</body></html>");
+	    //Close the output stream
+	    
+	    out.close();
+	    fstream.close();
+	}catch (Exception e){
+	    e.printStackTrace(printWriter);
+	}
+	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    }
+
+    
+    public void generateOverallStatisticsPage(HashMap MetadataSetMap){
+
+    	String fileName = "Overall";
+	ArrayList wholeList = new ArrayList();
+    	HashMap hp = mds.getMetadataSetMap();
+    	Collection c = hp.values();
+    	Iterator i = c.iterator();
+    	int counter = 0;
+    	String[][] MetadataData = new String[c.size()][2];
+    	
+	while(i.hasNext()){
+	    MetadataSet mds  = (MetadataSet)i.next();
+	    wholeList.add(mds);
+	    ArrayList newMDS  = new ArrayList();
+	    newMDS.add(mds);
+    		
+	    MetadataData[counter][0] = mds.getName();
+	    MetadataData[counter][1] = dm.getSingleMetadataSetCompleteness(newMDS)+"%";
+
+	    counter++;
+    	}
+
+    	try{
+    	    printWriter.write("<!-- generating all possible page-->");
+    	    printWriter.flush();
+    	    
+	    String str = sdf.format(cl.getTime());
+	    String collectionFolder = mds.getCollectionName();
+
+	    FileWriter fstream = new FileWriter(destination+fileName+".html");
+    	    BufferedWriter out = new BufferedWriter(fstream);
+    	    
+    	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+    	    out.write("<html>\r\n");
+    	    out.write("<head><title>Summary</title>\r\n");
+    	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<style type=\"text/css\">\r\n");
+    	    out.write(".tam{height:20; text-align:right}\r\n");
+    	    out.write("th{height:20; text-align:left}\r\n");
+    	    out.write("body{font-family: Arial;}\r\n");
+    	    out.write("</style>\r\n");    	    
+    	    out.write("<script type=\"text/javascript\" src=\"../script/status3.js\"></script>\r\n");
+    	    out.write("</head>\r\n");
+	    out.write("<body>\r\n");
+    	    out.write ("<td> <div style=\"float:left;\"> <a href=\"http://"+hostName+":"+port_number+"/greenstone3/mat\">Mat Home</a></div>");
+    	    out.write("<div style=\"float:right;\"><a href=\"http://chnm.gmu.edu/tools/surveys/4386/\">Please send feedback about the Mat tool</a></div></td><br>");
+    	    out.write("<h3 align=\"center\">Summary</h3>\r\n");
+    	    out.write("<table border=1 align=\"center\">\r\n");
+    	    out.write("<colgroup width=\"300\" span=\"2\">\r\n");
+    	    out.write("<tr>\r\n");
+    	    out.write("<th>OAI URL: </th>\r\n");
+	    
+    	    if(mds.getOAIURL().length()>50){
+    	    	out.write("<td class=\"tam\"><font size=\"2px\"><a href=\""+mds.getOAIURL()+"\">"+mds.getOAIURL().substring(0,47)+"....</a></font>\r\n");
+    	    }
+    	    else{
+    	    	out.write("<td class=\"tam\"><font size=\"2px\"><a href=\""+mds.getOAIURL()+"\">"+mds.getOAIURL()+"</a></font>\r\n");
+    	    }
+	    
+    	    out.write("</tr>\r\n");
+    	    out.write("<tr>\r\n");
+    	    out.write("<th>Number of Records:\r\n");
+    	    out.write("<td class=\"tam\">"+mds.getDocNum()+"\r\n");
+    	    out.write("</tr>\r\n");
+	    out.write("<table border=1 align=\"center\">\r\n");
+    	    out.write("<colgroup width=\"300\" span=\"2\">\r\n");
+    	    out.write("<tr>\r\n");
+    	    out.write("<th>Metadata:\r\n");
+    	    out.write("<td class=\"tam\"><b>Completeness</b>\r\n");
+    	    out.write("</tr><br>\r\n");
+
+    	    for(int a = 0; a<MetadataData.length; a++){
+    	    	out.write ("<tr>\r\n");
+    	    	if(MetadataData[a][0].equalsIgnoreCase("dublin")){
+		    out.write ("<th><a href =\""+MetadataData[a][0]+".html\">"+metadataSet1+"</a>");
+    	    	}
+    	    	else if (MetadataData[a][0].equalsIgnoreCase("extracted")){
+		    out.write ("<th><a href =\""+MetadataData[a][0]+".html\">"+metadataSet2+"</a>");
+    	    	}
+    	    	else{
+		    out.write ("<th><a href =\""+MetadataData[a][0]+".html\">"+MetadataData[a][0]+"</a>");
+    	    	}
+		out.write ("<td class=\"tam\"> "+MetadataData[a][1]);
+    	    	out.write ("</tr>\r\n");
+    	    }
+	    
+    	    out.write ("</table>");
+    	    out.write("<br>\r\n");
+    	    out.write("<FORM  name=\"test\" onsubmit=\"checkStatus()\" action=\"\">");
+    	    out.write("<table border=1 align=\"center\">\r\n");
+    	    out.write ("<colgroup width=\"606\" span=\"1\">\r\n");
+    	    out.write("<tr>\r\n<th>Customize Visualization");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"H1\"><INPUT align=\"left\" TYPE=\"checkbox\" ID=\"H1\">Hide Empty Metadata Elements</LABEL>");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"H2\"><INPUT align=\"left\" TYPE=\"checkbox\" ID=\"H2\">Hide Completed Metadata Elements</LABEL>");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"H3\"><INPUT align=\"left\" TYPE=\"checkbox\" ID=\"H3\">Hide Documents with Empty Metadata Elements</LABEL>");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"H4\"><INPUT align=\"left\" TYPE=\"checkbox\" ID=\"H4\">Hide Documents with  Completed Metadata Elements</LABEL>");
+    	    out.write("<tr>\r\n<th>Metadata: ");
+    	        	    
+    	    if(MetadataData.length==1){
+		if(MetadataData[0][0].equalsIgnoreCase("dublin")){
+		    out.write("<tr>\r\n<td><LABEL FOR=\"C1\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C1\" checked> Dublin Core </LABEL>");
+		}
+		else {
+		    out.write("<tr>\r\n<td><LABEL FOR=\"C4\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C4\" checked>"+MetadataData[0][0]+"</LABEL>");
+		}
+    	    }
+    	    else{
+    	    	for(int a = 0; a<MetadataData.length; a++){
+		    if(a==0){
+			if(MetadataData[a][0].equalsIgnoreCase("dublin")){
+			    out.write("<tr>\r\n<td><LABEL FOR=\"C1\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C1\" checked>Dublin Core</LABEL>");
+			}
+			else {
+			    out.write("<tr>\r\n<td><LABEL FOR=\"C4\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C4\" checked>"+MetadataData[a][0]+"</LABEL>");
+			}
+		    }
+		    else{
+			if(MetadataData[a][0].equalsIgnoreCase("dublin")){
+			    out.write("<tr>\r\n<td><LABEL FOR=\"C1\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C1\">"+MetadataData[a][0]+"</LABEL>");
+			}
+			else{
+			    out.write("<tr>\r\n<td><LABEL FOR=\"C4\"><INPUT align=\"left\" TYPE=\"radio\" name = \"col\" ID=\"C4\">"+MetadataData[a][0]+"</LABEL>");
+			}
+		    }
+    	    	}
+    	    }
+    	    
+    	    out.write("<tr>\r\n<th>Order By Completeness : ");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"R1\"><INPUT align=\"left\" TYPE=\"radio\" name = \"order\" ID=\"R1\">Best Case to Worst Case</LABEL>");
+    	    out.write("<tr>\r\n<td><LABEL FOR=\"R2\"><INPUT align=\"left\" TYPE=\"radio\" name = \"order\" ID=\"R2\" checked >Worst Case to Best Case</LABEL>");
+    	    out.write("</table><p align=\"center\"><INPUT TYPE=\"button\" VALUE=\"Show Visualization\" onClick=\"checkStatus()\"> </FORM>");
+    	    out.write("<p align=\"center\">"+str+"</p>");
+    	    out.write("</body></html>");
+    	    
+    	    out.close();
+    	    fstream.close();
+    	}catch (Exception e){
+	    System.err.println("Error: " + e.getMessage());
+    	}
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+    }
+    
+    public void WriteMetadataSetDetailHTML(MetadataSet mdset){
+    
+	String fileName = mdset.getName();
+
+    	try{
+    	    printWriter.write("<!-- generating all possible page-->");
+    	    printWriter.flush();
+    	    
+	    String str = sdf.format(cl.getTime());
+	    String collectionFolder = mds.getCollectionName();
+
+	    FileWriter fstream = new FileWriter(destination+fileName+".html");
+    	    BufferedWriter out = new BufferedWriter(fstream);
+    	    
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+    	    out.write("<html>\r\n");
+    	    out.write("<head>\r\n<title>Metadata Detail</title>\r\n");
+    	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+     	    out.write("<style type=\"text/css\">\r\n");
+    	    out.write("td{height:20;text-align:right;}\r\n");
+    	    out.write("th{height:20;text-align:left;}\r\n");
+    	    out.write("body{font-family: Arial;}\r\n");
+    	    out.write("</style>\r\n");
+    	    out.write( "</head>\r\n");
+    	    out.write ("<body>\r\n");
+    	    out.write ("<td> <div style=\"float:left;\"> <a href=\" Overall.html \">Summary</a></div>");
+    	    out.write("<div style=\"float:right;\"><a href=\"http://chnm.gmu.edu/tools/surveys/4386/\">Please send feedback about the Mat tool</a></div></td><br>");
+	    if(fileName.equalsIgnoreCase("dublin")){
+		out.write ("<h3 align=\"center\">Metadata Detail: "+metadataSet1+"</h3>\r\n");
+	    }
+	    else if (fileName.equalsIgnoreCase("extracted")){
+		out.write ("<h3 align=\"center\">Metadata Detail: "+metadataSet2+"</h3>\r\n");
+	    }
+	    else{
+		out.write ("<h3 align=\"center\">Metadata Detail: "+fileName+"</h3>\r\n");
+	    }
+	    
+	    out.write ("<table border=\"1\" align=\"center\">\r\n");
+	    out.write ("<colgroup width=\"200\" span=\"2\">\r\n");
+	    out.write ("<tr>\r\n");
+	    out.write ("<th>Elements:\r\n");
+	    out.write ("<td><b>Completeness</b>\r\n");
+	    out.write ("</tr>\r\n");
+	    
+	    ArrayList elementList = mdset.getIndexsList();
+	    HashMap hp = new HashMap();
+	    int num = elementList.size();
+	    
+	    for(int i = 0; i<num; i++){
+		String elementName = (String)elementList.get(i);
+		Double elementValue = new Double(dm.Mean(elementName));
+		hp.put(elementName, elementValue);
+	    }
+    		
+	    elementList = new ArrayList();
+	    elementList = dm.sortMap(hp);
+	    num = elementList.size();
+	    
+	    for(int i = 0; i<num; i++){
+		Map.Entry entry = (Map.Entry) elementList.get(i);
+		String elementName = (String) entry.getKey();
+		out.write ("<tr>\r\n");
+		out.write ("<th><a href=\""+ elementName +".html\">"+elementName+"</a>\r\n");
+		out.write ("<td>"+ dm.Mean(elementName)+"%\r\n");
+		out.write ("</tr>\r\n");
+	    }
+    	    
+    	    out.write("</table>\r\n");
+	    out.write ("<p align=\"center\"> <a href=\" Overall.html \">Summary</a></p>");
+    	    out.write("<p align=\"center\">"+str+"</p>");
+	    out.write ("</body></html>\r\n");
+	    
+	    out.close();
+    	    fstream.close();
+    	}catch (Exception e){
+	    e.printStackTrace(printWriter);
+    	}
+    	
+	printWriter.write("<!-- generating all possible page-->");
+	printWriter.flush();
+	
+        }
+    
+    public void generateMetadataElementDetailPage(MetadataSet mds,PrintWriter out){
+    	
+    	ArrayList nameList = mds.getIndexsList();
+    	
+	for(int i = 0; i<nameList.size();i++){
+	    WriteMetadataElementDetailHTML((String)nameList.get(i),mds.getName(),out);
+    	}
+    }
+    
+    public void WriteMetadataElementDetailHTML(String name, String linkName,PrintWriter outx){
+    	
+    	String fileName = name;
+    	
+    	try{
+    	    printWriter.write("<!-- generating all possible page-->");
+    	    printWriter.flush();
+
+	    String collectionFolder = mds.getCollectionName();
+
+	    FileWriter fstream = new FileWriter(destination+fileName+".html");
+    	    BufferedWriter out = new BufferedWriter(fstream);
+
+    	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+    	    out.write ("<html>\r\n");
+    	    out.write("<head>\r\n<title> "+ name +" </title>\r\n");
+    	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+    	    out.write("<style type=\"text/css\">\r\n");
+    	    out.write("td{height:20; text-align:left;}\r\n");
+    	    out.write(".tam{height:20; text-align:center}\r\n");
+    	    out.write("th{height:20; text-align:left;}\r\n");
+    	    out.write("body{font-family: Arial;}\r\n");
+    	    out.write("</style>\r\n");
+    	    out.write("<script type=\"text/javascript\" src=\"http://"+hostName+":"+port_number+"/greenstone3/mat/script/status3.js\"></script>");
+    	    out.write("</head>\r\n");
+	    out.write("<body>\r\n");
+	    out.write ("<td> <div style=\"float:left;\"> <a href=\" Overall.html \">Summary</a></div>");
+    	    out.write("<div style=\"float:right;\"><a href=\"http://chnm.gmu.edu/tools/surveys/4386/\">Please send feedback about the Mat tool</a></div></td>");
+	    out.write("&raquo;");
+
+	    if(linkName.equalsIgnoreCase("dublin")){
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+	    }
+	    else if (linkName.equalsIgnoreCase("extracted")){
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+	    }
+	    else{
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+linkName+")</a>"); 	  
+	    }
+	    
+	    if(name.startsWith("dc.")){
+		int dot = name.lastIndexOf('.');
+		dot++;
+		String nameReplace = name.substring(dot);
+		
+		if(url.containsKey(nameReplace)){
+		    out.write ("<h3 align=\"center\">Metadata Element Detail:<a href=\""+ url.get(nameReplace) +"\">"+ name +"</a></h3>\r\n");
+		}
+		else{
+		    out.write ("<h3 align=\"center\">Metadata Element Detail: "+ name +"</a></h3>\r\n");
+		}
+	    }
+	    else{
+		out.write ("<h3 align=\"center\">Metadata Element Detail: "+ name +"</h3>\r\n");
+	    }
+	        	   
+    	    out.write ("<table border=\"1\" align=\"center\" width=1000>\r\n");
+    	    out.write ("<colgroup width=\"500\" span=\"2\">\r\n");
+	    out.write ("<tr>\r\n");
+    	    out.write ("<td><b>Total Number of Records</b>\r\n");
+    	    out.write ("<td class=\"tam\">"+mds.getDocNum()+"\r\n");
+	    out.write ("</tr>\r\n");
+	    out.write ("<tr>\r\n");
+    	    out.write ("<td><b>Unique Values</b>\r\n");
+    	    out.write ("<td class=\"tam\">"+dm.getDistinctNumber(name)+"\r\n");
+	    out.write ("</tr>\r\n");
+	    out.write ("<tr>\r\n");
+    	    out.write ("<td><b>Total times element used</b>\r\n");
+    	    out.write ("<td class=\"tam\">"+dm.getFrequency(name) +"\r\n");
+	    out.write ("</tr>\r\n");
+	    out.write ("<tr>\r\n");
+    	    out.write ("<td><b>No. of records containing element</b>\r\n");
+    	    out.write ("<td class=\"tam\"> "+dm.getDocumentUsedElement(name)+"\r\n");
+	    out.write ("</tr>\r\n");
+	    out.write ("<tr>\r\n");
+
+    	    double percentage = dm.Mean(name);
+	    
+	    out.write ("<th title = \"The completeness means the arithmetic average \"> <div style=\"float:left;\"> Completeness</div>");
+	    out.write ("<td class=\"tam\"> "+dm.Mean(name) +"%\r\n");
+	    out.write ("</tr>\r\n");
+	    out.write ("<tr>\r\n");
+	    out.write ("<td><b><div style=\"float:left;\">Minimum "+name +" usage in any record</div></b><div style=\"float:right;\"> <a href=\"#\" onClick=\"helpWindow('Minimum','"+name+"')\" style=\"font-size:small;\">What's this?</a></div>\r\n");
+	    out.write ("<td class=\"tam\"> "+dm.getMinRange(name) +"\r\n");
+	    out.write ("</tr>\r\n");
+    		
+	    out.write ("<tr>\r\n");
+	    out.write ("<td><b><div style=\"float:left;\">Maximum "+name +" usage in any record</div></b><div style=\"float:right;\"> <a href=\"#\" onClick=\"helpWindow('Maximum','"+name+"')\" style=\"font-size:small;\">What's this?</a></div>\r\n");
+	    out.write ("<td class=\"tam\"> "+dm.getMaxRange(name) +"\r\n");
+	    out.write ("</tr>\r\n");
+    		
+	    out.write ("<tr>\r\n");
+	    out.write ("<td><b><div style=\"float:left;\">Average "+name +" usage/record</div></b><div style=\"float:right;\"> <a href=\"#\" onClick=\"helpWindow('Average','"+name+"')\" style=\"font-size:small;\">What's this?</a></div>\r\n");
+	    out.write ("<td class=\"tam\"> "+dm.Average(name) +"\r\n");
+	    out.write ("</tr>\r\n");
+    		
+	    out.write ("<tr>\r\n");
+	    out.write ("<td><b><div style=\"float:left;\">Mode of "+name +" usage/record</div></b><div style=\"float:right;\"><a href=\"#\" onClick=\"helpWindow('Mode','"+name+"')\" style=\"font-size:small;\">What's this?</a></div>\r\n");
+	    out.write ("<td class=\"tam\"> "+dm.getMode(name) +"\r\n");
+	    out.write ("</tr>\r\n");
+	    
+	    out.write ("<tr>\r\n");
+	    out.write ("<td><b><div style=\"float:left;\">Coverage of the mode of "+name +" usage/record</div></b><div style=\"float:right;\"><a href=\"#\" onClick=\"helpWindow('Mode Frequency','"+name+"')\" style=\"font-size:small;\">What's this?</a></div>\r\n");
+	    out.write ("<td class=\"tam\">"+dm.ModeFrequency(name) +"%\r\n");
+	    out.write ("</tr>\r\n");
+    		   		
+	    if(mds.getOaiPrefix().equalsIgnoreCase("oai_dc")){
+
+    		HashMap suggestionMap = generateMetadataElementSortList(fileName,"ASCII",linkName,outx);
+    		
+    		if(suggestionMap.size()>0){
+    			boolean status = compareElement(fileName,collectionFolder,suggestionMap,linkName);
+    			if(status){
+			    out.write("<tr><td class=\"tam\"><a href =\""+fileName+"_Suggestion.html\">View Potential Duplicate List</a>");
+			    if(percentage<100){
+				out.write ("<td class=\"tam\"><a href =\""+fileName+"_IncompletedList.html\">Records missing "+fileName+"</a></td>");
+				createIncompletedList(fileName,linkName,collectionFolder,outx);       
+			    }
+			    else{
+				out.write("<td class=\"tam\">No Records Missing "+ fileName);
+			    }
+    			}
+    			else{
+			    out.write("<tr><td class=\"tam\">No Potential Duplicates");
+			    if(percentage<100){
+				out.write ("<td class=\"tam\"><a href =\""+fileName+"_IncompletedList.html\">Records missing "+fileName+"</a></td>");
+				createIncompletedList(fileName,linkName,collectionFolder,outx);       
+			    }
+			    else{
+				out.write("<td class=\"tam\">No Records Missing "+ fileName);
+			    }
+    			}
+    		}
+    		else{
+		    out.write("<tr><td class=\"tam\">No Potential Duplicates");
+		    if(percentage<100){
+			out.write ("<td class=\"tam\"><a href =\""+fileName+"_IncompletedList.html\">Records missing "+fileName+"</a></td>");
+			createIncompletedList(fileName,linkName,collectionFolder,outx);       
+		    }
+		    else{
+			out.write("<td class=\"tam\">No Records Missing "+ fileName);
+		    }
+    		}
+		generateMetadataElementSortList(fileName,"Frequency-based",linkName,outx);
+	    }
+	    else{
+		HashMap suggestionMap = generateMetadataElementSortListNoIdentifier(fileName,"ASCII",linkName,outx);
+		generateMetadataElementSortListNoIdentifier(fileName,"Frequency-based",linkName,outx);
+    	   }
+	    
+	    out.write ("<tr>\r\n");
+	    out.write ("<td class=\"tam\"><a href =\""+fileName+"_Frequency-based.html\">"+"View Full Frequency Sorted list</a>" +
+		       " <td class=\"tam\"><a href =\""+fileName+"_ASCII.html\">"+"View Full ASCII Sorted list</a></td>");
+	    out.write ("</tr>\r\n");
+	    out.write ("</table><br>\r\n");
+	    out.write ("<table border=\"1\" align=\"center\" width=1000>\r\n");
+	    out.write ("<colgroup width=\"500\" span=\"2\">\r\n");
+	    out.write ("<tr>\r\n");
+	    out.write ("<th>ASCII-Based\r\n");
+	    out.write ("<th>First Five\r\n");
+	    out.write ("</tr>\r\n");
+	    
+	    String[] temp = dm.getSortList(name,"ASCII");
+	    String[] temp2 = {"&nbsp;","&nbsp;","&nbsp;","&nbsp;","&nbsp;"};
+	    int length = 0;
+	    if(temp.length>=5){
+		length=5;
+	    }
+	    else if(temp.length<5){
+		length = temp.length;
+	    }
+	    
+	    for(int i =0; i<length; i++){
+		temp2[i] = temp[i];
+	    }
+	    
+	    int x = temp2.length;
+	    for(int a = 0; a<x; a++){
+		out.write ("<tr>\r\n");
+		if(!temp2[a].equals("&nbsp;")){
+		    out.write ("<th>"+(a+1)+"\r\n");
+		}
+		else{
+		    out.write ("<th>&nbsp;\r\n");
+		}
+		if(temp2[a].startsWith("http://")){
+		    if(temp2[a].length()>60){
+			out.write ("<td><a href = \""+temp2[a]+"\">"+temp2[a].substring(0, 60)+"...</a>");
+		    }
+		    else {
+			out.write ("<td><a href = \""+temp2[a]+"\">"+temp2[a]+"</a>");
+		    }
+		}
+		else{
+		    char singleChar = temp2[a].charAt(0);
+		    if(temp2[a].length()>61 ){
+			if(temp2[a].startsWith(" ") && ((int)singleChar!=65279)){
+			    temp2[a] = temp2[a].substring(1);
+			    out.write ("<td>"+spaceLeft+space+spaceRight+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#topFive\"> ... </a>\r\n");
+			}
+			else if (((int)singleChar==65279)){
+			    temp2[a] = temp2[a].substring(1);
+			    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#topFive\"> ... </a>\r\n");
+			}
+			else{
+			    out.write ("<td>"+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#topFive\"> ... </a>\r\n");
+			}
+		    }
+		    else {
+			if(temp2[a].startsWith(" ")&& ((int)singleChar!=65279)){
+			    temp2[a] = temp2[a].substring(1);
+			    out.write ("<td>"+spaceLeft+space+spaceRight+temp2[a]+"\r\n");
+			}
+			else if (((int)singleChar==65279)){
+			    temp2[a] = temp2[a].substring(1);
+			    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp2[a]+"\r\n");
+			}
+			else{
+			    out.write ("<td>"+temp2[a]+"\r\n");
+			}
+		    }
+		}
+		out.write ("</tr>\r\n");
+	    }
+	    
+	    String[] temp3 ={"&nbsp;","&nbsp;","&nbsp;","&nbsp;","&nbsp;"};
+	    length = 0;
+	    int start = temp.length;
+	    
+	    if(temp.length>=5){
+		length= 5;
+	    }
+	    else if(temp.length<5){
+		length = temp.length;
+	    }
+                
+	    for(int i = length; i>0; i--){
+		temp3[i-1] = temp[start-1];
+		start--;
+	    }
+	    
+	    out.write ("<tr>\r\n");
+	    out.write ("<th>......\r\n");
+	    out.write ("<th>Last Five\r\n");
+	    out.write ("</tr>\r\n");
+	    
+	    int counter = temp.length;
+	    start = temp.length;
+	    x = temp3.length;
+	    
+	    for(int a = 0; a<x; a++){
+		out.write ("<tr>\r\n");
+		
+		if(!temp3[a].equals("&nbsp;")){
+		    out.write ("<th>"+(start-length+1+a));
+		}
+		else{
+		    out.write ("<th>&nbsp;");
+		}
+		if(temp3[a].startsWith("http://")){
+		    if(temp3[a].length()>60){
+			out.write ("<td><a href = \""+temp3[a]+"\">"+temp3[a].substring(0, 60)+"...</a>");
+		    }
+		    else {
+			out.write ("<td><a href = \""+temp3[a]+"\">"+temp3[a]+"</a>");
+		    }
+		}
+		else{
+		    char singleChar = temp3[a].charAt(0);
+		    if(temp3[a].length()>61){
+			if(temp3[a].startsWith(" ") && (int)singleChar!=65279){
+			    temp3[a] = temp3[a].substring(1);
+			    out.write ("<td>"+spaceLeft+space+spaceRight+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#lastFive\"> ... </a>\r\n");
+			}
+			else if((int)singleChar==65279){
+			    temp3[a] = temp3[a].substring(1);
+			    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#lastFive\"> ... </a>\r\n");
+			    
+			}
+			else{
+			    out.write ("<td>"+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_ASCII.html#lastFive\"> ... </a>\r\n");
+			}
+		    }
+		    else{
+			if(temp3[a].startsWith(" ")&& (int)singleChar!=65279){
+			    temp3[a] = temp3[a].substring(1);
+			    out.write ("<td>"+spaceLeft+space+spaceRight+temp3[a]+"\r\n");
+			}
+			else if((int)singleChar==65279){
+			    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp3[a]+"\r\n");
+			}
+			else{
+			    out.write ("<td>"+temp3[a]+"\r\n");
+			}
+		    }
+		    out.write ("</tr>\r\n");
+		}
+	    }
+	    out.write ("</table><br>\r\n");
+	    out.write ("<table border=\"1\" align=\"center\" width=1000>\r\n");
+	    out.write ("<colgroup width=\"500\" span=\"2\">\r\n");
+	    out.write ("<tr>\r\n");
+	    out.write ("<th>Frequency-Based:\r\n");
+	    out.write ("<th>First Five\r\n");
+	    out.write ("</tr>\r\n");
+	    
+	    HashMap xMap = dm.getDistinctValueMap(name);
+	    temp = dm.getSortList(name,"Frequency-based");
+	    temp2 = new String[] {"&nbsp;","&nbsp;","&nbsp;","&nbsp;","&nbsp;"};
+	    length = 0;
+	    
+	    if(temp.length>=5){
+		length=5;
+	    }
+	    else if(temp.length<5){length = temp.length;}
+            
+	    for(int i =0; i<length; i++){
+		temp2[i] = temp[i];
+	    }
+	    
+	    ArrayList tempList = new ArrayList();
+	    
+	    for(int i =0; i<length; i++){
+		if(((Integer)xMap.get(temp2[i])).intValue()==1){
+		    tempList.add(temp2[i]);
+		}
+	    }
+	    
+	    x = temp2.length;
+	    for(int a = 0; a<x; a++){
+		out.write ("<tr>\r\n");
+		
+		if(!temp2[a].equals("&nbsp;")){
+		    char singleChar = temp2[a].charAt(0);
+		    out.write ("<th>"+(a+1)+". (No. of occurrences: "+((Integer)xMap.get(temp2[a])).toString()+")\r\n");
+		    if(temp2[a].startsWith("http://")  && (int)singleChar != 65279){
+			if(temp2[a].length()>61){
+			    out.write ("<td><a href = \""+temp2[a]+"\">"+temp2[a].substring(0, 60)+"...</a>");
+			}
+			else {
+			    out.write ("<td><a href = \""+temp2[a]+"\">"+temp2[a]+"</a>");
+			}
+		    }
+		    
+		    else{
+			if(temp2[a].length()>61){
+			    
+			    if(temp2[a].startsWith(" ")){
+				temp2[a] = temp2[a].substring(1);	
+				out.write ("<td>"+spaceLeft+space+spaceRight+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#topFive\"> ... </a>\r\n");
+			    }
+			    
+			    else if((int)singleChar == 65279){
+				temp2[a] = temp2[a].substring(1);	
+				out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#topFive\"> ... </a>\r\n");
+			    }
+			    else{
+				out.write ("<td>"+temp2[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#topFive\"> ... </a>\r\n");
+			    }
+			    
+			}
+			else{
+			    if(temp2[a].startsWith(" ")){
+				temp2[a] = temp2[a].substring(1);
+				out.write ("<td>"+spaceLeft+space+spaceRight+temp2[a]+"\r\n");
+			    }
+			    else if((int)singleChar == 65279){
+				temp2[a] = temp2[a].substring(1);	
+				out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp2[a]+"\r\n");
+			    }
+			    else{
+				out.write ("<td>"+temp2[a]+"\r\n");
+			    }
+			}
+		    }
+		}
+		else{
+		    out.write ("<th>&nbsp;\r\n");
+		    out.write ("<td>\r\n");
+		}
+		out.write ("</tr>\r\n");
+	    }
+
+	    temp3 = new String[]{"&nbsp;","&nbsp;","&nbsp;","&nbsp;","&nbsp;"};
+	    length = 0;
+	    start = temp.length;
+	    if(temp.length>=5){length= 5;}
+	    else if(temp.length<5){length = temp.length;}
+            
+	    for(int i = length; i>0; i--){
+		temp3[i-1] = temp[start-1];
+		start--;
+	    }
+
+    		out.write ("<tr>\r\n");
+    		out.write ("<th>......\r\n");
+    		out.write ("<th>Last Five\r\n");
+    		out.write ("</tr>\r\n");
+    		
+    		x = temp3.length;
+    		start = temp.length;
+        	for(int a = 0; a<x; a++){
+		    out.write ("<tr>\r\n");
+		    if(!temp3[a].equals("&nbsp;")){
+
+        		
+			out.write ("<th>"+(start-length+1+a)+". (No. of occurrences: "+((Integer)xMap.get(temp3[a])).toString()+")\r\n");
+        		
+	    	char singleChar = temp3[a].charAt(0);
+            		if(temp3[a].startsWith("http://")){
+			    if(temp3[a].length()>60){
+				out.write ("<td><a href = \""+temp3[a]+"\">"+temp3[a].substring(0, 60)+"...</a>");          			
+			    }
+			    else {
+				out.write ("<td><a href = \""+temp3[a]+"\">"+temp3[a]+"</a>");
+			    }
+            		}
+            		else{
+			    if(temp3[a].length()>61){
+            			
+				if(temp3[a].startsWith(" ")){
+				    temp3[a] = temp3[a].substring(1);
+				    out.write ("<td>"+spaceLeft+space+spaceRight+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#lastFive\"> ... </a>\r\n");
+				}
+				else if((int)singleChar == 65279){
+					temp3[a] = temp3[a].substring(1);
+				    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#lastFive\"> ... </a>\r\n");
+				
+				}
+				else{
+				    out.write ("<td>"+temp3[a].substring(0, 60)+"<a href=\""+fileName+"_Frequency-based.html#lastFive\"> ... </a>\r\n");
+				}
+            			
+			    }
+			    else{
+				if(temp3[a].startsWith(" ")){
+				    out.write ("<td>"+spaceLeft+space+spaceRight+temp3[a]+"\r\n");
+				}
+				
+				else if((int)singleChar == 65279){
+					temp3[a] = temp3[a].substring(1);
+				    out.write ("<td>"+spaceLeft+oddChar+spaceRight+temp3[a]+"\r\n");
+				
+				}
+				
+				else{
+				    out.write ("<td>"+temp3[a]+"\r\n");
+				}
+			    }
+            		}
+            		
+        		}
+		    else{
+			//out.write ("<th>"+(a+1)+"\r\n");
+			out.write ("<th>&nbsp;\r\n");
+			out.write ("<th>\r\n");
+		    }
+		    out.write ("</tr>\r\n");
+        	}
+    		
+    		
+    	    out.write ("</table>\r\n");
+
+    	    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    	    out.write ("<p> <a href=\" Overall.html \">Summary</a>");
+    	    out.write ("&raquo;");
+    	    
+	    if(linkName.equalsIgnoreCase("dublin")){
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+
+	    }
+	    else if (linkName.equalsIgnoreCase("extracted")){
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+
+	    }
+	    else{
+		out.write ("<a href=\""+linkName+".html\">Metadata Detail ("+linkName+")</a>"); 	  
+	    }
+    	    //out.write(str+"</p>");
+    	    out.write ("</body></html>\r\n");
+
+    	    //Close the output stream
+    	    out.close();
+    	    fstream.close();
+    	}catch (Exception e){
+
+	    e.printStackTrace(printWriter);
+    	}
+    	
+	    printWriter.write("<!-- generating all possible page-->");
+	    printWriter.flush();
+    	//System.out.println("detail ends");
+    	
+    }
+    
+    
+    private HashMap generateMetadataElementSortList(String title, String sort, String metadataSetName,PrintWriter outx){
+
+    	String fileName = title+"_"+sort;
+    	String collectionFolder = mds.getCollectionName();
+
+    	String IDENTIFIER = linkIdentifier;
+    	SearchLink sl = new SearchLink(mds.StatsDirectory);
+    	HashMap suggestionMap = new HashMap();
+    	
+    	HashMap valueMap = sl.createValueMap(title,collectionFolder);
+
+    	HashMap linkMap = sl.createLinkMap(IDENTIFIER,collectionFolder);
+    	HashMap internalIDMap = dm.getInternalIdentifier(title);	
+    	boolean status = false;
+ 
+    	try{
+    	    printWriter.write("<!-- generating all possible page-->");
+    	    printWriter.flush();
+    	
+	    FileWriter fstream = new FileWriter(destination+fileName+".html");
+	    BufferedWriter out = new BufferedWriter(fstream);
+	    
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+	    out.write ("<html>\r\n");
+	    out.write("<head>\r\n<title>Metadata Element Sort List</title>\r\n");
+	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"../script/getInfomation.js\"></script>\r\n");   	    
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/yahoo/yahoo-min.js\"></script>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/event/event-min.js\"></script>\r\n");
+	    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/connection/connection-min.js\"></script>\r\n");
+
+	    out.write("<style type=\"text/css\">\r\n");
+	    out.write("td{height:20; text-align:left;padding-left:5px;}\r\n");
+	    out.write(".tam{height:20; text-align:center}\r\n");
+	    out.write("th{height:20; text-align:center;}\r\n");
+	    out.write("body{font-family: Arial;}\r\n");
+	    out.write("</style>\r\n");
+	    out.write("</head><body id=\""+collectionFolder+"\">\r\n");
+    	
+	    out.write("<td><div style=\"float:left;\"><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    ////////////////////////////////////////////////////////////////////////////////////
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+	    }
+	    else if (metadataSetName.equalsIgnoreCase("extracted")){
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+	    }
+	    else{
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+	    }
+	    out.write("&raquo;");
+	    out.write("<a href=\""+title+".html\">"+title+"</a></div>"); 
+	    out.write("<div style=\"float:right;\"><a href=\"http://chnm.gmu.edu/tools/surveys/4386/\">Please send feedback about the Mat tool</a></div></td><br>");
+ 	  
+	  	
+	    String[] list = dm.getSortList(title, sort);
+    
+       	
+	    if(list.length>=1){
+    		
+    		
+    		if(sort.equals("ASCII")){
+    			
+            	out.write("<table border=\"1\" align=\"center\" width=\"800\">\r\n");
+            	out.write ("<h2 align=\"center\">"+ title+"</h2>\r\n");      	
+            	out.write ("<th class=\"tam\">ASCII Sort\r\n");
+            	out.write ("<th class=\"tam\">Element Values\r\n");
+            	out.write ("<th class=\"tam\">Source Documents\r\n");   
+            	out.write ("<th class=\"tam\">Internal Link\r\n"); 
+            	
+		out.write ("<a name='topFive'>\r\n");
+        		
+		int counter = 0;
+
+		for(int i = 0; i<list.length; i++){
+		    if(list.length<=5 && i==0){
+			out.write ("<a name='lastFive'>\r\n");
+		    }
+		    else if((list.length>5) && (list.length-5==i)){
+			out.write ("<a name='lastFive'>\r\n");
+		    }
+		    
+		    
+        	InternalLink il= (InternalLink)internalIDMap.get(list[i]);
+        	ArrayList alist2 = il.retrieveList();
+        	String id = (String)alist2.get(0);
+        	id = id.substring(4);
+        	out.write("<tr id=\""+id+"\" >\r\n");
+
+    				
+		    if(list[i].length()>=201){
+			if(list[i].startsWith("http://")){
+
+				if(title.equals(IDENTIFIER)){
+					if(list[i].length()>=100){
+						out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+	
+					}
+					else{
+						out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+						out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+				}
+				else{
+					ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+					
+					if(list[i].length()>=100){
+						String url = (String)alist.get(0);
+						if(alist.size()==1){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+							if(url.startsWith("http://")){
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+							}else{
+								out.write("<td>Source\r\n");
+							}
+						}
+						else if(alist.size()>1){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+							if(url.startsWith("http://")){
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+							}else{
+								out.write("<td>Source\r\n");
+							}
+
+						}
+						else{
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td>No Source Available\r\n");	
+						}
+						suggestionMap.put(list[i], url);
+					}
+					else{
+
+						String url = (String)alist.get(0);
+						if(alist.size()==1){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							//out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+							if(url.startsWith("http://")){
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+							}else{
+								out.write("<td>Source\r\n");
+							}
+						}
+						else if(alist.size()>1){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							//out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");	
+							if(url.startsWith("http://")){
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+							}else{
+								out.write("<td>Source\r\n");
+							}
+						}
+						else{
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							out.write("<td>No Source Available\r\n");	
+						}
+						suggestionMap.put(list[i], url);
+					}
+				}
+
+				
+			   
+			}
+			else{
+
+				 ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+				    
+			    //////////////////////////////////////
+    						
+			    if(list[i].startsWith(" ")){
+			    String elements = list[i];
+				list[i] = list[i].substring(1);
+    				
+				if(alist.size()==1){
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i].substring(0, 200)+"...<td>"+
+					      "<a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+				    		suggestionMap.put(elements, (String)alist.get(0));
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i].substring(0, 200)+"...<td>"+
+						      "<a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+				    		suggestionMap.put(elements, (String)alist.get(0));
+				}
+				else{
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i].substring(0, 200)+"...<td>No Source Available \r\n");
+					suggestionMap.put(elements, "No Source Available");
+				}
+			    }
+			    else{
+    				
+				if(alist.size()==1){
+				    out.write("<td>"+(counter+1)+"<td>"+list[i].substring(0, 200)+"...<td>" +
+					      "<a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+				    suggestionMap.put(list[i], (String)alist.get(0));
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+(counter+1)+"<td>"+list[i].substring(0, 200)+"...<td>" +
+						      "<a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+				    suggestionMap.put(list[i], (String)alist.get(0));
+				}	
+				else{
+				    out.write("<td>"+(counter+1)+"<td>"+list[i].substring(0, 200)+"... <td>No Source Available\r\n");
+					suggestionMap.put(list[i],"No Source Available");
+				}
+			    }
+			}
+    			
+		    }
+		    else{
+				if(list[i].startsWith("http://")){
+
+					if(title.equals(IDENTIFIER)){
+						if(list[i].length()>=100){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+
+						}
+						else{
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+						}
+							out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+					}
+					else{
+						ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+						
+						if(list[i].length()>=100){
+							if(alist.size()==1){
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+							}
+							else if(alist.size()>1){
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");						
+							}
+							else{
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+								out.write("<td>No Source Available\r\n");	
+							}
+						}
+						else{
+							if(alist.size()==1){
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+								suggestionMap.put(list[i], (String)alist.get(0));
+							}
+							else if(alist.size()>1){
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+								out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");	
+								suggestionMap.put(list[i], (String)alist.get(0));
+							}
+							else{
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+								out.write("<td>No Source Available\r\n");	
+								suggestionMap.put(list[i],"No Source Available");
+							}
+						}
+					}
+
+				}
+		    
+    					
+			else{
+
+				 ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+				   
+			    if(list[i].startsWith(" ")){
+			    	String elements = list[i];
+				list[i] = list[i].substring(1);
+    				
+				if(alist.size()==1){
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i]+
+					      "<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+				    suggestionMap.put(list[i], (String)alist.get(0));
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i]+
+					      "<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+				    suggestionMap.put(list[i], (String)alist.get(0));
+				}  				
+				else{
+				    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i]+"<td>No Source Available\r\n");
+				    suggestionMap.put(elements,"No Source Available");
+				}
+
+			    }
+			    else{
+				if(alist.size()==1){
+				    out.write("<td>"+(counter+1)+"<td>"+list[i]+"<td><a href=\""+alist.get(0)+"\">Source</a>\r\n");
+				    suggestionMap.put(list[i],(String)alist.get(0));
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+(counter+1)+"<td>"+list[i]+"<td><a href=\""+alist.get(0)+"\">Source</a>...\r\n");
+				    suggestionMap.put(list[i],(String)alist.get(0));
+				}
+				else{
+				    out.write("<td>"+(counter+1)+"<td>"+list[i]+"<td>No Source Available\r\n");
+				    suggestionMap.put(list[i],"No Source Available");
+				}
+				//out.write("<td>"+(counter+1)+"<td>"+list[i]+"\r\n");
+			    }
+			}
+		    }
+		    counter++;
+		    out.write("<td onclick=\"GD(this)\">View");
+		}
+		if(suggestionMap.size()>1 && !title.equals(IDENTIFIER)){
+    		status = true;
+    	}
+    		}
+    		else{
+    			
+    			
+            	out.write("<table border=\"1\" align=\"center\" width=\"800\">\r\n");
+            	out.write ("<h2 align=\"center\">"+ title+"</h2>\r\n");
+            	out.write ("<th class=\"tam\">&nbsp;\r\n");
+            	out.write ("<th class=\"tam\">Frequency\r\n");
+            	out.write ("<th class=\"tam\">Element Values\r\n");    
+            	out.write ("<th class=\"tam\">Source Documents\r\n");   
+            	out.write ("<th class=\"tam\">Internal Link\r\n"); 
+		out.write ("<a name='topFive'>\r\n");
+        		
+		HashMap xMap = dm.getDistinctValueMap(title);
+		int counter = 0;
+		for(int i = 0; i<list.length; i++){
+		    if(list.length<=5 && i==0){
+			out.write ("<a name='lastFive'>\r\n");
+		    }
+		    else if((list.length>5) && (list.length-5==i)){
+			out.write ("<a name='lastFive'>\r\n");
+		    }
+
+        	InternalLink il= (InternalLink)internalIDMap.get(list[i]);
+        	ArrayList alist2 = il.retrieveList();
+        	String id = (String)alist2.get(0);
+        	id = id.substring(4);
+        	out.write("<tr id=\""+id+"\" >\r\n");
+        	
+		    out.write("<td>"+(counter+1)+"<th> "+((Integer)xMap.get(list[i])).toString()+"\r\n");
+    				
+		    if(list[i].length()>=201){
+		    	
+			if(list[i].startsWith("http://")){
+				if(title.equals(IDENTIFIER)){
+					if(list[i].length()>=100){
+						out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+					}
+					else{
+						out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+					
+				    out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+				}
+				else{
+					ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+					if(list[i].length()>=100){
+						if(alist.size()==1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td><a href=\""+alist.get(0)+"\">Source</a>\r\n");
+						}
+						else if (alist.size()>1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td><a href=\""+alist.get(0)+"\">Source</a>...\r\n");
+						}
+						else{
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td>No Source Available\r\n");	
+						}
+					
+					}
+					else{
+						out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+				}
+			}
+			
+			
+			
+			
+			else{
+			    ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+    						
+			    if(list[i].startsWith(" ")){
+				list[i]=list[i].substring(1);
+					if(alist.size()==1){
+						out.write("<td>"+spaceLeft+space+spaceRight+list[i].substring(0,200)+"...<td>"+
+						"<a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+					}
+					else if(alist.size()>1){
+						out.write("<td>"+spaceLeft+space+spaceRight+list[i].substring(0,200)+"...<td>"+
+					      "<a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+					}
+					else{
+						out.write("<td>"+spaceLeft+space+spaceRight+list[i].substring(0,200)+"...<td>No Source Available\r\n");
+					}
+			    }
+			    else{
+			    	
+			    	if(alist.size()==1){
+			    		out.write("<td>"+list[i].substring(0,200)+"...<td>" +
+					      "<a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+			    	}
+			    	else if(alist.size()>1){
+			    		out.write("<td>"+list[i].substring(0,200)+"...<td>"+
+					      "<a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+			    	}
+			    	else{
+			    		out.write("<td>"+list[i].substring(0,200)+"...<td>No Source Available\r\n");
+			    	}
+			    }
+			}
+		    }
+		    else{
+			if(list[i].startsWith("http://")){
+				if(title.equals(IDENTIFIER)){
+					if(list[i].length()>=100){
+						out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+					}
+					else{
+						out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+				    out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+				}
+				else{
+					//ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],"dc.Identifier", collectionFolder,valueMap,linkMap);
+					ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+					
+					if(list[i].length()>=100){
+						if(alist.size()==1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+						}
+						else if(alist.size()>1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+						}
+						else{
+							out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+							out.write("<td>No Source Available\r\n");
+						}
+					}
+					else{
+						if(alist.size()==1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+						}
+						else if(alist.size()>1){
+							out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							out.write("<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+						}
+						else{
+							out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+							out.write("<td>No Source Available\r\n");
+						}
+					}
+					
+				}
+				
+			}
+			else{
+			    
+			    ArrayList alist= sl.CreateIndentifierLinkPage(title, list[i],IDENTIFIER, collectionFolder,valueMap,linkMap);
+    						
+			    if(list[i].startsWith(" ")){
+				list[i]=list[i].substring(1);
+				if(alist.size()==1){
+				    out.write("<td>"+spaceLeft+space+spaceRight+list[i]+
+					      "<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+spaceLeft+space+spaceRight+list[i]+
+					      "<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+				}
+				else{
+				    out.write("<td>"+spaceLeft+space+spaceRight+list[i]+"<td>No Source Available\r\n");
+				}
+			    }
+			    else{
+				if(alist.size()==1){
+				    out.write("<td>"+list[i]+"<td><a href=\""+(String)alist.get(0)+"\">Source</a>\r\n");
+				}
+				else if(alist.size()>1){
+				    out.write("<td>"+list[i]+"<td><a href=\""+(String)alist.get(0)+"\">Source</a>...\r\n");
+				}
+				else{
+				    out.write("<td>"+list[i]+"<td>No Source Available\r\n");
+				}
+			    }
+			}
+		    }
+		    counter++;
+		    out.write("<td onclick=\"GD(this)\">View");
+		}   		  	
+    		}
+    		out.write("</table>\r\n");
+	    }
+	    else{
+    		out.write("<p>Sorry! The list is empty</p>");
+    	}
+	    out.write("<tr>\r\n");
+    	
+    	
+	    out.write("<p><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    ////////////////////////////////////////////////////////////////////////////////////
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+	    }
+	    else if (metadataSetName.equalsIgnoreCase("extracted")){
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+	    }
+	    else{
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+	    }
+	    out.write("&raquo;");
+	    out.write("<a href=\""+title+".html\">"+title+"</a></p>"); 
+	    out.write("</body></html>\r\n");
+	    out.close();
+	    fstream.close();
+      	}catch(Exception e){
+
+	    e.printStackTrace(outx);
+    	}
+
+  	if(status){
+  		return suggestionMap;
+  	}
+  	else{
+  		return new HashMap();
+  	}
+    }
+    
+    
+    private void setupMetadataLink(String fileName){
+    	
+    	String destination = globalProperty.getGSDL3Home()+fileSeparator+"mat"+fileSeparator+"script"+fileSeparator+fileName+".xml";
+    	   
+    	try{
+	    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+	    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+	    Document doc = docBuilder.newDocument();
+	    //doc = docBuilder.parse (new File("/research/cc108/greenstone3/web/mat/script/"+fileName+".xml"));
+	    doc = docBuilder.parse (destination);
+	    Element rootNode = doc.getDocumentElement();
+    		
+	    NodeList listOfName = rootNode.getElementsByTagName("metadataElement");
+	    url = new HashMap();
+	    for(int i = 0; i<listOfName.getLength(); i++){
+		Node NameNode = listOfName.item(i);
+		Element docElement = (Element)NameNode;
+		NodeList valueList = docElement.getElementsByTagName("URL");
+		Node urlParentNode = valueList.item(0);
+		String urlText = urlParentNode.getChildNodes().item(0).getNodeValue();
+    		
+		valueList = docElement.getElementsByTagName("name");
+		Node urlNameParentNode = valueList.item(0);
+		String urlNameText = urlNameParentNode.getChildNodes().item(0).getNodeValue();
+    		
+		url.put(urlNameText, urlText);
+    		
+	    }
+    		
+    		
+    	}catch(Exception e){
+	    e.printStackTrace(printWriter);
+    	}
+   }
+     
+    private void createIncompletedList(String fileName, String metadataSetName, String collectionFolder,PrintWriter outx){
+
+    	HashMap hp = dm.getIdentifierLink(linkIdentifier);
+    	String[] ids = dm.getDocumentIDList(fileName);
+
+    	for(int i = 0; i<ids.length; i++){
+    		hp.remove(ids[i]);
+    	}
+
+    	try{	
+	    FileWriter fstream = new FileWriter(destination+fileName+"_IncompletedList.html");
+	    BufferedWriter out = new BufferedWriter(fstream);
+	    
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+	    out.write ("<html>\r\n");
+	    out.write("<head>\r\n<title>Incompleted Document List</title>\r\n");
+	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<style type=\"text/css\">\r\n");
+	    out.write("td{height:20; text-align:left;padding-left:5px;}\r\n");
+	    out.write(".tam{height:20; text-align:center}\r\n");
+	    out.write("th{height:20; text-align:center;}\r\n");
+	    out.write("body{font-family: Arial;}\r\n");
+	    out.write("</style>\r\n");
+	    out.write("</head><body>\r\n");
+	    
+	    out.write("<p><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+    	}
+    	else if (metadataSetName.equalsIgnoreCase("extracted")){
+        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+    	}
+    	else{
+        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+    	}
+    	out.write("&raquo;");
+	    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+	    
+	    out.write("<h3 align=center>"+fileName+" does not appear in the following documents</h3>");
+	    int counter = 0;
+	    Set s = hp.keySet();
+	    Iterator i = s.iterator();
+	    out.write("<table border=1 align=center><tr><th>Document ID<th>Source Link\r\n");
+	    while(i.hasNext()){
+	    	counter++;
+	    	String keys = (String)i.next();
+	    	InternalLink il = (InternalLink) hp.get(keys);
+	    	ArrayList alist = il.retrieveList();
+	    	String url = (String)alist.get(0);
+	    	out.write("<tr><td>"+counter+"<td><a href=\""+url+"\">"+url+"</a>\r\n");
+	    }
+	    out.write("</table></body></html>\r\n");
+	    
+	    
+	    out.write("<p><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+    	}
+    	else if (metadataSetName.equalsIgnoreCase("extracted")){
+        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+    	}
+    	else{
+        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+    	}
+    	out.write("&raquo;");
+	    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+	    
+	    out.close();
+	    fstream.close();
+    	}catch(IOException ex){ex.printStackTrace();}
+    	
+    	
+    }
+    
+    public boolean compareElement(String fileName,String collectionFolder,HashMap suggestionMap, String metadataSetName){
+
+    	boolean status = false;
+
+	    Set kset = suggestionMap.keySet();
+		ArrayList arrayList = new ArrayList();
+		Iterator is = kset.iterator();
+		while(is.hasNext()){
+			arrayList.add((String)is.next());
+		}
+		HashMap distanceMap = new HashMap();
+	
+		int totalLength = 0;
+
+		int arrayListLength = arrayList.size();
+		int counter = 0;
+
+		printWriter.write("<!-- generating all possible page-->");
+		printWriter.flush();
+		double distance ;
+		try{
+		for(int i = 0; i<arrayListLength; i++){
+			String keyword = (String)arrayList.get(i);
+
+			int spaceCounterFront1 = 0;
+			int spaceCounterEnd1 = 0;
+			String keywordClone = keyword;
+			String _keywordClone = keyword;
+			String bkKeyWord = keyword;
+
+			keywordClone = removeUnusedCharacter(keywordClone);
+
+			while(true){
+
+				if(_keywordClone.length()>1){
+				if(_keywordClone.charAt(0)==' '){
+					_keywordClone = _keywordClone.substring(1);
+					spaceCounterFront1++;
+				}
+				else{
+					if(_keywordClone.charAt(_keywordClone.length()-1)==' '){
+						_keywordClone = _keywordClone.substring(0,_keywordClone.length()-1);
+						spaceCounterEnd1++;
+					}
+					else{
+						break;
+					}
+				}
+				}
+				else{
+					break;
+				}
+			}
+
+			int keywordLength = keywordClone.length();
+			totalLength = totalLength + keywordLength;
+			for(int j = i+1; j<arrayListLength; j++){
+				String keyword2 = (String)arrayList.get(j);
+				counter ++;
+				double pre_cost = 0;
+				String keywordClone2 = keyword2;
+				String _keywordClone2 = keyword2;
+				String bkKeyWord2 = keyword2;
+				costModel cm = new costModel();
+				cm = removeUnusedCharacter(keywordClone2,pre_cost);
+				keywordClone2 = cm.getString();
+				pre_cost = cm.getCost();
+				
+				int spaceCounterFront2 = 0;
+				int spaceCounterEnd2 = 0;
+				
+				while(true){
+					if(_keywordClone2.length()>1){
+					if(_keywordClone2.charAt(0)==' '){
+						_keywordClone2 = _keywordClone2.substring(1);
+						spaceCounterFront2++;
+					}
+					else{
+						if(_keywordClone2.charAt(_keywordClone2.length()-1)==' '){
+							_keywordClone2 = _keywordClone2.substring(0,_keywordClone2.length()-1);
+							spaceCounterEnd2++;
+						}
+						else{
+							break;
+						}
+					}}
+				else{
+					break;
+				}
+				}
+
+				if(counter == 50000){
+					printWriter.write("<!-- generating all possible page-->");
+					printWriter.flush();
+					counter = 0;
+				}
+				int keyword2Length = keywordClone2.length();
+
+				if(keywordLength>(keyword2Length+2) || (keywordLength+2)<keyword2Length ){}
+				else{
+
+					distance = calculateEditDistance(keywordClone.toLowerCase().toCharArray(),keywordClone2.toLowerCase().toCharArray());
+
+					distance = distance + pre_cost;
+
+					if(distance<=2){
+						if(distanceMap.containsKey(_keywordClone)){
+							InternalLink il = (InternalLink)distanceMap.get(_keywordClone);
+							
+							String keywordHolder = _keywordClone2;
+							
+							while(true){
+								if(keywordHolder.contains("  ")){
+									keywordHolder = keywordHolder.replaceFirst("  ",spaceLeft+space+spaceRight+" ");
+								}
+								else{
+									break;
+								}
+							}
+							
+							for(int a = 0; a<spaceCounterFront2; a++){
+								keywordHolder = spaceLeft+space+spaceRight+keywordHolder;
+							}
+							for(int a = 0; a<spaceCounterEnd2; a++){
+								keywordHolder = keywordHolder+spaceLeft+space+spaceRight;
+							}
+							
+							InternalLink il2 = new InternalLink();
+							il2.setValue(keywordHolder);
+							if(suggestionMap.containsKey(bkKeyWord2)){
+								il2.increaseElement((String)suggestionMap.get(bkKeyWord2));
+								//System.out.println("input this url(3):"+(String)alistkw2.get(0));
+							}
+							else{
+								il2.increaseElement("&nbsp;");
+							}
+							
+							il.increaseNode(il2);
+
+							distanceMap.put(_keywordClone, il);
+
+						}
+						else{
+							InternalLink il = new InternalLink();
+							String keywordHolder = _keywordClone;
+							
+							while(true){
+								if(keywordHolder.contains("  ")){
+									keywordHolder = keywordHolder.replaceFirst("  ",spaceLeft+space+spaceRight+" ");
+								}
+								else{
+									break;
+								}
+							}
+							
+							for(int a = 0; a<spaceCounterFront1; a++){
+								keywordHolder = spaceLeft+space+spaceRight+keywordHolder;
+							}
+							for(int a = 0; a<spaceCounterEnd1; a++){
+								keywordHolder = keywordHolder+spaceLeft+space+spaceRight;
+							}
+							
+							InternalLink il2 = new InternalLink();
+							il2.setValue(keywordHolder);
+							if(suggestionMap.containsKey(bkKeyWord)){
+								il2.increaseElement((String)suggestionMap.get(bkKeyWord));
+
+							}
+							else{
+								il2.increaseElement("&nbsp;");
+							}
+							
+							il.increaseNode(il2);
+
+							
+							keywordHolder = _keywordClone2;
+							
+							while(true){
+								if(keywordHolder.contains("  ")){
+									keywordHolder = keywordHolder.replaceFirst("  ",spaceLeft+space+spaceRight+" ");
+								}
+								else{
+									break;
+								}
+							}
+							for(int a = 0; a<spaceCounterFront2; a++){
+								keywordHolder = spaceLeft+space+spaceRight+keywordHolder;
+							}
+							for(int a = 0; a<spaceCounterEnd2; a++){
+								keywordHolder = keywordHolder+spaceLeft+space+spaceRight;
+							}
+							
+							InternalLink il3 = new InternalLink();
+							il3.setValue(keywordHolder);
+							if(suggestionMap.containsKey(bkKeyWord2)){
+								il3.increaseElement((String)suggestionMap.get(bkKeyWord2));
+
+							}
+							else{
+								il3.increaseElement("&nbsp;");
+							}
+
+							il.increaseNode(il3);
+
+							distanceMap.put(_keywordClone, il);
+						}
+					}
+				}
+
+			}
+
+		}
+
+		
+		printWriter.write("<!-- generating all possible page-->");
+		printWriter.flush();
+
+		if(distanceMap.size()!=0){
+			generateHTML(distanceMap,fileName,metadataSetName);
+			status = true;
+		}
+		}catch(Exception ex){ex.printStackTrace();}
+		return status;
+
+	}
+	
+	
+	private void generateHTML(HashMap distanceMap, String fileName,String metadataSetName){
+		printWriter.write("<!-- generating all possible page-->");
+		printWriter.flush();
+		
+		String collectionFolder = mds.getCollectionName();
+		
+		try{	
+		    FileWriter fstream = new FileWriter(destination+fileName+"_Suggestion.html");
+		    BufferedWriter out = new BufferedWriter(fstream);
+		    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+		    out.write ("<html>\r\n");
+		    out.write("<head>\r\n<title>Potential Duplicate List</title>\r\n");
+		    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+		    out.write("<style type=\"text/css\">\r\n");
+		    out.write("td{height:20; text-align:left;padding-left:5px;}\r\n");
+		    out.write(".tam{height:20; text-align:center}\r\n");
+		    out.write("th{height:20; text-align:left;}\r\n");
+		    out.write("body{font-family: Arial;}\r\n");
+		    out.write("</style>\r\n");
+		    out.write("</head><body>\r\n");
+		    out.write("<p><a href=\" Overall.html \">Summary</a>");
+		    out.write("&raquo;");
+		    if(metadataSetName.equalsIgnoreCase("dublin")){
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+	    	}
+	    	else if (metadataSetName.equalsIgnoreCase("extracted")){
+	        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+	    	}
+	    	else{
+	        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+	    	}
+	    	out.write("&raquo;");
+		    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+		    out.write("<h3 align=center>Potential Duplicate List</h3>");
+		    out.write("<table border=1 align=center>\r\n");
+
+			Set s = distanceMap.keySet();
+			Iterator i = s.iterator();
+			while(i.hasNext()){
+
+				String keyword = (String)i.next();
+				InternalLink il = (InternalLink)distanceMap.get(keyword);
+
+				ArrayList alist = il.retrieveNodeList();
+
+				out.write("<tr><th>Original Text<th>Source Link\r\n");
+				for(int a = 0; a<alist.size(); a++){
+					InternalLink il2 = new InternalLink();
+					il2 = (InternalLink)alist.get(a);
+					String url = (String)il2.retrieveList().get(0);
+					out.write("<tr><td>"+il2.getValue()+"<td><a href=\""+url+"\">"+url+"</a>");
+
+				}
+			    out.write("</tr>");
+
+			}
+		    out.write("</table>\r\n");
+		    
+		    out.write("<p><a href=\" Overall.html \">Summary</a>");
+		    out.write("&raquo;");
+		    if(metadataSetName.equalsIgnoreCase("dublin")){
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+	    	}
+	    	else if (metadataSetName.equalsIgnoreCase("extracted")){
+	        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+	    	}
+	    	else{
+	        		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+	    	}
+	    	out.write("&raquo;");
+		    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+		    out.write("</body></html>\r\n");
+		    out.close();
+		    fstream.close();
+		}catch(IOException ex){ex.printStackTrace();}
+
+	}
+	
+	public int calculateEditDistance(char[] args1, char[] args2){
+
+        int n = args1.length; 
+        int m = args2.length; 
+        if (n == 0) {
+            return m;
+        }
+        else if (m == 0) {
+            return n;
+        }
+        
+        int[] p = new int[n + 1];
+    	int[] d = new int[n + 1]; 
+        int[] _d; 
+        
+
+        int i; 
+        int j; 
+
+        int cost; // cost
+
+        for (i = 0; i <= n; i++) {
+            p[i] = i;
+        }
+        for (j = 1; j <= m; j++) {
+
+            d[0] = j;
+            for (i = 1; i <= n; i++) {
+                cost = (args1[i-1] == args2[j-1]) ? 0 : 1;
+                d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1),
+                        p[i - 1] + cost);
+            }
+
+            // copy current distance counts to 'previous row' distance counts
+            _d = p;
+            p = d;
+            d = _d;
+        }
+        // our last action in the above loop was to switch d and p, so p now
+        // actually has the most recent cost counts
+        return p[n];
+	}
+
+    private String removeUnusedCharacter(String target){
+    	//printWriter.write("<!-- generating all possible page-->");
+    	//printWriter.flush();
+
+    	// remove \n 
+    	while(true){
+    	    if(target.contains("\\n")){
+    	    	target = target.replaceFirst("\\\\n","");
+    	    }
+    	    else{
+    	    	break;
+    	    }
+    	}
+    	
+    	//remove spaces at the end of string
+    	if(target.length()>1){
+    	while(true){
+    	   	if(target.length()>1){
+    	    if(target.charAt(target.length()-1)==' '){
+    	    	target = target.substring(0,target.length()-1);
+    	    }
+    	    else{
+    	    	break;
+    	    }
+    	   	}
+    	   	else{
+    	   		break;
+    	   	}
+        }
+    	}
+    	//remove leading spaces
+    	if(target.length()>1){
+        while(true){
+
+        	if(target.length()>1){
+    	   if(target.charAt(0)==' '){
+    		   target = target.substring(1,target.length());
+    	    }
+    	    else{
+    	    	break;
+    	    }}
+        	else{
+        		break;
+        	}
+        }
+    	}	 
+        //remove multiple spaces between words
+        while(true){
+    	   if(target.contains("  ")){
+    		   target = target.replaceFirst("  "," ");
+    	   }
+    	   else{
+    		   break;
+    	   }
+        }
+
+    	 return target;
+   }
+    	
+    	
+   private costModel removeUnusedCharacter(String target, double cost){
+
+    		
+    	costModel cm = new costModel();
+    	 
+    	// remove \n plus cost 
+    	while(true){
+
+    	    if(target.contains("\\n")){
+    	    	target = target.replaceFirst("\\\\n","");
+    	    	cost = cost + 0.2;
+    	    }
+    	    else{
+    	    	break;
+    	    }
+    	}
+    	
+    	//remove spaces at the end of string
+    	if(target.length()>1){
+    	while(true){
+    		if(target.length()>1){
+    	    if(target.charAt(target.length()-1)==' '){
+    	    	target = target.substring(0,target.length()-1);
+    	    	cost = cost + 0.2;
+    	    }
+    	    else{
+    	    	break;
+    	    }}
+    		else{
+    			break;
+    		}
+    	}
+    	}
+    	//remove leading spaces  
+    	if(target.length()>1){
+    	while(true){
+
+    		if(target.length()>1){
+    	    if(target.charAt(0)==' '){
+    	    	target = target.substring(1,target.length());
+    	    	cost = cost + 0.2;
+    	    }
+    	    else{
+    	    	break;
+    	    }}
+    		else{
+    			break;
+    		}
+    	}
+    	}
+        //remove multiple spaces between words
+    	while(true){
+
+    	    if(target.contains("  ")){
+    	    	target = target.replaceFirst("  "," ");
+    	    	cost = cost + 0.2;
+    	    }
+    	    else{
+    	    	break;
+    	    }
+    	}
+
+    		cm.setCost(cost);
+    		cm.setString(target);
+
+    	return cm;
+        }
+   
+
+   
+   private HashMap generateMetadataElementSortListNoIdentifier(String title, String sort, String metadataSetName,PrintWriter outx){
+	
+	    	String fileName = title+"_"+sort;
+	    	String collectionFolder = mds.getCollectionName();
+	    	System.out.println("level 3");
+	    	SearchLink sl = new SearchLink(mds.StatsDirectory);
+	    	System.out.println("level 4");
+	    	HashMap suggestionMap = new HashMap();
+	    	
+		    HashMap internalIDMap = dm.getInternalIdentifier(title);	
+	    	boolean status = false;
+
+	    	try{
+	    	printWriter.write("<!-- generating all possible page-->");
+	    	printWriter.flush();
+	    	
+		    FileWriter fstream = new FileWriter(destination+fileName+".html");
+		    BufferedWriter out = new BufferedWriter(fstream);
+		    
+		    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+		    out.write ("<html>\r\n");
+		    out.write("<head>\r\n<title>Metadata Element Sort List</title>\r\n");
+		    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+		    out.write("<script type=\"text/javascript\" src=\"../script/getInfomation.js\"></script>\r\n");   	    
+		    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/yahoo/yahoo-min.js\"></script>\r\n");
+		    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/event/event-min.js\"></script>\r\n");
+		    out.write("<script type=\"text/javascript\" src=\"http://yui.yahooapis.com/2.4.1/build/connection/connection-min.js\"></script>\r\n");
+
+		    out.write("<style type=\"text/css\">\r\n");
+		    out.write("td{height:20; text-align:left;padding-left:5px;}\r\n");
+		    out.write(".tam{height:20; text-align:center}\r\n");
+		    out.write("th{height:20; text-align:center;}\r\n");
+		    out.write("body{font-family: Arial;}\r\n");
+		    out.write("</style>\r\n");
+		    out.write("</head><body id=\""+collectionFolder+"\">\r\n");
+	    	
+		    out.write("<td><div style=\"float:left;\"><a href=\" Overall.html \">Summary</a>");
+		    out.write("&raquo;");
+
+		    if(metadataSetName.equalsIgnoreCase("dublin")){
+			out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+		    }
+		    else if (metadataSetName.equalsIgnoreCase("extracted")){
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+		    }
+		    else{
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+		    }
+		    out.write("&raquo;");
+		    out.write("<a href=\""+title+".html\">"+title+"</a></div>"); 
+		  	
+		    out.write("<div style=\"float:right;\"><a href=\"http://chnm.gmu.edu/tools/surveys/4386/\">Please send feedback about the Mat tool</a></div></td><br>");
+	 	  
+		  	
+		    String[] list = dm.getSortList(title, sort);
+
+	      
+	       	
+		    if(list.length>=1){
+	    		
+	    		
+	    		if(sort.equals("ASCII")){
+	    			
+	            	out.write("<table border=\"1\" align=\"center\" width=\"800\">\r\n");
+	            	out.write ("<h2 align=\"center\">"+ title+"</h2>\r\n");      	
+	            	out.write ("<th class=\"tam\">ASCII Sort\r\n");
+	            	out.write ("<th class=\"tam\">Element Values\r\n");
+	            	out.write ("<th class=\"tam\">Source Documents\r\n");   
+	            	out.write ("<th class=\"tam\">Internal Link\r\n"); 
+	            	
+			out.write ("<a name='topFive'>\r\n");
+	        		
+			int counter = 0;
+
+			for(int i = 0; i<list.length; i++){
+
+			    if(list.length<=5 && i==0){
+				out.write ("<a name='lastFive'>\r\n");
+			    }
+			    else if((list.length>5) && (list.length-5==i)){
+				out.write ("<a name='lastFive'>\r\n");
+			    }
+			    
+			            	
+	        	InternalLink il= (InternalLink)internalIDMap.get(list[i]);
+	        	ArrayList alist2 = il.retrieveList();
+	        	String id = (String)alist2.get(0);
+	        	id = id.substring(4);
+	        	out.write("<tr id=\""+id+"\" >\r\n");
+	    				
+			    if(list[i].length()>=201){
+				if(list[i].startsWith("http://")){
+
+						if(list[i].length()>=100){
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+								out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+						}
+						else{
+								out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+								out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+						}
+					
+				}
+				else{
+
+				    if(list[i].startsWith(" ")){
+				    String elements = list[i];
+					list[i] = list[i].substring(1);
+					    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i].substring(0, 200)+"...<td>No Source Available \r\n");
+				    }
+				    else{
+					    out.write("<td>"+(counter+1)+"<td>"+list[i].substring(0, 200)+"... <td>No Source Available\r\n");
+				    }
+				}
+	    			
+			    }
+			    else{
+	    					
+
+					if(list[i].startsWith("http://")){
+
+						if(list[i].length()>=100){
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"...</a>");
+						}
+						else{
+							out.write("<td>"+(counter+1)+"<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+						}
+							out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");				
+					   
+					}
+			    
+
+	    					
+				else{
+					   
+				    if(list[i].startsWith(" ")){
+				    String elements = list[i];
+					list[i] = list[i].substring(1);
+
+					    out.write("<td>"+(counter+1)+"<td>"+spaceLeft+space+spaceRight+list[i]+"<td>No Source Available\r\n");
+					    suggestionMap.put(elements,"No Source Available");
+
+				    }
+				    else{
+					    out.write("<td>"+(counter+1)+"<td>"+list[i]+"<td>No Source Available\r\n");
+				    }
+				}
+			    }
+			    counter++;
+			    out.write("<td onclick=\"GD(this)\">View");
+			}
+
+	    		}
+	    		else{
+	    			
+	    			
+	            	out.write("<table border=\"1\" align=\"center\" width=\"800\">\r\n");
+	            	out.write ("<h2 align=\"center\">"+ title+"</h2>\r\n");
+	            	out.write ("<th class=\"tam\">&nbsp;\r\n");
+	            	out.write ("<th class=\"tam\">Frequency\r\n");
+	            	out.write ("<th class=\"tam\">Element Values\r\n");    
+	            	out.write ("<th class=\"tam\">Source Documents\r\n");   
+	            	out.write ("<th class=\"tam\">Internal Link\r\n"); 
+			out.write ("<a name='topFive'>\r\n");
+	        		
+			HashMap xMap = dm.getDistinctValueMap(title);
+			int counter = 0;
+			for(int i = 0; i<list.length; i++){
+			    if(list.length<=5 && i==0){
+				out.write ("<a name='lastFive'>\r\n");
+			    }
+			    else if((list.length>5) && (list.length-5==i)){
+				out.write ("<a name='lastFive'>\r\n");
+			    }
+
+	        	InternalLink il= (InternalLink)internalIDMap.get(list[i]);
+	        	ArrayList alist2 = il.retrieveList();
+	        	String id = (String)alist2.get(0);
+	        	id = id.substring(4);
+	        	out.write("<tr id=\""+id+"\" >\r\n");
+	        	
+			    out.write("<td>"+(counter+1)+"<th> "+((Integer)xMap.get(list[i])).toString()+"\r\n");
+	    				
+			    if(list[i].length()>=201){
+			    	
+				if(list[i].startsWith("http://")){
+					if(list[i].length()>=100){
+						out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+					}
+					else{
+						out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+					
+				    out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+				}
+				
+				
+				
+				
+				else{   						
+				    if(list[i].startsWith(" ")){
+					list[i]=list[i].substring(1);
+
+							out.write("<td>"+spaceLeft+space+spaceRight+list[i].substring(0,200)+"...<td>No Source Available\r\n");
+					
+				    }
+				    else{
+
+				    		out.write("<td>"+list[i].substring(0,200)+"...<td>No Source Available\r\n");
+				    	
+				    }
+				}
+			    }
+			    else{
+				if(list[i].startsWith("http://")){
+					if(list[i].length()>=100){
+						out.write("<td><a href=\""+list[i]+"\">"+list[i].substring(0,100)+"</a>");
+					}
+					else{
+						out.write("<td><a href=\""+list[i]+"\">"+list[i]+"</a>");
+					}
+				    out.write("<td><a href=\""+list[i]+"\">Source</a>\r\n");
+					
+				}
+				else{
+	    						
+				    if(list[i].startsWith(" ")){
+					list[i]=list[i].substring(1);
+
+
+					    out.write("<td>"+spaceLeft+space+spaceRight+list[i]+"<td>No Source Available\r\n");
+					
+				    }
+				    else{
+
+					    out.write("<td>"+list[i]+"<td>No Source Available\r\n");
+					
+				    }
+				}
+			    }
+			    counter++;
+			    out.write("<td onclick=\"GD(this)\">View");
+			}   		  	
+	    		}
+	    		out.write("</table>\r\n");
+		    }
+		    else{
+	    		out.write("<p>Sorry! The list is empty</p>");
+	    	}
+		    out.write("<tr>\r\n");
+	    	
+	    	
+		    out.write("<p><a href=\" Overall.html \">Summary</a>");
+		    out.write("&raquo;");
+
+		    if(metadataSetName.equalsIgnoreCase("dublin")){
+			out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+		    }
+		    else if (metadataSetName.equalsIgnoreCase("extracted")){
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+		    }
+		    else{
+	    		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+		    }
+		    out.write("&raquo;");
+		    out.write("<a href=\""+title+".html\">"+title+"</a></p>"); 
+		    out.write("</body></html>\r\n");
+		    out.close();
+		    fstream.close();
+	      	}catch(Exception e){
+
+		    e.printStackTrace(outx);
+	    	}
+	      	
+	      	return new HashMap();
+	    }
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   private void createIncompletedListNoIdentifier(String fileName, String metadataSetName, String collectionFolder,PrintWriter outx){
+   	//HashMap hp = dm.getIdentifierLink("dc.Identifier");
+   	HashMap hp = dm.getIdentifierLinkNoIdentifier();
+   	String[] ids = dm.getDocumentIDList(fileName);
+   	System.out.println("Length: "+ids.length+" "+fileName+ " "+hp.size());
+   	
+   	Set st = hp.keySet();
+   	Iterator it = st.iterator();
+   	while(it.hasNext()){
+   		System.out.println(it.next());
+   	}
+   	for(int i = 0; i<ids.length; i++){
+   		System.out.println("IDS:" +ids[i]);
+   		hp.remove(ids[i]);
+   	}
+   	System.out.println("Length: "+ids.length+" "+fileName+ " "+hp.size());
+     	//String collectionFolder = mds.getCollectionName();
+   	//System.out.println(hp.size());
+   	try{	
+	    FileWriter fstream = new FileWriter(destination+fileName+"_IncompletedList.html");
+	    BufferedWriter out = new BufferedWriter(fstream);
+	    
+	    out.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">");
+	    out.write ("<html>\r\n");
+	    out.write("<head>\r\n<title>Incompleted Document List</title>\r\n");
+	    out.write("<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\r\n");
+	    out.write("<style type=\"text/css\">\r\n");
+	    out.write("td{height:20; text-align:left;padding-left:5px;}\r\n");
+	    out.write(".tam{height:20; text-align:center}\r\n");
+	    out.write("th{height:20; text-align:center;}\r\n");
+	    out.write("body{font-family: Arial;}\r\n");
+	    out.write("</style>\r\n");
+	    out.write("</head><body>\r\n");
+	    
+	    out.write("<p><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+   		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+   	}
+   	else if (metadataSetName.equalsIgnoreCase("extracted")){
+       		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+   	}
+   	else{
+       		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+   	}
+   	out.write("&raquo;");
+	    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+	    
+	    out.write("<h3 align=center>"+fileName+" does not appear in the following documents</h3>");
+	    int counter = 0;
+	    System.out.println("link.....");
+	    Set s = hp.keySet();
+	    Iterator i = s.iterator();
+	    System.out.println("link2.....");
+	    out.write("<table border=1 align=center><tr><th>Document ID<th>Source Link\r\n");
+	    while(i.hasNext()){
+	    	//printWriter.write("<!-- generating all possible page-->");
+	    	System.out.println("link.....3");
+	    	counter++;
+	    	String keys = (String)i.next();
+	    	  System.out.println("link.....4"+keys);
+	    	InternalLink il = (InternalLink) hp.get(keys);
+	    	
+	    	ArrayList alist = il.retrieveList();
+	    	  System.out.println("link.....5"+alist.size());
+	    	String url = (String)alist.get(0);
+	    	out.write("<tr><td>"+counter+"<td><a href=\""+url+"\">"+url+"</a>\r\n");
+	    }
+	    out.write("</table></body></html>\r\n");
+	    
+	    
+	    out.write("<p><a href=\" Overall.html \">Summary</a>");
+	    out.write("&raquo;");
+	    if(metadataSetName.equalsIgnoreCase("dublin")){
+   		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet1+")</a>"); 
+   	}
+   	else if (metadataSetName.equalsIgnoreCase("extracted")){
+       		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSet2+")</a>"); 
+   	}
+   	else{
+       		out.write ("<a href=\""+metadataSetName+".html\">Metadata Detail ("+metadataSetName+")</a>"); 	  
+   	}
+   	out.write("&raquo;");
+	    out.write("<a href=\""+fileName+".html \">"+fileName+"</a>");
+	    
+	    out.close();
+	    fstream.close();
+   	}catch(IOException ex){ex.printStackTrace();}
+   	
+   	
+   }
+   ///////////////////////////////////////////////////////////////////////////////////////
+}
+
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/SearchLink.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/SearchLink.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/SearchLink.java	(revision 18104)
@@ -0,0 +1,215 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class SearchLink {
+
+	private String path = "";
+	private boolean status = true;
+    protected final String DocumentNode = "Document";
+    protected final String ActualValueNode = "ActualValue";
+	
+	public SearchLink(String arg1){
+		path = arg1;
+	}
+	
+	public SearchLink(){}
+	
+	public boolean getStatus(){
+		return status;
+	}
+	
+	public ArrayList CreateIndentifierLinkPage(String metadataElement, String text, String identifier, String col, HashMap valueMap, HashMap linkMap){
+
+		if(valueMap.size()==0){
+			status = false;
+			return new ArrayList();
+		}
+		else{
+			ArrayList alist = retrieveHASHID(valueMap,text);
+			if(alist.size()==0){
+				status = false;
+				return new ArrayList();
+			}
+
+			if(linkMap.size()==0){	
+				status = false;
+				return new ArrayList();
+			}
+			else{
+			ArrayList alist2 = retrieveHASHID(linkMap,alist);
+			return alist2;
+			}
+		}
+	}
+	
+	
+	private Element openfile(String core_element, String collection){
+		
+		try{
+			DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
+		    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+		    Document doc = docBuilder.newDocument();
+		  
+		    doc = docBuilder.parse (new File(path+core_element+".xml"));
+		    Element rootNode = doc.getDocumentElement();
+		    return rootNode;  
+		    }catch (Exception e) {
+		    	System.out.println(core_element + " does not exist");
+	    	    return null;
+		    }
+	}
+	
+	private ArrayList retrieveHASHID (HashMap hp, String text){
+	    if(hp.containsKey(text)){
+	    	InternalLink il = (InternalLink) hp.get(text);
+	    	return il.retrieveList();
+	    }
+	    else{
+	    	return new ArrayList();
+	    }
+	}
+	
+	private ArrayList retrieveHASHID (HashMap hp, ArrayList arg){
+     
+		ArrayList alist = new ArrayList();
+		for(int i = 0; i<arg.size(); i++){
+			String text = (String)arg.get(i);
+			if(hp.containsKey(text)){
+				InternalLink il = (InternalLink) hp.get(text);
+				ArrayList arrayList = il.retrieveList();
+				if(arrayList.size()!=0){
+					alist.add(il.retrieveList().get(0));
+				}
+				else{
+					alist.add("&nbsp;");
+				}
+			}
+		}
+		return alist;
+	}
+
+	public HashMap createValueMap(String core_element, String collection){
+
+ 	HashMap hp = new HashMap();
+ 	Element ex = openfile(core_element,collection);
+  	NodeList valueList = ex.getElementsByTagName(ActualValueNode);
+  	
+    for(int s=0; s<valueList.getLength() ; s++){
+      	   Element docNode = (Element)valueList.item(s);
+      	   NodeList NodeIDMap = docNode.getChildNodes();
+      	   Node DocNodeID = NodeIDMap.item(0);
+      	   String actualValue = DocNodeID.getNodeValue();
+      	   String DocID = DocNodeID.getParentNode().getParentNode().getAttributes().item(0).getNodeValue();
+  		 
+      	   if(hp.containsKey(actualValue)){
+      		 InternalLink il = (InternalLink) hp.get(actualValue);
+      		 il.increaseElement(DocID);
+      		 hp.put(actualValue, il);
+      	   }
+      	   else{
+      		   InternalLink il = new InternalLink();
+      		   il.setValue(actualValue);
+      		   il.increaseElement(DocID);
+      		   hp.put(actualValue, il);
+      	   }
+    	}
+
+    return hp;
+	}
+
+	
+	public HashMap createLinkMap (String core_element, String collection){
+
+		HashMap hp = new HashMap();
+		Element ex = openfile(core_element,collection);
+      	NodeList listOfFrequency = ex.getElementsByTagName(DocumentNode);
+      	
+      	if(listOfFrequency.getLength()==0){
+      		return hp;
+      	}
+      	
+      	for(int s=0; s<listOfFrequency.getLength() ; s++){
+      	   Node docNode = listOfFrequency.item(s);
+      	   NamedNodeMap NodeIDMap = docNode.getAttributes();
+      	   Node DocNodeID = NodeIDMap.item(0);
+      	   String DocID = DocNodeID.getNodeValue();
+
+
+      	   if(hp.containsKey(DocID)){
+
+      		   Element xNode = (Element)docNode;
+          	   NodeList valueList = xNode.getElementsByTagName(ActualValueNode);
+          	   
+          	   ArrayList alist = new ArrayList();
+          	   for(int y = 0; y<valueList.getLength(); y++){
+          		   Element valueElement = (Element)valueList.item(y);
+          		   NodeList textFNList = valueElement.getChildNodes();
+          		   String textValue = ((Node)textFNList.item(0)).getNodeValue();
+
+          		   if(textValue.startsWith("http://")){
+          			 InternalLink il = (InternalLink)hp.get(DocID);
+          			 il.increaseElement(textValue);
+          			 break;
+          		   }
+       		    }
+      	   }
+      	   
+      	   
+      	   else {
+      		   InternalLink il = new InternalLink();
+      		   il.setValue(DocID);
+      		   
+    		   Element xNode = (Element)docNode;
+          	   NodeList valueList = xNode.getElementsByTagName(ActualValueNode);
+          	   
+          	   for(int y = 0; y<valueList.getLength(); y++){
+          		   Element valueElement = (Element)valueList.item(y);
+          		   NodeList textFNList = valueElement.getChildNodes();
+          		   String textValue = ((Node)textFNList.item(0)).getNodeValue();
+
+          		   if(textValue.startsWith("http://")){
+          			 il.increaseElement(textValue);
+          			 break;
+          		   }
+       		    }
+          	   hp.put(DocID, il);
+      	   }
+        }
+		return hp;
+	}
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////
+}
Index: other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/costModel.java
===================================================================
--- other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/costModel.java	(revision 18104)
+++ other-projects/trunk/greenstone3-extension/mat/src/org/greenstone/gsdl3_extension/mat/servlet/costModel.java	(revision 18104)
@@ -0,0 +1,23 @@
+package org.greenstone.gsdl3_extension.mat.servlet;
+
+public class costModel {
+	double cost = 0;
+	String target ="";
+	
+	public String getString(){
+		return target;
+	}
+	
+	public double getCost(){
+		return cost;
+	}
+	
+	public void setCost(double arg){
+		cost = arg;
+	}
+	
+	public void setString(String arg){
+		target = arg;
+	}
+	
+}
