package org.greenstone3.ms;


import org.w3c.dom.*;
import javax.xml.parsers.*;
import javax.xml.transform.*;
import javax.xml.transform.dom.*;
import javax.xml.transform.stream.*;


import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;

import org.greenstone.gsdl3.core.MessageRouter;
import org.greenstone.gsdl3.util.DBInfo;
import org.greenstone.gsdl3.util.GDBMWrapper;
import org.greenstone.gsdl3.util.GSFile;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.XMLConverter;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;


public class MetadataStats {
    
    private Document doc=null;
    private MessageRouter mr = null;
    private XMLConverter converter=null;
    private GDBMWrapper gdbm_src = null;
    private String site_name = "localsite";
    private String node_id = "browselist";
    private String DLS_SET = "dls";
    private String DC_SET = "dublin";
    private String OAI_URL = "";
    private boolean status = false;
    private String oai_Prefix ="";
    //private PrintWriter pw;
    // servlet
    final String destination = "/home/cc108/MRWks1/Greenstone3Project/";
    private int TotalDoc = 0;
    //private int TotalElement = 0;

    private HashMap MDS_list = new HashMap();
    
    private String collection = null;
    public ArrayList metadataNameList = new ArrayList();
    public String myNewDir;
    private String collection_Name = null;

    
    MetadataElement me;
    /*
      The constructor connects to the database and retrieve 
      information for the collection
    */
    
    private static boolean deleteDir(File dir) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i=0; i<children.length; i++) {
                boolean success = deleteDir(new File(dir, children[i]));
                if (!success) {
                    return false;
                }
            }
        }
    
        return dir.delete();
    } 

    public MetadataStats(String site_home, String collection,String url,String oaiPrefix){
    	OAI_URL = url;
    	collection_Name = collection; 
    	oai_Prefix = oaiPrefix;
    	//pw = out;
    	
	//myNewDir = collection+"/metadataStats";
    	myNewDir =destination+collection+"/metadataStats";
    	
    	if(new File(myNewDir).exists()){deleteDir(new File(myNewDir));}
    	new File(myNewDir).mkdirs();
	    

    	String gdbm = GSFile.collectionDatabaseFile(site_home,collection,collection,"gdbm");

	mr = new MessageRouter();
	mr.setSiteName(this.site_name);
	mr.configure();
	
	this.collection = collection;
	this.gdbm_src = new GDBMWrapper();
	this.converter = new XMLConverter();
	this.doc = this.converter.newDOM();
	Setup(gdbm);
    }
    
    public void getAvailableMetadataSets(String SetName,HashMap UsedMap, String SetAbbr){
	
	File metadata_directory = new File("/research/cc108/greenstone3/gli/metadata");
	//File metadata_directory = new File("/greenstone3/gli/metadata");
	//File metadata_directory = new File("C://Program Files//Greenstone3//gli//metadata");
	
	if (metadata_directory.exists()) {
	    // Load just those .mds files in this directory, and return them
	    File[] directory_files = metadata_directory.listFiles();

	    for (int i = 0; i < directory_files.length; i++) {
		File child_file = directory_files[i];
		
		if (!child_file.isDirectory() && child_file.getName().endsWith("mds")) {
                    String fileName = child_file.getName();
		    
		    /////////////////////////
		    //LOADING DLS CORE
		    //---if(fileName.equals(SetName)&& fileName.equals("dls.mds") && DLS_STATUS){
		    if(!MDS_list.containsKey(SetName)&& fileName.equals(SetName+".mds")){
            		    
			MetadataSet ms = new MetadataSet();
			ms.setName(SetName);
			ms.setAbb(SetAbbr);
			
			converter.newDOM();
			Document d = converter.getDOM(child_file);
			     
			NodeList e = d.getElementsByTagName("Element");
			     
			int length = e.getLength();
			
			for(int y = 0; y<length; y++){
			    Node temp = e.item(y);
			    NamedNodeMap mmp = temp.getAttributes();
			    
			    if(!mmp.item(0).getNodeValue().equals("dc.Description")){
			    	MetadataElement me = new MetadataElement();
			    	//me.setMetadataName((SetAbbr+"."+mmp.item(0).getNodeValue().toLowerCase()));
			    	me.setMetadataName((SetAbbr+"."+mmp.item(0).getNodeValue()));
			    	//System.out.println(me.getMetadataName());
			    	//.child_file............child_file
			    	if(!metadataNameList.contains(me.getMetadataName())){
			    		metadataNameList.add(me.getMetadataName());
			    		ms.addIndex(me.getMetadataName());
			    	}
			    UsedMap.put(me.getMetadataName(),me);
			    }
			}

			MDS_list.put(ms.getName(), ms);

		    }// DLS FINISH
		}
    	    }
	}   	
    }
    
    // doc_list is the list of document ids
    //MetadataElement represents which metadata element we want to explore
    private void parseMetadata(ArrayList doc_list, String MetadataElement){
    	//MetadataElement = "dc.contributor";
	//System.out.println("parse metadata starts");
	int counter = 0;
	//int x = 0;
	int docSize = doc_list.size();
    	
	System.out.print("<!-- prepare request -->");
	while(true){
    		
	    Node message = this.doc.createElement(GSXML.MESSAGE_ELEM);
	
	    Node request = GSXML.createBasicRequest(
						       this.doc, GSXML.REQUEST_TYPE_PROCESS,collection+"/DocumentMetadataRetrieve","en", "");
	    
	    Node param_list = this.doc.createElement(
							GSXML.PARAM_ELEM+GSXML.LIST_MODIFIER);
	    
	    param_list.appendChild(GSXML.createParameter(
							 this.doc, "metadata", "all"));
	//metadatalist-dc
	    Node documentNode_list = this.doc.createElement(
							       GSXML.DOC_NODE_ELEM+GSXML.LIST_MODIFIER);
	
	    if(docSize>=300){
		docSize = docSize - 300;
		for(int i = 0; i<300; i++){
		    Element documentNode = this.doc.createElement(GSXML.DOC_NODE_ELEM);
		    documentNode.setAttribute("nodeID",(String)doc_list.get(counter));
		    documentNode_list.appendChild(documentNode);
		    counter++;
		}
		request.appendChild(param_list);
		request.appendChild(documentNode_list);
		message.appendChild(request);
		Node response = mr.process(message);
		getAttributes(response);
		
		response  = null;
		request = null;
		param_list = null;
		documentNode_list = null;
		//System.gc();	
		}
	
	    else if(docSize<300){
		for(int i = 0; i<docSize; i++){
		    Element documentNode = this.doc.createElement(GSXML.DOC_NODE_ELEM);
		    documentNode.setAttribute("nodeID",(String)doc_list.get(counter));
		    documentNode_list.appendChild(documentNode);
		    counter++;
		}
		request.appendChild(param_list);
		request.appendChild(documentNode_list);
		message.appendChild(request);

		Node response = mr.process(message);
		getAttributes(response);
		response  = null;
		request = null;
		param_list = null;
		documentNode_list = null;

		break;
	    }
	}
    }
    
    public boolean getStatus(){
    	return this.status;
    }
    

    private void Setup(String gdbm){

	
	if (!this.gdbm_src.openDatabase(gdbm,GDBMWrapper.READ)) {
	    System.out.println("Could not open GDBM database!");
	    
	}
	
	else{
		
		  String info = this.gdbm_src.getValue("browselist");
			
		    if (info == null) {
		    	System.out.println("cannot locate the list");
		    }
		    else{
			
			if (info == null) {
				System.out.println("the db does not contain any info");
			}
			
			ArrayList children = new ArrayList();
			StringTokenizer st = new StringTokenizer(info, ";");
			
			while (st.hasMoreTokens()) {
				String part = st.nextToken(";");
		
				
				
				if(part.contains("<contains>")){
					part = part.replace("<contains>", "");
				}
				else if(part.contains("<thistype>")){

					int location = part.indexOf("<thistype>");
					part = part.substring(0,location-1);
				}
				children.add(part);
				System.out.println("..."+part+".....2");

			}
			
			gdbm_src.closeDatabase();
			parseMetadata(children,"all");
			TotalDoc = children.size();
			System.out.println("Total Doc22222:"+TotalDoc);
			status = true;
	}
	}
    }

    
    public void getAttributes(Node f){
	
	HashMap UsedMap = new HashMap();
    Element e = (Element) f;
	NodeList metadataNode =  e.getElementsByTagName("metadata");
	int length = metadataNode.getLength();
	
	for (int j = 0; j < length; j++){
	    Node aNode = metadataNode.item(j);
	    NamedNodeMap NodeMap = aNode.getAttributes();
	    Node AttributeNode = NodeMap.item(0);
	    String att_name = AttributeNode.getNodeValue();
	    
	    if(att_name.indexOf("dls.")!=-1){getAvailableMetadataSets(DLS_SET,UsedMap,"dls");}
	    else if(att_name.indexOf("dc.")!=-1){getAvailableMetadataSets(DC_SET,UsedMap,"dc");}
	    
	}

	adjust(UsedMap,e);
	UsedMap = null;
	e = null;
	System.gc();
    }
    

    /* 
     * MATCH UP THE ELEMENTS FROM BOTH DOCUMENTS AND PRE-DEFINED METADATA SET
     * MOVE THE ELEMENT FROM NOUSEDMAP TO USEDMAP IF THE ELEMENT HAS NOT BEEN DISCOVERED BEFORE
     * INCREASE THE COUNTER ONCE THE ELEMENT IS RECONGNIZED
     * CALCULATE THE TIMES OF ELEMENTS USED AND STORE IT INTO THE LIST
     */
    
    private void adjust(HashMap UsedMap, Element response){


	String DocID;;
	NodeList nList = response.getElementsByTagName("documentNode");
	int length = nList.getLength();
	for (int j = 0; j <length;  j++){
	    
	    Node aNode = nList.item(j);
	    NamedNodeMap NodeMap = aNode.getAttributes();
	    Node AttributeNode = NodeMap.item(0);
	    String att_name = AttributeNode.getNodeValue();
	    DocID = att_name;

	    NodeList childList = aNode.getChildNodes();
	    Node nNode = childList.item(0);
	    NodeList grandChildList = nNode.getChildNodes();
	    
	    int length1 = grandChildList.getLength();
	    
	    for(int e = 0; e<length1 ; e++){

		Node xNode = grandChildList.item(e);
		NamedNodeMap xNodeMap = xNode.getAttributes();
		Node xAttributeNode = xNodeMap.item(0);
		String xatt_name = xAttributeNode.getNodeValue();

		NodeList cList = xNode.getChildNodes();
		
		String ActValue = cList.item(0).getNodeValue();


		char firstchart = xatt_name.charAt(0);

		
		if(xatt_name.indexOf('^')!=-1){
			xatt_name = xatt_name.replace('^', '.');
		}
		
		if(UsedMap.containsKey(xatt_name)){

		    MetadataElement met = (MetadataElement) UsedMap.get(xatt_name);
		    met.IncreaseFrequency();
		    HashMap metadataMap = met.getMetadataList();
		    
		    if(metadataMap.containsKey(DocID)){
		
			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
			dc.IncreaseFrequence();
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }
		    else{
	
			DocumentInfo dc = new DocumentInfo();
			dc.IncreaseFrequence();
			dc.setDocumentID(DocID);
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }
		    
		    HashMap valueMap = met.getValueList();
		    if(valueMap.containsKey(ActValue)){
			Integer f = (Integer)valueMap.get(ActValue);
			int fx = f.intValue();
			fx++;
			valueMap.put(ActValue,new Integer(fx));
		    }
		    else{
			valueMap.put(ActValue, new Integer(1));
		    }
		    
		}
		//else if ((!UsedMap.containsKey(xatt_name)) && (xatt_name.indexOf(".")==-1) && (!xatt_name.equals("prettymd")) && (!xatt_name.equals("Description")) && (Character.isLetter(firstchart)) && (xatt_name.indexOf("-")==-1)){
		else if( (!UsedMap.containsKey(xatt_name)) && xatt_name.equals("archivedir")){   
		    //pw.write("NEW EX element");
		    String SetAbbr = "ex";
		    String SetName = "extracted";
		    
		    MetadataSet ms = new MetadataSet();
		    ms.setName(SetName);
		    ms.setAbb(SetAbbr);
		    
		    MetadataElement me = new MetadataElement();
		    me.setMetadataName(xatt_name);
		    me.IncreaseFrequency();
		    HashMap metadataMap = me.getMetadataList();
	    	
		    if(metadataMap.containsKey(DocID)){
			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
			dc.IncreaseFrequence();
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }
		    else{
			DocumentInfo dc = new DocumentInfo();
			dc.IncreaseFrequence();
			dc.setDocumentID(DocID);
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }

		    HashMap valueMap = me.getValueList();
		    if(valueMap.containsKey(ActValue)){
			Integer f = (Integer)valueMap.get(ActValue);
			int fx = f.intValue();
			fx++;
			valueMap.put(ActValue,new Integer(fx));
		    }
		    else{

			valueMap.put(ActValue, new Integer(1));
		    }
		    if(!metadataNameList.contains(me.getMetadataName())){
		    	metadataNameList.add(me.getMetadataName());
		    	ms.addIndex(me.getMetadataName());
		    }

		    UsedMap.put(me.getMetadataName(), me);
		    if(!MDS_list.containsKey(ms.getName())){MDS_list.put(ms.getName(), ms);}
		    else{
			MetadataSet msx = (MetadataSet)MDS_list.get(ms.getName());
			msx.addIndex(me.getMetadataName());
			MDS_list.put(ms.getName(), msx);
		    }
		}
		
		
		else if((!UsedMap.containsKey(xatt_name)) && xatt_name.indexOf(".")!=-1 && xatt_name.indexOf("dc.")==-1){   
		    //pw.write("NEW EX element");
			int dotLocation = xatt_name.indexOf(".");
		    String SetAbbr = xatt_name.substring(0,dotLocation);
		    String SetName = SetAbbr;
		    
		    MetadataSet ms = new MetadataSet();
		    ms.setName(SetName);
		    ms.setAbb(SetAbbr);
		    
		    MetadataElement me = new MetadataElement();
		    me.setMetadataName(xatt_name);
		    me.IncreaseFrequency();
		    HashMap metadataMap = me.getMetadataList();
	    	
		    if(metadataMap.containsKey(DocID)){
			DocumentInfo dc = (DocumentInfo)metadataMap.get(DocID);
			dc.IncreaseFrequence();
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }
		    else{
			DocumentInfo dc = new DocumentInfo();
			dc.IncreaseFrequence();
			dc.setDocumentID(DocID);
			dc.addActualValue(ActValue);
			metadataMap.put(DocID, dc);
		    }

		    HashMap valueMap = me.getValueList();
		    if(valueMap.containsKey(ActValue)){
			Integer f = (Integer)valueMap.get(ActValue);
			int fx = f.intValue();
			fx++;
			valueMap.put(ActValue,new Integer(fx));
		    }
		    else{

			valueMap.put(ActValue, new Integer(1));
		    }
		    if(!metadataNameList.contains(me.getMetadataName())){
		    	metadataNameList.add(me.getMetadataName());
		    	ms.addIndex(me.getMetadataName());
		    }
		    //pw.write("adding element");
		    UsedMap.put(me.getMetadataName(), me);
		    if(!MDS_list.containsKey(ms.getName())){MDS_list.put(ms.getName(), ms);}
		    else{
			MetadataSet msx = (MetadataSet)MDS_list.get(ms.getName());
			msx.addIndex(me.getMetadataName());
			MDS_list.put(ms.getName(), msx);
		    }
		}
		
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		else if ((!UsedMap.containsKey(xatt_name)) && (xatt_name.indexOf("dc.")==0) && xatt_name.indexOf("dc.Description")!=0){
		    
		    //pw.write("QUALIFIED DC element");
			
		    String SetName = "dublin";
		    xatt_name = xatt_name.replace('^', '.');
		    
		    MetadataElement me = new MetadataElement();
		    me.setMetadataName(xatt_name);
		    me.IncreaseFrequency();
		    HashMap metadataMap = me.getMetadataList();
		    
		    DocumentInfo dc = new DocumentInfo();
		    dc.IncreaseFrequence();
		    dc.setDocumentID(DocID);
		    dc.addActualValue(ActValue);
		    metadataMap.put(DocID, dc);
		    
		    HashMap valueMap = me.getValueList();
		    valueMap.put(ActValue, new Integer(1));
			
		    if(!metadataNameList.contains(me.getMetadataName())){
		    	metadataNameList.add(me.getMetadataName());
		    }
		    
		    //pw.write("adding element");
		    UsedMap.put(me.getMetadataName(), me);
		    
		    MetadataSet msx = (MetadataSet)MDS_list.get(SetName);
		    msx.addIndex(me.getMetadataName());
		    MDS_list.put(SetName, msx);
		}
		/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	    }// element
	    
	}//doc id
	////////////////////
	//write file
	int counter = 0;

	
	Set s = UsedMap.keySet();
	Iterator is = s.iterator();

	while(is.hasNext()){

	    String fileName = (String)is.next();
	    MetadataElement me = (MetadataElement)UsedMap.get(fileName);
	    HashMap hp = me.getMetadataList();
	    Collection ks = hp.values();
	    Iterator iks = ks.iterator();
		
	    try{
		
		DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
		DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
		Document doc = docBuilder.newDocument();
		Element root = doc.createElement("root");
		boolean fileExist = (new File(myNewDir+"/"+fileName+".xml")).exists();
		
		if(!fileExist){
		    root = doc.createElement("root");
		}
		else{
		    doc = docBuilder.parse (new File(myNewDir+"/"+fileName+".xml"));
		    root = doc.getDocumentElement();
		}
		
		while(iks.hasNext()){
		    
		    DocumentInfo dc = (DocumentInfo)iks.next();
		    Element docID = doc.createElement("Document");
		    docID.setAttribute("id", dc.getDocumentID());
		    //pw.write("doc id"+dc.getDocumentID());
            	
		    Element freq = doc.createElement("Frequency");
		    Text text = doc.createTextNode(dc.getFrequence()+"");
		    freq.appendChild(text);
		    docID.appendChild(freq);
            	
		    Element actValue = doc.createElement("ActualValue");
		    ArrayList alist = dc.getActualValue();
		    if(alist.size()==0){
            		text = doc.createTextNode(" ");
            		actValue = doc.createElement("ActualValue");
            		actValue.appendChild(text);
            		docID.appendChild(actValue);
            		}
            
		    for(int i = 0; i<alist.size(); i++){
            		actValue = doc.createElement("ActualValue");
            		String utf8String = new String(((String)alist.get(i)).getBytes(),"UTF-8");
            		Text text1 = doc.createTextNode(utf8String);
            		actValue.appendChild(text1);
            		docID.appendChild(actValue);
		    }	
            	
		    root.appendChild(docID);
		    docID = null;
		    counter++;
            }
 
		TransformerFactory   tf=TransformerFactory.newInstance();     
		Transformer   transformer=tf.newTransformer();     
		DOMSource   source=new   DOMSource(root);     
		transformer.setOutputProperty(OutputKeys.INDENT,"yes");     
		
		Writer pwx= new  BufferedWriter(new OutputStreamWriter(new FileOutputStream(myNewDir+"/"+fileName+".xml"),"UTF-8"));     
		StreamResult   result= new   StreamResult(pwx);     
		transformer.transform(source,result);  
		pwx.close();
  			
		root = null;
		docBuilderFactory = null;
		docBuilder = null;
		doc = null;
		
	    }catch (Exception e) {
		System.out.println(e);
		//e.printStackTrace(pw);
	    }
	}
    }
    
    public int getDocNum(){
	return TotalDoc;
    }
        
    public HashMap getMetadataSetMap(){
    	MDS_list.remove("extracted");
    	return (HashMap)MDS_list.clone();
    }
    
    public String getCollectionName(){
    	return collection_Name;
    }
    
    public void setOAIURL(String url){
    	OAI_URL = url;
    }
    
    public String getOAIURL(){
    	return OAI_URL;
    }
    
    public String getOaiPrefix(){
    	return oai_Prefix;
    }
    
    public ArrayList getMetadataNameList(){
    	return (ArrayList)metadataNameList.clone();
    }
    private String constructUpdateMessgae (){
    	String message = "<message><request type='system' to=''><system type='configure' subset=''/></request></message>";
    	return message;
    }
}