source: trunk/gli/src/org/greenstone/gatherer/msm/GDMManager.java@ 6318

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.msm;

import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.util.*;
import javax.swing.*;
import javax.swing.event.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.file.FileNode;
import org.greenstone.gatherer.gui.GLIButton;
import org.greenstone.gatherer.gui.MetaEditPrompt;
import org.greenstone.gatherer.msm.ElementWrapper;
import org.greenstone.gatherer.msm.GDMDocument;
import org.greenstone.gatherer.msm.GDMParser;
import org.greenstone.gatherer.msm.Metadata;
import org.greenstone.gatherer.msm.MSMEvent;
import org.greenstone.gatherer.msm.MSMListener;
import org.greenstone.gatherer.util.DOMTree;
import org.greenstone.gatherer.util.HashMap3D;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.w3c.dom.*;

/** This object manages the metadata attached to file records. By storing all of the metadata in one place you garner several advantages. Firstly only one copy of each metadata object is retained, all actual entries are converted to references. Next you can immediately determine what metadata is assigned to an entire directory, thus the metadata.xml files can be built more effeciently (whereas the current 'optimal' method uses recursion through the tree contents). Finally, and perhaps most importantly, it allows for dynamic 'on demand' lookup of metadata. This is especially necessary with large collections, where the raw, unconnected metadata files could range into the tens of megabytes of memory and require hundreds of megabytes to read by in using serialization. Dynamic loading allows you to connect the metadata objects on load, reducing value node paths (possibly of hundreds of characters) down to a single reference pointer! At the very worst this object uses far less memory than the current method, and given that the current method is completely incapable of handling large collections, is necessary. The trade off of course is in time needed to load metadata.xml on demand, the worst possible case being the user selecting the root node of the collection tree of a large collection immediately after opening the collection. The subsequent attempt to build the metadata table will result in the metadata being loaded for every single file. But since this process is sequential and a small cache of metadata.xml files is implemented, and given that the table will actually be build on a separate thread, the wait should not be too arduous.<BR>
    As for the size of the GDMParser cache, I was at first tempted to put around five. However after analysis of cache usage, I determined that no gain occured because of caching (in fact if everythings working as it should there should only ever be one call for a certain metadata.xml).
*/
public class GDMManager
    extends LinkedHashMap
    implements MSMListener {
    /** A list of the known metadata instances, thus we only store one of each unique metadata, and reference the rest. */
    static public HashMap3D metadata_cache = null;
    private DOMTree tree = null;
    private JComboBox documents_in_cache_combobox = null;
    /** The root file node. */
    private TreeNode root = null;
    /** The threaded object responsible for loading all of the existing metadata.xml files prior to any save action. This is necessary so that hierarchy indexes within the metadata.xml files stay fresh. */
    //private GDMLoader gdm_loader = null;
    /** The maximum number of GDMDocuments to load at any one time */
    static final private int MAX_DOCUMENTS = 25;
    /** Constructor. */
    public GDMManager() {
    super();
    this.metadata_cache = new HashMap3D(Gatherer.c_man.getCollection().msm.getSize());
    // Connect
    Gatherer.c_man.getCollection().msm.addMSMListener(this);
    // We also have a debug dialog
    if(Gatherer.debug != null) {
        display();
    }
    }

    /** This may seem a little odd but this method doesn't add the given metadata directly, instead calling fireMetadataChanged in MetadataSetManager so as to recursively add metadata if necessary, and to ensure that all listeners who are interested in data change (such as the Metadata Table and Save listeners) can be up to date. */
    public void addMetadata(FileNode node, ArrayList metadatum) {
    for(int i = 0; i < metadatum.size(); i++) {
        Metadata metadata = (Metadata) metadatum.get(i);
        Gatherer.c_man.getCollection().msm.fireMetadataChanged(node, (Metadata)null, metadata);
    }
    }

   /** Destructor necessary for clean exit, subsequent to saving of metadata.xml files.
     * @see org.greenstone.gatherer.Gatherer
     * @see org.greenstone.gatherer.collection.CollectionManager
     * @see org.greenstone.gatherer.msm.GDMParser
     * @see org.greenstone.gatherer.msm.MetadataSetManager
     */
    public void destroy() {
    // Destroy all the cached documents.
    /*
      Iterator iterator = keySet().iterator();
      while(iterator.hasNext()) {
      File file = (File) iterator.next();
      GDMDocument document = (GDMDocument) get(file);
      document.destroy(file);
      }
    */
    // Deregister as listener
    Gatherer.c_man.getCollection().msm.removeMSMListener(this);
    // Deallocate all data members
    metadata_cache.clear();
    metadata_cache = null;
    // Finally clear self
    clear();
    // Done!
    }

    /** This debug facility shows the currently loaded collect.cfg or CollectConfig.xml file as a DOM tree. */
    public void display() {
    JDialog dialog = new JDialog(Gatherer.g_man, "Greenstone Metadata Document Manager", false);
    dialog.setSize(400,400);
    JPanel content_pane = (JPanel) dialog.getContentPane();
    GDMDocument metadata_xml = new GDMDocument();
    tree = new DOMTree(metadata_xml.getDocument());
    metadata_xml = null;
    documents_in_cache_combobox = new JComboBox();
    documents_in_cache_combobox.addItemListener(new DOMItemListener());
    JButton refresh_button = new GLIButton("Refresh Tree");
    refresh_button.setMnemonic(KeyEvent.VK_R);
    refresh_button.addActionListener(new ActionListener() {
        public void actionPerformed(ActionEvent event) {
            File metadata_file = (File) documents_in_cache_combobox.getSelectedItem();
            if(metadata_file != null) {
            GDMDocument document = (GDMDocument) get(metadata_file);
            tree.setDocument(document.getDocument());
            document = null;
            metadata_file = null;
            }
        }
        });
    content_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
    content_pane.setLayout(new BorderLayout());
    content_pane.add(documents_in_cache_combobox, BorderLayout.NORTH);
    content_pane.add(new JScrollPane(tree), BorderLayout.CENTER);
    content_pane.add(refresh_button, BorderLayout.SOUTH);
    dialog.show();
    }

    private class DOMItemListener
    implements ItemListener {
    
    public void itemStateChanged(ItemEvent event) {
        if(event.getStateChange() == ItemEvent.SELECTED) {
        File metadata_file = (File) documents_in_cache_combobox.getSelectedItem();
        if(metadata_file != null) {
            GDMDocument document = (GDMDocument) get(metadata_file);
            tree.setDocument(document.getDocument());
            document = null;
            metadata_file = null;
        }
        }
    }
    }

    /** Method that is called whenever an element within a set is changed or modified. Ensure all cached GDMDocuments are marked as stale.
     * @param event A <strong>MSMEvent</strong> containing details of the event that caused this message to be fired.
     */
    public void elementChanged(MSMEvent event) {
    for(Iterator values = values().iterator(); values.hasNext(); ) {
        GDMDocument document = (GDMDocument) values.next();
        document.setUpToDate(false);
        document = null;
    }
    }
    /** Retrieve the GreenstoneMetadataDocument that is associated with a certain file. If the document is in cache returns it. If the document exists but isn't in cache loads, caches, then returns it. Otherwise it creates a brand new document, caches it, then returns it.
     * @see org.greenstone.gatherer.msm.GDMParser
     */
    public GDMDocument getDocument(File file) {
    ///ystem.err.println("Get the GDMDocument for " + file.getAbsolutePath());
    GDMDocument metadata_xml = null;
    // Determine the name of the target files metadata.xml file.
    File metadata_file = null;
    if(file.isFile()) {
        metadata_file = new File(file.getParentFile(), StaticStrings.METADATA_XML);
    }
    else {
        metadata_file = new File(file, StaticStrings.METADATA_XML);
    }
    // Then try to retrieve it from cache. First we consider the case of a cache hit.
    if(containsKey(metadata_file)) {
        metadata_xml = (GDMDocument) get(metadata_file);
        ///ystem.err.println("Get the GDMDocument for " + metadata_file.getAbsolutePath());
    }
    else {
        // Now the two potential cache misses. The first requires us to load an existing metadata.xml
        if(metadata_file.exists()) {
        ///ystem.err.println("Reload the GDMDocument for " + metadata_file.getAbsolutePath());
        metadata_xml = new GDMDocument(metadata_file);
        }
        // The final case is where no current metadata.xml exists. Create a new one just by creating a new GDMDocument.
        else {
        ///ystem.err.println("Create a new GDMDocument for " + metadata_file.getAbsolutePath());
        metadata_xml = new GDMDocument();
        }
        put(metadata_file, metadata_xml);
        if(documents_in_cache_combobox != null) {
        documents_in_cache_combobox.addItem(metadata_file);
        }
        //Gatherer.println("[0ms]\tCached " + metadata_file);
    }
    return metadata_xml;
    }

    /*
    // returns metadata file from cache or creates a new one
    public GDMDocument getDummyDocument(File file) {
    ///ystem.err.println("Get the GDMDocument for " + file.getAbsolutePath());
    GDMDocument metadata_xml = null;
    // Determine the name of the target files metadata.xml file.
    File metadata_file = null;
    if(file.isFile()) {
        metadata_file = new File(file.getParentFile(), StaticStrings.METADATA_XML);
    }
    else {
        metadata_file = new File(file, StaticStrings.METADATA_XML);
    }
    // Then try to retrieve it from cache. First we consider the case of a cache hit.
    if(containsKey(metadata_file)) {
        metadata_xml = (GDMDocument) get(metadata_file);
    }
    else {
        metadata_xml = new GDMDocument();
        put(metadata_file, metadata_xml);
    }
    return metadata_xml;
    }
    */

    /*
    public synchronized void dummyGetMetadata(File file) {
    String filename = null;
    if(file.isFile()) {
        filename = file.getName();
        file = file.getParentFile();
    }
    GDMDocument document = getDummyDocument(file);
    }
    */

    /** Recover the metadata associated with a particular file. Note that this call is synchronized, so that all of the data holders don't need to be. */
    public synchronized ArrayList getMetadata(File file) {
    return getMetadata(file, false, true);
    }

/** Recover the metadata associated with a particular file excluding folder level metadata. Note that this call is synchronized, so that all of the data holders don't need to be. */
    public synchronized ArrayList getMetadataOnly(File file) {
    return getMetadata(file, false, false);
    }

    public synchronized ArrayList getMetadata(File file, ElementWrapper element) {
    ArrayList metadata = getMetadata(file, false, true);
    ArrayList values = new ArrayList();
    for(int i = 0; i < metadata.size(); i++) {
        Metadata data = (Metadata) metadata.get(i);
        if(element.equals(data.getElement())) {
        values.add(data.getValue());
        }
    }
    if(values.size() > 0) {
        Collections.sort(values);
    }
    return values;
    }

    private ArrayList getMetadata(File file, boolean remove, boolean append_folder_level) {
    ArrayList metadata = null;
    String filename = null;
    if(file.isFile()) {
        filename = file.getName();
        file = file.getParentFile();
    }
    GDMDocument document = getDocument(file);
    if(file != null && document != null) {
        metadata = document.getMetadata(filename, remove, metadata, file, append_folder_level);
        document = null;
    }
    return metadata;
    }

    public synchronized ArrayList getAllMetadata(File file) { // boolean remove) {
    ///ystem.err.println("getMetadata(" + file.getAbsolutePath() + ")");
    ArrayList metadata = null;
    // Build up a list of all the metadata xml files we have to check for metadata.
    ArrayList search_files = new ArrayList();
    String filename = null;
    File start_file = file;
    if(file.isFile()) {
        filename = file.getName();
        start_file = file.getParentFile();
    }
    File collection_dir = new File(Gatherer.c_man.getCollectionDirectory());
    ///ystem.err.println("Collection directory = " + collection_dir.getAbsolutePath());
    ///ystem.err.println("Start directory = " + start_file.getAbsolutePath());
    while(!start_file.equals(collection_dir)) {
        ///ystem.err.println("Blip!");
        search_files.add(0, new MetadataXMLFileSearch(start_file, filename));
        if(filename != null) {
        filename = start_file.getName() + "/" + filename;
        }
        else {
        filename = start_file.getName() + "/";
        }
        start_file = start_file.getParentFile();
        ///ystem.err.println("Start directory = " + start_file.getAbsolutePath());
    }
    // Now search each of these metadata xml for metadata, remembering to accumulate or overwrite as we go along.
    for(int i = 0; i < search_files.size(); i++) {
        MetadataXMLFileSearch a_search = (MetadataXMLFileSearch) search_files.get(i);
        ///ystem.err.println("Search " + a_search.file.getAbsolutePath() + File.separator + "metadata.xml for " + (a_search.filename != null ? a_search.filename : "directory metadata"));
        // Retrieve the document
        GDMDocument document = getDocument(a_search.file);
        if(document != null) {
        // There is one piece of slight of hand here. You can never remove metadata during a get all metadata.
        metadata = document.getMetadata(a_search.filename, false, metadata, a_search.file, true);
        ///ystem.err.println("Current metadata: " + toString(metadata));
        document = null;
        }
        a_search = null;
    }
    start_file = null;
    collection_dir = null;
    filename = null;
    search_files.clear();
    search_files = null;
    return metadata;
    }

    public String toString(ArrayList list) {
    StringBuffer text = new StringBuffer("(");
    for(int i = 0; list != null && i < list.size(); i++) {
        text.append((list.get(i)).toString());
        if(i < list.size() - 1) {
        text.append(", ");
        }
    }
    text.append(")");
    return text.toString();
    }

    private class MetadataXMLFileSearch {
    public File file;
    public String filename;
    public MetadataXMLFileSearch(File file, String filename) {
        this.file = file;
        this.filename = filename;
    }
    }

    /** Called whenever the metadata value changes in some way, such as the addition of a new value. This is the only event type we care about, but we care about it a lot. It tells us what metadata to add, remove, etc from the cached metadata.xml files. Note that this method is synchronized so that the data objects don't need to be.
     * @param event A <strong>MSMEvent</strong> containing details of the event that caused this message to be fired.
     * @see org.greenstone.gatherer.msm.GDMDocument
     * @see org.greenstone.gatherer.msm.Metadata
     * @see org.greenstone.gatherer.util.HashMap3D
     */
    public synchronized void metadataChanged(MSMEvent event) {
    ///ystem.err.println("Recieved Event: " + event.toString());
    File file = event.getFile();
    if(file == null) {
        FileNode record = event.getRecord();
        file = record.getFile();
    }
    Metadata new_metadata = event.getNewMetadata();
    Metadata old_metadata = event.getOldMetadata();
    // These metadata objects may be new instances of metadata objects that already exist. Replace them if they are.
    new_metadata = checkCache(new_metadata);
    old_metadata = checkCache(old_metadata);
    // Now apply the change to the document in question.
    GDMDocument metadata_xml = getDocument(file);
    if(metadata_xml != null) {
        if(old_metadata != null) {
        // File level
        if(file.isFile()) {
            metadata_xml.removeMetadata(file.getName(), old_metadata);
        }
        // Folder level
        else {
            metadata_xml.removeMetadata(null, old_metadata);
        }
        }
        if(new_metadata != null) {
        // File level
        if(file.isFile()) {
            metadata_xml.addMetadata(file.getName(), new_metadata, event.getAction() == MetaEditPrompt.ACCUMULATE);
        }
        else {
            metadata_xml.addMetadata(null, new_metadata, event.getAction() == MetaEditPrompt.ACCUMULATE);
        }
        }
    }
    }

    public void removeExtractedMetadata() {
    try {
        // Remove all of the extracted metadata in the collection
        removeExtractedMetadata(new File(Gatherer.c_man.getCollectionImport()));
    }
    catch (Exception exception) {
        Gatherer.println("Exception in GDMManager.removeExtractedMetadata - unexpected");
        Gatherer.printStackTrace(exception);
    }
    }

    private void removeExtractedMetadata(File file) {
    // Retrieve the gdm document for this file
    GDMDocument metadata_xml_document = getDocument(file);
    // Remove the extracted metadata
    ///ystem.err.println("Removing the extracted metadata from the file: " + file);
    metadata_xml_document.removeExtractedMetadata();
    metadata_xml_document = null;
    // Then recurse down the directory structure looking for other metadata.xml files
    File child_files[] = file.listFiles();
    for(int i = 0; i < child_files.length; i++) {
        if(child_files[i].isDirectory()) {
        removeExtractedMetadata(child_files[i]);
        }
    }
    child_files = null;
    }

    public ArrayList removeMetadata(File file) {
    return getMetadata(file, true, false);
    }

    /** Causes all currently loaded GDMDocuments to write themselves out.
     * @see org.greenstone.gatherer.msm.GDMDocument
     */
    public void save() {
    Iterator iterator = keySet().iterator();
    while(iterator.hasNext()) {
        File file = (File) iterator.next();
        GDMDocument document = (GDMDocument) get(file);
        if(!document.isUpToDate()) {
        //ystem.err.println("Saving: " + file.getAbsolutePath());
        // First purge any old references.
        document.getMetadata(null, false, null, null, false, true);
        // If there is no metadata in this document then don't write out a file. In fact delete any file that already exists.
        int count = document.countMetadata();
        if(count > 0) {
            // Now write the xml
            Utility.export(document.getDocument(), file);
            document.setUpToDate(true);
        }
        else if(file.exists()) {
            file.delete();
        }
        }
    }
    }
    /** Used to cause the document associated with a particular file to write the latest copy of itself to disk. */
    public void save(FileNode node) {
    File file = node.getFile();
    if(file != null && file.isFile()) {
        GDMDocument document = getDocument(file);
        File xml_file;
        if(file.isFile()) {
        xml_file = new File(file.getParentFile(), "metadata.xml");
        }
        else {
        xml_file = new File(file, "metadata.xml");
        }
        if(document != null && !document.isUpToDate()) {
        // First purge any old references.
        document.getMetadata(null, false, null, null, true);
        // Now write the xml
        Utility.export(document.getDocument(), xml_file);
        document.setUpToDate(true);
        }
        xml_file = null;
        document = null;
    }
    file = null;
    }

    /** Write out the latest copy of a certain document. */
    public void save(File file, GDMDocument document) {
        if(!document.isUpToDate()) {
            // First purge any old references.
            document.getMetadata(null, false, null, null, true);
            // Now write the xml
            Utility.export(document.getDocument(), file);
            document.setUpToDate(true);
        }
    }


    /** Method that is called whenever the metadata set collection changes in some way, such as the addition of a new set or the merging of two sets. If a set changes, mark all cached GDMDocuments as being stale.
     * @param event A <strong>MSMEvent</strong> containing details of the event that caused this message to be fired.
     */
    public void setChanged(MSMEvent event) {
    for(Iterator values = values().iterator(); values.hasNext(); ) {
        GDMDocument document = (GDMDocument) values.next();
        document.setUpToDate(false);
        document = null;
    }
    }
    /** Called whenever the value tree of an metadata element changes in some way, such as the addition of a new value. --While the comments below are now obsolete, I'll keep them just to remind me of how easy it is to back yourself into a corner with issues such as caching and persisitant references--. Such an action would require us to painstakingly reload every metadata.xml file using the value model prior to the change, then painstakingly write out each metadata.xml file again using the modified model, but I'm a glutton for punishment so thats ok. The alternative is to not do this and watch in horror as heirarchy references quickly fall into disarray, pointing to the wrong place. This task gets even more complicated by three facts:<br>1. We want to do this is a seperate thread, as we don't want the program to come to a screaming halt while we're updating metadata.xml files.<br>2. We have to prevent any metadata.xml files being removed from cache while we're doing this, as if we encounter these more recently written files their heirarchy references will already be correct and that will balls up our little process. Note that this means the saving process may have to block while pending metadata heirarchy updates are in progress.<br>3. Regarding (2) we don't have to rewrite any metadata.xml files already in cache as they will be correctly written out whenever they happen to be dumped from cache.<br>4. We need the ability to pre-empt the general update to load a user demanded metadata.xml and store it in cache, using the old value tree model as per usual, and<br>5. We have to store a cue of these events, and process them one at a time. Perhaps one day when I'm feeling masacistic I'll figure out someway to merge several updates into one, but for now we have to change the tree one node at a time in order for references to remain correct.<br>Ok, so thats five facts, but you get the gist. Not an easy task, but crucial for accurate storage and recall of metadata heirarchies.
     * @param event A <strong>MSMEvent</strong> containing details of the event that caused this message to be fired.
     */
    public void valueChanged(MSMEvent event) {}

    public void waitUntilComplete() {
    //if(gdm_loader != null) {
    //  gdm_loader.waitUntilComplete();
    //}
    }

    private Metadata checkCache(Metadata metadata) {
    if(metadata != null) {
        ///ystem.err.println("Search for " + metadata.toString());
        if(metadata_cache.contains(metadata.getElement(), metadata.getValueNode())) {
        metadata = (Metadata) metadata_cache.get(metadata.getElement(), metadata.getValueNode());
        }
    }
    return metadata;
    }

    /** A separately threaded class to load all of the current metadata.xml files. Note that files can still be loaded on demand if they're not already in the cache. Also provides the functionality to block any other thread until the loading is complete, such as is necessary when moving values about in the value tree hierarchy. */
    /*
    private class GDMLoader
    extends Thread {
    private boolean complete = false;
    private boolean dummy_load = false;

    GDMLoader(boolean dummy_load) {
        super("blarg");
        this.dummy_load = dummy_load;
    }


    public void run() {
        // Can't open a collections metadata when the collection isn't open!
        while(!Gatherer.c_man.ready()) {
        try {
            wait(100);
        }
        catch(Exception error) {
        }
        }
        // Now for each non-file directory in the tree, ask it to load its metadata
        ArrayList remaining = new ArrayList();
        remaining.add((FileNode)Gatherer.c_man.getRecordSet().getRoot());
        int remaining_size = 0;
        while((remaining_size = remaining.size()) > 0) {
        FileNode record = (FileNode) remaining.remove(remaining_size - 1);
        if(!record.isLeaf()) {
            ///atherer.println("Retrieving metadata.xml for " + record);
            if (this.dummy_load) {
            dummyGetMetadata(record.getFile());
            } else {
            getMetadata(record.getFile());
            }
            for(int i = 0; i < record.getChildCount(); i++) {
            remaining.add(record.getChildAt(i));
            }

            ///atherer.println("Retrieving metadata.xml for " + record);
            getMetadata(record.getFile());

            record.unmap();
        }
        record = null;
        }
        remaining = null;
        complete = true;
    }
    public void waitUntilComplete() {
        try {
        while(!complete) {
            sleep(100); // 1 second hopefully.
        }
        }
        catch(Exception error) {
        Gatherer.printStackTrace(error);
        }
    }
    }
    */

    protected boolean removeEldestEntry(Map.Entry eldest) {
        if(size() > MAX_DOCUMENTS) {
        // Save the oldest document before its dumped
        File file = (File) eldest.getKey();
        ///ystem.err.println("Dumping oldest Document: " + file.getAbsolutePath());
        GDMDocument document = (GDMDocument) eldest.getValue();
        save(file, document);
        
        if(documents_in_cache_combobox != null) {
        documents_in_cache_combobox.removeItem(file);
        }
        // And then dump it
        return true;
    }
    else {
        return false;
    }
    }
}