source: trunk/gli/src/org/greenstone/gatherer/msm/parsers/GreenstoneMetadataParser.java@ 8022

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.msm.parsers;

/**************************************************************************************
 * Written:      ??/??/02
 * Revised:      ??/??/02 - Commented
 *               25/07/03 - Fix to allow any valid greenstone metadata.xml to be imported from, not just those that occur within a collection. This functionality is accidental and caused by GLI attempting to find a collect.cfg to extract hierarchy file information from, and failing.
 **************************************************************************************/
import java.io.*;
import java.lang.ref.*;
import java.net.*;
import java.util.*;
import java.util.regex.*;
import javax.swing.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.cdm.CommandTokenizer;
import org.greenstone.gatherer.file.FileNode;
import org.greenstone.gatherer.gui.WarningDialog;
import org.greenstone.gatherer.msm.ElementWrapper;
import org.greenstone.gatherer.msm.Metadata;
import org.greenstone.gatherer.msm.MetadataParser;
import org.greenstone.gatherer.msm.MSMUtils;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.valuetree.GValueModel;
import org.greenstone.gatherer.valuetree.GValueNode;
import org.w3c.dom.*;
/** Provides a metadata parser implementation that knows how to locate, prepare for, then import metadata from a previous Greenstone collection. Is aware of such factors as the presence of Metadata Set files and hierarchy files. Updates the profiler where possible to allow for faster subsequent imports from a certain collection. Caches all the information about encountered collections in CollectCFG objects which are softly cached (ie are cached, but are reclaimed before an OutOfMemory exception would be thrown). 
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.3
 */
public class GreenstoneMetadataParser
    extends LinkedHashMap
    implements MetadataParser {

    static final private int MAX_CFG_CACHE_SIZE = 10;
    static final private int MAX_GDM_CACHE_SIZE = 10;
    /** The default name and location for a collection configuration file (presuming that a collection file prefix will be added). */
    static final private String CONFIG_FILENAME = "etc" + File.separator + "collect.cfg";
    /** The pattern to match when searching for directory level assignments. */
    static final private String DIRECTORY_FILENAME = ".*";
    static final private String DIRECTORY_FILENAME_SUFFIX = "/.*";
    static final private String DESCRIPTION_ELEMENT = "Description";
    static final private String FILENAME_ELEMENT = "FileName";
    static final private String FILESET_ELEMENT = "FileSet";
    /** The name of a gdm file. */
    static final private String GIMPORT = "gimport";
    static final private String IMPORT = "import";
    static final private String METADATA_ELEMENT = "Metadata";
    static final private String METADATA_XML_FILENAME = "metadata.xml";
    static final private String MODE_ATTRIBUTE = "mode";
    static final private String NAME_ATTRIBUTE = "name";
    static final private String SEPARATOR = "/";

    /** A list of the collect.cfg paths that we should ignore. */
    private ArrayList ignore_list = new ArrayList();
    /** Has this process been cancelled. */
    private boolean dialog_cancelled = false;
    /** A cache of previously parsed collection configuration files. */
    private CollectCFGCache cfg_cache = new CollectCFGCache();
    /** A mapping from BasicMetadata to their fully enabled Metadata incarnation. */
    private HashMap transform = new HashMap();

    /** Default constructor needed for dynamic class loading. */
    public GreenstoneMetadataParser() {
    }
    /** Locate and import any metadata parsed by this metadata parser given the file involved and its previous incarnation. */
    public boolean process(FileNode destination, FileNode origin, boolean folder_level, boolean dummy_run) {
    Gatherer.println("GreenstoneMetadataParser: Process " + origin + ": ");
    int counter = 0;
    dialog_cancelled = false;

    // 1. Determine what collection the file is in, and load/parse the appropriate collect.cfg. Cache collect.cfg object.
    ///ystem.err.print("1 ");
    // Start at the origin node file. If its a file get its parent directory.
    File collection_dir = origin.getFile();
    if(collection_dir.isFile()) {
        collection_dir = collection_dir.getParentFile();
    }
    // We're currently in the importing directory so we'll go one more step up.
    collection_dir = collection_dir.getParentFile();
    // We are looking for a directory which contains a etc/collect.cfg file and either an import or a gimport directory.
    boolean found = false;
    while(!found && collection_dir != null) {
        File possible_cfg_file = new File(collection_dir, CONFIG_FILENAME);
        File possible_gimport_directory = new File(collection_dir, GIMPORT);
        File possible_import_directory = new File(collection_dir, IMPORT);
        if(possible_cfg_file.exists() && (possible_gimport_directory.exists() || possible_import_directory.exists())) {
        found = true;
        Gatherer.println("Found greenstone collection at " + collection_dir.getAbsolutePath());
        }
        else {
        collection_dir = collection_dir.getParentFile();
        }
    }

    // Now retrieve the configuration file if there is one.
    CollectCFG collect_cfg = null;
    if(collection_dir != null) {
        File collect_cfg_file = new File(collection_dir, CONFIG_FILENAME);
        if(collect_cfg_file.exists()) {
        collect_cfg = cfg_cache.get(collect_cfg_file);
        }
    }

    // 2. If a collection configuration file was found, attempt to merge in any mdses and make note of those that are successfully imported (by removing reference from collect.cfg).
    Gatherer.println("Merging in any metadata sets found.");
    if(collect_cfg != null) {
        ArrayList mdses = collect_cfg.getMetadataSets();
        for(int i = 0; i < mdses.size(); i++) {
        File mds_file = (File) mdses.get(i);
        Gatherer.c_man.getCollection().msm.importMDS(mds_file, false);
        }
        mdses.clear();
        mdses = null;
    }

    // 3. Locate all of the metadata.xml files that may have an affect on the origin file. Make sure the metadata.xml closest to the origin files directory is last (to ensure property inheritance regarding accumulate/overwrite).
    Gatherer.println("Searching for metadata.xml files");
    ArrayList search_files = new ArrayList();
    File file = origin.getFile();
    String filename = null;
    boolean file_level;
    if(file.isFile()) {
        file_level = false;
        filename = file.getName();
        file = file.getParentFile();
    }
    else {
        file_level = true;
    }
    while(file != null && (collection_dir == null || !file.equals(collection_dir))) {
        File test_file = new File(file, Utility.METADATA_XML);
        if(test_file.exists()) {
        search_files.add(0, new MetadataXMLFileSearch(test_file, filename));
        }
        if(filename != null) {
        filename = file.getName() + SEPARATOR + filename;
        }
        else {
        filename = file.getName();
        }
        file = file.getParentFile();
    }
    filename = null;
    file = null;
    // Start with an initially empty ArrayList of metadata
    ArrayList metadata = new ArrayList();
    // Now search each of these metadata xml for metadata, remembering to accumulate or overwrite as we go along.
    for(int i = 0; i < search_files.size(); i++) {
        MetadataXMLFileSearch a_search = (MetadataXMLFileSearch) search_files.get(i);
        Gatherer.println("Search " + a_search.file.getAbsolutePath() + " for " + (a_search.filename != null ? a_search.filename : ".*"));
        Gatherer.println("Search at the " + (folder_level ? "Folder" : "Filename") + " level");
        // Retrieve the document
        BasicGDMDocument document = getDocument(a_search.file);
        if(document != null) {
        // If this is a dummy run, our original source file is actually the metadata.xml file and we retrieve all metadata for this collection, as if accumulated!
        if(dummy_run) {
            metadata = document.getAllMetadata();
        }
        else {
            metadata = document.getMetadata(a_search.filename, metadata, folder_level);
        }
        document = null;
        }
        a_search = null;
    }
    search_files = null;
    // Finally assign the metadata
    Gatherer.println("Found " + metadata.size() + " pieces of metadata for " + destination);
    if(metadata.size() > 0) {
        addMetadata(origin, destination, metadata, collection_dir, collect_cfg, dummy_run);
    }
    return dialog_cancelled;
    }

    protected boolean removeEldestEntry(java.util.Map.Entry entry) {
    return (size() > MAX_GDM_CACHE_SIZE);
    }

    private void addMetadata(FileNode origin, FileNode destination, ArrayList metadata, File collection_dir, CollectCFG collect_cfg, boolean dummy_run) {
    // before we try to addMetadata, we need to check that there are some metadata sets for the collection - otherwise we cant add or import
    Vector meta_sets = Gatherer.c_man.getCollection().msm.getSets(false);
    if (meta_sets.size()==0) {
        ///ystem.out.println("GreenstoneMetadataParser:Error: we have been asked to add metadata but there are no existing sets");
        // print the warning dialog
        if(Gatherer.f_man.complain_if_no_sets) {
        WarningDialog dialog = new WarningDialog("warning.MissingMDS", true);
        if (dialog.display() == JOptionPane.CANCEL_OPTION) {
            // the user has cancelled
            dialog_cancelled = true;
            Gatherer.f_man.complain_if_no_sets = true;
        }
        else {
            Gatherer.f_man.complain_if_no_sets = false;
        }
        }
        return;
    }
    ///ystem.err.print("6 ");
    // Used in a complicated test later on.
    for(int i = 0; !dialog_cancelled && i < metadata.size(); i++) {
        BasicMetadata basic_metadata = ((BasicMetadata) metadata.get(i)).copy();
        BasicMetadata metadatum = (BasicMetadata) metadata.get(i);
        metadatum.collection = collection_dir; // May be null. Doesn't matter.
        Metadata final_metadata = null;
        // If this BasicMetadata already exists in the transform cache then we can save ourselves a lot of work.
        SoftReference reference = (SoftReference) transform.get(basic_metadata);
        if(reference != null) {
        final_metadata = (Metadata) reference.get();
        }
        if(final_metadata == null) {
        ///ystem.err.println("No existing Metadata object for BasicMetadata: " + basic_metadata);
        // 6a. Check if an hfile is associated with this metadata, and if so load it, cache it in the collection.cfg object, then resolve metadata value index. Of course we can only do this if a collection configuration file was found in the first place.
        if(collect_cfg != null) {
            HFile h_file = collect_cfg.getHFile(metadatum.element);
            if(h_file != null && !dummy_run) {
            ///ystem.err.print(metadata.value + " maps to ");
            metadatum.value = h_file.getValue(metadatum.value);
            ///ystem.err.println(metadatum.value);
            }
            h_file = null;
        }
        // 6b. Check if there is a profile regarding the current metadata. The profile may be stored for the collection directory, or if no such directory is available, then try the ancestor folders of the origin file.
        ///ystem.err.println("Retrieve existing action: " + collection_dir.getAbsolutePath() + ", " + metadatum.element);
        if(collection_dir != null) {
            // Note that the first test is whether a profile action exist, while the 'getAction' can return null as the profile action. 
            if(Gatherer.c_man.getCollection().msm.profiler.containsAction(collection_dir.getAbsolutePath(), metadatum.element)) {
            String new_element_name = Gatherer.c_man.getCollection().msm.profiler.getAction(collection_dir.getAbsolutePath(), metadatum.element);
            ///ystem.err.println("Profile result = " + new_element_name);
            if(new_element_name == null) {
                metadatum = null;
            }
            else {
                metadatum.element = new_element_name;
            }
            new_element_name = null;
            }
        }
        else {
            boolean found = false;
            File current_folder = origin.getFile().getParentFile();
            while(!found && metadatum != null && current_folder != null) {
            if(Gatherer.c_man.getCollection().msm.profiler.containsAction(current_folder.getAbsolutePath(), metadatum.element)) {
                found = true;
                String new_element_name = Gatherer.c_man.getCollection().msm.profiler.getAction(current_folder.getAbsolutePath(), metadatum.element);
                ///ystem.err.println("Profile result = " + new_element_name);
                if(new_element_name == null) {
                metadatum = null;
                }
                else {
                metadatum.element = new_element_name;
                }
                new_element_name = null;
            }
            else {
                current_folder = current_folder.getParentFile();
            }
            }
            current_folder = null;
        }
        ///atherer.println("Assigning metadatum.");
        if(metadatum != null) {
           ElementWrapper element = null;
           // 6c. Try to add metadata.
           // If we just happen to be importing files from our base collection, then we can assume that any non-namespaced elements are actually extracted metadata
           if(Utility.isParentFolderOf(Gatherer.c_man.getCollection().getBaseCollection(), origin.getFile().getParentFile())) {
              element = Gatherer.c_man.getCollection().msm.getElement(metadatum.element);// Perfect Match not required
           }
           // Typical case
           else {
              element = Gatherer.c_man.getCollection().msm.getElement(metadatum.element, true); // Perfect match
           }
           
           ///ystem.err.println("Closest match: " + element);
            // 6ci. If no match exists, prompt the user to add/merge with specific metadata element. The user can also choose to ignore this metadata.
            if(element == null) {
            element = selectElement(metadatum.element);
            if(!dialog_cancelled) {
                // 6ciii. If either of the above work, remember to add to profile.
                if(element == null) {
                ///ystem.err.println("Adding profile action: " + collection_dir.getAbsolutePath() + ", " + metadatum.element + ", null");
                if(collection_dir != null) {
                    Gatherer.c_man.getCollection().msm.profiler.addAction(collection_dir.getAbsolutePath(), metadatum.element, null);
                }
                else {
                    Gatherer.c_man.getCollection().msm.profiler.addAction(origin.getFile().getParentFile().getAbsolutePath(), metadatum.element, null);
                }
                }
                else {
                ///ystem.err.println("Adding profile action: " + collection_dir.getAbsolutePath() + ", " + metadatum.element + ", " + element.getName());
                if(collection_dir != null) {
                    Gatherer.c_man.getCollection().msm.profiler.addAction(collection_dir.getAbsolutePath(), metadatum.element, element.getName());
}
                else {
                    Gatherer.c_man.getCollection().msm.profiler.addAction(origin.getFile().getParentFile().getAbsolutePath(), metadatum.element, element.getName());
                }
                }
            }
            }
            // - Add metadata
            if(!dummy_run && element != null && !dialog_cancelled) {
            ///ystem.err.println("Retrieve the value tree for " + element.toString());
            GValueModel model = Gatherer.c_man.getCollection().msm.getValueTree(element);
            if(model != null) {
                // One little 'fix' for importing from the demo or dls files. The Title metadata found in the metadata.xml isn't used in preference for the automatically extracted Titles. However we want to use them, so we should remove '.*(<filename>)$' for a certain file <filename>.
                String raw_value = metadatum.value.trim();
                String filename_munged = destination.getFile().getName();
                int index = -1;
                if((index = filename_munged.indexOf(".")) != -1) {
                filename_munged = filename_munged.substring(0, index);
                }
                filename_munged = "(" + filename_munged + ")";
                if(raw_value.endsWith(filename_munged)) {
                raw_value = (raw_value.substring(0, raw_value.length() - filename_munged.length())).trim();
                }
                GValueNode node = model.addValue(raw_value);
                final_metadata = new Metadata(element, node);
                ///ystem.err.println("Adding final metadata: " + metadatum.toString());
                node = null;
            }
            model = null;
            }
            element = null;
        }
        // If we have successfully created a Metadata from the BasicMetadata, store it
        if(final_metadata != null && !dialog_cancelled) {
            transform.put(basic_metadata, new SoftReference(final_metadata));
            ///ystem.err.println("Add a Metadata object for BasicMetadata: " + basic_metadata);
        }
        }
        else {
        ///ystem.err.println("Found a Metadata object for BasicMetadata: " + basic_metadata);
        }
        if(!dummy_run && final_metadata != null && !dialog_cancelled) {
        final_metadata.setAccumulate(metadatum.accumulates);
        // Now we can finally add the metadata.
        ///ystem.err.println("Adding Metadata: " + final_metadata);
        Gatherer.c_man.getCollection().msm.fireMetadataChanged(0, destination, null, final_metadata);
        }
                // Otherwise there is no way to add this metadata. No value model no metadata value.
        final_metadata = null;
        metadatum = null;
    }
    }

    /** Determine the different suffix between two string.
     * @param base_str The base <strong>String</strong>, expected to be the short of the two strings provided.
     * @param target_str The target <strong>String</strong>, whose differing suffix is returned.
     * @return A <strong>String</strong> containing the suffix from target which is different from base.
     */
    private String diff(String base_str, String target_str) {
    StringTokenizer base_tokenizer = new StringTokenizer(base_str, File.separator);
    StringTokenizer target_tokenizer = new StringTokenizer(target_str, File.separator);
    String base = null;
    String target = null;
    while(base_tokenizer.hasMoreTokens() && (base = base_tokenizer.nextToken()).equals((target = target_tokenizer.nextToken()))) {
    }
    StringBuffer result = new StringBuffer(target);
    while(target_tokenizer.hasMoreTokens()) {
        result.append(File.separator);
        result.append(target_tokenizer.nextToken());
    }
    return result.toString();
    }

    /** Retrieve the BasicGDMDocument found at the given file, or null if there is no such file or if it isn't a valid BasicGDMDocument. */
    private BasicGDMDocument getDocument(File file) {
    ///ystem.err.println("Get Document at: " + file.getAbsolutePath());
    BasicGDMDocument document = null;
    if(!ignore_list.contains(file) && file.exists()) {
                // Check cache
        SoftReference reference = (SoftReference) get(file);
        if(reference != null) {
        ///ystem.err.println("Hit!!");
        document = (BasicGDMDocument) reference.get();
        reference = null;
        }
                // If that didn't work try to parse in the document
        if(document == null) {
        ///ystem.err.println("Miss or stale reference.");
        document = new BasicGDMDocument(file);
        if(document.isValid()) {
            put(file, new SoftReference(document));
        }
        else {
            ///ystem.err.println(file.getAbsolutePath() + " is not a valid GDM XML file.");
            ignore_list.add(file);
            document = null;
        }
        }
    }
    else {
                ///ystem.err.println("Ignoring file or file doesn't exists.");
    }
    return document;
    }


    /** Display a prompt allowing a user to select a metadata element to attempt to force add/merge or ignore a metadata element to. For instance an old version of a metadata.xml from the DLS collection might have an assigned metadata value "Publisher=EC Courier", however Publisher won't automatically match to any metadata set. This prompt will be displayed, and some effort will be made to systematically locate the appropriate set. In this case this should be the DLS metadata set as dls.Publisher should be the closest match. Regardless the element selected is returned.
     * @param element_name The name of the element we are trying to add, as a <strong>String</strong>.
     * @return The <strong>ElementWrapper</strong> choosen by the user, or <i>null</i> to skip this metadata element.
     */
    private ElementWrapper selectElement(String element_name) {
    ElementWrapper result = Gatherer.c_man.getCollection().msm.prompt.selectElement(element_name);
    dialog_cancelled = Gatherer.c_man.getCollection().msm.prompt.wasDialogCancelled();
    return result;
    }

    /** A 'basic' version of the more complete GDMDocument used elsewhere, this object provides the same functionality except that it doesn't use Metadata objects. These objects require live references to elements within the MetadataSetManager and GValueModels, but these may not yet exist (and indeed may never exist) for metadata parsed from metadata.xml's outside of our current collection. Thus this class returns a String (or an ArrayList of Strings) when asked for the metadata associated with a certain file. Also notice that this class provides no constructor method for creating a blank document, nor does it ever need a reference to the Gatherer.*/
    private class BasicGDMDocument 
    extends HashMap {
    /** The document this class sources its data from. */
    private Document base_document;
    /** This constructor takes the original document and parsed out and stores metadata with its association to filenames. */
    public BasicGDMDocument(File file) {
        ///ystem.err.println("New BasicGDMDocument: " + file.getAbsolutePath());
        base_document = Utility.parse(file.getAbsolutePath(), false);
    }
    /** Retrieve all of the metadata in this file. */
    public ArrayList getAllMetadata() {
        ArrayList metadatum = new ArrayList();
        // Don't search the cache as this would never have been added.
        try {
        // Retrieve the document element.
        Element directorymetadata_element = base_document.getDocumentElement();
        // Iterate through the filesets, checking the FileName child element against    the target file's name using regular expression matching.
        NodeList fileset_elements = directorymetadata_element.getElementsByTagName(FILESET_ELEMENT);
        for(int i = 0; i < fileset_elements.getLength(); i++) {
            Element fileset_element = (Element) fileset_elements.item(i);
            NodeList filename_elements = fileset_element.getElementsByTagName(FILENAME_ELEMENT);
            for(int j = 0; j < filename_elements.getLength(); j++) {
            Element filename_element = (Element) filename_elements.item(j);
            // If they match add all of the metadata found in the Description child element, overwriting any metadata with the same element
            NodeList description_elements = fileset_element.getElementsByTagName(DESCRIPTION_ELEMENT);
            for(int k = 0; k < description_elements.getLength(); k++) {
                Element description_element = (Element) description_elements.item(k);
                NodeList metadata_elements = description_element.getElementsByTagName(METADATA_ELEMENT);
                for(int l = 0; l < metadata_elements.getLength(); l++) {
                Element metadata_element = (Element) metadata_elements.item(l);
                String element = metadata_element.getAttribute(NAME_ATTRIBUTE);
                BasicMetadata metadata = new BasicMetadata(element, Utility.METADATA_XML, true);
                // Remove any previous values for this metadata element.
                for(int m = metadatum.size() - 1; m >= 0; m--) {
                    BasicMetadata old_metadata = (BasicMetadata) metadatum.get(m);
                    if(old_metadata.element.equals(element)) {
                    metadatum.remove(m);
                    }
                    old_metadata = null;
                }
                // Add the completed metadata and clean up
                metadatum.add(metadata);
                metadata = null;
                element = null;
                metadata_element = null;
                }
                metadata_elements = null;
                description_element = null;
            }
            description_elements = null;
            filename_element = null;
            }
            filename_elements = null;
            fileset_element = null;
        }
        fileset_elements = null;
        directorymetadata_element = null;
        }
        catch (Exception error) {
        Gatherer.self.printStackTrace(error);
        }
        return metadatum;
    }
          
    /** Retrieve any metadata associated with a certain file. If filename is null we are attempting to find directory level metadata. */
    public ArrayList getMetadata(String filename, ArrayList metadatum_so_far, boolean folder_level) {
        ///ystem.err.println("Retrieving metadata for: " + filename + " [" + folder_level + "]");
        ArrayList metadatum = null;
        // We start by attempting to retrieve this metadata from the cache.
        if(filename != null) {
        metadatum = (ArrayList) get(filename);
        }
        else {
        metadatum = (ArrayList) get(DIRECTORY_FILENAME);
        }
        // If that failed we consult the document for metadata.
        if(metadatum == null) {
        metadatum = new ArrayList();
        if(metadatum_so_far == null) {
            metadatum = new ArrayList();
        }
        else {
            metadatum = metadatum_so_far;
        }
        try {
            // Retrieve the document element.
            Element directorymetadata_element = base_document.getDocumentElement();
            // Iterate through the filesets, checking the FileName child element against    the target file's name using regular expression matching.
            NodeList fileset_elements = directorymetadata_element.getElementsByTagName(FILESET_ELEMENT);
            for(int i = 0; i < fileset_elements.getLength(); i++) {
            Element fileset_element = (Element) fileset_elements.item(i);
            NodeList filename_elements = fileset_element.getElementsByTagName(FILENAME_ELEMENT);
            for(int j = 0; j < filename_elements.getLength(); j++) {
                Element filename_element = (Element) filename_elements.item(j);
                String filename_text = MSMUtils.getValue(filename_element);
                if(isMatchingFileSet(filename, filename_text, folder_level)) {
                ///ystem.err.println("Match: " + (filename != null ? filename : ".*") + " => " + filename_text);
                // If they match add all of the metadata found in the Description child element, remembering to abide by desired mode (accumulate vs. overwrite).
                NodeList description_elements = fileset_element.getElementsByTagName(DESCRIPTION_ELEMENT);
                for(int k = 0; k < description_elements.getLength(); k++) {
                    Element description_element = (Element) description_elements.item(k);
                    NodeList metadata_elements = description_element.getElementsByTagName(METADATA_ELEMENT);
                    for(int l = 0; l < metadata_elements.getLength(); l++) {
                    Element metadata_element = (Element) metadata_elements.item(l);
                    String element = metadata_element.getAttribute(NAME_ATTRIBUTE);
                    ///ystem.err.println("Found element: " + element);
                    //String language = metadata_element.getAttribute("language");
                    String mode = metadata_element.getAttribute(MODE_ATTRIBUTE);
                    // Add the new metadata to our list of metadata for this target file.
                    String value = Utility.stripNL(MSMUtils.getValue(metadata_element));
                    ///ystem.err.println("Found value: " + element);
                    BasicMetadata metadata = new BasicMetadata(element, value, mode.equals("accumulate"));
                    // If mode is overwrite, then remove any previous values for this metadata element.
                    if(!metadata.accumulates) {
                        for(int m = metadatum.size() - 1; m >= 0; m--) {
                        BasicMetadata old_metadata = (BasicMetadata) metadatum.get(m);
                        if(old_metadata.element.equals(element)) {
                            metadatum.remove(m);
                        }
                        old_metadata = null;
                        }
                    }
                    mode = null;
                                                     
                    // Add the completed metadata and clean up
                    metadatum.add(metadata);
                    metadata = null;
                    value = null;
                    element = null;
                    metadata_element = null;
                    }
                    metadata_elements = null;
                    description_element = null;
                }
                description_elements = null;
                }
                else {
                ///ystem.err.println("No Match!");
                }
                filename_text = null;
                filename_element = null;
            }
            filename_elements = null;
            fileset_element = null;
            }
            fileset_elements = null;
            directorymetadata_element = null;
        }
        catch (Exception error) {
            Gatherer.self.printStackTrace(error);
        }
        // Cache the result, given that these external metadata.xmls are taken to be static at the time of reading (if you happen to be sourcing information from a opened collection that someone is working on, too bad.
        if(filename != null) { 
            put(filename, metadatum);
        }
        else {
            put(DIRECTORY_FILENAME, metadatum);
        }
        }
        return metadatum;
    }

    private boolean isMatchingFileSet(String filename, String filename_text, boolean folder_level) {
        // Crappy. There are apparently two ways of assigning, say, directory level metadata to anything in the ac01ne directory from a parent directories metadata.xml.
        // The developers guide way: ac01ne/.*
        // The dls way: ac01ne
        // So the three tests are:
        // 1. Check for an exact match i.e "ac01ne/ac01ne.htm" matches "ac01ne/".*
        // 2. Check for a parent folder match, in the absence of further pattern i.e "ac01ne/ac01ne.htm" matches "ac01ne"
        // 3. Check for a folder level match if thats what we are looking for i.e "null" matches ".*"

        ///ystem.err.println("Check for: " + (filename != null ? filename : ".*"));
        ///ystem.err.println("Folder level = " + folder_level);
        ///ystem.err.println("filename != null && '" + filename + "'.matches('" + filename_text + "') = " + (filename != null ? filename.matches(filename_text) && !filename_text.equals(DIRECTORY_FILENAME) : false));
        ///ystem.err.println("filename != null && '" + filename + "'.matches('" + filename_text + DIRECTORY_FILENAME_SUFFIX + "') [folder level = " + folder_level + "] = " + (filename != null ? filename.matches(filename_text + DIRECTORY_FILENAME_SUFFIX) && folder_level: false));
        ///ystem.err.println("filename == null && '" + filename_text + "'.equals('.*') = " + (filename == null ? filename_text.equals(DIRECTORY_FILENAME) : false));
        if (filename != null) {
        if(folder_level) {
            return filename.matches(filename_text) || filename.matches(filename_text + DIRECTORY_FILENAME_SUFFIX);
        }
        else {
            return filename.matches(filename_text) && !filename_text.equals(DIRECTORY_FILENAME);
        }
        }
        else {
        return filename_text.equals(DIRECTORY_FILENAME);
        }
    }
          
    /** Determine is this is a valid Greenstone Directory Metadata file. It may of course just be some xml file with the name metadata.xml. */
    public boolean isValid() {
                // Just determine if the doctype is GreenstoneDirectoryMetadata and root node is called DirectoryMetadata.
        String doctype_name = base_document.getDoctype().getName();
        String root_name = base_document.getDocumentElement().getTagName();
        return ((doctype_name.equals("DirectoryMetadata") || doctype_name.equals("GreenstoneDirectoryMetadata")) && (root_name.equals("DirectoryMetadata") || root_name.equals("GreenstoneDirectoryMetadata")));
    }

    /** Decode a string that was previously made Perl safe.
     * @param safe The encoded <strong>String</strong> where dangerous characters have been escaped.
     * @return A <strong>String</strong> with all the escaping removed.
     */
    private String decode(String safe) {
        String dangerous = safe.replaceAll("\\\\.",".");
        return dangerous;
    }
    }
    /** A simplistic version of metadata, with no live references. */
    private class BasicMetadata 
    implements Comparable {
    public boolean accumulates;
    /** The collection this metadata was extracted from. Important when attempting to map BasicMetadata to its Metadata incarnation. */
    public File collection;
    /** The metadata element. */
    public String element = null;
    /** The value. */
    public String value = null;
    /** Constructor takes initial values for element and value. 
     * @param element The metadata element as a <strong>String</strong>.
     * @param value The value as a <strong>String</strong>.
     */
    public BasicMetadata(String element, String value, boolean accumulates) {
        this.accumulates = accumulates;
        this.element = element;
        this.value = value;
    }

    public BasicMetadata copy() {
        return new BasicMetadata(element, value, accumulates);
    }

    public int compareTo(Object other) {
        return toString().compareTo(other.toString());
    }
    /** Compare two BasicMetadata objects for equality.
     * @param object The other <strong>Object</strong>.
     * @return <i>true</i> if this BasicMetadata matches the given object, <i>false</i> otherwise.
     */
    public boolean equals(Object object) {
        BasicMetadata other = (BasicMetadata) object;
        if(collection != null && other.collection != null) {
        return (collection.equals(other.collection) && element.equals(other.element) && value.equals(other.value));
        }
        return (element.equals(other.element) && value.equals(other.value)); 
    }
    public String toString() {
        return element + " = " + value;
    }
    }

    /** This class provides a cache for the instances of parsed collect.cfg files and their associated data. Assures that the most recently cached CollectCFG will remain available. Older objects are maintained as soft references and are freed at the JVM implementations descretion, but are gareunteed to be garbage collected before an OutOfMemory exception is thrown. */
    private class CollectCFGCache
    extends LinkedHashMap {
    /** Retrieve the CollectCFG object that matches the given collection file path.
     * @param collect_cfg_file The <strong>File</strong> that references the collection's directory.
     * @return The <strong>CollectCFG</strong> that belongs to this collection, or <i>null</i> if no such file exists (so we probably aren't in a collection!).
     */
    public CollectCFG get(File collect_cfg_file) {
                ///ystem.err.println("Retrieve the collection configuration file at: " + collect_cfg_file);
        CollectCFG collect_cfg = null;
                // Attempt to load from cache.
        SoftReference reference = (SoftReference) super.get(collect_cfg_file);
                // If is doesn't exist, either because its never been loaded, or thats its cache reference has gone stale, attempt to load it again.
        if(reference == null || (collect_cfg = (CollectCFG)reference.get()) == null) {
        try {
            collect_cfg = new CollectCFG(collect_cfg_file);
            put(collect_cfg_file, new SoftReference(collect_cfg));
        }
        catch(Exception error) {
            Gatherer.printStackTrace(error);
            collect_cfg = null;
        }
        }
        return collect_cfg;
    }

    protected boolean removeEldestEntry(java.util.Map.Entry entry) {
        return (size() > MAX_CFG_CACHE_SIZE);
    }
    }

    /** The CollectCFG object encapsulates important metadata information extracted from a collect.cfg file, such as required metadata sets, and hfile associations. As the former are merged, their references are removed from this object, whereas the for the later references are replaced a representation of the hfile itself. */
    private class CollectCFG {
    /** A list of the metadata sets associated with the collect.cfg file. */
    private ArrayList metadatasets = null;
    /** A hash mapping from metadata element name to hierarchy file, or possibly hierarchy object. */
    private HashMap hfiles = null;
    /** The token at the start of a classify command line within the collect.cfg. */
    static final private String CLASSIFY_COMMAND = "classify";
    /** The token at the start of a metadataset command line within the collect.cfg. */
    static final private String METADATASET_COMMAND = "metadataset";
    /** Constructor which takes a file assumed to be the location of a collect.cfg file belonging to a Greenstone Collection.
     * @param file A <strong>File</strong> referencing a collect.cfg file.
     */
    public CollectCFG(File file) 
        throws Exception {
                ///atherer.println("Loading a new collection configuration file: " + file.getAbsolutePath());
        File etc_directory = file.getParentFile();
        hfiles = new HashMap();
        metadatasets = new ArrayList();
        FileReader reader = new FileReader(file);
        BufferedReader in = new BufferedReader(reader);
        String command = null;
        while((command = in.readLine()) != null) {
        CommandTokenizer tokenizer = new CommandTokenizer(command);
        if(tokenizer.hasMoreTokens()) {
            String token = tokenizer.nextToken().toLowerCase();
            if(token.equals(METADATASET_COMMAND)) {
            String family_name = tokenizer.nextToken(); 
            String file_str = tokenizer.nextToken();
            if(file_str.startsWith("\"") && file_str.endsWith("\"") && !file_str.equals("\"\"")) {
                file_str = file_str.substring(1, file_str.length() - 1);
            }
            // If the file str is -only- the filename then we add <col_dir>/metadata/
            File mds_file = null;
            if(file_str.indexOf(File.separator) == -1) {
                mds_file = new File(file.getParentFile().getParentFile(), File.separator + "metadata" + File.separator + file_str);
            }
            else {
                mds_file = new File(file_str);
            }
            ///ystem.err.println("Attempting to file mds file at " + file.getAbsolutePath());
            if(mds_file.exists()) {
                metadatasets.add(mds_file);
            }
            mds_file = null;
            file_str = null;
            family_name = null;
            }
            // Also look for any classify commands that include an hfile and element
            else if(token.equals(CLASSIFY_COMMAND)) {
            String hfile_name = null;
            String element_name = null;
            // Drop the classifier name
            tokenizer.nextToken();
            while(tokenizer.hasMoreTokens()) {
                token = tokenizer.nextToken().toLowerCase();
                if(token.equals("-hfile")) {
                hfile_name = tokenizer.nextToken();
                }
                else if(token.equals("-metadata")) {
                element_name = tokenizer.nextToken();
                }
            }
            if(hfile_name != null && element_name != null) {
                // If hfile_name has no path, append the etc directories one. Either way create a file reference
                File hfile = null;
                hfile_name = hfile_name.replace('\\', File.separatorChar);
                hfile_name = hfile_name.replace('/', File.separatorChar);
                if(hfile_name.indexOf(File.separator) == -1) {
                hfile = new File(etc_directory, hfile_name);
                }
                else {
                hfile = new File(hfile_name);
                }
                // Add to hfiles
                ///atherer.println("Adding hfile reference: " + element_name + " -> " + hfile);
                hfiles.put(element_name, hfile);
                hfile = null;
            }
            element_name = null;
            hfile_name = null;
            }
            tokenizer = null;
        }
        }
        command = null;
        in.close();
        reader.close();
        in = null;
        reader = null;
                // Now we search the etc directory for *.txt files which we attempt to parse as hfiles
        File children[] = etc_directory.listFiles(); // We are sure there is at least one, collect.cfg
        for(int i = 0; i < children.length; i++) {
        // If this is a text file, extract the element name and process
        String name = children[i].getName();
        if(children[i].isFile() && name.endsWith(".txt")) {
            String element_name = name.substring(0, name.lastIndexOf("."));
            if(!hfiles.containsKey(element_name)) {
            ///atherer.println("Adding hfile reference: " + element_name + " -> " + children[i]);
            hfiles.put(element_name, children[i]);
            }
            element_name = null;
        }
        name = null;
        }
        children = null;
        etc_directory = null;
        file = null;
    }
    /** Attempts to retrieve the HFile object associated with a certain metadata element. This may have already been cached, or may need to be loaded. Then again it may not even be necessary.
     * @param element The fully qualified name of a metadata element, as a <strong>String</strong>.
     * @return The <strong>HFile</strong> associated with the given element, or <i>null</i> if its unnecessary.
     * @see org.greenstone.gatherer.cdm.CommandTokenizer
     */
    public HFile getHFile(String element) {
        HFile result = null;
        Object target = hfiles.get(element);
                // If target is non-null
        if(target != null) {
        // If we haven't already load and parse the file.
        if(target instanceof File) {
            ///ystem.err.println("\nHFILE-MISS!! Loading " + target.toString());
            result = new HFile();
            try {
            FileReader in_filereader = new FileReader((File)target);
            BufferedReader in = new BufferedReader(in_filereader);
            String line = null;
            while((line = in.readLine()) != null) {                             
                CommandTokenizer tokenizer = new CommandTokenizer(line);
                String alias = Utility.decodeGreenstone(tokenizer.nextToken());
                String index = tokenizer.nextToken();
                String value = Utility.decodeGreenstone(tokenizer.nextToken());
                ///ystem.err.println("Read " + index + ", " + alias + ", " + value);
                if(alias.startsWith("\"") && alias.endsWith("\"") && !alias.equals("\"\"")) {
                alias = alias.substring(1, alias.length() - 1);
                }
                if(value.startsWith("\"") && value.endsWith("\"") && !value.equals("\"\"")) {
                value = value.substring(1, value.length() - 1);
                }
                result.add(index, alias, value);
                value = null;
                alias = null;
                index = null;
                tokenizer = null;
            }
            line = null;
            in.close();
            in = null;
            in_filereader = null;
            hfiles.put(element, result);
            }
            catch (Exception error) {
            error.printStackTrace();
            hfiles.remove(element);
            }
        }
        else {
            ///ystem.err.print("HFILE-HIT!!");
            result = (HFile) target;
        }
        }
                // Else no hfile is needed for this element
        target = null;
        return result;
    }
    /** Retrieve the list of metadata sets associated with this collection.
     * @return An <strong>ArrayList</strong> of metadata set Files.
     */
    public ArrayList getMetadataSets() {
        return metadatasets;
    }
    }

    /** The HFile object provides a container for the mappings from indexes, of the form 1.1.1, to alias-value pairs. It also provides method to retrieving the alias and value for a certain element, remembering that values must be expressed in terms of their absolute subject heirarchy path. */
    private class HFile 
    extends HashMap {
    /** Construct a new HFile object with no initial values. */
    public HFile() {
        super();
    }
    /** Add a new (index,(alias, value)) mapping.
     * @param index The index of this mapping as a <strong>String</strong>.
     * @param alias The alias of this mapping as a <strong>String</strong>.
     * @param value And finally the value of this mapping as a, you guessed it, <strong>String</strong>.
     */
    public void add(String index, String alias, String value) {
        Entry entry = new Entry(index, alias, value);
        ///ystem.err.println("Adding entry: " + index + " \"" + alias + "\" \"" + value + "\"");
        put(index, entry);
        put(alias, entry);
    }
    public String getAlias(String index) {
        String alias = "";
        Entry entry = (Entry) get(index);
        if(entry != null) {
        alias = entry.alias;
        }
        entry = null;
        return alias;
    }
    /** Retrieve the value associated with a certain index. This is harder than it first sounds as you must take into account the parent indexes of this one.
     * @param index The index whose value you wish to calculate, as a <strong>String</strong>.
     * @return The fully quantified path to the value that matches index, also as a <strong>String</strong>. Delimitiation between subject layers is denoted by the string "|"
     */
    public String getValue(String index) {
        ///ystem.err.println("Retrieve value for the alias/index: '" + index + "'");
        StringBuffer value = new StringBuffer("");
        // If index isn't the index, it must be the alias. Replace it with the index dammit.
        Entry entry = null;
        if(!Utility.isIndex(index)) {
        ///ystem.err.println("\tThis is an alias.");
        // Store this for later, as its exactly the same entry we'd get had we found the last component of a proper index.
        entry = (Entry) get(index);
        index = entry.index;
        ///ystem.err.println("\tIndex is actually: " + index);
        }
        // Now build the hierarchy if necessary.
        int dot_index = -1;
        if((dot_index = index.indexOf(".")) != -1) {
        ///ystem.err.println("\tHierarchy information required -->");
        value.append(getValue(index.substring(0, dot_index)));
        value.append(StaticStrings.PIPE_STR);
        ///ystem.err.println("\t<-- Hierarchy information complete");
        }
        if(entry == null) {
        entry = (Entry) get(index);
        }
        if(entry != null) {
        value.append(entry.value);
        }
        entry = null;
        ///ystem.err.println("\tFinal value is: '" + value.toString() + "'\n");
        return value.toString();
    }

    private class Entry {
        public String alias = null;
        public String index = null;
        public String value = null;
        public Entry(String index, String alias, String value) {
        this.alias = alias;
        this.index = index;
        this.value = value;
        }
    }
    }

    private class MetadataXMLFileSearch {
    public File file;
    public String filename;
    public MetadataXMLFileSearch(File file, String filename) {
        this.file = file;
        this.filename = filename;
    }
    }
}