source: trunk/gli/src/org/greenstone/gatherer/msm/LegacyCollectionImporter.java@ 6318

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.msm;
/**************************************************************************************
 * Written:      26/11/03
 * Revised:      
 **************************************************************************************/
import java.io.*;
import java.util.*;
import javax.swing.*;
import org.w3c.dom.*;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.cdm.Argument;
import org.greenstone.gatherer.cdm.Classifier;
import org.greenstone.gatherer.cdm.ClassifierManager;
import org.greenstone.gatherer.cdm.CollectionDesignManager;
import org.greenstone.gatherer.cdm.CommandTokenizer;
import org.greenstone.gatherer.collection.CollectionManager;
import org.greenstone.gatherer.file.FileManager;
import org.greenstone.gatherer.file.FileQueue;
import org.greenstone.gatherer.msm.ElementWrapper;
import org.greenstone.gatherer.msm.MetadataSetManager;
import org.greenstone.gatherer.msm.MSMProfiler;
import org.greenstone.gatherer.msm.MSMPrompt;
import org.greenstone.gatherer.util.Codec;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.valuetree.GValueModel;
import org.greenstone.gatherer.valuetree.GValueNode;
/** Importing metadata from legacy collections requires three distinct steps:
 * 1. Import hfiles and build value tree for the appropriate element. This is quite tricky as the only place to determine the relation between hfiles and the elements is the collect.cfg file. Fortunately we can reuse a lot of code from SearchForMetadata.
 * 2. Recurses through the import folder of a legacy collection, and prompts the user how to transform old skool metadata names into the new and improved namespace standard. Because of the optimization of only opening each metadata.xml file once, this method is so different from GreenstoneMetadataParser that I decided not to try to hang to two together, but instead to create this new class. GLI's becoming bloatware. 
 * 3. At the same time watch for values which come from subject hierarchies, and replace with the new, confirmably unique, value.
 * 4. Update collection configuration as we should now be able to fix the classify commands anyway
 * Ok so that is four. Four distinct steps. Definately four. Couldn't possibly be more than six.
 */
public class LegacyCollectionImporter 
    extends HashMap {

    private CollectionDesignManager cdm;
    private File collection_folder;

    /** Constructor. The legacy collection importer object itself is a HashMap of element wrappers to HFile objects. Actually it starts off as a mapping of raw_element_name_str's to Files, but we hope to remedy that as we progress through importing metadata.
     * @param collection_folder the base File folder for the collection, mostly used for profile matching
     * @param cdm the CollectionDesignManager so we ca determine what elements map to what hierarchy file
     */
    public LegacyCollectionImporter(File collection_folder, CollectionDesignManager cdm) {
    super();
    this.cdm = cdm;
    this.collection_folder = collection_folder;
    // Prefill the HashMap with raw_element_name_str mappings to Files
    // Retrieve all of the hierarchy classifiers
    ArrayList hierarchy_classifiers_list = cdm.classifier_manager.getHierarchyClassifiers();
    ///ystem.err.println("When Legacy importing found: " + hierarchy_classifiers_list.size() + " hierarchies");
    for(int i = 0; i < hierarchy_classifiers_list.size(); i++) {
        Classifier classifier = (Classifier) hierarchy_classifiers_list.get(i);
        // Get the element name
        Argument metadata_name_argument = classifier.getArgument(StaticStrings.METADATA_ARGUMENT);
        String metadata_name_str = metadata_name_argument.getValue();
        // Because of the way of things the elements may already have had ex. put infront of them
        if(metadata_name_str.startsWith(StaticStrings.EXTRACTED_NAMESPACE)) {
        metadata_name_str = metadata_name_str.substring(StaticStrings.EXTRACTED_NAMESPACE.length());
        }
        metadata_name_argument = null;
        // Generate the file
        Argument hfile_filename_argument = classifier.getArgument(StaticStrings.HFILE_ARGUMENT);
        String hfile_filename_str = hfile_filename_argument.getValue();
        hfile_filename_argument = null;
        File hfile_file = new File(collection_folder, StaticStrings.ETC_FOLDER + File.separator + hfile_filename_str);
        put(metadata_name_str, hfile_file);
        ///ystem.err.println("Queued hierarchy information for later parsing: " + metadata_name_str + "->" + hfile_file.getAbsolutePath());
        classifier = null;
    }
    }

    public void updateCDM() {
       
    }

    public void importMetadata() {
    importMetadata(new File(collection_folder, StaticStrings.IMPORT_FOLDER));
    }

    /** Processes the files in the import tree, editing metadata.xml files so they contain namespaced element names.
     * @param collection_folder the base folder of this collection, needed for storing importing profiles
     * @param file the current file we are inspecting
     */
    private void importMetadata(File file) {
    // If its a directory we recurse into it searching for metadata.xml files
    if(file.isDirectory()) {
        File[] files = file.listFiles();
        for(int i = 0; files != null && i < files.length; i++) {
        importMetadata(files[i]);
        }
        files = null;
    }
    else {
        // We only care about metadata.xml files
        if(file.getName().equals(StaticStrings.METADATA_XML)) {
        boolean changed = false; // only set if we change something
        // Parse in the metadata.xml document
        try {
            Document document = Utility.parse(file.getAbsolutePath(), false);
            // Now, if we haven't currently got any metadata sets available to the collection, then this is as far as we need go. Rename the metadata.xml file and then move on the to next one. Remember we always have one set, the extracted metadata set.
            if(Gatherer.c_man.getCollection().msm.getSets().size() <= 1) {
            // First of all we create a backup of the current metadata.xml
            // I'll start by trying to rename it - but we all know how successful thats been in the past
            boolean done = false;
            try {
                File old_file = new File(file.getAbsolutePath());
                File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
                // Remove any existing backup
                if(new_file.exists()) {
                new_file.delete();
                }
                old_file.renameTo(new_file);
                // Then test if the file has been renamed
                if(new_file.exists()) {
                ///ystem.err.println("Rename Gooooood");
                done = true;
                }
            }
            catch(Exception exception) {
                ///ystem.err.println("Rename Baaaaaad");
            }
            
            // Failing that I'll copy it.
            if(!done) {
                File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
                Gatherer.f_man.getQueue().copyFile(file, new_file, null);
                if(new_file.exists()) {
                ///ystem.err.println("Copy Goooooood");
                }
                else {
                ///ystem.err.println("Copy Baaaaaaad");
                }
            }
            }
            // Otherwise me have some metadata sets, and so can carry on importing
            else {
            // For each metadata DOM Element
            NodeList metadata_DOM_elements = document.getDocumentElement().getElementsByTagName(StaticStrings.METADATA_ELEMENT);
            for(int i = 0; i < metadata_DOM_elements.getLength(); i++) {
                Element metadata_DOM_element = (Element) metadata_DOM_elements.item(i);
                // Extract the metadata element name, then check profiles to see if we already know how to handle this element. Once done try to retrieve the element from msm. If that fails, prompt the user for how to import this element.
                String raw_element_name_str = metadata_DOM_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
                // Check profiles
                if(Gatherer.c_man.getCollection().msm.profiler.containsAction(collection_folder.getAbsolutePath(), raw_element_name_str)) {
                String element_name_str = Gatherer.c_man.getCollection().msm.profiler.getAction(collection_folder.getAbsolutePath(), raw_element_name_str);
                // Update the DOM
                if(element_name_str != null) {
                    metadata_DOM_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, element_name_str);
                    // Now to ensure GLI correctly shows this element as assigned we'll increment its occurances
                    ElementWrapper element_ew = Gatherer.c_man.getCollection().msm.getElement(element_name_str, true);
                    element_ew.inc();
                    // We now check if this element has a hierarchy, and if so we update the value
                    checkForHierarchy(element_ew, metadata_DOM_element);
                    element_ew = null;
                }
                // If the raw name is null now, we have been instructed to ignore this metadata, so we can delete it from the metadata.xml (GDMDocument would do this anyway when it tried to save)
                else {
                    // Retrieve the parent node
                    Node parent_DOM_node = metadata_DOM_element.getParentNode();
                    parent_DOM_node.removeChild(metadata_DOM_element);
                    parent_DOM_node = null;
                }
                changed = true;
                element_name_str = null;
                }
                // No profile, lets hope it either matches straight away, or the user can decide what to do with it
                else {
                // Try to retrieve an element with this name from the msm. Only a perfect match is acceptable
                ElementWrapper element_ew = Gatherer.c_man.getCollection().msm.getElement(raw_element_name_str, true);
                // If an element was found, we increment the element count but thats it. Welcome to the only case where the DOM isn't changed
                if(element_ew != null) {
                    element_ew.inc();
                    // We now check if this element has a hierarchy, and if so we update the value
                    checkForHierarchy(element_ew, metadata_DOM_element);
                }
                // If no match was found prompt the user as to how to proceed
                else {
                    boolean confirmed = false;
                    boolean dialog_cancelled = false;
                    while(!confirmed) {
                    element_ew = Gatherer.c_man.getCollection().msm.prompt.selectElement(raw_element_name_str);
                    dialog_cancelled = Gatherer.c_man.getCollection().msm.prompt.wasDialogCancelled();
                    // If the user chooses something then add to profile, and process the DOM as appropriate
                    if(!dialog_cancelled) {
                        // Choosen an element to append or merge
                        if(element_ew != null) {
                        String element_name_str = element_ew.getName();
                        Gatherer.c_man.getCollection().msm.profiler.addAction(collection_folder.getAbsolutePath(), raw_element_name_str, element_name_str);
                        metadata_DOM_element.setAttribute(StaticStrings.NAME_ATTRIBUTE, element_name_str);
                        element_name_str = null;
                        element_ew.inc();
                        
                        // We can now build the hfile properly is necessary
                        File hfile_file = (File) get(raw_element_name_str);
                        if(hfile_file != null) {
                            ///ystem.err.println("Processing the mapping for: " + raw_element_name_str);
                            // Remove the current mapping
                            remove(raw_element_name_str);
                            // Create the HFile
                            try {
                            HFile hfile = new HFile(element_ew, hfile_file);
                            // Store the mapping
                            ///ystem.err.println("Adding a hfile mapping for: " + element_ew);
                            put(element_ew, hfile);
                            // Now update the value
                            String raw_value_str = MSMUtils.getValue(metadata_DOM_element);
                            String new_value_str = hfile.getFullValue(raw_value_str);
                            MSMUtils.setValue(metadata_DOM_element, new_value_str);
                            raw_value_str = null;
                            new_value_str = null;
                            hfile = null;
                            }
                            catch (Exception exception) {
                            Gatherer.println("***** Exception in org.greenstone.gatherer.msm.LegacyCollectionImporter *****");
                            Gatherer.printStackTrace(exception);
                            }
                        }
                        element_ew = null;
                        }
                        // Choosen to ignore the element
                        else {
                        Gatherer.c_man.getCollection().msm.profiler.addAction(collection_folder.getAbsolutePath(), raw_element_name_str, null);
                        // Retrieve the parent node
                        Node parent_DOM_node = metadata_DOM_element.getParentNode();
                        parent_DOM_node.removeChild(metadata_DOM_element);
                        parent_DOM_node = null;
                        }
                        confirmed = true;
                        changed = true;
                    } 
                    // Cancel only skips the current metadata.xml file - actually I'll ignore the file on a cancel, as otherwise it will bugger up GDMManager when it starts up
                    else {
                        // Confirm the cancel, informing the user that the metadata.xml will be completely ignored if they continue
                        confirmed = (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("MSM.Legacy.Delete_Metadata_XML", file.getAbsolutePath()), Dictionary.get("General.Warning"), JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION);
                    }
                    }
                }
                }
                raw_element_name_str = null;
                metadata_DOM_element = null;
            }
            // If the DOM has saved, write it back out again
            if(changed) {
                // First of all we create a backup of the current metadata.xml
                // I'll start by trying to rename it - but we all know how successful thats been in the past
                boolean done = false;
                try {
                File old_file = new File(file.getAbsolutePath());
                File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
                // Remove any existing backup
                if(new_file.exists()) {
                    new_file.delete();
                }
                old_file.renameTo(new_file);
                // Then test if the file has been renamed
                if(new_file.exists()) {
                ///ystem.err.println("Rename Gooooood");
                    done = true;
                }
                }
                catch(Exception exception) {
                ///ystem.err.println("Rename Baaaaaad");
                }
                
                // Failing that I'll copy it.
                if(!done) {
                File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
                Gatherer.f_man.getQueue().copyFile(file, new_file, null);
                if(new_file.exists()) {
                ///ystem.err.println("Copy Goooooood");
                }
                else {
                ///ystem.err.println("Copy Baaaaaaad");
                }
                }
                Utility.export(document, file);
            }
            document = null;
            }
        }
        catch(Exception exception) {
            // Dump a stack trace to debug
            Gatherer.println("***** Exception in org.greenstone.gatherer.msm.LegacyCollectionImporter *****");
            Gatherer.printStackTrace(exception);
            // Then display a message complaining about this file
            JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("MSM.Legacy.Corrupt_Metadata_XML", file.getAbsolutePath()), Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
        }
        }
        // Else you may go to the Devil's dam. Yours gifts are so good, heres none will hold you. Their love is not so great, Hortensio, but we may blow our nails together, and fast it fairly out.
    }
    }

    private void checkForHierarchy(ElementWrapper element, Element metadata_DOM_element) {
    // Iterator through the keys in the HashMap looking for a match to our element. Why, oh why, doesn't HashMap use element.equals(element) like its meant to?
    HFile hfile = null;
    for(Iterator keys = keySet().iterator(); hfile == null && keys.hasNext(); ) {
        Object key_object = keys.next();
        if(key_object instanceof ElementWrapper) {
        ElementWrapper key = (ElementWrapper) key_object;
        ///ystem.err.print("Does " + key + " equal " + element + "? ");
        if(key.equals(element)) {
            ///ystem.err.println("Yes!");
            hfile = (HFile) get(key);
        }
        key = null;
        }
        key_object = null;
    }
    if(hfile != null) {
        String raw_value_str = MSMUtils.getValue(metadata_DOM_element);
        String new_value_str = hfile.getFullValue(raw_value_str);
        MSMUtils.setValue(metadata_DOM_element, new_value_str);
        raw_value_str = null;
        new_value_str = null;
        hfile = null;
    }
    }

    /** Another basic HFile wrapper. This one expects you to provide an element when you create it, then as it is built it generates the value tree as well. Later it allows you to provide an alias and retrieve the full path string (delimited by pipes) */
    private class HFile 
    extends HashMap {

    public HFile(ElementWrapper element, File file) 
        throws Exception {
        super();
        HashMap index_to_entry = new HashMap();
        // Read in the hfile, line by line, creating entry mappings
        FileReader in_filereader = new FileReader(file);
        BufferedReader in = new BufferedReader(in_filereader);
        String line = null;
        while((line = in.readLine()) != null) {
        CommandTokenizer tokenizer = new CommandTokenizer(line);
        String alias = tokenizer.nextToken();
        String index = tokenizer.nextToken();
        String value = Utility.decodeGreenstone(tokenizer.nextToken());
        ///ystem.err.println("Read " + index + ", " + alias + ", " + value);
        if(alias.startsWith("\"") && alias.endsWith("\"") && !alias.equals("\"\"")) {
            alias = alias.substring(1, alias.length() - 1);
        }
        if(value.startsWith("\"") && value.endsWith("\"") && !value.equals("\"\"")) {
            value = value.substring(1, value.length() - 1);
        }
        ///ystem.err.println("Storing:\nindex:" + index + "\nalias:" + alias + "\nvalue:" + value);
        index_to_entry.put(index, new Entry(alias, value));
        value = null;
        alias = null;
        index = null;
        tokenizer = null;
        }
        in.close();
        in_filereader.close();
        in = null;
        in_filereader = null;
        // Now iterate through the HashMap creating ValueNodes and mappings as we go
        for(Iterator keys = index_to_entry.keySet().iterator(); keys.hasNext(); ) {
        String index = (String) keys.next();
        Entry entry = (Entry) index_to_entry.get(index);
        String alias = entry.alias;
        String value = entry.value;
        // Chop the last reference off index, as we already have it
        if(index.indexOf(StaticStrings.STOP_CHARACTER) > -1) {
            index = index.substring(0, index.lastIndexOf(StaticStrings.STOP_CHARACTER));
            // Then while theres still futher indexes left, retrieve them
            while(index.length() > 0) {
            // Retrieve that value (if any).
            entry = (Entry) index_to_entry.get(index);
            if(entry != null) {
                // Precatenate with the current value separating with a pipe
                value = entry.value + StaticStrings.PIPE_CHAR + value;
            }
            // Then trim the index down
            if(index.indexOf(StaticStrings.STOP_CHARACTER) > -1) {
                index = index.substring(0, index.lastIndexOf(StaticStrings.STOP_CHARACTER));
            }
            else {
                index="";
            }
            }
        }
        // Create the value node
        GValueModel value_model = Gatherer.c_man.getCollection().msm.getValueTree(element);
        GValueNode value_node = value_model.addValue(value);
        // And place the mapping
        put(alias, value_node);
        }
        index_to_entry.clear();
        index_to_entry = null;

        // Now we rename the old hfile
        boolean done = false;
        try {
        File old_file = new File(file.getAbsolutePath());
        File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
        // Remove any existing backup
        if(new_file.exists()) {
            new_file.delete();
        }
        old_file.renameTo(new_file);
        // Then test if the file has been renamed
        if(new_file.exists()) {
            ///ystem.err.println("Rename Gooooood");
            done = true;
        }
        }
        catch(Exception exception) {
        ///ystem.err.println("Rename Baaaaaad");
        }
        
        // Failing that I'll copy it.
        if(!done) {
        File new_file = new File(file.getParentFile(), StaticStrings.METADATA_BAK);
        Gatherer.f_man.getQueue().copyFile(file, new_file, null);
        if(new_file.exists()) {
            ///ystem.err.println("Copy Goooooood");
        }
        else {
            ///ystem.err.println("Copy Baaaaaaad");
        }
        }
        // Right. Get rid of original
        file.delete();
    }
    
    public String getFullValue(String alias) {
        ///ystem.err.println("Searching for the value for alias: " + alias);
        for(Iterator keys = keySet().iterator(); keys.hasNext(); ) {
        String key = (String) keys.next();
        if(alias.equals(key)) {
            GValueNode value_node = (GValueNode) get(key);
            if(value_node != null) {
            ///ystem.err.println("Found value: " + value_node.getFullPath(false));
            return Codec.transform(value_node.getFullPath(false), Codec.TEXT_TO_DOM);
            }
            else {
            ///ystem.err.println("Error!");
            return alias; // Can't do any better
            }
        }
        }
        ///ystem.err.println("Error!");
        return alias;
    }

    private class Entry {
        public String alias;
        public String value;
        public Entry(String alias, String value) {
        this.alias = alias;
        this.value = value;
        }
    }
    }
}