source: trunk/gli/src/org/greenstone/gatherer/msm/GreenstoneArchiveParser.java@ 7743

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.msm;

import java.io.*;
import java.net.*;
import java.util.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.collection.Collection;
import org.greenstone.gatherer.collection.CollectionManager;
import org.greenstone.gatherer.file.FileNode;
import org.greenstone.gatherer.msm.ElementWrapper;
import org.greenstone.gatherer.msm.MetadataSet;
import org.greenstone.gatherer.msm.MetadataSetManager;
import org.greenstone.gatherer.msm.MSMUtils;
import org.greenstone.gatherer.shell.GShell;
import org.greenstone.gatherer.shell.GShellProgressMonitor;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.valuetree.GValueModel;
import org.greenstone.gatherer.valuetree.GValueNode;
import org.w3c.dom.*;

public class GreenstoneArchiveParser {

    private GShell shell;
    private String file_path;

    static final String ignore_list[] = { "assocfilepath", "gsdl", "Identifier", "SourceSegment", "URL" };

    public GreenstoneArchiveParser(GShellProgressMonitor progress, GShell shell) {
    // We can only extract metadata if an extracted metadata set exists in our collection.
    if(Gatherer.c_man.msm.getSet(Utility.EXTRACTED_METADATA_NAMESPACE) != null) {
        this.shell = shell;
        // The very firstist thing we do is remove any existing extracted metadata
        if(Gatherer.c_man != null && Gatherer.c_man.getCollection() != null) {
        Gatherer.c_man.getCollection().getGDM().removeExtractedMetadata();
        }

        // Determine the collection archive directory.
        File archive_directory = new File(Gatherer.c_man.getCollectionArchive());
        // Start the extraction process
        processDirectory(archive_directory, progress);
    }
    // All done. Outta here like a bald man.
    }
    
    private void processDirectory(File directory, GShellProgressMonitor progress) {
    // look for a doc.xml file here
    File document_file = new File(directory, "doc.xml");
    // Then extract the metadata from it.
    if(document_file.exists()) {
        int count = extractMetadata(document_file);
        // Display a pretty progress message.
        String[] args = new String[2];
        args[0] = directory.getName();
        args[1] = String.valueOf(count);
        this.shell.fireMessage(GShell.IMPORT, shell.typeAsString(GShell.IMPORT) + "> " + Dictionary.get("GShell.Extracted", args), GShell.OK, null);
        args = null;
        progress.increment();
    }
    
    // for each directory here, process it
    File sub_directories[] = directory.listFiles();
    for(int i = 0; i < sub_directories.length; i++) {
        // Find the doc.xml file within
        if(sub_directories[i].isDirectory()) {
        processDirectory(sub_directories[i], progress);
        }
    }
    }

    private int extractMetadata(File file) {
    int count = 0;
    // Retrieve the DOM of the file.
    Document document = Utility.parse(file, false);

    // If we successfully parsed the document, then it is time to search through the DOM for the Metadata tags.
    if (document != null) {
        file_path = null;
        Element archive_element = document.getDocumentElement();
        // Retrieve the initial Section element
        NodeList section_elements = archive_element.getElementsByTagName(StaticStrings.SECTION_ELEMENT);
        // It may be the case that we have some sort of mutant freak archive file which doesn't have any section elements
        if (section_elements.getLength() < 1) {
        return count; 
        }
        Element section_element = (Element) section_elements.item(0);       
        section_elements = null;

        // Retrieve all of the Metadata sections
        NodeList metadata_elements = section_element.getElementsByTagName(StaticStrings.METADATA_ELEMENT);
        section_element = null;

        // Zip through the retrieved metadata checking for SourceSegment elements
        // These are a good sign of bibliographic files, which we must handle specially
        boolean ignore_values = false;
        for (int i = 0; i < metadata_elements.getLength(); i++) {
        Element metadata_element = (Element) metadata_elements.item(i);
        String name = metadata_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
        if (name.equalsIgnoreCase(StaticStrings.SOURCESEGMENT_VALUE)) {
            ignore_values = true;
            break;
        }
        }

        // Now for each Metadata entry retrieved...
        for (int i = 0; i < metadata_elements.getLength(); i++) {
        Element metadata_element = (Element) metadata_elements.item(i);
        if (processMetadataElement(metadata_element, ignore_values) == true) {
            count++;
        }
        }
    }

    return count;
    }


    private boolean processMetadataElement(Element metadata_element, boolean ignore_values)
    {
    String name = metadata_element.getAttribute(StaticStrings.NAME_ATTRIBUTE);
    // There is also a special case when the metadata name is gsdlsourcefilename, as we use this to find the FileRecord we want to add metadata to.
    if (name.equals("gsdlsourcefilename")) {
        file_path = MSMUtils.getValue(metadata_element);
        if (!file_path.startsWith(File.separator)) {
        String collection_dir = Utility.getCollectionDir();
        file_path = collection_dir + file_path;
        }

        return false;
    }

    // Check if its name starts with, or is equal to, one of the values in our ignore list, and if so ignore this metadata
    for (int j = 0; j < ignore_list.length; j++) {
        if (name.startsWith(ignore_list[j])) {
        return false;
        }
    }

    // namespaced metadata, we don't extract at the moment
    if (name.indexOf(MSMUtils.NS_SEP)!=-1) {
        return false;
    }

    if (file_path == null) {
        return false;
    }

    // We now retrieve the appropriate element. If no such element exists we create a new one in the greenstone mds. Remember that no element in the greenstone mds has an associated value tree, so it is perfect for metadata elements with a small number of repeated values but where the values have no relation between files (such as encoding, where many files will be iso_8859_1, but if you change one you don't intend to change them all).
    ElementWrapper element = Gatherer.c_man.msm.getElement(name);
    if (element == null) {
        MetadataSet extracted_mds = Gatherer.c_man.msm.getSet(Utility.EXTRACTED_METADATA_NAMESPACE);
        if (extracted_mds != null) {
        element = extracted_mds.addElement(name, Gatherer.config.getLanguage());
        }
    }

    // If ignore_values is set (bibliographic records) we don't care about the values
    if (ignore_values == true) {
        return false;
    }

    // Retrieve the metadata for the current file
    File target_file = new File(file_path);
    String value = "";
    try {
        value = Utility.decodeGreenstone(URLDecoder.decode(MSMUtils.getValue(metadata_element), "UTF-8"));
    }
    catch (IllegalArgumentException error) { // ****
        value = Utility.decodeGreenstone(MSMUtils.getValue(metadata_element));
    }
    catch (UnsupportedEncodingException error) {
        Gatherer.printStackTrace(error);
    }

    if (value == null) {
        return false;
    }
    // Create a new metadata object.
    GValueModel value_tree = Gatherer.c_man.msm.getValueTree(element);
    GValueNode value_node = null;
    if (value_tree != null) {
        value_node = value_tree.getValue(value);
    }
    else {
        value_node = new GValueNode(element.toString(), value);
    }

    Metadata metadata = new Metadata(element, value_node);
    element.inc();
    ///ystem.err.println("Adding extracted metadata: " + metadata);
    Gatherer.c_man.getCollection().msm.fireMetadataChanged(new MSMEvent(this, System.currentTimeMillis(), target_file, null, metadata));

    return true;
    }


    static final String metadata_ignore_list[] = {"assocfilepath", "gsdl", "Identifier","URL"};

    static public ArrayList extractMetadataElements(File archive_directory) {
    ArrayList extracted_metadata_elements = new ArrayList();
    File document_directories[] = archive_directory.listFiles();
    for(int i = 0; i < document_directories.length; i++) {
        // Find the doc.xml file within
        if(document_directories[i].isDirectory()) {
        File document_file = new File(document_directories[i], "doc.xml");
        // Then extract the metadata from it.
        if(document_file.exists()) {
            try {
            Document document = Utility.parse(document_file, false);
            // Retrieve all of the Metadata sections.
            Element archive_element = document.getDocumentElement();
            NodeList metadata_elements = archive_element.getElementsByTagName("Metadata");
            // Now for each Metadata entry retrieved...
            for(int j = 0; j < metadata_elements.getLength(); j++) {
                Element metadata_element = (Element) metadata_elements.item(j);
                String name = metadata_element.getAttribute("name");
                // Check if its name starts with, or is equal to, one of the values in our ignore list, and if so ignore this metadata.
                boolean ignore = false;
                for(int k = 0; !ignore && k < metadata_ignore_list.length; k++) {
                ignore = name.startsWith(metadata_ignore_list[k]);
                }
                if(!ignore && !extracted_metadata_elements.contains(name)) {
                extracted_metadata_elements.add(name);
                }
                name = null;
                metadata_element = null;
            }
            metadata_elements = null;
            archive_element = null;
            document = null;
            }
            catch (Exception error) {
            Gatherer.printStackTrace(error);
            }
        }
        document_file = null;
        }
    }
    document_directories = null;
    return extracted_metadata_elements;
    }
}