Changeset 18362

Timestamp:

2009-01-12T11:19:23+13:00 (15 years ago)

Author:

kjdon

Message:

updated the rtl-gli branch with files from trunk. Result of a merge 14807:18318

Location:

gli/branches/rtl-gli/src/org/greenstone/gatherer/metadata

Files:

• DocGAFile.java (copied) (copied from gli/trunk/src/org/greenstone/gatherer/metadata/DocGAFile.java )
• DocMetsXMLFile.java (copied) (copied from gli/trunk/src/org/greenstone/gatherer/metadata/DocMetsXMLFile.java )
• DocXMLFile.java (modified) (18 diffs)
• DocXMLFileManager.java (modified) (3 diffs)
• MetadataXMLFileManager.java (modified) (1 diff)
• ProfileXMLFile.java (modified) (1 diff)

gli/branches/rtl-gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java

@@ -30 +30 @@
 import java.io.*;
 import java.util.*;
+import java.net.URLDecoder;
 import org.greenstone.gatherer.DebugStream;
 import org.greenstone.gatherer.util.Utility;
@@ -35 +36 @@
 
 /** This class represents one doc.xml file */
-public class DocXMLFile
-    extends File
+
+public abstract class DocXMLFile extends File
 {
-    private HashMap source_file_name_to_description_elements_mapping = new HashMap();
-
-
-    public DocXMLFile(String doc_xml_file_path)
+    protected HashMap source_file_name_to_description_elements_mapping = new HashMap();
+
+    protected final String MetadataWrap;
+    protected final String MetadataItem;
+
+    public DocXMLFile(String doc_xml_file_path, String metaWrap, String metaItem)
     {
     super(doc_xml_file_path);
+    this.MetadataWrap = metaWrap;
+    this.MetadataItem = metaItem;
     }
 
@@ -58 +63 @@
     }
 
-    // Check whether this doc.xml file contains extracted metadata for the specified file
+    // Check whether this file (i.e. doc.xml or docmets.xml on inheritance) file contains extracted metadata for the specified file
     ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
     if (description_elements_list == null) {
@@ -67 +72 @@
     MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
 
-    // Parse the doc.xml file
-    DebugStream.println("Applicable doc.xml file: " + this);
+    // Parse the file
+    DebugStream.println("Applicable file: " + this);
     try {
         BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
@@ -78 +83 @@
         String line = null;
         for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
-        // Check if this line contains the start of a relevant Description element
+        // Check if this line contains the start of a relevant "Description" element
+        // (mets:xmlData in METS parlance, Description in GreenstoneArchive format) 
         if (line_num == next_description_element_start) {
             in_relevant_description_element = true;
@@ -90 +96 @@
 
         // Check if this line contains the end of the relevant Description element
-        if (line.indexOf("</Description>") != -1) {
+        if (line.indexOf("</"+MetadataWrap+">") != -1) {
             description_element_num++;
             if (description_element_num == description_elements_list.size()) {
@@ -102 +108 @@
 
         // If this line doesn't contain a complete Metadata element, we're not interested
-        if (line.indexOf("<Metadata ") == -1 || line.indexOf("</Metadata>") == -1) {
+        if (line.indexOf("<"+MetadataItem+" ") == -1 || line.indexOf("</"+MetadataItem+">") == -1) {
             continue;
         }
@@ -134 +140 @@
         // Value trees are not stored for extracted metadata, so create a new value tree node now
         int value_index = line.indexOf(">", name_index) + ">".length();
-        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</Metadata>"));
+        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</"+MetadataItem+">"));
 
         metadata_element.addMetadataValue(metadata_element_value);
@@ -157 +163 @@
 
 
+
+
     /**
-     * Every doc.xml file must be skimmed when a collection is opened, for two reasons:
+     * Every file must be skimmed when a collection is opened, for two reasons:
      *   - To build a mapping from source file to its corresponding doc.xml file
      *   - To get a complete list of all extracted metadata elements
@@ -166 +174 @@
     MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
 
-    // Skim the doc.xml file as quickly as possible (don't parse as XML), looking at the Metadata elements
-    DebugStream.println("Skimming doc.xml file " + this + "...");
+    // Skim the file as quickly as possible (don't parse as XML), looking at the Metadata elements
+    DebugStream.println("Skimming " + this + "...");
     try {
         BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
@@ -174 +182 @@
         String line = null;
         for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
-        // This line contains the start of a Description element
-        if (line.indexOf("<Description>") != -1) {
+        // This line contains the start of a "MetadataWrap" element 
+        // (mets:xmlData in METS parlance, Description in GreenstoneArchive format) 
+        if (line.indexOf("<"+MetadataWrap+">") != -1) {
             if (description_element_start != -1) {
-            System.err.println("Parse error: previous Description element unfinished!");
+            System.err.println("Parse error: previous " + MetadataWrap + " element unfinished!");
             }
             description_element_start = line_num;
@@ -183 +192 @@
         }
 
-        // This line contains the end of a Description element
-        if (line.indexOf("</Description>") != -1) {
+        // This line contains the end of a "MetadataWrap" element
+        if (line.indexOf("</"+MetadataWrap+">") != -1) {
             if (description_element_start == -1) {
-            System.err.println("Parse error: Description element unstarted!");
+            System.err.println("Parse error: "+MetadataWrap+" element unstarted!");
             }
             description_element_start = -1;
@@ -192 +201 @@
         }
 
-        // If we're not in a Description element there shouldn't be any Metadata elements
+        // If we're not in a"MetadataWrap" element there shouldn't be any Metadata elements
         if (description_element_start == -1) {
             continue;
@@ -198 +207 @@
 
         // This line doesn't contain a Metadata element, so we're not interested
-        if (line.indexOf("<Metadata ") == -1) {
-            DebugStream.println("Warning: Description element line doesn't contain Metadata element.");
+        if (line.indexOf("<"+MetadataItem+" ") == -1) {
+            DebugStream.println("Warning: "+MetadataWrap+" element line doesn't contain Metadata element.");
             continue;
         }
@@ -216 +225 @@
         String metadata_element_name = metadata_element_name_full;
 
-        // Note which file this doc.xml is for
+        // Note which file this is for
         if (metadata_element_name.equals("gsdlsourcefilename")) {
             // Extract the gsdlsourcefilename element value
@@ -229 +238 @@
             boolean is_unix_path = gsdlsourcefilename_value.startsWith("/");
             gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
+
+            // URL decode gsdlsourcefilename. Need to set the decoder to use the default system encoding
+            // This is stored in the System's file.encoding property.
+            gsdlsourcefilename_value = URLDecoder.decode(gsdlsourcefilename_value, System.getProperty("file.encoding")); 
 
             // Make sure the path matches the OS that is running
@@ -248 +261 @@
             }
 
+            // Warn about an odd gsdlsourcefilename, except if it is the Greenstone "tmp" directory or
+            // (as in the case of using FLI) if it is the etc/collect.cfg or etc/collectionConfig.xml file
+            // which are the gsdlsourcefilenames for the fedora digital object representing a collection.
+            // This (tmp dir) is true when the source files come from a zip file processed by ZIPPlug, for example
+            else if (gsdlsourcefilename_value.indexOf("tmp") == -1 
+                 && !gsdlsourcefilename_value.endsWith("collect.cfg")
+                 && !gsdlsourcefilename_value.endsWith("collectionConfig.xml")) {
+            // We don't really know what is going on...
+            System.err.println("Warning: Could not understand gsdlsourcefilename " + gsdlsourcefilename_value);
+            }
+        }
+
+        // Ignore metadata starting with gsdl (used to be lowercase metadata and /metadata)
+        if (metadata_element_name.startsWith("gsdl")) {
+            continue;
+        }
+
+        MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+        if (metadata_element == null) {
+            // This element isn't defined in ex.mds, so create it for this session
+            DebugStream.println("Extracted metadata element not defined: " + metadata_element_name);
+            extracted_metadata_set.addMetadataElementForThisSession(metadata_element_name);
+        }
+        }
+
+        buffered_reader.close();
+    }
+    catch (FileNotFoundException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    catch (IOException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    }
+
+
+    /*
+    public ArrayList getMetadataExtractedFromFile(File file)
+    {
+    // Build up a list of metadata extracted from this file
+    ArrayList metadata_values = new ArrayList();
+
+    String file_relative_path = file.getAbsolutePath();
+    int import_index = file_relative_path.indexOf("import");
+    if (import_index != -1) {
+        file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
+    }
+
+    // Check whether this doc.xml file contains extracted metadata for the specified file
+    ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
+    if (description_elements_list == null) {
+        // ...it doesn't
+        return metadata_values;
+    }
+
+    MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+    // Parse the doc.xml file
+    DebugStream.println("Applicable doc.xml file: " + this);
+    try {
+        BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+
+        int description_element_num = 0;
+        int next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+        boolean in_relevant_description_element = false;
+
+        String line = null;
+        for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+        // Check if this line contains the start of a relevant Description element
+        if (line_num == next_description_element_start) {
+            in_relevant_description_element = true;
+            continue;
+        }
+
+        // If we're not in a relevant Description element we don't care about anything
+        if (in_relevant_description_element == false) {
+            continue;
+        }
+
+        // Check if this line contains the end of the relevant Description element
+        if (line.indexOf("</Description>") != -1) {
+            description_element_num++;
+            if (description_element_num == description_elements_list.size()) {
+            break;
+            }
+
+            next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+            in_relevant_description_element = false;
+            continue;
+        }
+
+        // If this line doesn't contain a complete Metadata element, we're not interested
+        if (line.indexOf("<Metadata ") == -1 || line.indexOf("</Metadata>") == -1) {
+            continue;
+        }
+
+        // Extract the metadata element name
+        int name_index = line.indexOf(" name=\"") + " name=\"".length();
+        String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+        // If the metadata has a namespace it isn't extracted metadata, so we're not interested
+        String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+        if (!metadata_set_namespace.equals("")) {
+            continue;
+        }
+
+        // Extracted metadata!
+        String metadata_element_name = metadata_element_name_full;
+
+        // We completely ignore bibliographic data
+        if (metadata_element_name.equals("SourceSegment")) {
+            buffered_reader.close();
+            return new ArrayList();
+        }
+
+        // Ignore metadata starting with gsdl (used to be lowercase metadata and /metadata)
+        if (metadata_element_name.startsWith("gsdl")) {
+            continue;
+        }
+
+        MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+
+        // Value trees are not stored for extracted metadata, so create a new value tree node now
+        int value_index = line.indexOf(">", name_index) + ">".length();
+        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</Metadata>"));
+
+        metadata_element.addMetadataValue(metadata_element_value);
+        MetadataValueTreeNode metadata_value_tree_node = metadata_element.getMetadataValueTreeNode(metadata_element_value);
+
+        // Add the new metadata value to the list
+        MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+        metadata_values.add(metadata_value);
+        }
+
+        buffered_reader.close();
+    }
+    catch (FileNotFoundException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    catch (IOException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+
+    return metadata_values;
+    }
+
+    */
+
+    /**
+     * Every doc.xml file must be skimmed when a collection is opened, for two reasons:
+     *   - To build a mapping from source file to its corresponding doc.xml file
+     *   - To get a complete list of all extracted metadata elements
+     */
+    /*
+    public void skimFile()
+    {
+    MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+    // Skim the doc.xml file as quickly as possible (don't parse as XML), looking at the Metadata elements
+    DebugStream.println("Skimming " + this + "...");
+    try {
+        BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+        int description_element_start = -1;
+
+        String line = null;
+        for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+        // This line contains the start of a Description element
+        if (line.indexOf("<Description>") != -1) {
+            if (description_element_start != -1) {
+            System.err.println("Parse error: previous Description element unfinished!");
+            }
+            description_element_start = line_num;
+            continue;
+        }
+
+        // This line contains the end of a Description element
+        if (line.indexOf("</Description>") != -1) {
+            if (description_element_start == -1) {
+            System.err.println("Parse error: Description element unstarted!");
+            }
+            description_element_start = -1;
+            continue;
+        }
+
+        // If we're not in a Description element there shouldn't be any Metadata elements
+        if (description_element_start == -1) {
+            continue;
+        }
+
+        // This line doesn't contain a Metadata element, so we're not interested
+        if (line.indexOf("<Metadata ") == -1) {
+            DebugStream.println("Warning: Description element line doesn't contain Metadata element.");
+            continue;
+        }
+
+        // Extract the metadata element name
+        int name_index = line.indexOf(" name=\"") + " name=\"".length();
+        String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+        // If the metadata has a namespace it isn't extracted metadata, so we're not interested
+        String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+        if (!metadata_set_namespace.equals("")) {
+            continue;
+        }
+
+        // Extracted metadata!
+        String metadata_element_name = metadata_element_name_full;
+
+        // Note which file this doc.xml is for
+        if (metadata_element_name.equals("gsdlsourcefilename")) {
+            // Extract the gsdlsourcefilename element value
+            int value_index = line.indexOf(">", name_index) + ">".length();
+            String gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
+
+            // We're only interested in the path relative to the import folder
+            int import_index = gsdlsourcefilename_value.indexOf("import");
+            if (import_index != -1) {
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
+
+            boolean is_unix_path = gsdlsourcefilename_value.startsWith("/");
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
+
+            // URL decode gsdlsourcefilename. Need to set the decoder to use the default system encoding
+            // This is stored in the System's file.encoding property.
+            gsdlsourcefilename_value = URLDecoder.decode(gsdlsourcefilename_value, System.getProperty("file.encoding")); 
+
+            // Make sure the path matches the OS that is running
+            if (is_unix_path && Utility.isWindows()) {
+                // Convert path from Unix to Windows
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
+            }
+            else if (!is_unix_path && !Utility.isWindows()) {
+                // Convert path from Windows to Unix
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
+            }
+
+            // Remember this for quick access later
+            if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
+                source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, new ArrayList());
+            }
+
+            ((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
+            }
+
             // Warn about an odd gsdlsourcefilename, except if it is the Greenstone "tmp" directory
             // This is true when the source files come from a zip file processed by ZIPPlug, for example
@@ -278 +535 @@
     }
     }
+    */
+
 }

gli/branches/rtl-gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java

@@ -60 +60 @@
 
 
-    static public void loadDocXMLFiles(File directory)
+    static public void loadDocXMLFiles(File directory, String filename_match)
     {
     // Make sure the directory (archives) exists
@@ -72 +72 @@
         File child_file = directory_files[i];
         if (child_file.isDirectory()) {
-        loadDocXMLFiles(child_file);
+        loadDocXMLFiles(child_file,filename_match); 
         }
-        else if (child_file.getName().equals("doc.xml")) {
-        loadDocXMLFile(child_file);
+        else if (child_file.getName().equals(filename_match)) {
+        // e.g. doc.xml (for regular Greenstone, docmets.xml for Fedora)
+
+        loadDocXMLFile(child_file,filename_match);
         }
     }
@@ -81 +83 @@
 
 
-    static private void loadDocXMLFile(File doc_xml_file_file)
+    static private void loadDocXMLFile(File doc_xml_file_file,String filename_match)
     {
-    DocXMLFile doc_xml_file = new DocXMLFile(doc_xml_file_file.getAbsolutePath());
-    try {
+    String file = doc_xml_file_file.getAbsolutePath();
+
+    // Need to do typecasts in the following to keep Java 1.4 happy
+    DocXMLFile doc_xml_file 
+        = (filename_match.equals("docmets.xml")) 
+        ? (DocXMLFile) new DocMetsXMLFile(file) 
+        : (DocXMLFile) new DocGAFile(file);
+
+    try {       
         doc_xml_file.skimFile();
         doc_xml_files.add(doc_xml_file);

gli/branches/rtl-gli/src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java

@@ -368 +368 @@
 
         // Upload the files modified since last time, then reset the list
-        RemoteGreenstoneServer.uploadCollectionFiles(CollectionManager.getLoadedCollectionName(), (File[]) modified_metadata_xml_files.toArray(new File[0]));
+        Gatherer.remoteGreenstoneServer.uploadCollectionFiles(
+                  CollectionManager.getLoadedCollectionName(), (File[]) modified_metadata_xml_files.toArray(new File[0]));
         modified_metadata_xml_files.clear();
     }

gli/branches/rtl-gli/src/org/greenstone/gatherer/metadata/ProfileXMLFile.java

@@ -110 +110 @@
     // This is inefficient but for simplicity we'll just upload the file every time it is changed
     if (Gatherer.isGsdlRemote) {
-        RemoteGreenstoneServer.uploadCollectionFile(CollectionManager.getLoadedCollectionName(), this);
+        Gatherer.remoteGreenstoneServer.uploadCollectionFile(CollectionManager.getLoadedCollectionName(), this);
     }