Changeset 17009

Timestamp:

2008-08-26T17:23:26+12:00 (16 years ago)

Author:

davidb

Message:

Generalisation of skimming doc.xml files to find extracted metadata. Can now read docmets.xml files in export folder instead of doc.xml in archives folder.

Location:

gli/trunk/src/org/greenstone/gatherer

Files:

• collection/CollectionManager.java (modified) (2 diffs)
• metadata/DocXMLFile.java (modified) (15 diffs)
• metadata/DocXMLFileManager.java (modified) (3 diffs)
• shell/GShell.java (modified) (1 diff)

gli/trunk/src/org/greenstone/gatherer/collection/CollectionManager.java

@@ -1315 +1315 @@
 
         // Read through the doc.xml files in the archives directory
-        File collection_archives_directory = new File(getLoadedCollectionArchivesDirectoryPath());
+
         DocXMLFileManager.clearDocXMLFiles();
-        DocXMLFileManager.loadDocXMLFiles(collection_archives_directory);
+
+        if (Configuration.fedora_info.isActive()) { // FLI case
+        File collection_export_directory = new File(getLoadedCollectionExportDirectoryPath());
+        DocXMLFileManager.loadDocXMLFiles(collection_export_directory,"docmets.xml");
+        }
+        else {
+        File collection_archives_directory = new File(getLoadedCollectionArchivesDirectoryPath());
+        DocXMLFileManager.loadDocXMLFiles(collection_archives_directory,"doc.xml");
+        }
+
 
         // Get a list of the collection specific classifiers and plugins
@@ -1358 +1367 @@
      * more situations. */
     public static boolean canDoScheduling() {
+    // Would be nice to support more of these, rather than returning false
     if(Gatherer.isGsdlRemote) {
         return false;
     } 
     if(Gatherer.GS3) {
+        return false;
+    }
+    if (Configuration.fedora_info.isActive()) {
         return false;
     }

gli/trunk/src/org/greenstone/gatherer/metadata/DocXMLFile.java

@@ -36 +36 @@
 
 /** This class represents one doc.xml file */
-public class DocXMLFile
-    extends File
+
+public abstract class DocXMLFile extends File
 {
-    private HashMap source_file_name_to_description_elements_mapping = new HashMap();
-
+    protected HashMap source_file_name_to_description_elements_mapping = new HashMap();
+
+    protected static String MetadataWrap = null;
+    protected static String MetadataItem = null;
 
     public DocXMLFile(String doc_xml_file_path)
@@ -59 +61 @@
     }
 
-    // Check whether this doc.xml file contains extracted metadata for the specified file
+    // Check whether this file (i.e. doc.xml or docmets.xml on inheritance) file contains extracted metadata for the specified file
     ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
     if (description_elements_list == null) {
@@ -68 +70 @@
     MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
 
-    // Parse the doc.xml file
-    DebugStream.println("Applicable doc.xml file: " + this);
+    // Parse the file
+    DebugStream.println("Applicable file: " + this);
     try {
         BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
@@ -79 +81 @@
         String line = null;
         for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
-        // Check if this line contains the start of a relevant Description element
+        // Check if this line contains the start of a relevant "Description" element
+        // (mets:xmlData in METS parlance, Description in GreenstoneArchive format) 
         if (line_num == next_description_element_start) {
             in_relevant_description_element = true;
@@ -91 +94 @@
 
         // Check if this line contains the end of the relevant Description element
-        if (line.indexOf("</Description>") != -1) {
+        if (line.indexOf("</"+MetadataWrap+">") != -1) {
             description_element_num++;
             if (description_element_num == description_elements_list.size()) {
@@ -103 +106 @@
 
         // If this line doesn't contain a complete Metadata element, we're not interested
-        if (line.indexOf("<Metadata ") == -1 || line.indexOf("</Metadata>") == -1) {
+        if (line.indexOf("<"+MetadataItem+" ") == -1 || line.indexOf("</"+MetadataItem+">") == -1) {
             continue;
         }
@@ -135 +138 @@
         // Value trees are not stored for extracted metadata, so create a new value tree node now
         int value_index = line.indexOf(">", name_index) + ">".length();
-        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</Metadata>"));
+        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</"+MetadataItem+">"));
 
         metadata_element.addMetadataValue(metadata_element_value);
@@ -158 +161 @@
 
 
+
+
     /**
-     * Every doc.xml file must be skimmed when a collection is opened, for two reasons:
+     * Every file must be skimmed when a collection is opened, for two reasons:
      *   - To build a mapping from source file to its corresponding doc.xml file
      *   - To get a complete list of all extracted metadata elements
@@ -167 +172 @@
     MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
 
-    // Skim the doc.xml file as quickly as possible (don't parse as XML), looking at the Metadata elements
-    DebugStream.println("Skimming doc.xml file " + this + "...");
+    // Skim the file as quickly as possible (don't parse as XML), looking at the Metadata elements
+    DebugStream.println("Skimming " + this + "...");
     try {
         BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
@@ -175 +180 @@
         String line = null;
         for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
-        // This line contains the start of a Description element
-        if (line.indexOf("<Description>") != -1) {
+        // This line contains the start of a "MetadataWrap" element 
+        // (mets:xmlData in METS parlance, Description in GreenstoneArchive format) 
+        if (line.indexOf("<"+MetadataWrap+">") != -1) {
             if (description_element_start != -1) {
-            System.err.println("Parse error: previous Description element unfinished!");
+            System.err.println("Parse error: previous " + MetadataWrap + " element unfinished!");
             }
             description_element_start = line_num;
@@ -184 +190 @@
         }
 
-        // This line contains the end of a Description element
-        if (line.indexOf("</Description>") != -1) {
+        // This line contains the end of a "MetadataWrap" element
+        if (line.indexOf("</"+MetadataWrap+">") != -1) {
             if (description_element_start == -1) {
-            System.err.println("Parse error: Description element unstarted!");
+            System.err.println("Parse error: "+MetadataWrap+" element unstarted!");
             }
             description_element_start = -1;
@@ -193 +199 @@
         }
 
-        // If we're not in a Description element there shouldn't be any Metadata elements
+        // If we're not in a"MetadataWrap" element there shouldn't be any Metadata elements
         if (description_element_start == -1) {
             continue;
@@ -199 +205 @@
 
         // This line doesn't contain a Metadata element, so we're not interested
-        if (line.indexOf("<Metadata ") == -1) {
-            DebugStream.println("Warning: Description element line doesn't contain Metadata element.");
+        if (line.indexOf("<"+MetadataItem+" ") == -1) {
+            DebugStream.println("Warning: "+MetadataWrap+" element line doesn't contain Metadata element.");
             continue;
         }
@@ -217 +223 @@
         String metadata_element_name = metadata_element_name_full;
 
-        // Note which file this doc.xml is for
+        // Note which file this is for
         if (metadata_element_name.equals("gsdlsourcefilename")) {
             // Extract the gsdlsourcefilename element value
@@ -283 +289 @@
     }
     }
+
+
+    /*
+    public ArrayList getMetadataExtractedFromFile(File file)
+    {
+    // Build up a list of metadata extracted from this file
+    ArrayList metadata_values = new ArrayList();
+
+    String file_relative_path = file.getAbsolutePath();
+    int import_index = file_relative_path.indexOf("import");
+    if (import_index != -1) {
+        file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
+    }
+
+    // Check whether this doc.xml file contains extracted metadata for the specified file
+    ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
+    if (description_elements_list == null) {
+        // ...it doesn't
+        return metadata_values;
+    }
+
+    MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+    // Parse the doc.xml file
+    DebugStream.println("Applicable doc.xml file: " + this);
+    try {
+        BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+
+        int description_element_num = 0;
+        int next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+        boolean in_relevant_description_element = false;
+
+        String line = null;
+        for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+        // Check if this line contains the start of a relevant Description element
+        if (line_num == next_description_element_start) {
+            in_relevant_description_element = true;
+            continue;
+        }
+
+        // If we're not in a relevant Description element we don't care about anything
+        if (in_relevant_description_element == false) {
+            continue;
+        }
+
+        // Check if this line contains the end of the relevant Description element
+        if (line.indexOf("</Description>") != -1) {
+            description_element_num++;
+            if (description_element_num == description_elements_list.size()) {
+            break;
+            }
+
+            next_description_element_start = ((Integer) description_elements_list.get(description_element_num)).intValue();
+            in_relevant_description_element = false;
+            continue;
+        }
+
+        // If this line doesn't contain a complete Metadata element, we're not interested
+        if (line.indexOf("<Metadata ") == -1 || line.indexOf("</Metadata>") == -1) {
+            continue;
+        }
+
+        // Extract the metadata element name
+        int name_index = line.indexOf(" name=\"") + " name=\"".length();
+        String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+        // If the metadata has a namespace it isn't extracted metadata, so we're not interested
+        String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+        if (!metadata_set_namespace.equals("")) {
+            continue;
+        }
+
+        // Extracted metadata!
+        String metadata_element_name = metadata_element_name_full;
+
+        // We completely ignore bibliographic data
+        if (metadata_element_name.equals("SourceSegment")) {
+            buffered_reader.close();
+            return new ArrayList();
+        }
+
+        // Ignore metadata starting with gsdl (used to be lowercase metadata and /metadata)
+        if (metadata_element_name.startsWith("gsdl")) {
+            continue;
+        }
+
+        MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+
+        // Value trees are not stored for extracted metadata, so create a new value tree node now
+        int value_index = line.indexOf(">", name_index) + ">".length();
+        String metadata_element_value = line.substring(value_index, line.lastIndexOf("</Metadata>"));
+
+        metadata_element.addMetadataValue(metadata_element_value);
+        MetadataValueTreeNode metadata_value_tree_node = metadata_element.getMetadataValueTreeNode(metadata_element_value);
+
+        // Add the new metadata value to the list
+        MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
+        metadata_values.add(metadata_value);
+        }
+
+        buffered_reader.close();
+    }
+    catch (FileNotFoundException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    catch (IOException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+
+    return metadata_values;
+    }
+
+    */
+
+    /**
+     * Every doc.xml file must be skimmed when a collection is opened, for two reasons:
+     *   - To build a mapping from source file to its corresponding doc.xml file
+     *   - To get a complete list of all extracted metadata elements
+     */
+    /*
+    public void skimFile()
+    {
+    MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
+
+    // Skim the doc.xml file as quickly as possible (don't parse as XML), looking at the Metadata elements
+    DebugStream.println("Skimming " + this + "...");
+    try {
+        BufferedReader buffered_reader = new BufferedReader(new FileReader(this));
+        int description_element_start = -1;
+
+        String line = null;
+        for (int line_num = 0; (line = buffered_reader.readLine()) != null; line_num++) {
+        // This line contains the start of a Description element
+        if (line.indexOf("<Description>") != -1) {
+            if (description_element_start != -1) {
+            System.err.println("Parse error: previous Description element unfinished!");
+            }
+            description_element_start = line_num;
+            continue;
+        }
+
+        // This line contains the end of a Description element
+        if (line.indexOf("</Description>") != -1) {
+            if (description_element_start == -1) {
+            System.err.println("Parse error: Description element unstarted!");
+            }
+            description_element_start = -1;
+            continue;
+        }
+
+        // If we're not in a Description element there shouldn't be any Metadata elements
+        if (description_element_start == -1) {
+            continue;
+        }
+
+        // This line doesn't contain a Metadata element, so we're not interested
+        if (line.indexOf("<Metadata ") == -1) {
+            DebugStream.println("Warning: Description element line doesn't contain Metadata element.");
+            continue;
+        }
+
+        // Extract the metadata element name
+        int name_index = line.indexOf(" name=\"") + " name=\"".length();
+        String metadata_element_name_full = line.substring(name_index, line.indexOf("\"", name_index));
+
+        // If the metadata has a namespace it isn't extracted metadata, so we're not interested
+        String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
+        if (!metadata_set_namespace.equals("")) {
+            continue;
+        }
+
+        // Extracted metadata!
+        String metadata_element_name = metadata_element_name_full;
+
+        // Note which file this doc.xml is for
+        if (metadata_element_name.equals("gsdlsourcefilename")) {
+            // Extract the gsdlsourcefilename element value
+            int value_index = line.indexOf(">", name_index) + ">".length();
+            String gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
+
+            // We're only interested in the path relative to the import folder
+            int import_index = gsdlsourcefilename_value.indexOf("import");
+            if (import_index != -1) {
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
+
+            boolean is_unix_path = gsdlsourcefilename_value.startsWith("/");
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
+
+            // URL decode gsdlsourcefilename. Need to set the decoder to use the default system encoding
+            // This is stored in the System's file.encoding property.
+            gsdlsourcefilename_value = URLDecoder.decode(gsdlsourcefilename_value, System.getProperty("file.encoding")); 
+
+            // Make sure the path matches the OS that is running
+            if (is_unix_path && Utility.isWindows()) {
+                // Convert path from Unix to Windows
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
+            }
+            else if (!is_unix_path && !Utility.isWindows()) {
+                // Convert path from Windows to Unix
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
+            }
+
+            // Remember this for quick access later
+            if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
+                source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, new ArrayList());
+            }
+
+            ((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
+            }
+
+            // Warn about an odd gsdlsourcefilename, except if it is the Greenstone "tmp" directory
+            // This is true when the source files come from a zip file processed by ZIPPlug, for example
+            else if (gsdlsourcefilename_value.indexOf("tmp") == -1) {
+            // We don't really know what is going on...
+            System.err.println("Warning: Could not understand gsdlsourcefilename " + gsdlsourcefilename_value);
+            }
+        }
+
+        // Ignore metadata starting with gsdl (used to be lowercase metadata and /metadata)
+        if (metadata_element_name.startsWith("gsdl")) {
+            continue;
+        }
+
+        MetadataElement metadata_element = extracted_metadata_set.getMetadataElementWithName(metadata_element_name);
+        if (metadata_element == null) {
+            // This element isn't defined in ex.mds, so create it for this session
+            DebugStream.println("Extracted metadata element not defined: " + metadata_element_name);
+            extracted_metadata_set.addMetadataElementForThisSession(metadata_element_name);
+        }
+        }
+
+        buffered_reader.close();
+    }
+    catch (FileNotFoundException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    catch (IOException exception) {
+        DebugStream.printStackTrace(exception);
+    }
+    }
+    */
+
 }

gli/trunk/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java

@@ -60 +60 @@
 
 
-    static public void loadDocXMLFiles(File directory)
+    static public void loadDocXMLFiles(File directory, String filename_match)
     {
     // Make sure the directory (archives) exists
@@ -72 +72 @@
         File child_file = directory_files[i];
         if (child_file.isDirectory()) {
-        loadDocXMLFiles(child_file);
+        loadDocXMLFiles(child_file,filename_match); 
         }
-        else if (child_file.getName().equals("doc.xml")) {
-        loadDocXMLFile(child_file);
+        else if (child_file.getName().equals(filename_match)) {
+        // e.g. doc.xml (for regular Greenstone, docmets.xml for Fedora)
+
+        loadDocXMLFile(child_file,filename_match);
         }
     }
@@ -81 +83 @@
 
 
-    static private void loadDocXMLFile(File doc_xml_file_file)
+    static private void loadDocXMLFile(File doc_xml_file_file,String filename_match)
     {
-    DocXMLFile doc_xml_file = new DocXMLFile(doc_xml_file_file.getAbsolutePath());
-    try {
+    String file = doc_xml_file_file.getAbsolutePath();
+
+    DocXMLFile doc_xml_file 
+        = (filename_match.equals("docmets.xml")) ? new DocMetsXMLFile(file) : new DocGAFile(file);
+
+    try {       
         doc_xml_file.skimFile();
         doc_xml_files.add(doc_xml_file);

gli/trunk/src/org/greenstone/gatherer/shell/GShell.java

@@ -405 +405 @@
         fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Start"), status, null);
         DocXMLFileManager.clearDocXMLFiles();
-        DocXMLFileManager.loadDocXMLFiles(new File(CollectionManager.getLoadedCollectionArchivesDirectoryPath()));
+        if (Configuration.fedora_info.isActive()) { // FLI case
+            File collection_export_directory = new File(CollectionManager.getLoadedCollectionExportDirectoryPath());
+            DocXMLFileManager.loadDocXMLFiles(collection_export_directory,"docmets.xml");
+        }
+        else {
+            File collection_archives_directory = new File(CollectionManager.getLoadedCollectionArchivesDirectoryPath());
+            DocXMLFileManager.loadDocXMLFiles(collection_archives_directory,"doc.xml");
+        }
+
+
         fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Complete"), status, null);
         }