Changeset 23433

Timestamp:

2010-12-09T22:27:33+13:00 (13 years ago)

Author:

ak19

Message:

GLI now has a gs.FilenameEncoding metadata field which appears like all the others in GLI's EnrichPane, but is unique in that this metadata (once set, changed or removed) must be applied to the affected filenames in the Collection Tree. More importantly, the changes made for this are to allow GLI's java code to interact with the recent changes to Perl where strings were made unicode-aware (for proper regex matching) but which required other changes elsewhere. To still support filenames with different encodings Perl used URL encoded versions of filenames representing characters' code point values in URL encoding. This required that GLI write out URL encoded filenames to the metadata.xml files that are associated with each folder level of a collection, so that Perl can read them. In this way, they can both speak of the same filenames. Only works on unicode 16 (such as latin-1), non-UTF8 systems. The latter is a requirement since Java uses the filesystem encoding from startup. If it is UTF8, non-recognised characters are replaced by the invalid char for UTF8. This process being destructive, we can't get the original filenames' bytecodes back. The changes made to GLI will work on Windows which is UTF-16 (windows codepage 1252), presumably also Macs (some kind of UTF-16) and also works on Native Latin 1 Linux systems. UTF-8 Linux systems need to be reconfigured to Native Latin-1, or if not installed, an administrator can install it easily.

Location:

main/trunk/gli

Files:

• classes/dictionary.properties (modified) (1 diff)
• metadata/greenstone.mds (modified) (1 diff)
• src/org/greenstone/gatherer/Gatherer.java (modified) (4 diffs)
• src/org/greenstone/gatherer/collection/CollectionManager.java (modified) (2 diffs)
• src/org/greenstone/gatherer/collection/CollectionTreeNode.java (modified) (3 diffs)
• src/org/greenstone/gatherer/file/FileNode.java (modified) (4 diffs)
• src/org/greenstone/gatherer/gui/EnrichPane.java (modified) (2 diffs)
• src/org/greenstone/gatherer/gui/GUIManager.java (modified) (2 diffs)
• src/org/greenstone/gatherer/metadata/FilenameEncoding.java (added)
• src/org/greenstone/gatherer/metadata/MetadataValueTableModel.java (modified) (3 diffs)
• src/org/greenstone/gatherer/metadata/MetadataXMLFile.java (modified) (18 diffs)
• src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java (modified) (9 diffs)
• src/org/greenstone/gatherer/util/SynchronizedTreeModelTools.java (modified) (2 diffs)

main/trunk/gli/classes/dictionary.properties

@@ -652 +652 @@
 General.Warning:Warning
 General.Yes:Yes
+General.MultipleFileNamesNotSupported.Message:Your locale seems to be in UTF-8. Non-UTF-8 filenames are not supported by your setup.
+General.MultipleFileNamesNotSupported.Title:Filename Encoding Support
 #****************************
 #

main/trunk/gli/metadata/greenstone.mds

@@ -43 +43 @@
     <Language code="en">
       <Attribute name="label">Filename Encoding</Attribute>
-      <Attribute name="definition">TODO</Attribute>
+      <Attribute name="definition">The encoding of the filename. If this is known, it can be manually set here.</Attribute>
+      <Attribute name="comment">If not manually specified, Greenstone will try to guess the encoding of the filename upon building, which may or may not be correct.</Attribute>
     </Language>
   </Element>

main/trunk/gli/src/org/greenstone/gatherer/Gatherer.java

@@ -56 +56 @@
 import org.greenstone.gatherer.gui.WarningDialog;
 import org.greenstone.gatherer.gui.FedoraLogin;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
 import org.greenstone.gatherer.util.JarTools;
@@ -471 +472 @@
         }
 
+        // Check that the local can support multiple filename encodings
+            //System.err.println("#### Java identifies current Locale as (file.encoding): "
+            //  + System.getProperty("file.encoding"));
+        if(System.getProperty("file.encoding").equals("UTF-8")){
+            // If the locale is UTF-8, Java will interpret all filename bytes as UTF-8, 
+            // which is a destructive process as it will convert characters not recognised
+            // by UTF-8 into the invalid character, rather than preserving the bytecodes.
+            // This has the effect that non-UTF8 encoded filenames on a system set to a 
+            // UTF-8 locale are not 'seen' by Java (if they contain non-ASCII characters).
+            multipleFilenameEncodingsNotSupported();        
+            FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED = false; 
+            FilenameEncoding.URL_FILE_SEPARATOR = File.separator;
+        } else {
+            FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED = true;
+            FilenameEncoding.URL_FILE_SEPARATOR = "/"; // URL file separator is always "/" 
+        }
+        
         // Set the default font for all Swing components.
         FontUIResource default_font = Configuration.getFont("general.font", true);
@@ -852 +870 @@
                 + File.separator + "cgi" + File.separator + "gsdl3site.cfg";
         } else { // cgi-bin/gsdlsite.cfg
-            return Configuration.gsdl_path + File.separator 
+            return Configuration.gsdl_path /* + File.separator */
                 + "cgi-bin" + File.separator + "gsdlsite.cfg";
         }
@@ -1258 +1276 @@
     }
 
+    /** Prints a warning message about the OS not supporting multiple filename encodings.  */
+    static private void multipleFilenameEncodingsNotSupported() {   
+        WarningDialog dialog = new WarningDialog("warning.NoEncodingSupport", 
+            Dictionary.get("General.MultipleFileNamesNotSupported.Title"), 
+            Dictionary.get("General.MultipleFileNamesNotSupported.Message"), null, false);
+        dialog.display();
+        dialog.dispose();
+        dialog = null;
+    }
 
     /** Sets up the proxy connection by setting JVM Environment flags and creating a new Authenticator.

main/trunk/gli/src/org/greenstone/gatherer/collection/CollectionManager.java

@@ -62 +62 @@
 import org.greenstone.gatherer.gui.WarningDialog;
 import org.greenstone.gatherer.metadata.DocXMLFileManager;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.metadata.MetadataChangedListener;
 import org.greenstone.gatherer.metadata.MetadataSet;
@@ -1552 +1553 @@
     if (collection != null) {
         collection.setMetadataChanged(true);
+        
+        // we're only going to refresh the tree's already visible nodes (and reselect them) IFF 
+        // gs.FilenameEncoding meta was set on any files/folders and the filenames of the
+        // affected CollectionTreeNodes need to be recalculated
+        if(FilenameEncoding.isRefreshRequired()) {
+            
+            // refreshes the relevant part of the tree
+            TreePath[] paths = collection_tree.getSelectionPaths();        
+            if(paths != null) {
+                for(int i = 0; i < paths.length; i++) {
+                collection_tree_model.refresh(paths[i]);
+                collection_tree.setSelectionPath(paths[i]);
+                }
+            }
+            // refreshed the tree, so turn off the requirement to refresh
+            FilenameEncoding.setRefreshRequired(false);     
+        }
     }
     }

main/trunk/gli/src/org/greenstone/gatherer/collection/CollectionTreeNode.java

@@ -32 +32 @@
 import org.greenstone.gatherer.cdm.CollectionDesignManager;
 import org.greenstone.gatherer.file.FileNode;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.util.JarTools;
 import java.util.Set;
 import java.util.Iterator;
-import java.nio.charset.Charset;
 
 
@@ -54 +54 @@
     {
     super(file);
+        // the super call will additionally call calcDisplayString() which will get any
+        // applicable the filename encoding and apply it to the file's name for display.
 
     this.is_explodable = CollectionDesignManager.plugin_manager.isFileExplodable(file);
@@ -81 +83 @@
     return is_srcreplaceable;
     }
+
+    /** This method returns a string representation of the filenodes in the 
+    * <i>Collection</i> Tree, that can then be displayed in the tree. If the 
+    * filename encoding is specified as metadata with the file, then this
+    * method will apply that to the filename's bytes to get the displayName.
+    */
+    protected String calcDisplayString() {      
+        // metadata.xml files in collections don't get displayed anyway
+        if(file.getName().equals("metadata.xml")) { 
+            return super.calcDisplayString();
+        }
+        
+        if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            return super.calcDisplayString();
+        }
+
+        // Else: try to encode the display string from the 
+        // file_to_encoding map. If that fails, just use the locale 
+        String displayName = null;
+        String urlEncodedPath = getURLEncodedFilePath();
+        String encoding = FilenameEncoding.findFilenameEncoding(file, urlEncodedPath, false);
+
+        // if it isn't the same as the current encoding already applied, reapply
+        // it which will set the filenameEncoding again as well
+        if(!encoding.equals(getFilenameEncoding())) {
+            displayName = reencodeDisplayName(encoding); // may return null
+        }
+        
+        if(displayName == null) {
+            if(FilenameEncoding.DEBUGGING) {
+                displayName = getURLEncodedFileName(); 
+            } else {
+                displayName = super.calcDisplayString();
+            }
+        }
+        
+        return displayName;
+    }
+
+    /** Call this if the filename encoding has changed and needs to be 
+    * recalculated and re-applied to the filename. This is only for display,
+    * so it only gets applied to the urlEncodedFileName (not urlEncodedFilePath). 
+    * Note that the filenameEncoding may not be the same as the given encoding
+    * at the end of this method (in case it was a charset alias of the encoding). 
+    * This method both sets and returns the displayFileName member variable. 
+    * @return the reencoded display name, if the encoding was a recognised alias
+    * in which case the filenameEncoding will store the canonical name.  */
+    public String reencodeDisplayName(String encoding) {
+        if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            return displayFileName; // may still be null
+        }
+
+        filenameEncoding = encoding; // clear previous value
+
+        if(filenameEncoding.equals("")) {
+            displayFileName = super.calcDisplayString(); // *FileNode* calcDisplayString
+            return displayFileName;
+        }
+
+        try{
+            displayFileName = new String(file.getName().getBytes(), filenameEncoding);
+        } catch(Exception e) { 
+            // IllegalCharsetName-, UnsupportedCharset- or UnsupportedEncoding-Exception
+            // Store the unsupported encoding, but display with filesystem (or URL) encoding
+            filenameEncoding = encoding;
+            
+            if(FilenameEncoding.DEBUGGING) {
+                displayFileName = getURLEncodedFileName();
+            } else {
+                displayFileName = super.calcDisplayString(); 
+            }
+        } 
+        return displayFileName;
+    }
+        
+    /** Can call this upon refreshing a FileNode (this CollectionTreeNode). It makes
+    * sure the display names of the visible nodes in the CollectionTree aren't stale */
+    public void refreshDescendantEncodings() {
+        // now recalculate encodings for all visible children
+        
+        // Don't bother with the encoding stuff when multiple filename encodings 
+        // are not supported because the system is UTF-8 (not Native-Latin-1) and 
+        // Java interprets all filename bytes as UTF-8, which destructively 
+        // converts unrecognised characters (for UTF-8) into the invalid character. 
+
+        if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED 
+                || child_nodes == null) 
+        {
+            return;
+        }       
+        
+        // get and apply the filename encoding, if any
+        for(int i = 0; i < child_nodes.size(); i++) {
+            CollectionTreeNode child_node = (CollectionTreeNode) child_nodes.get(i);            
+            String urlEncodedPath = child_node.getURLEncodedFilePath();     
+            
+            String encoding = FilenameEncoding.findFilenameEncoding(
+                    child_node.getFile(), urlEncodedPath, false);
+            
+            // if current encoding is different from the existing one, re-apply encoding
+            if(!child_node.getFilenameEncoding().equals(encoding)) {
+            child_node.reencodeDisplayName(encoding);
+            }
+        }
+    }
+    
+    // Unused at present
+    /** Call when the filename encoding of this folder level fileNode has changed
+    * and therefore the filename encodings of the descendant fileNodes have to be
+    * reset (their entries in the file_to_encoding hashmap cleared), so that we know
+    * to recalculate these later.  Call when deleting, moving or renaming col nodes */
+    public void resetDescendantEncodings() {
+        if(FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            resetDescendantEncodings(this.file, this.getURLEncodedFilePath());
+            
+            // for an actual file/folder delete, want to remove all corresponding 
+            // entries from the hashtable without recalculating any encodings 
+            // (files have been deleted, so no there will be no filenames to display).
+            // So don't do this here: refreshDescendantEncodings();
+            // It's done near the end of FileNode.map().
+        }
+    }
+    
+    // Together with the above, unused at present
+    private static void resetDescendantEncodings(File f, String urlEncodedPath) 
+    {
+        // remove this file f's urlencoded path name from the file_to_encoding Map
+        FilenameEncoding.map.remove(urlEncodedPath);
+            
+        if(f.isDirectory()) {
+            File[] children = f.listFiles();        
+            for(int i = 0; i < children.length; i++) {
+                urlEncodedPath = FilenameEncoding.fileToURLEncoding(children[i]);
+                resetDescendantEncodings(children[i], urlEncodedPath); 
+                        // sets filenameEncoding var
+            }
+        }
+    }
+
 }

main/trunk/gli/src/org/greenstone/gatherer/file/FileNode.java

@@ -8 +8 @@
 import javax.swing.tree.*;
 import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.util.ArrayTools;
 
@@ -19 +20 @@
     protected File file = null;
     protected FileSystemModel model = null;
-    protected MutableTreeNode parent = null;
+    protected MutableTreeNode parent = null;    
+
+    protected String urlEncodedFileName = "";
+    protected String urlEncodedFilePath = "";
+    protected String filenameEncoding = "";
     /** The string that is displayed as the filename. Attempts to be in the correct encoding. */
     protected String displayFileName = null;
-
+    
 
     public FileNode(File file)
     {
-    this.file = file;
-
-    // Files cannot have children
-    if (file != null && !file.isDirectory()) { //file.isFile()) {
-        // Cache this result to prevent unceasing missing disk messages being thrown if the
-        // removable media was, um, removed after directory mapped
-        this.allows_children = false;
-        displayFileName = calcDisplayString();
-    }   
-    }
-
-
-     /** This method returns a string representation of the filenodes in the Collection
-     * Tree, that can then be displayed in the tree. 
-     * We'll initially assume that the filenames are utf8 encoded and so convert the
-     * filename into utf8 for proper presentation in the Collection tree pane.
-     * If the filenames are not utf8, then the conversion would have introduced funny
-     * characters. Therefore, when converting to utf8, if the converted filename 
-     * contains the special character '\ufffd', then we know the conversion did not work
-     * and we return the original string which may or may not be properly presented by
-     * default.
-     * See http://java.sun.com/j2se/1.4.2/docs/api/java/nio/charset/CharsetDecoder.html
-     * which says "How a decoding error is handled depends upon the action requested for 
-     * that type of error, which is described by an instance of the CodingErrorAction class.
-     * The possible error actions are to ignore the erroneous input, report the error to
-     * the invoker via the returned CoderResult object, or replace the erroneous input with
-     * the current value of the replacement string. The replacement has the initial value 
-     * "\uFFFD"; its value may be changed via the replaceWith method."
-     * The following made me think that String(byte[], String charsetName) constructor may
-     * use the replacement value \uFFFD.
-     * http://www.experts-exchange.com/Programming/Programming_Languages/Java/Q_20512969.html
-     * mentions the following which made me think of this: 
-     * convertedStr = convertedStr.replace('\ufffd', ' ');
-     */
-    protected String calcDisplayString() {
-    String filename = file.getName();
-    try{
-        String utf8filename = new String(filename.getBytes(), "UTF8");
-        if(utf8filename.indexOf('\ufffd') == -1) {
-        return utf8filename;
-        } else { // contains the character indicating that it's invalid utf8
-        // return the original string
-        return filename;
-        }
-    } catch(java.io.UnsupportedEncodingException e) {
-        return filename;
-    }
+        this.file = file;
+
+        if (file != null) {
+            // Files cannot have children
+            if(file.isFile()) {
+                // Cache this result to prevent unceasing missing disk messages being thrown if the
+                // removable media was, um, removed after directory mapped
+                this.allows_children = false;
+            }
+            filenameEncoding = "";
+            urlEncodedFilePath = FilenameEncoding.calcURLEncodedFilePath(file);
+            urlEncodedFileName = FilenameEncoding.calcURLEncodedFileName(urlEncodedFilePath);
+
+            // work out the display string (extra special processing for CollectionTreeNodes)
+            displayFileName = calcDisplayString();      
+        }
+    }
+
+    public String getURLEncodedFileName() { return urlEncodedFileName; }
+
+    public String getURLEncodedFilePath() { return urlEncodedFilePath; }
+
+    public String getFilenameEncoding() { return filenameEncoding; }
+
+
+    /** This method returns a string representation of the filenodes in the tree, 
+    * that can then be displayed in the tree. Overridden in subclass CollectionTreeNode.
+    * Turn FilenameEncoding.DEBUGGING on to see URLEncoded filenames.
+    */
+    protected String calcDisplayString() {  
+        if(FilenameEncoding.DEBUGGING) {
+            return getURLEncodedFileName();
+        } else {
+            return file.getName();
+        }
     }
 
@@ -314 +307 @@
             }
         }
+        
+            // in case any filename encodings had gone stale, 
+            // (recalculate these and) refresh the display name
+            refreshDescendantEncodings();           
+                    
         }
 
@@ -327 +325 @@
     }
 
-
+    // overridden in subclass CollectionTreeNode to reset and reencode display strings
+    public void resetDescendantEncodings() {}
+    public void refreshDescendantEncodings() {}
+    
+    
     public void setModel(FileSystemModel model) {
     this.model = model;

main/trunk/gli/src/org/greenstone/gatherer/gui/EnrichPane.java

@@ -44 +44 @@
 import org.greenstone.gatherer.collection.CollectionTreeNode;
 import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.metadata.MetadataElement;
 import org.greenstone.gatherer.metadata.MetadataValue;
@@ -287 +288 @@
   public void valueChanged(TreeSelectionEvent event)
     {
+    if(FilenameEncoding.isRefreshRequired()) { 
+        // The CollectionTree is in the process of being re-built to deal with filename-encodings, 
+        // so don't mess up the table during that time.
+        // (CollectionTreeNode.refreshDescendantEncodings() doesn't cope well if this method
+        // responds on Tree selection changes while the tree is updating its encodings, resulting in lost
+        // and misplaced metadata values in the table and thereby in the metadata.xml files themselves).
+        return;
+    }
+    
+    
     // If we haven't got focus then it must have been a selection in the Gather pane, so don't bother rebuilding
     if (has_focus == false) {

main/trunk/gli/src/org/greenstone/gatherer/gui/GUIManager.java

@@ -59 +59 @@
 import org.greenstone.gatherer.gui.metaaudit.MetaAuditFrame;
 import org.greenstone.gatherer.gui.tree.DragTree;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
 import org.greenstone.gatherer.metadata.MetadataSet;
 import org.greenstone.gatherer.metadata.MetadataXMLFileManager;
@@ -295 +296 @@
     tab_pane.setSelectedComponent(gather_pane);
     Gatherer.c_man.closeCollection();
+    FilenameEncoding.closeCollection(); // clear filename-to-encodings map
     }
 

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataValueTableModel.java

@@ -139 +139 @@
 
     MetadataValueTableEntry metadata_value_table_entry = (MetadataValueTableEntry) metadata_value_table_entries.get(row);
+    
+    if(metadata_value_table_entry == null) {
+        System.err.println("\n@@@@@ MetadataValueTableModel.getValueAt(): metadata_value_table_entry is unexpectedly null!\n");
+        return null;
+    }
+    
     if (col == 0 && metadata_value_table_entry.isInheritedMetadata()) {
         return metadata_value_table_entry.getFolderMetadataInheritedFrom();
@@ -213 +219 @@
     {
     MetadataValueTableEntry metadata_value_table_entry = getMetadataValueTableEntry(row);
-
+    if(metadata_value_table_entry == null) {
+        System.err.println("\n@@@@@ MetadataValueTableModel.setValueAt(): metadata_value_table_entry is unexpectedly null!\n");
+        return;
+    }
+    
     // If nothing has changed no action is necessary
     String old_metadata_value = metadata_value_table_entry.getFullValue();
@@ -252 +262 @@
         MetadataValueTreeNode metadata_value_tree_node = metadata_element.addMetadataValue((String) new_metadata_value);
         MetadataValue metadata_value = new MetadataValue(metadata_element, metadata_value_tree_node);
-        metadata_value.setIsAccumulatingMetadata(true);
+        metadata_value.setIsAccumulatingMetadata(true);     
         (new AppendMetadataTask(metadata_value)).run();
     }

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java

@@ -31 +31 @@
 import java.util.*;
 import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
 import org.greenstone.gatherer.util.XMLTools;
 import org.w3c.dom.*;
@@ -45 +46 @@
     static final private String METADATA_ELEMENT = "Metadata";
 
+    /** Special metadata field: the filename encoding is a unique sort of metadata in
+     * that it is not just information stored with a collection file, but also needs to
+     * be applied in real-time to the collection file (to its filename) for display. */
+    static final public String FILENAME_ENCODING_METADATA = "gs.filenameEncoding";
+    
     // To speed things up a bit we keep the last accessed metadata.xml file in memory
     static private File loaded_file = null;
@@ -57 +63 @@
 
 
-    public void addMetadata(File file, ArrayList metadata_values)
+    public void addMetadata(CollectionTreeNode file_node, ArrayList metadata_values)
     {
     // If this metadata.xml file isn't the one currently loaded, load it now
@@ -76 +82 @@
 
     // Determine the file's path relative to the location of the metadata.xml file
-    String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
-    String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
-    if (file_relative_path.startsWith(File.separator)) {
-        file_relative_path = file_relative_path.substring(File.separator.length());
+    String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
+    String file_relative_path = file_node.getURLEncodedFilePath().substring(metadata_xml_file_directory_path.length());
+    if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
+        file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
     }
 
@@ -127 +133 @@
         appropriate_fileset_element.appendChild(new_description_element);
 
+        // add the fileset element for .* at the top: especially important for
+        // non-accumulating (and override mode) meta. Other type fileset elements can be appended
         if(file_path_regexp.equals(DIRECTORY_FILENAME)) {
-          loaded_file_document.getDocumentElement().insertBefore(appropriate_fileset_element, loaded_file_document.getDocumentElement().getFirstChild());
+            loaded_file_document.getDocumentElement().insertBefore(appropriate_fileset_element,
+                    loaded_file_document.getDocumentElement().getFirstChild());
         } else {
           loaded_file_document.getDocumentElement().appendChild(appropriate_fileset_element);
@@ -147 +156 @@
         metadata_value_string = metadata_value_string.replaceAll("\\]", "]");
 
+        // the gs.filenameEncoding metadata is unique in that, when added, removed or
+        // changed, it must be applied on the file(name) whose metadata has been adjusted
+        if(metadata_element_name_full.equals(FILENAME_ENCODING_METADATA)) {
+            metadata_value_string = processFilenameEncoding(file_path_regexp, 
+                                        file_node, metadata_value_string, false);
+                              // true only if removing meta
+        }
+
         // Check if this piece of metadata has already been assigned to this FileSet element
         boolean metadata_already_assigned = false;
@@ -156 +173 @@
         String current_metadata_element_name_full = current_metadata_element.getAttribute("name");
         if (current_metadata_element_name_full.equals(metadata_element_name_full)) {
-          // if the metadata must not accumulate, then edit teh current value
+          // if the metadata must not accumulate, then edit the current value
           if (!metadata_value.isAccumulatingMetadata()) {
             XMLTools.setNodeText(current_metadata_element, metadata_value_string);
@@ -196 +213 @@
 
 
-    public ArrayList getMetadataAssignedToFile(File file)
+    public ArrayList getMetadataAssignedToFile(File file, boolean fileEncodingOnly)
     {
     // If this metadata.xml file isn't the one currently loaded, load it now
@@ -214 +231 @@
     }
 
-    // Determine the file's path relative to the location of the metadata.xml file
+    // Determine the file's path relative to the location of the metadata.xml file  
+    String file_relative_path = FilenameEncoding.fileToURLEncoding(file);
     File metadata_xml_file_directory = getParentFile();
-    String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory.getAbsolutePath().length());
-    if (file_relative_path.startsWith(File.separator)) {
-        file_relative_path = file_relative_path.substring(File.separator.length());
+    String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(metadata_xml_file_directory);
+    file_relative_path = file_relative_path.substring(metadata_xml_file_directory_path.length());
+
+    if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
+        file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
     }
 
@@ -255 +275 @@
 
         // This fileset specifies metadata for the folder the file is in
-        if (file_relative_path.startsWith(current_filename_element_value + File.separator)) {
+        if (file_relative_path.startsWith(current_filename_element_value + FilenameEncoding.URL_FILE_SEPARATOR)) {
             current_fileset_matches = true;
             folder_metadata_inherited_from = new File(metadata_xml_file_directory, current_filename_element_value);
@@ -272 +292 @@
         Element current_metadata_element = (Element) metadata_elements_nodelist.item(k);
         String metadata_element_name_full = current_metadata_element.getAttribute("name");
+        // if we're only looking for fileEncoding metadata and this isn't it, skip to the next
+        if(fileEncodingOnly && !metadata_element_name_full.equals(FILENAME_ENCODING_METADATA)) {
+            continue;
+        }       
         String metadata_set_namespace = MetadataTools.getMetadataSetNamespace(metadata_element_name_full);
 
@@ -335 +359 @@
 
 
-    public void removeMetadata(File file, ArrayList metadata_values)
+    public void removeMetadata(CollectionTreeNode file_node, ArrayList metadata_values)
     {
     // If this metadata.xml file isn't the one currently loaded, load it now
@@ -354 +378 @@
 
     // Determine the file's path relative to the location of the metadata.xml file
-    String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
-    String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
-    if (file_relative_path.startsWith(File.separator)) {
-        file_relative_path = file_relative_path.substring(File.separator.length());
+    String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
+    String file_relative_path = file_node.getURLEncodedFilePath().substring(metadata_xml_file_directory_path.length());
+    if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
+        file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
     }
 
@@ -422 +446 @@
             String current_metadata_value_string = XMLTools.getElementTextValue(current_metadata_element);
             if (current_metadata_value_string.equals(metadata_value_string)) {
+            
             // Remove this Metadata element
             current_metadata_element.getParentNode().removeChild(current_metadata_element);
-
+            
+            // the gs.filenameEncoding metadata is unique in that, when added, removed or
+            // changed, it must be applied on the file(name) whose metadata has been adjusted
+            if(current_metadata_element_name_full.equals(FILENAME_ENCODING_METADATA)) {
+            
+                // metadata_value_string will hereafter be the inherited gs.FilenameEncoding 
+                // metadata (if any), now that the value at this level has been removed
+                metadata_value_string = processFilenameEncoding(file_path_regexp,
+                        file_node, "", true); // true only if *removing* this meta  
+            }
+            
             // If there are no Metadata elements left now, remove the (empty) FileSet element
             if (metadata_elements_nodelist.getLength() == 0) {
@@ -441 +476 @@
 
 
-    public void replaceMetadata(File file, MetadataValue old_metadata_value, MetadataValue new_metadata_value)
+    public void replaceMetadata(CollectionTreeNode file_node, MetadataValue old_metadata_value, MetadataValue new_metadata_value)
     {
     // If this metadata.xml file isn't the one currently loaded, load it now
@@ -460 +495 @@
 
     // Determine the file's path relative to the location of the metadata.xml file
-    String metadata_xml_file_directory_path = getParentFile().getAbsolutePath();
-    String file_relative_path = file.getAbsolutePath().substring(metadata_xml_file_directory_path.length());
-    if (file_relative_path.startsWith(File.separator)) {
-        file_relative_path = file_relative_path.substring(File.separator.length());
+    String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
+    String file_relative_path = file_node.getURLEncodedFilePath().substring(metadata_xml_file_directory_path.length());
+    if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
+        file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
     }
 
@@ -541 +576 @@
         // If the new metadata value already existed, remove the original value
         if (new_metadata_value_already_exists) {
-        metadata_element_to_edit.getParentNode().removeChild(metadata_element_to_edit);
+            if(metadata_element_to_edit != null) { //?????????
+                metadata_element_to_edit.getParentNode().removeChild(metadata_element_to_edit);
+            } else {
+                System.err.println("ERROR MetadataXMLFile: metadata_element_to_edit is null");
+            }
         }
         // Otherwise replace the old value with the new value
         // Ensure metadata_element_to_edit isn't null (may occur when multiple files are selected)
         else if (metadata_element_to_edit != null) {
+        
+        // the gs.filenameEncoding metadata is unique in that, when added, removed or
+        // changed, it must be applied on the file(name) whose metadata has been adjusted
+        if(metadata_element_name_full.equals(FILENAME_ENCODING_METADATA)) {
+            new_metadata_value_string = processFilenameEncoding(file_path_regexp, file_node, new_metadata_value_string, false);
+            // true only if removing meta       
+        }       
         XMLTools.setElementTextValue(metadata_element_to_edit, new_metadata_value_string);
         }
@@ -649 +695 @@
     }
     }
+
+    /** 
+     * The gs.filenameEncoding metadata is unique in that, when added, removed or
+     * replaced, it must be applied on the file(name) whose metadata has been 
+     * adjusted. 
+     * This method handles all that, given the regular expression or filepath name 
+     * to match on (.* matches subdirectories), the affected fileNode, the new 
+     * encoding value and whether a new encoding value has been added/an existing
+     * one has been replaced or whether the encoding metadata has been removed.
+     * The new adjusted value for the encoding metadata is returned.
+     *
+     * MetadataXMLFileManager maintains a hashmap of (URL-encoded filepaths, encoding) 
+     * to allow fast access to previously assigned gs.filenameEncoding metadata (if
+     * any) for each file. This hashmap also needs to be updated, but this update
+     * is complicated by the fact that it concerns regular expressions that could
+     * affect multiple filenames.
+     */
+    public String processFilenameEncoding(String file_path_regexp, CollectionTreeNode file_node, 
+                    String encoding_metadata_value, boolean removingMetadata) 
+    {
+        if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            return encoding_metadata_value;
+        }
+
+        // Work out this filenode's new encoding and apply it:
+        
+        if(removingMetadata) { // encoding_metadata_value = ""
+            // gs.filenameEncoding metadata being removed, work out 
+            // any inherited metadata to replace it with in the meta-table
+            encoding_metadata_value = FilenameEncoding.getInheritedFilenameEncoding(
+                        file_node.getURLEncodedFilePath(), file_node.getFile());
+            // should be canonical encoding already
+        } 
+        else if(!encoding_metadata_value.equals("")) { 
+            // if adding or replacing filename encoding,
+            // get the canonical encoding name for this alias
+            encoding_metadata_value = FilenameEncoding.canonicalEncodingName(encoding_metadata_value);
+        }   
+        // Reencode the display of this filenode only as any affected 
+        // childnodes will be reencoded on FileNode.refreshDescendantEncodings()
+        file_node.reencodeDisplayName(encoding_metadata_value);
+            
+
+        // Whether removing or adding/replacing the file's gs.filename encoding meta,
+        // store this in the file-to-encoding map for fast access, since the map stores
+        // empty string values when no meta has been assigned at this file level.
+        // In the case of removingMetadata, the value stored will be the fallback value
+        
+        String urlpath = file_node.getURLEncodedFilePath();
+        if(removingMetadata) { 
+            // remove it from the map instead of inserting "", so that when folders in the collectiontree 
+            // are being deleted or shifted, the removemetada (and addmetadata) calls that get fired 
+            // for each affected filenodes does not cause the undesirable effect of multiple "" to be 
+            // entered into the filename-to-encoding map for filepaths that no longer exist .
+            FilenameEncoding.map.remove(urlpath); 
+        } else { // for adding and replacing, put the encoding into the map (also replaces any existing encoding for it)
+            FilenameEncoding.map.put(urlpath, encoding_metadata_value);         
+        }
+        
+        // If new folder-level metadata (or metadata for a set of files fitting a pattern) has been
+        //  assigned, the file_to_encodings map will be cleared for all descendant folders and files,
+        //  so that these can be re-calculated upon refreshing the visible parts of the CollectionTree. 
+        // Mark the state as requiring a refresh of the CollectionTree. 
+        // This next step also serves to prevent the MetadataValueTableModel from trying to update
+        // itself while a refresh (involving re-encoding of filenames of visible nodes) is in progress.
+        FilenameEncoding.setRefreshRequired(true);
+        
+        return encoding_metadata_value;
+    }
 }

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFileManager.java

@@ -90 +90 @@
         if (current_file_directory_path.equals(metadata_xml_file.getParentFile().getAbsolutePath())) {
             applicable_metadata_xml_file_found = true;
-            metadata_xml_file.addMetadata(current_file, metadata_values);
+            metadata_xml_file.addMetadata(file_nodes[i], metadata_values);
             if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
             modified_metadata_xml_files.add(metadata_xml_file);
@@ -107 +107 @@
 
         // ...and add the metadata
-        new_metadata_xml_file.addMetadata(current_file, metadata_values);
+        new_metadata_xml_file.addMetadata(file_nodes[i], metadata_values);
         if (!modified_metadata_xml_files.contains(new_metadata_xml_file)) {
             modified_metadata_xml_files.add(new_metadata_xml_file);
@@ -160 +160 @@
 
     // Get the metadata assigned to the specified file from the applicable metadata.xml files
-    ArrayList assigned_metadata = getMetadataAssignedToFile(file, applicable_metadata_xml_files);
+    ArrayList assigned_metadata = getMetadataAssignedToFile(file, applicable_metadata_xml_files, false);
 
     // Remove any folder-level metadata
@@ -173 +173 @@
 
 
+    /** Returns the metadata assigned to a file inside the collection, excluding folder-level/inherited metadata. */
+    static public ArrayList getMetadataAssignedDirectlyToFile(File file)
+    {
+        return getMetadataAssignedDirectlyToFile(file, false);
+    }
+    
     /** Returns the metadata assigned to a file inside the collection, excluding folder-level/inherited metadata. */
-    static public ArrayList getMetadataAssignedDirectlyToFile(File file)
-    {
-    // Get all the metadata assigned to the specified file...
-    ArrayList assigned_metadata = getMetadataAssignedToFile(file);
-
-    // ...then remove any folder-level metadata
-    for (int i = assigned_metadata.size() - 1; i >= 0; i--) {
-        if (((MetadataValue) assigned_metadata.get(i)).isInheritedMetadata()) {
-        assigned_metadata.remove(i);
-        }
-    }
-
-    return assigned_metadata;
-    }
-
+    static public ArrayList getMetadataAssignedDirectlyToFile(File file, boolean filenameEncodingMetaOnly)
+    {
+        
+        // Get all the metadata assigned to the specified file...
+        ArrayList assigned_metadata = getMetadataAssignedToFile(file, filenameEncodingMetaOnly);
+
+        // ...then remove any folder-level metadata
+        for (int i = assigned_metadata.size() - 1; i >= 0; i--) {
+            if (((MetadataValue) assigned_metadata.get(i)).isInheritedMetadata()) {
+            assigned_metadata.remove(i);
+            }
+        }
+
+        return assigned_metadata;
+        /*
+        // Get all the metadata assigned to the specified file...
+        // Build up a list of applicable metadata.xml files - which in this case 
+        // is exclusively the metadata file at this file/folder's own level.
+        ArrayList applicable_metadata_xml_files = new ArrayList();
+
+        // Find the metadata.xml file (if any) that is at the same level as the file
+        String file_directory_path = (file.isDirectory() ? file : file.getParentFile()).getAbsolutePath() + File.separator;
+        for (int i = 0; i < metadata_xml_files.size(); i++) {
+            MetadataXMLFile metadata_xml_file = (MetadataXMLFile) metadata_xml_files.get(i);
+            
+            if (file_directory_path.equals(metadata_xml_file.getParentFile().getAbsolutePath() + File.separator)) {
+                //System.err.println("Found metadata_xml_file: " + metadata_xml_file);
+                applicable_metadata_xml_files.add(metadata_xml_file);
+            }
+        }
+        
+        if(applicable_metadata_xml_files.size() == 0) {
+            return new ArrayList(0);
+        }
+
+        // Return the metadata assigned to the specified file from the applicable metadata.xml files
+        return getMetadataAssignedToFile(file, applicable_metadata_xml_files, filenameEncodingMetaOnly);
+        */
+    }
+    
 
     /** Returns all the metadata assigned to a file inside the collection. */
     static public ArrayList getMetadataAssignedToFile(File file)
+    {
+    return getMetadataAssignedToFile(file, false);
+    }
+    
+    /** Returns all the metadata assigned to a file inside the collection (or 
+     *  just gs.filenameEncoding if parameter filenameEncodingMetaOnly is true),
+     *  including folder-level/inherited metadata.
+     */    
+    static public ArrayList getMetadataAssignedToFile(File file, boolean filenameEncodingMetaOnly)
     {
     // Build up a list of applicable metadata.xml files
@@ -207 +247 @@
     }
 
-    // sort the metadataxml files in order starting from those in the 
+    // Sort the metadataxml files in order starting from those in the 
     // topmost folders down to the one in the lowest level folder.
     Collections.sort(applicable_metadata_xml_files, metadataXMLFileComparator);
     // Return the metadata assigned to the specified file from the applicable metadata.xml files
-    return getMetadataAssignedToFile(file, applicable_metadata_xml_files);
-    }
-
-
-    static private ArrayList getMetadataAssignedToFile(File file, ArrayList applicable_metadata_xml_files)
+    return getMetadataAssignedToFile(file, applicable_metadata_xml_files, filenameEncodingMetaOnly);
+    }
+
+    // package access method
+    static ArrayList getMetadataAssignedToFile(File file, ArrayList applicable_metadata_xml_files,
+                               boolean filenameEncodingMetaOnly)    
     {
     // Build up a list of metadata values assigned to this file
@@ -225 +266 @@
         DebugStream.println("Applicable metadata.xml file: " + metadata_xml_file);
 
-        ArrayList metadata_values = metadata_xml_file.getMetadataAssignedToFile(file);
+        ArrayList metadata_values = metadata_xml_file.getMetadataAssignedToFile(file, filenameEncodingMetaOnly);
         for (int j = 0; j < metadata_values.size(); j++) {
         MetadataValue metadata_value = (MetadataValue) metadata_values.get(j);
@@ -316 +357 @@
         // This metadata.xml file is only potentially applicable if it is above or at the same level as the file
         if (current_file_directory_path.startsWith(metadata_xml_file.getParentFile().getAbsolutePath())) {
-            metadata_xml_file.removeMetadata(current_file, metadata_values);
+            metadata_xml_file.removeMetadata(file_nodes[i], metadata_values);
             if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
             modified_metadata_xml_files.add(metadata_xml_file);
@@ -350 +391 @@
         // This metadata.xml file is only applicable if it is at the same level as the file
         if (current_file_directory_path.equals(metadata_xml_file.getParentFile().getAbsolutePath())) {
-            metadata_xml_file.replaceMetadata(current_file, old_metadata_value, new_metadata_value);
+            metadata_xml_file.replaceMetadata(file_nodes[i], old_metadata_value, new_metadata_value);
             if (!modified_metadata_xml_files.contains(metadata_xml_file)) {
             modified_metadata_xml_files.add(metadata_xml_file);
@@ -399 +440 @@
 
   
-  /**
-  * Comparator to order MetadataXMLFiles in ascending order from
-  * those in a higher level folder to those in a lower level folder
-  * It is based on the assumption that all MetadataXMLFiles sent to
-  * it to compare will be linear descendants of one toplevel folder
-  * E.g. /A/metadata.xml, /A/B/metadata.xml, /A/B/C/D/metadata.xml.
-  * In other words, that each is a substring of one other until we
-  * get to the toplevel folder.
-  */
-  private static class MetadataXMLFileComparator implements Comparator {
-    
-    public int compare(Object o1, Object o2) {
-      if(!(o1 instanceof MetadataXMLFile)) {
-    return -1;
-      } else if (!(o2 instanceof MetadataXMLFile)) {
-    return 1;
-      } 
-
-      // Both are MetadataXMLFiles objects. Remove the terminating
-      // "metadata.xml" from their filenames to get their containing folder
-      String filename1 = ((MetadataXMLFile)o1).getParentFile().getAbsolutePath();
-      String filename2 = ((MetadataXMLFile)o2).getParentFile().getAbsolutePath();
-
-      // if 1 is a prefix 2, then 1 < 2 in the ordering (1 comes before 2)
-      if(filename2.startsWith(filename1)) {
-    return -1;
-      } else if(filename1.startsWith(filename2)) {
-    return 1;
-      } else { 
-    // unlikely that the metadata.xml files will be the same
-    // or that neither is a prefix of the other
-    return filename1.compareTo(filename2); // sorts in ascending order
-      }      
-      
-    }
-
-    public boolean equals(Object obj) {
-      if(!(obj instanceof MetadataXMLFileComparator)) {
-    return false;
-      } 
-     
-      // else it is the same sort of comparator
-      return true;
-    }
-
-  }
+    /**
+      * Comparator to order MetadataXMLFiles in ascending order from
+      * those in a higher level folder to those in a lower level folder
+      * It is based on the assumption that all MetadataXMLFiles sent to
+      * it to compare will be linear descendants of one toplevel folder
+      * E.g. /A/metadata.xml, /A/B/metadata.xml, /A/B/C/D/metadata.xml.
+      * In other words, that each is a substring of one of the others until we
+      * the toplevel folder is reached.
+    */
+    private static class MetadataXMLFileComparator implements Comparator {
+        
+        public int compare(Object o1, Object o2) {
+            if(!(o1 instanceof MetadataXMLFile)) {
+                return -1;
+            } else if (!(o2 instanceof MetadataXMLFile)) {
+                return 1;
+            } 
+
+            // Both are MetadataXMLFiles objects. Remove the terminating
+            // "metadata.xml" from their filenames to get their containing folder
+            String filename1 = ((MetadataXMLFile)o1).getParentFile().getAbsolutePath();
+            String filename2 = ((MetadataXMLFile)o2).getParentFile().getAbsolutePath();
+
+            // if 1 is a prefix of 2, then 1 < 2 in the ordering (1 comes before 2)
+            if(filename2.startsWith(filename1)) {
+                return -1;
+            } else if(filename1.startsWith(filename2)) {
+                return 1;
+            } else { 
+                // unlikely that the metadata.xml files will be the same
+                // or that neither is a prefix of the other
+                return filename1.compareTo(filename2); // sorts in ascending order
+            }      
+
+        }
+
+        public boolean equals(Object obj) {
+            if(!(obj instanceof MetadataXMLFileComparator)) {
+                return false;
+            } 
+
+            // else it is the same sort of comparator
+            return true;
+        }
+
+    }
 
 }

main/trunk/gli/src/org/greenstone/gatherer/util/SynchronizedTreeModelTools.java

@@ -31 +31 @@
 import javax.swing.tree.*;
 import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.metadata.FilenameEncoding;
+import org.greenstone.gatherer.collection.CollectionTreeNode;
 
 /** Due to the TreeModel objects not having any synchronization, certain assumptions, such as the model state remaining constant during a repaint, don't always hold - especially given that I'm changing the tree model on a different thread. In order to get around this I will use the latest swing paradigm wherein you flag a section of code to be executed by the AWT GUI Event queue, as soon as other gui tasks have finished. This way I shouldn't have tree redraws throwing NPEs because the array size of the children of a certain node has changed -while- the repaint call was made, i.e. repaint() calls getChildCount() = 13, removeNodeFromParent() called, repaint calls getChildAt(12) = ArrayIndexOutOfBoundsException. 
@@ -104 +106 @@
     final Runnable doRemoveNodeFromParent = new Runnable() {
         public void run() {
-            model.removeNodeFromParent(target_node);
+            // If we're dealing with a collection tree node, it may have 
+            // gs.FilenameEncoding assigned, so we remove its entry from the map. 
+            // Needs to be done here because the tree is constantly changing 
+            // when nodes are being removed, renamed and deleted, and this 
+            // affects lookup queries sent to the map. 
+            // Don't need to do a recursive reset on this coltreenode, because 
+            // Delete/Move/Rename FileJobs were created for *each* node         
+            if(target_node instanceof CollectionTreeNode) {
+                CollectionTreeNode colNode = (CollectionTreeNode)target_node;
+                FilenameEncoding.map.remove(colNode.getURLEncodedFilePath());
+            }
+            
+            model.removeNodeFromParent(target_node);            
         }
         };