Changeset 34507

Timestamp:

2020-10-22T01:48:03+13:00 (3 years ago)

Author:

ak19

Message:

Redoing work of commit revision 34394: Redoing Bugfix 1 for GLI doc.xml metadata slowdown resulting from earlier bugfix to help GLI cope with filenames and assigned meta that have non-ASCII chars in them. The slowdown happened when gathered files got selected in GLI and was fixed in commit 34394, but the fix was not ideal for 2 reasons. 1. A new form of filename encoding (hexed unicode) going into doc.xml, instead of existing encodings like URL and base64, though those existing encodings weren't the right ones for my first solution. 2. The solution was specific to Windows to cope with special chars in filenames and relied on a new meta field gsdlfullsourcepath being written out to doc.xml by doc.pm. So a built collection moved from Linux to Windows won't show up doc.xml meta in GLI, as it won't have the new doc.xml meta field that Windows is expecting. Have a better solution for 1 that doesn't require the new field. But still can't fix all of point 2, as the existing gsdlsourcefilename meta field in doc.xml can contain Windows Short filenames when the coll is built on Windows and this won't be backwards compatible on Linux anyway. This problem existed before too, except I didn't realise it until now. But the new solution fixes more issues. Second step: modified DocXMLFile to no longer use the new field gsdlfullsourcepath, but return to using gsdlsourcefilename field. This time however, the code is optimised to detect a filename match between doc.xml and any file selected in GLI by storing gsdlsourcefilename in its Long filename form whenever doc.xml had stored it in Win 8.3 Short filename form. The Long filename can be obtained for any file that exists by calling getCanonicalPath(). Of course, the full filename was not stored in gsdlsourcefilename, rather the filename from import folder onwards. So to ensure a file by that filename in long form has a chance of existing, first prefixed the current collection folder and then checked for existence before obtaining the canonical form for it. This is then stored in the hashmap in place of any win short filename. Now a match is more readily found without using any hex encoded unicode filenames stored by doc.pm, and without using the older and inefficient method of making cmd calls to DOS to calculate the Win 8.3 Short filename for each selected file.

Location:

main/trunk/gli/src/org/greenstone/gatherer/metadata

Files:

• DocXMLFile.java (modified) (8 diffs)
• DocXMLFileManager.java (modified) (2 diffs)

main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java

@@ -32 +32 @@
 import java.net.URLDecoder;
 import org.greenstone.gatherer.DebugStream;
+import org.greenstone.gatherer.Gatherer;
 import org.greenstone.gatherer.util.Utility;
 
@@ -42 +43 @@
 {
     static boolean isWin = Utility.isWindows();
-    // For Linux, we continue using gsdlsourcefilename as key to the metadata mapping
-    // For Windows, we use the hex encoded long file paths as key
-    static String GSDL_SOURCE_FILE_METANAME = isWin ? "gsdlfullsourcepath" : "gsdlsourcefilename";
     
     protected HashMap source_file_name_to_description_elements_mapping = new HashMap();
@@ -64 +62 @@
     /** On Windows, file_relative_path will be hex-encoded for codepts beyond ASCII.
      * But keys into the source_file_name_to_description_elements_mapping will then also match on Windows */
-    public ArrayList getMetadataExtractedFromFile(File file, String file_relative_path)                                             
+    public ArrayList getMetadataExtractedFromFile(File file, String file_relative_path)
     {
     // Build up a list of metadata extracted from this file
@@ -249 +247 @@
         
         // Note which file this is for
-        //else if (metadata_element_name.equals("gsdlsourcefilename")) {    
-        else if (metadata_element_name.equals(GSDL_SOURCE_FILE_METANAME)) {
-            // On Unix, GSDL_SOURCE_FILE_METANAME is the gsdlsourcefilename metadata field
-            // which may be encoded by the encoding denoted in fileRenameMethod (and will need decoding)
-            // On Windows, GSDL_SOURCE_FILE_METANAME is a different metadata field that
-            // will be hex encoded for non-ASCII chars
+        else if (metadata_element_name.equals("gsdlsourcefilename")) {
+            // the gsdlsourcefilename metadata field may be encoded by the encoding denoted
+            // in fileRenameMethod (and will need decoding)
             
             // Extract the gsdlsourcefilename element value
             int value_index = line.indexOf(">", name_index) + ">".length();
             gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
-
-            // We're only interested in the path relative to the import folder
-            int import_index = gsdlsourcefilename_value.indexOf("import");
+            
+            // We're only interested in the path relative to the import folder          
+            int import_index = gsdlsourcefilename_value.indexOf("import");
             if (import_index != -1) {
-            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
-
-            is_unix_path = gsdlsourcefilename_value.startsWith("/");
-            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
-
-            // (Will decode gsdlsourcefilename at end of this method, once we know
-            // for certain the fileRenameMethod that was used to encode it.)
-
-            // Make sure the path matches the OS that is running
-            if (is_unix_path && isWin) {
-                // Convert path from Unix to Windows
-                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
-            }
-            else if (!is_unix_path && !isWin) {
-                // Convert path from Windows to Unix
-                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
-            }
             
-            ///System.err.println("@@@@ Found gsdlsourcefilename: " + gsdlsourcefilename_value);
-            // Remember this for quick access later
-            if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
-                source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, new ArrayList());
-            }
-
-            // Would be better to store hex src file name decoded? But how do we know what encoding the filename is in
-            // https://stackoverflow.com/questions/13990941/how-to-convert-hex-string-to-java-string
-            
-            
-            ((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
+                ///System.err.println("@@@@ Found gsdlsourcefilename: " + gsdlsourcefilename_value);                
             }
 
@@ -301 +269 @@
                  && !gsdlsourcefilename_value.endsWith("collect.cfg")
                  && !gsdlsourcefilename_value.endsWith("collectionConfig.xml")) {
-            // We don't really know what is going on...
-            System.err.println("Warning: Could not understand gsdlsourcefilename " + gsdlsourcefilename_value);
+                // We don't really know what is going on...
+                System.err.println("Warning: Could not understand gsdlsourcefilename " + gsdlsourcefilename_value);
             }
         }
@@ -321 +289 @@
         buffered_reader.close();
 
-        // ON WINDOWS, we're working with hex encoded full file path instead of with gsdlsourcefilename,
-        // so needn't bother decoding gsdlsourcefilename as it's unused.
-        // On UNIX, continue decoding gsdlsourcefilename as before
-        if(!isWin) {
+
         // Now that we're done skimming, we actually need to decode gsdlsourcefilename
         // based on whatever fileRenameMethod was used to encode it, so that we can
-        // at last properly compare properly against filenames on the file system
+        // at last properly compare against filenames on the file system
         // in order to load the correct ex.meta for the file.
         // Now that we should have both gsdlsourcefilename AND fileRenameMethod set,
@@ -337 +302 @@
         // If gsdlsourcefilename was encoded, we remove it from the map under its encoded
         // filename, decode it and add it back into map using its decoded filename.
-        if(!fileRenameMethod.equals(FILE_RENAME_METHOD_NONE)) {
-            ArrayList value_list = (ArrayList) source_file_name_to_description_elements_mapping.remove(gsdlsourcefilename_value);
-            gsdlsourcefilename_value = decodeSourceFilename(gsdlsourcefilename_value, fileRenameMethod, is_unix_path);
-            source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, value_list);
-        }
-        }
-        
+        if(!fileRenameMethod.equals(FILE_RENAME_METHOD_NONE)) {         
+            gsdlsourcefilename_value = decodeSourceFilename(gsdlsourcefilename_value, fileRenameMethod, is_unix_path);          
+        }       
+        
+        // Next, if Windows, check if dealing with Win 8.3 Short Filename
+        // In that case, convert short file name to full name - works only if the file exists       
+        if(isWin && gsdlsourcefilename_value.indexOf("~") != -1) {
+            // gsdlsourcefilename is stored from import folder onwards: import/opt_subdir/filename.ext
+            // This may contain Win 8.3 shortening. To get Win Long filename, prefix current collection dir
+            // and if resulting file exists, getCanonicalPath() which produces Win Long filename.
+            File currentCollectionFolder = Gatherer.c_man.getCollection().getCollectionDirectory();
+            File f = new File(currentCollectionFolder, /*"import" + File.separator +*/ gsdlsourcefilename_value);
+            ///System.err.println("### file: " + f.getAbsolutePath());
+            
+            if(f.exists()) {
+                gsdlsourcefilename_value = f.getCanonicalPath();
+                ///System.err.println("### canon: " + gsdlsourcefilename_value);
+                
+            } // else couldn't find a version of the filename stored in doc.xml that exists, giving up, leave gsdlsourcefilename_value as is    
+        }
+        
+        // We're only interested in the path relative to the import folder
+        int import_index = gsdlsourcefilename_value.indexOf("import");
+        if (import_index != -1) {
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
+
+            is_unix_path = gsdlsourcefilename_value.startsWith("/");
+            gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
+
+            // (Will decode gsdlsourcefilename at end of this method, once we know
+            // for certain the fileRenameMethod that was used to encode it.)
+
+            // Make sure the path matches the OS that is running
+            if (is_unix_path && isWin) {
+                // Convert path from Unix to Windows
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
+            }
+            else if (!is_unix_path && !isWin) {
+                // Convert path from Windows to Unix
+                gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
+            }
+        
+            ///System.err.println("@@@ into map: " + gsdlsourcefilename_value);
+        
+            if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
+                source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, new ArrayList());
+            }
+            
+            ((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
+        }
     }
     catch (FileNotFoundException exception) {
@@ -354 +362 @@
     }
     }
+    
     
     protected String decodeSourceFilename(String relative_sourcefile_path,

main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java

@@ -53 +53 @@
         file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
     }
-    String searchFileName = DocXMLFile.isWin ? Utility.stringToHex(file_relative_path) : file_relative_path;
-        
+    
     // Build up a list of metadata values extracted from this file
     ArrayList metadata_values = new ArrayList();
@@ -62 +61 @@
         DocXMLFile doc_xml_file = (DocXMLFile) doc_xml_files.get(i);
         ///System.err.println("@@@@ Looking at doc.xml file: " + doc_xml_files.get(i));
-        metadata_values.addAll(doc_xml_file.getMetadataExtractedFromFile(file, searchFileName));
+        metadata_values.addAll(doc_xml_file.getMetadataExtractedFromFile(file, file_relative_path));
     }