Changeset 33756 for main

Timestamp:

2019-12-05T21:58:02+13:00 (4 years ago)

Author:

ak19

Message:

Attempted bugfix for ex meta not always loading in gli for docs that are in subdirs when filenames are base64 encoded. This commit only testedand works on linux for my basic tests with subdirs and without. 1. Perl now encodes all subdirs and the filename in gsdlsourcefilename (but as before, not file extension). Can't encode entire relative path starting with import in one go, as other parts of the perl code do comparisons and remove file GSDLIMPORTDIR prefixes. 2. Perl now also writes out the file rename method used, which can be none, url or base64, into doc.xml. 3. GLI now decodes each part of the gsdlsourcefilename relative path based on the file rename method. e.g. for import/subdir/filename.ext the import, subdir and filename are decoded to reconstitute the filename as it originally was, with file extension stuck back on. This has allowed GLI to finally detect the ex meta associated with a gsdlsourcefilename in cases of subdirs in import or when dealing with base64 encoded filenames. Still need to test more complex cases on linux, then windows too.

Location:

main/trunk

Files:

• gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java (modified) (9 diffs)
• greenstone2/perllib/doc.pm (modified) (3 diffs)

main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java

@@ -34 +34 @@
 import org.greenstone.gatherer.util.Utility;
 
+import org.apache.commons.codec.binary.Base64;
+
+//import org.greenstone.gatherer.feedback.Base64;
 
 /** This class represents one doc.xml file */
@@ -43 +46 @@
     protected final String MetadataWrap;
     protected final String MetadataItem;
+
+    protected final String FILE_RENAME_METHOD_NONE = "none";
+    protected final String FILE_RENAME_METHOD_URL = "url";
+    protected final String FILE_RENAME_METHOD_BASE64 = "base64";
 
     public DocXMLFile(String doc_xml_file_path, String metaWrap, String metaItem)
@@ -63 +70 @@
     }
 
+    ///for (Object relFilename : source_file_name_to_description_elements_mapping.keySet()) {
+    ///    System.err.println("@@@ relFilename: " + relFilename);
+    ///}
+    
     // Check whether this file (i.e. doc.xml or docmets.xml on inheritance) file contains extracted metadata for the specified file
     ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
@@ -175 +186 @@
     public void skimFile()
     {
+    String fileRenameMethod = null;
+    String gsdlsourcefilename_value = null;
+    boolean is_unix_path = false;
+        
     MetadataSet extracted_metadata_set = MetadataSetManager.getMetadataSet(MetadataSetManager.EXTRACTED_METADATA_NAMESPACE);
 
@@ -227 +242 @@
         // Extracted metadata! May have ex. so make sure we remove that
         String metadata_element_name = MetadataTools.getMetadataElementName(metadata_element_name_full);
+        if(metadata_element_name.equals("gsdlsourcefilerenamemethod")) {
+            // Extract the element value
+            int value_index = line.indexOf(">", name_index) + ">".length();
+            fileRenameMethod = line.substring(value_index, line.indexOf("<", value_index));         
+        }
+        
         // Note which file this is for
-        if (metadata_element_name.equals("gsdlsourcefilename")) {
+        else if (metadata_element_name.equals("gsdlsourcefilename")) {
             // Extract the gsdlsourcefilename element value
             int value_index = line.indexOf(">", name_index) + ">".length();
-            String gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
+            gsdlsourcefilename_value = line.substring(value_index, line.indexOf("<", value_index));
 
             // We're only interested in the path relative to the import folder
@@ -238 +259 @@
             gsdlsourcefilename_value = gsdlsourcefilename_value.substring(import_index + "import".length());
 
-            boolean is_unix_path = gsdlsourcefilename_value.startsWith("/");
+            is_unix_path = gsdlsourcefilename_value.startsWith("/");
             gsdlsourcefilename_value = gsdlsourcefilename_value.substring(1);
 
-            // URL decode gsdlsourcefilename. Need to set the decoder to use the default system encoding
-            // This is stored in the System's file.encoding property.
-            gsdlsourcefilename_value = URLDecoder.decode(gsdlsourcefilename_value, System.getProperty("file.encoding")); 
+            // (Will decode gsdlsourcefilename at end of this method, once we know
+            // for certain the fileRenameMethod that was used to encode it.)
 
             // Make sure the path matches the OS that is running
@@ -254 +274 @@
                 gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
             }
-
+            
+            ///System.err.println("@@@@ Found gsdlsourcefilename: " + gsdlsourcefilename_value);
             // Remember this for quick access later
             if (source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value) == null) {
@@ -289 +310 @@
 
         buffered_reader.close();
+
+        // Now that we're done skimming, we actually need to decode gsdlsourcefilename
+        // based on whatever fileRenameMethod was used to encode it, so that we can
+        // at last properly compare properly against filenames on the file system
+        // in order to load the correct ex.meta for the file.
+        // Now that we should have both gsdlsourcefilename AND fileRenameMethod set,
+        // we can finally perform the decoding of gsdlsourcefilename.       
+        if(fileRenameMethod == null) {
+        fileRenameMethod = FILE_RENAME_METHOD_URL; // default for building
+        }
+        // If gsdlsourcefilename was encoded, we remove it from the map under its encoded
+        // filename, decode it and add it back into map using its decoded filename.
+        if(!fileRenameMethod.equals(FILE_RENAME_METHOD_NONE)) {     
+        ArrayList value_list = (ArrayList) source_file_name_to_description_elements_mapping.remove(gsdlsourcefilename_value);       
+        gsdlsourcefilename_value = decodeSourceFilename(gsdlsourcefilename_value, fileRenameMethod, is_unix_path);
+        source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, value_list);
+        }
     }
     catch (FileNotFoundException exception) {
@@ -295 +333 @@
     catch (IOException exception) {
         DebugStream.printStackTrace(exception);
-    }
+    } catch (Exception exception) { // e.g. exception decoding gsdlsourcefilename
+        DebugStream.printStackTrace(exception);
+    }
     }
-
-
+    
+    protected String decodeSourceFilename(String relative_sourcefile_path,
+                      String encodingMethod, boolean is_unix_path)
+    throws Exception
+    {   
+
+    ///System.err.println("*** relative_sourcefile_path: " + relative_sourcefile_path);
+
+    // First get the file extension. Both in Base64 and URL encoded strings,
+    // the full-stop character (.) doesn't get encoded.
+    // That means getting the file extension is straightforward.
+    
+    // Valid base64: "The 64 characters (hence the name Base64) are 10 digits,
+    // 26 lowercase characters, 26 uppercase characters as well as the
+    // Plus sign (+) and the Forward Slash (/).
+    int fullstop = relative_sourcefile_path.indexOf(".");
+    String file_ext = "";
+    if(fullstop != -1) {
+        file_ext = relative_sourcefile_path.substring(fullstop);
+        relative_sourcefile_path = relative_sourcefile_path.substring(0, fullstop);
+    }
+    
+    String[] importFilePathParts = DocXMLFile.getFilePathParts(relative_sourcefile_path, is_unix_path);
+    
+    String decoded_gsdlsourcefilename = "";
+
+    String separator = is_unix_path ? "/" : "\\";
+    for(int i = 0; i < importFilePathParts.length; i++) {
+        String decoded_filePathPart = "";
+        if(encodingMethod.equals(FILE_RENAME_METHOD_URL)) {
+        // URL decode each part of gsdlsourcefilename.
+        // Need to set the decoder to use the default system encoding
+        // This is stored in the System's file.encoding property.
+        decoded_filePathPart = URLDecoder.decode(importFilePathParts[i], System.getProperty("file.encoding"));
+        }
+        else{ // if(encodingMethod.equals(FILE_RENAME_METHOD_BASE64)) {
+        // Decoding with org.greenstone.gatherer.feedback.Base64 didn't work
+        //byte[] bytes = org.greenstone.gatherer.feedback.Base64.decode(importFilePathParts[i]);
+        // Using org.apache.commons.codec.binary.Base64 instead
+        // https://commons.apache.org/proper/commons-codec/archives/1.7/apidocs/org/apache/commons/codec/binary/Base64.html
+        // General info: https://stackoverflow.com/questions/43089541/difference-between-basic-and-url-base64-encoding-in-java-8
+        byte[] bytes = Base64.decodeBase64(importFilePathParts[i].getBytes());
+        ///System.err.println("Got base64 string: " + importFilePathParts[i]);
+        ///System.err.println("Decoded from base64 to bytes: " + bytes);
+        // Using system file.encoding to interpret the resulting bytestring as a String,
+        // just as we always did with URL decoding method
+        decoded_filePathPart = (bytes == null) ? importFilePathParts[i] : new String(bytes, System.getProperty("file.encoding"));
+        }       
+        
+        if(i == 0) {
+        decoded_gsdlsourcefilename = decoded_filePathPart;
+        } else {
+        decoded_gsdlsourcefilename = decoded_gsdlsourcefilename + separator + decoded_filePathPart;
+        }
+        ///System.err.println("Built up: " + decoded_gsdlsourcefilename);
+    }
+
+    // add the file extension back in
+    decoded_gsdlsourcefilename += file_ext;
+    
+    ///System.err.println("@@@@ decoded_gsdlsourcefilename: " + Utility.debugUnicodeString(decoded_gsdlsourcefilename));
+
+    return decoded_gsdlsourcefilename;
+    }    
+    
+    /**
+     * Given a filepath, returns the parts between each file separator as an array.
+     * For example, "/Users/me/pinky.txt" should return {"Users", "me", "pinky.txt"};
+     */
+    private static String[] getFilePathParts(String filepath, boolean is_unix_path) {   
+    StringTokenizer tok;
+    if(is_unix_path) {
+        tok = new StringTokenizer(filepath, "/");           
+    } else {
+        tok = new StringTokenizer(filepath, "\\");          
+    }
+    String[] parts;
+    int count = tok.countTokens();      
+    if(count <= 0) {
+        parts = new String[]{filepath};         
+    } else {
+        int i = 0;
+        parts = new String[count];
+        while(tok.hasMoreTokens()) {
+        parts[i] = tok.nextToken();
+        //System.err.println("Next part: " + parts[i]);
+        i++;
+        }       
+    }
+    return parts;       
+    }
+    
     /*
     public ArrayList getMetadataExtractedFromFile(File file)

main/trunk/greenstone2/perllib/doc.pm

@@ -261 +261 @@
     
 #    print STDERR "******URL/base64 encoding the gsdl_source_filename $source_filename ";
+##    print STDERR "******URL/base64 encoding the gsdl_source_filename $source_filename\n";
 
     # URLencode just the gsdl_source_filename, not the directory. Then prepend dir 
@@ -270 +271 @@
 #    $source_filename = &FileUtils::filenameConcatenate($dirname, $srcfilename);
 #    print STDERR "$source_filename\n";
-    
+
+##    print STDERR "AFTER URL/base64 encoding gsdlsourcefilename: $source_filename\n"; 
+    
     $self->set_utf8_metadata_element ($self->get_top_section(), 
                  "gsdlsourcefilename", 
                  $source_filename);
+
+    # If we set the file renaming method, at least GLI can decode gsdlsourcefilename to get the
+    # original filename (relative path to file) back for GLI to notice what ex Meta is associated with it
+    $self->set_utf8_metadata_element($self->get_top_section(),
+                     "gsdlsourcefilerenamemethod",
+                     $rename_method);
 }
 
@@ -284 +293 @@
 #    print STDERR "-> $srcfilename -> ";
     $srcfilename = &util::rename_file($srcfilename.$suffix, $rename_method);
+
+    # encode any subdirs of "import" also, but not import itself
+    my $sep = ($ENV{'GSDLOS'} =~ /^windows$/i) ? "\\" : "/";   
+    my @dirs = ($sep eq "\\") ? split(/[\\\/]+/, $dirname) : split(/\//, $dirname);
+    
+    my $dirpath = "";    
+    foreach my $subdir (@dirs) {
+    ##print STDERR "@@@@ Found subdir: $subdir\n";
+    
+    # The import folder can be called anything, including in non-ASCII encodings.
+    # Don't need to avoid encoding default import folder called "import", as it gets
+    # URL/base64 encoded to ITSELF.
+    # But can't encode (URL/base64 encode) any $ENV{'GSDLIMPORTDIR'} though if it's set,
+    # as BasePlugout::get_doc_dir() removes any $ENV{'GSDLIMPORTDIR'} prefix. Because if
+    # the $ENV{'GSDLIMPORTDIR'} part of gsdlsourcefilename is encoded here, the prefix
+    # won't match with $ENV{'GSDLIMPORTDIR'}
+
+    unless ($ENV{'GSDLIMPORTDIR'} && $subdir eq $ENV{'GSDLIMPORTDIR'}) { 
+        $subdir = &util::rename_file($subdir, $rename_method);
+    }
+    $dirpath = &FileUtils::filenameConcatenate($dirpath, $subdir);
+    }
+    if($dirpath ne "") {
+    $dirname = $dirpath;
+    }
+    
     $source_filename = &FileUtils::filenameConcatenate($dirname, $srcfilename);