Changeset 31440 for main

Timestamp:

2017-02-23T14:49:41+13:00 (7 years ago)

Author:

kjdon

Message:

nearly there for handling russian etc subfolders in import. need to test on windows though. still one part to be worked out.

File:

• main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm (modified) (17 diffs)

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -186 +186 @@
     
     my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
-    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
-
-    # read in file ($text will be in utf8)
+    $self->{'store_content_encoding'}->{$filename_full_path} = [$content_encoding, $language];
+    
+
+    # read in file ($text will be in the filesystem encoding)
     my $raw_text = "";
     $self->read_file_no_decoding($filename_full_path, \$raw_text);
@@ -296 +297 @@
     &util::block_filename($block_hash,$url_original_filename) if $url_original_filename ne $html_fname;
 
-            # but only add the linked file to the blocklist if the current html file does not link to itself
+    # but only add the linked file to the blocklist if the current html file does not link to itself
         
     }
@@ -332 +333 @@
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
 
-    # Lookup content_encoding worked out in file_block pass for this file
-    # Store it under the local name 'content_encoding' so its nice and 
-    # easy to access
-    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
+    # Lookup content_encoding and language worked out in file_block pass for this file
+    # Store them under the local names they are nice and easy to access
+    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path}[0];
+    $self->{'language'} = $self->{'store_content_encoding'}->{$filename_full_path}[1];
 
     # get the input file
@@ -463 +464 @@
     my $url_encoded_file = &unicode::raw_filename_to_url_encoded($tailname);
     my $utf8_url_encoded_file = &unicode::raw_filename_to_utf8_url_encoded($tailname);
-
+    
     my $web_url = "http://";
     my $utf8_web_url = "http://";
+    
     if(defined $dirname) { # local directory
+
         # Check for "ftp" in the domain name of the directory
         #  structure to determine if this URL should be a ftp:// URL
@@ -481 +484 @@
     $dirname .= &util::get_dirsep() if $dirname ne ""; # if there's a directory, it should end on "/"
 
-    $web_url = $web_url.$dirname.$url_encoded_file; 
-    $utf8_web_url = $utf8_web_url.$dirname.$utf8_url_encoded_file; 
+    # this local directory in import may need to be URL encoded like the file
+    my $url_encoded_dir = &unicode::raw_filename_to_url_encoded($dirname);
+    my $utf8_url_encoded_dir =  &unicode::raw_filename_to_utf8_url_encoded($dirname);
+    
+    # changed here
+    $web_url = $web_url.$url_encoded_dir.$url_encoded_file; 
+    $utf8_web_url = $utf8_web_url.$utf8_url_encoded_dir.$utf8_url_encoded_file; 
     } else {
     $web_url = $web_url.$url_encoded_file;
@@ -493 +501 @@
     print STDERR "*******DEBUG: upgraded_file:       $upgraded_file\n";
     print STDERR "*******DEBUG: adding URL metadata: $utf8_url_encoded_file\n";
+    print STDERR "*******DEBUG: web url:             $web_url\n";
+    print STDERR "*******DEBUG: utf8 web url:        $utf8_web_url\n";
     }
 
@@ -847 +857 @@
     }
 
+    # can't remember adding this :-( must have had a reason though...
+    if ($link =~ /^\_http/ || $link =~ /^\_libraryname\_/) {
+    # assume it is a greenstone one and leave alone
+    return $front . $link . $back;
+    }
+
     # attempt to sort out targets - frames are not handled 
     # well in this plugin and some cases will screw things
@@ -912 +928 @@
         $href = encode($content_encoding,$href);
     }
-
-    $href = &unicode::raw_filename_to_utf8_url_encoded($href);  
+    
+    $href = &unicode::raw_filename_to_utf8_url_encoded($href); 
     $href = &unicode::filename_to_url($href);
-
+    
     &ghtml::urlsafe ($href);
-
+    
     if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
         print STDERR "******DEBUG: href=$href\n";    
     }
-
+    
+    #TODO here
+#   if ($rl ==1) {
+        # have a relative link, we need to do URL encoding etc so it matches what has happened for that file
+        #$href = &util::rename_file($href, $self->{'file_rename_method'});
+#       $href = &unicode::raw_filename_to_url_encoded($href);
+        # then, this might be url encoded, so we replace % with %25
+#       $href = &unicode::filename_to_url($href);
+#       print STDERR "DEBUG: url encoded href = $href\n";
+#   }
 
     return $front . "_httpextlink_&rl=" . $rl . "&href=" . $href . $hash_part . $back;
@@ -1022 +1047 @@
     # exists on the file system
     $filename = encode($content_encoding, $opt_decode_unicode_filename);
+    
     }
     elsif ($ENV{'GSDLOS'} =~ /^darwin$/i) {
@@ -1051 +1077 @@
     # some special processing if the intended filename was converted to utf8, but
     # the actual file still needs to be renamed
-    #if (!&util::fd_exists($filename)) {
     if (!&FileUtils::fileExists($filename)) {
     # try the original filename stored in map
@@ -1071 +1096 @@
         }
         $filename = $original_filename;
-    }
+    } 
     }
     
@@ -1160 +1185 @@
     my $self = shift (@_);
     my ($link, $base_dir, $file) = @_;
-
+ 
     # strip off hash part, e.g. #foo, but watch out for any entities, e.g. α
     my ($before_hash, $hash_part) = $link =~ m/^(.*?[^&])(\#.*)?$/;
@@ -1168 +1193 @@
         my $outhandle = $self->{'outhandle'};
         print $outhandle "HTMLPlugin: ERROR - badly formatted tag ignored ($link)\n"
-            if $self->{'verbosity'};
+            if $self->{'verbosity'};
         return ($link, "", 0);
     }
     
+#    my $dirname;
     if ($before_hash =~ s@^((?:http|https|ftp|file|mms)://)@@i) {
-        my $type = $1;
-        my $before_hash_file = $before_hash;
+    my $type = $1;
+    my $before_hash_file = $before_hash;
         
     if ($link =~ m/^(http|ftp):/i) {
@@ -1187 +1213 @@
     my $before_hash_url = $before_hash_file;
     if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-        $before_hash_url =~ s@\\@\/@g;
-    }
-    
+        $before_hash_url =~ s@\\@\/@g;
+    }
+    
+    ######## TODO need to check this for encoding stufff
     my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash_file);
-    
+    print STDERR "chekcing for existence whether relative link or not $linkfilename\n";
     my $rl = 0;
     $rl = 1 if (-e $linkfilename);
-
+    if (-e $linkfilename) {
+        
+        print STDERR "DOES exist $linkfilename\n";
+    } else {
+        print STDERR "DOESN'T exist $linkfilename\n";
+    }
     # make sure there's a slash on the end if it's a directory
     if ($before_hash_url !~ m/\/$/) {
@@ -1203 +1235 @@
     } elsif ($link !~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i && $link !~ m/^\//) {
 
+    #### TODO whst is this test doing???
     if ($before_hash =~ s@^/@@ || $before_hash =~ m/\\/) {
 
@@ -1240 +1273 @@
         }
     } else {
-        # Turn relative file path into full path
+
+        # Turn relative file path into full path (inside import dir)
         my $dirname = &File::Basename::dirname($file);
-        $before_hash = &FileUtils::filenameConcatenate($dirname, $before_hash);
+
+        # we want to add dirname (which is raw filesystem path) onto $before_hash, (which is perl unicode aware string). Convert dirname to perl string
+
+        my $unicode_dirname ="";
+        #my $content_encoding = $self->{'content_encoding'};
+        #my $language = $self->{'language'};
+
+        # actually I think this is wrong. why should we use content encoding?
+        #$self->decode_text($dirname, $content_encoding, $language, \$unicode_dirname);
+        #my $filename_encoding = $self->{'filename_encoding'};
+        # filename_encoding might be auto...
+
+        # TODO what is the best thing to do here?????
+        # try and guess default filesystem encoding, similar to deduce_filename_encoding, but without a file?
+        my $filename_encoding = "utf8";
+        # copied this from set_Source_metadata in BasePlugin
+        if ((defined $filename_encoding) && ($filename_encoding ne "ascii")) {
+        # Use filename_encoding to map raw filename to a Perl unicode-aware string 
+        $unicode_dirname = decode($filename_encoding,$dirname);     
+        }
+        else {
+        # otherwise generate %xx encoded version of filename for char > 127
+        $unicode_dirname = &unicode::raw_filename_to_url_encoded($dirname);
+        }
+ 
+        $before_hash = &FileUtils::filenameConcatenate($unicode_dirname, $before_hash);
         $before_hash = $self->eval_dir_dots($before_hash);   
-        $before_hash =~ s@\\@/@g; # for windows         
-    }
-
-    my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash); 
-
-
-#   print STDERR "**** linkfilename = $linkfilename\n";
-#   if (!&util::fd_exists($linkfilename)) {
-#       print STDERR "***** Warning: Could not find $linkfilename\n";
-#   }
-
+        $before_hash =~ s@\\@/@g; # for windows         
+    }
+
+    my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash);    
 
     # make sure there's a slash on the end if it's a directory
@@ -1260 +1312 @@
         $before_hash .= "/" if (-d $linkfilename);
     }
-
-#   print STDERR "*** returning: $before_hash\n";
-
     return ("http://" . $before_hash, $hash_part, 1);
     } else {