Changeset 23335 for main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

Timestamp:

2010-11-19T13:29:29+13:00 (13 years ago)

Author:

davidb

Message:

Work done on improving handing of filenames when the actualy filename encoding used is not necesarrily known. Tested for Linux. Work currently includes some debug statements that will be removed once testing for Windows and Mac is done.

File:

• main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm (modified) (10 diffs)

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -181 +181 @@
     my @file_blocks;
     
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
 
     # read in file ($text will be in utf8)
@@ -229 +230 @@
     # Convert the url_original_filename into its utf8 version. Store the utf8 link along with the url_original_filename
     my $utf8_link = "";
-    $self->decode_text($link,$encoding,$language,\$utf8_link);
+    $self->decode_text($link,$content_encoding,$language,\$utf8_link);
 
     $self->{'utf8_to_original_filename'}->{$utf8_link} = $url_original_filename;
@@ -272 +273 @@
         
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+    # Lookup content_encoding worked out in file_block pass for this file
+    # Store it under the local name 'content_encoding' so its nice and 
+    # easy to access
+    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
+
     # get the input file
     my $input_filename = $file;
@@ -311 +318 @@
     $self->set_Source_metadata($doc_obj, $filename_no_path);
     }
+
+    delete $self->{'store_content_encoding'}->{$filename_full_path};
+    $self->{'content_encoding'} = undef;
+
     return ($process_status,$doc_obj);
 }
@@ -367 +378 @@
     # links, so even if 'file_is_url' is off, still need to store info
 
-    my ($tailname,$dirname,$suffix) = &File::Basename::fileparse($file, "\\.[^\\.]+\$");
-    my $utf8_file = $self->filename_to_utf8_metadata($file);
-    $utf8_file =~ s/&\#095;/_/g;
+    my ($tailname,$dirname) = &File::Basename::fileparse($file);
+    print STDERR "***!! file = $file\n";
+#    my $utf8_file = $self->filename_to_utf8_metadata($file);
+#    $utf8_file =~ s/&\#095;/_/g;
+    my $utf8_file = &unicode::raw_filename_to_url_encoded($tailname);
+    print STDERR "***!! utf8_file = $utf8_file\n";
+
     my $web_url = "http://";
     if(defined $dirname) { # local directory
@@ -535 +550 @@
     $self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
     }
+
     return 1;
 }
@@ -737 +753 @@
     return $front . $link . $back if $href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/is;
 
-
     if (($rl == 0) || ($filename =~ m/$self->{'process_exp'}/) || 
     ($href =~ m/\/$/) || ($href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
+
+
+    # If web page didn't give encoding, then default to utf8
+    print "*************** looking up $file\n";
+
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+    $href = encode($content_encoding,$href);
+
+    $href = &unicode::raw_filename_to_url_encoded($href);
+    $href = &unicode::filename_to_url($href);
+
     &ghtml::urlsafe ($href);
+    print STDERR "***!!! href=$href\n";    
+
     return $front . "_httpextlink_&rl=" . $rl . "&href=" . $href . $hash_part . $back;
     } else {
-    # link is to some other type of file (eg image) so we'll
+    # link is to some other type of file (e.g., an image) so we'll
     # need to associate that file
     return $front . $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) . $back;
@@ -769 +797 @@
     $filename =~ s/([\\\/])tidytmp([\\\/])/$1import$2/;
     }
-    # Replace %XX's in URL with decoded value if required. Note that the filename may include the %XX in some
-    # situations. If the *original* file's name was in URL encoding, the following method will not decode it.
+
+    # Replace %XX's in URL with decoded value if required. Note that the
+    # filename may include the %XX in some situations. If the *original*
+    # file's name was in URL encoding, the following method will not decode
+    # it.
     my $utf8_filename = $filename;
-    $filename = $self->opt_url_decode($utf8_filename);
+    my $opt_decode_utf8_filename = $self->opt_url_decode($utf8_filename);
+
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+
+    # The filenames that come through the HTML file have been decoded
+    # into Unicode aware Perl strings.  Need to convert them back
+    # to their initial raw-byte encoding to match the file that
+    # exists on the file system
+    $filename = encode($content_encoding, $opt_decode_utf8_filename);
+
 
     # some special processing if the intended filename was converted to utf8, but
@@ -778 +818 @@
     if (!-e $filename) {
     # try the original filename stored in map
-    my $original_filename = $self->{'utf8_to_original_filename'}->{$filename};
+    print STDERR "***###!! orig filename did not exist: $filename\n";
+
+    my $original_filename = $self->{'utf8_to_original_filename'}->{$utf8_filename};
+
+    print STDERR "**** Trying for $original_filename\n";
+
     if (defined $original_filename && -e $original_filename) {
+        print STDERR "*** found match\n";
         $filename = $original_filename;
     }
@@ -1118 +1164 @@
     $title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
     $title =~ s/^\s+//s; # in case title_sub introduced any...
-    $doc_obj->add_utf8_metadata ($section, 'Title', $title);
+    print STDERR "**** adding Title: ", Encode::encode("utf8",$title), "\n";
+    $doc_obj->add_utf8_metadata ($section, "Title", $title);
     print $outhandle " extracted Title metadata \"$title\" from $from\n" 
         if ($self->{'verbosity'} > 2);