Changeset 23363 for main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

Timestamp:

2010-12-01T11:42:27+13:00 (13 years ago)

Author:

davidb

Message:

Plugin code upgrade to support Greenstone working with filenames under Windows when then go beyond Latin-1 and start turning up in their DOS abbreviated form (e.g. Test~1.txt)

File:

• main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm (modified) (12 diffs)

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -186 +186 @@
     # read in file ($text will be in utf8)
     my $raw_text = "";
-    $self->read_file_no_decoding ($filename_full_path, \$raw_text);
+    $self->read_file_no_decoding($filename_full_path, \$raw_text);
 
     my $textref = \$raw_text;
@@ -192 +192 @@
     my $closecom = '(?:-->|(?:—|—|--)>)';
     $$textref =~ s/$opencom(.*?)$closecom//gs;
+
+    # Convert entities to their UTF8 equivalents
+    $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
+    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,0)/gseo; # on this occassion, want it left as utf8
+    $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;
 
     my $attval = "\\\"[^\\\"]+\\\"|[^\\s>]+";
@@ -209 +214 @@
 
     # remove quotes from link at start and end if necessary
-    if ($link=~/^\"/) {
-        $link=~s/^\"//;
-        $link=~s/\"$//;
-    }
-
-    $link =~ s/\#.*$//s; # remove any anchor names, e.g. foo.html#name becomes foo.html
+    if ($link =~ m/^\"/) {
+        $link =~ s/^\"//;
+        $link =~ s/\"$//;
+    }
+
+    $link =~ s/\#.*$//s; # remove any anchor names, e.g. foo.html#name becomes foo.html 
     # some links may just be anchor names
     next unless ($link =~ /\S+/);
@@ -242 +247 @@
     }
 
+    
+    my $unicode_url_original_filename = decode("utf8",$url_original_filename);
+
+##  print STDERR "*****!!! Blocking url original filename = $unicode_url_original_filename\n";
+
+    # Allow for possibility of raw byte version (UTF8) and Unicode versions of file
     $block_hash->{'file_blocks'}->{$url_original_filename} = 1;
+    $block_hash->{'file_blocks'}->{$unicode_url_original_filename} = 1;
     }
 }
@@ -250 +262 @@
 # filename*, it does not URL decode any filename if a file by the name of the *URL-encoded*
 # string already exists in the local folder.
+#
+# Is the following still true??
 # Return the original filename corresponding to the parameter URL-encoded filename, and
 # a decoded flag that is set to true iff URL-decoding had to be applied.
@@ -312 +326 @@
     $doc_obj->set_source_filename ($collect_file, $self->{'file_rename_method'}); 
     ## set_source_filename does not set the doc_obj source_path which is used in archives dbs for incremental
-    # build. so set it manually.
-    $doc_obj->{'source_path'} = $filename_full_path;
+    # build. So set it manually.
+    $doc_obj->set_source_path($filename_full_path);
     my $collect_conv_file = &util::filename_within_collection($tidy_filename);
     $doc_obj->set_converted_filename($collect_conv_file);
@@ -387 +401 @@
     my $utf8_file = &unicode::raw_filename_to_url_encoded($tailname);
 
-    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-        print STDERR "***!! file = $file\n";
-        print STDERR "***!! utf8_file = $utf8_file\n";
-    }
+#   if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
+#       print STDERR "***!! file = $file\n";
+#       print STDERR "***!! utf8_file = $utf8_file\n";
+#   }
 
 
@@ -764 +778 @@
     ($href =~ m/\/$/) || ($href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
 
-
     # If web page didn't give encoding, then default to utf8
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+
     if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-        print STDERR "*** Web page didn't give encoding, defaulting to UTF8!\n";
-        print STDERR "*****  looking up $file\n";
-    }
-
-    my $content_encoding= $self->{'content_encoding'} || "utf8";
+        print STDERR "*** Encoding with $content_encoding href: $href\n";
+    }
+
     $href = encode($content_encoding,$href);
 
@@ -807 +820 @@
 
     $filename = &util::filename_cat($base_dir, $filename);
+
     if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'})) {
     # we are processing a tidytmp file - want paths to be in import
@@ -827 +841 @@
     $filename = encode($content_encoding, $opt_decode_utf8_filename);
 
-
     # some special processing if the intended filename was converted to utf8, but
     # the actual file still needs to be renamed
-    if (!-e $filename) {
+    if (!&util::fd_exists($filename)) {
     # try the original filename stored in map
     if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-        print STDERR "***###!! orig filename did not exist: $filename\n";
-    }
+        print STDERR "******!! orig filename did not exist: $filename\n";
+    }
+
+##  print STDERR "**** trying to look up utf8_filename: $utf8_filename\n";
 
     my $original_filename = $self->{'utf8_to_original_filename'}->{$utf8_filename};
 
     if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-        print STDERR "**** Trying for $original_filename\n";
+        print STDERR "******   From lookup utf8_filename, now trying for: $original_filename\n";
     }
 
     if (defined $original_filename && -e $original_filename) {
         if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-            print STDERR "*** found match\n";
+            print STDERR "******   Found match!\n";
         }
         $filename = $original_filename;
@@ -891 +906 @@
     $newname = &util::rename_file($newname, $self->{'file_rename_method'});
 
+### print STDERR "***** associating $filename (raw-byte/utf8)-> $newname\n";
     $doc_obj->associate_file($filename, $newname, undef, $section);
 
@@ -1296 +1312 @@
     $self->SUPER::read_file($filename, $encoding, $language, $textref);
 
-    # Convert entities to their UTF8 equivalents
+    # Convert entities to their Unicode code-point equivalents
     $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
     $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,1)/gseo;