Changeset 31477 for main/trunk

Timestamp:

2017-03-09T14:32:44+13:00 (7 years ago)

Author:

kjdon

Message:

blocking moved to EncodingUtil. debug stuff still in here. needs tidying up

File:

• main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm (modified) (6 diffs)

main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm

@@ -238 +238 @@
         foreach my $e (keys %$exts) {
         # block the file
-        &util::block_filename($block_hash,"$prefix$e");
+        $self->block_filename($block_hash,"$prefix$e");
         # set up as an associatd file
         print STDERR "  $self->{'plugin_type'}: Associating $prefix$e with $tie_to version\n";
@@ -254 +254 @@
 
 
-# do block exp OR special blocking ???
-
-sub file_is_blocked {
-    my $self = shift (@_);
-    my ($block_hash, $filename_full_path) = @_;
-
-    $filename_full_path = &util::upgrade_if_dos_filename($filename_full_path);
-
-    if (($ENV{'GSDLOS'} =~ m/^windows$/) && ($^O ne "cygwin")) {
-    # on windows, all block paths are lowercased.
-    my $lower_filename = lc ($filename_full_path);
-    if (defined $block_hash->{'file_blocks'}->{$lower_filename}) {
-        $self->{'num_blocked'} ++;
-        return 1;
-    }
-    }
-    else {
-    if (defined $block_hash->{'file_blocks'}->{$filename_full_path}) {
-        $self->{'num_blocked'} ++;
-        return 1;
-    }
-    }
-    # check Directory plugin's own block_exp 
-    if ($self->{'block_exp'} ne "" && $filename_full_path =~ /$self->{'block_exp'}/) {
-    $self->{'num_blocked'} ++;
-    return 1; # blocked
-    }
-    return 0;
-}
-
 
 
@@ -334 +304 @@
 
     # Recursively read each $raw_subfile
-    print $outhandle "DirectoryPlugin block recurring: ". Encode::decode("utf8", $raw_file_subfile) ."\n" if ($verbosity > 2);
-    print $outhandle "DirectoryPlugin block recurring: ". Encode::decode(locale =>$raw_file_subfile) ."\n" if ($verbosity > 2);
-    
+    print $outhandle "DirectoryPlugin block recurring: $raw_file_subfile\n" if ($verbosity > 2);
     #$count += &plugin::file_block_read ($pluginfo, $this_file_base_dir,
 
@@ -434 +402 @@
     # but with URL style slashes. 
     my $local_dirname = &util::filename_within_directory_url_format($dirname, $base_dir);
-    ### need it in perl unicode, not raw filesystem
-    my $filename_encoding =  $self->guess_filesystem_encoding();  
-        
-    # copied this from set_Source_metadata in BasePlugin
-    if ((defined $filename_encoding) && ($filename_encoding ne "ascii")) {
-    # Use filename_encoding to map raw filename to a Perl unicode-aware string 
-    $local_dirname = decode($filename_encoding,$local_dirname);     
-    }
-    else {
-    # otherwise generate %xx encoded version of filename for char > 127
-    $local_dirname = &unicode::raw_filename_to_url_encoded($local_dirname);
-    }
 
     # if we are in import folder, then local_dirname will be empty
     if ($local_dirname ne "") {
+    # convert to perl unicode
+    $local_dirname = $self->raw_filename_to_unicode($local_dirname);
+    
     # look for extra metadata passed down from higher folders   
     $local_dirname .= "/"; # closing slash must be URL type slash also and not $dirsep;
@@ -476 +435 @@
     my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
     my $raw_full_filename = &FileUtils::filenameConcatenate($this_file_base_dir, $raw_file_subfile);
-
-    if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
-        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for metadata_read\n" if ($verbosity > 2);
+    if ($self->raw_file_is_blocked($block_hash, $raw_full_filename)) {
+        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for metadata_read\n";# if ($verbosity > 2);
+        print STDERR "raw file was blocked for metadata read\n";
+        print STDERR &unicode::debug_unicode_string($raw_full_filename)."\n";
         next;
     }
-    
+
     # Recursively read each $raw_subfile
-    print $outhandle "DirectoryPlugin metadata recurring: $raw_subfile\n" if ($verbosity > 2);
+    print $outhandle "DirectoryPlugin metadata recurring: $raw_subfile\n"; # if ($verbosity > 2);
     
     &plugin::metadata_read ($pluginfo, $this_file_base_dir,
@@ -560 +520 @@
     my $raw_full_filename 
         = &FileUtils::filenameConcatenate($this_file_base_dir,$raw_file_subfile);
-
+    my $full_unicode_file = $self->raw_filename_to_unicode($raw_full_filename);
+    print STDERR "full unicode filename $full_unicode_file\n";
+    print STDERR &unicode::debug_unicode_string($full_unicode_file)."\n";
+    if ($self->file_is_blocked($block_hash,$full_unicode_file)) {
+        print STDERR "Actually, we have blocked the unicode version\n";
+        next;
+    }
     if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
-        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for read\n" if ($verbosity > 2);
+        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for read\n"; # if ($verbosity > 2);
         next;
     }
-
     # Follow Windows shortcuts
     if ($raw_subfile =~ m/(?i)\.lnk$/ && (($ENV{'GSDLOS'} =~ m/^windows$/i) && ($^O ne "cygwin"))) {