Index: main/trunk/greenstone2/perllib/plugins/BasePlugin.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 31456)
+++ main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 31457)
@@ -42,8 +42,8 @@
 use FileUtils;
 
-use PrintInfo;
+use EncodingUtil;
 
 BEGIN {
-    @BasePlugin::ISA = ( 'PrintInfo' );
+    @BasePlugin::ISA = ( 'EncodingUtil' );
 }
 
@@ -59,36 +59,5 @@
 	'hiddengli' => "yes" } ];
 
-our $encoding_list =
-    [ { 'name' => "ascii",
-	'desc' => "{BasePlugin.encoding.ascii}" },
-      { 'name' => "utf8",
-	'desc' => "{BasePlugin.encoding.utf8}" },
-      { 'name' => "unicode",
-	'desc' => "{BasePlugin.encoding.unicode}" } ];
-
-
-my $e = $encodings::encodings;
-foreach my $enc (sort {$e->{$a}->{'name'} cmp $e->{$b}->{'name'}} keys (%$e)) 
-{
-    my $hashEncode =
-    {'name' => $enc,
-     'desc' => $e->{$enc}->{'name'}};
-    
-    push(@{$encoding_list},$hashEncode);
-}
-
-our $encoding_plus_auto_list = 
-    [ { 'name' => "auto",
-	'desc' => "{BasePlugin.filename_encoding.auto}" },
-      { 'name' => "auto-language-analysis",
-	'desc' => "{BasePlugin.filename_encoding.auto_language_analysis}" }, # textcat
-      { 'name' => "auto-filesystem-encoding",
-	'desc' => "{BasePlugin.filename_encoding.auto_filesystem_encoding}" }, # locale
-      { 'name' => "auto-fl",
-	'desc' => "{BasePlugin.filename_encoding.auto_fl}" }, # locale followed by textcat
-      { 'name' => "auto-lf",
-	'desc' => "{BasePlugin.filename_encoding.auto_lf}" } ]; # texcat followed by locale 
-
-push(@{$encoding_plus_auto_list},@{$encoding_list});
+# here went encoding list stuff
 
 our $oidtype_list = 
@@ -160,10 +129,10 @@
 	'type' => "flag",
 	'reqd' => "no" },
-      { 'name' => "filename_encoding",
-	'desc' => "{BasePlugin.filename_encoding}",
-	'type' => "enum",
-	'deft' => "auto",
-	'list' => $encoding_plus_auto_list,
-	'reqd' => "no" },
+#      { 'name' => "filename_encoding",
+#	'desc' => "{BasePlugin.filename_encoding}",
+#	'type' => "enum",
+#	'deft' => "auto",
+#	'list' => $encoding_plus_auto_list,
+#	'reqd' => "no" },
       { 'name' => "smart_block",
         'desc' => "{common.deprecated}. {BasePlugin.smart_block}",
@@ -185,5 +154,5 @@
 		'desc'     => "{BasePlugin.desc}",
 		'abstract' => "yes",
-		'inherits' => "no",
+		'inherits' => "yes",
 		'args'     => $arguments };
 
@@ -197,5 +166,5 @@
     push(@{$hashArgOptLists->{"OptList"}},$options);
 
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
+    my $self = new EncodingUtil($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
     
     if ($self->{'info_only'}) {
@@ -203,5 +172,4 @@
         return bless $self, $class;
     }
-
     if ($self->{'smart_block'}) {
 	print STDERR "WARNING: -smart_block option has been deprecated and is no longer useful\n";
@@ -284,12 +252,6 @@
     my $self = shift (@_);
     my ($verbosity, $outhandle, $failhandle) = @_;
-
-    # verbosity is passed through from the processor
-    $self->{'verbosity'} = $verbosity;
-
-    # as are the outhandle and failhandle
-    $self->{'outhandle'} = $outhandle if defined $outhandle;
-    $self->{'failhandle'} = $failhandle;
-#    $self->SUPER::init(@_);
+    
+    $self->SUPER::init(@_);
     
     # set process_exp and block_exp to defaults unless they were
@@ -550,409 +512,4 @@
     return undef;
 }
-
-
-# just converts path as is to utf8.
-sub filepath_to_utf8 {
-    my $self = shift (@_);  
-    my ($file, $file_encoding) = @_;
-    my $filemeta = $file;
-
-    my $filename_encoding = $self->{'filename_encoding'}; # filename encoding setting
-
-    # Whenever filename-encoding is set to any of the auto settings, we
-    # check if the filename is already in UTF8. If it is, then we're done.
-    if($filename_encoding =~ m/auto/) {
-	if(&unicode::check_is_utf8($filemeta)) 
-	{
-	    $filename_encoding = "utf8";
-	    return $filemeta;
-	} 
-    }
-    
-    # Auto setting, but filename is not utf8
-    if ($filename_encoding eq "auto") 
-    {
-	# try textcat
-	$filename_encoding = $self->textcat_encoding($filemeta);
-	
-	# check the locale next
-	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
-	
-	
-	# now try the encoding of the document, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-
-    }
-
-    elsif ($filename_encoding eq "auto-language-analysis") 
-    {	
-	$filename_encoding = $self->textcat_encoding($filemeta);
-
-	# now try the encoding of the document, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	} 
-    }
-
-    elsif ($filename_encoding eq "auto-filesystem-encoding") 
-    {	
-	# try locale
-	$filename_encoding = $self->locale_encoding();
-    }
-
-    elsif ($filename_encoding eq "auto-fl") 
-    {
-	# filesystem-encoding (locale) then language-analysis (textcat)
-	$filename_encoding = $self->locale_encoding();
-	
-	# try textcat
-	$filename_encoding = $self->textcat_encoding($filemeta) if $filename_encoding eq "undefined";
-	
-	# else assume filename encoding is encoding of file content, if that's available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-    }
-    
-    elsif ($filename_encoding eq "auto-lf") 
-    {
-	# language-analysis (textcat) then filesystem-encoding (locale)
-	$filename_encoding = $self->textcat_encoding($filemeta);
-	
-	# guess filename encoding from encoding of file content, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-
-	# try locale
-	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
-    }
-    
-    # if still undefined, use utf8 as fallback
-    if ($filename_encoding eq "undefined") {
-	$filename_encoding = "utf8";
-    }
-
-    #print STDERR "**** UTF8 encoding the filename $filemeta ";
-    
-    # if the filename encoding is set to utf8 but it isn't utf8 already--such as when
-    # 1. the utf8 fallback is used, or 2. if the system locale is used and happens to
-    # be always utf8 (in which case the filename's encoding is also set as utf8 even 
-    # though the filename need not be if it originates from another system)--in such
-    # cases attempt to make the filename utf8 to match.
-    if($filename_encoding eq "utf8" && !&unicode::check_is_utf8($filemeta)) {
-	&unicode::ensure_utf8(\$filemeta);
-    }
-
-    # convert non-unicode encodings to utf8
-    if ($filename_encoding !~ m/(?:ascii|utf8|unicode)/) {
-	$filemeta = &unicode::unicode2utf8(
-					   &unicode::convert2unicode($filename_encoding, \$filemeta)
-					   );
-    }
-
-    #print STDERR " from encoding $filename_encoding -> $filemeta\n";
-    return $filemeta;
-}
-
-# gets the filename with no path, converts to utf8, and then dm safes it.
-# filename_encoding set by user
-sub filename_to_utf8_metadata
-{
-    my $self = shift (@_);  
-    my ($file, $file_encoding) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    print $outhandle "****!!!!**** BasePlugin::filename_to_utf8_metadata now deprecated\n";
-    my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(0);
-    print $outhandle "Calling method: $cfilename:$cline $cpackage->$csubr\n";
-
-    my ($filemeta) = $file =~ /([^\\\/]+)$/; # getting the tail of the filepath (skips all string parts containing slashes upto the end)
-    $filemeta = $self->filepath_to_utf8($filemeta, $file_encoding);
-
-    return $filemeta;
-}
-
-sub locale_encoding {
-    my $self = shift(@_);
-    
-    if (!defined $self->{'filesystem_encoding'}) {
-	$self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
-    }
-
-    #print STDERR "*** filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
-    return $self->{'filesystem_encoding'}; # can be the string "undefined"
-}
-
-sub textcat_encoding {
-    my $self = shift(@_);
-    my ($filemeta) = @_;
-
-    # analyse filenames without extensions and digits (and trimmed of
-    # surrounding whitespace), so that irrelevant chars don't confuse
-    # textcat
-    my $strictfilemeta = $filemeta;
-    $strictfilemeta =~ s/\.[^\.]+$//g;
-    $strictfilemeta =~ s/\d//g;
-    $strictfilemeta =~ s/^\s*//g;
-    $strictfilemeta =~ s/\s*$//g;
-    
-    my $filename_encoding = $self->encoding_from_language_analysis($strictfilemeta);
-    if(!defined $filename_encoding) {
-	$filename_encoding = "undefined";
-    }
-
-    return $filename_encoding; # can be the string "undefined"
-}
-
-# performs textcat
-sub encoding_from_language_analysis {
-    my $self = shift(@_);
-    my ($text) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $best_encoding = undef;
-    
-    # get the language/encoding of the textstring using textcat
-    require textcat;  # Only load the textcat module if it is required
-    $self->{'textcat'} = new textcat() unless defined($self->{'textcat'});
-    my $results = $self->{'textcat'}->classify_cached_filename(\$text);
-
-
-    if (scalar @$results < 0) { 
-	return undef;
-    }
-    
-    # We have some results, we choose the first
-    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
-    
-    $best_encoding = $encoding;
-    if (!defined $best_encoding) {
-	return undef;
-    } 
-    
-    if (defined $best_encoding && $best_encoding =~ m/^iso_8859/ && &unicode::check_is_utf8($text)) {
-	# the text is valid utf8, so assume that's the real encoding (since textcat is based on probabilities)
-	$best_encoding = 'utf8';
-    }
-    
-    
-    # check for equivalents where textcat doesn't have some encodings...
-    # eg MS versions of standard encodings
-    if (defined $best_encoding && $best_encoding =~ /^iso_8859_(\d+)/) {
-	my $iso = $1; # which variant of the iso standard?
-	# iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
-	if ($text =~ /[\x80-\x9f]/) {
-	    # Western Europe
-	    if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
-	    elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
-	    elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
-	    elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
-	    elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
-	    elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
-	    elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
-	}
-    }
-    
-    if (defined $best_encoding && $best_encoding !~ /^(ascii|utf8|unicode)$/ &&
-	!defined $encodings::encodings->{$best_encoding}) 
-    {
-	if ($self->{'verbosity'}) { 
-	    gsprintf($outhandle, "BasePlugin: {ReadTextFile.unsupported_encoding}\n", $text, $best_encoding, "undef");
-	}
-	$best_encoding = undef;
-    }
-    
-    return $best_encoding;
-}
-
-# uses locale
-sub get_filesystem_encoding 
-{
-
-    my $self = shift(@_);
-
-    my $outhandle = $self->{'outhandle'};
-    my $filesystem_encoding = undef;
-
-    eval {
-	# Works for Windows as well, returning the DOS code page in use	
-	use POSIX qw(locale_h);
-	
-	# With only one parameter, setlocale retrieves the 
-	# current value
-	my $current_locale = setlocale(LC_CTYPE);
-	
-	my $char_encoding = undef;
-	if ($current_locale =~ m/\./) {
-	    ($char_encoding) = ($current_locale =~ m/^.*\.(.*?)$/);
-	    $char_encoding = lc($char_encoding);
-	}
-	else {
-	    if ($current_locale =~ m/^(posix|c)$/i) {
-		$char_encoding = "ascii";
-	    }
-	}
-
-	if (defined $char_encoding) {
-	    if ($char_encoding =~ m/^(iso)(8859)-?(\d{1,2})$/) {
-		$char_encoding = "$1\_$2\_$3";
-	    }
-
-	    $char_encoding =~ s/-/_/g;
-	    $char_encoding =~ s/^utf_8$/utf8/;
-	    
-	    if ($char_encoding =~ m/^\d+$/) {
-		if (defined $encodings::encodings->{"windows_$char_encoding"}) {
-		    $char_encoding = "windows_$char_encoding";
-		}
-		elsif (defined $encodings::encodings->{"dos_$char_encoding"}) {
-		    $char_encoding = "dos_$char_encoding";
-		}
-	    }
-	    
-	    if (($char_encoding =~ m/(?:ascii|utf8|unicode)/) 
-		|| (defined $encodings::encodings->{$char_encoding})) {
-		$filesystem_encoding = $char_encoding;
-	    }
-	    else {
-		print $outhandle "Warning: Unsupported character encoding '$char_encoding' from locale '$current_locale'\n";
-	    }
-	}
-	
-
-    };
-    if ($@) {
-	print $outhandle "$@\n";
-	print $outhandle "Warning: Unable to establish locale.  Will assume filesystem is UTF-8\n";
-	
-    }
-
-    return $filesystem_encoding;
-}
-
-
-sub guess_filesystem_encoding
-{
-   my $self = shift (@_); 
-	# Look to file system to provide a character encoding
-   my $deduced_filename_encoding = "";
-	# If Windows NTFS, then -- assuming we work with long file names got through
-	# Win32::GetLongFilePath() -- then the underlying file system is UTF16
-
-	if (($ENV{'GSDLOS'} =~ m/^windows$/i) && ($^O ne "cygwin")) {
-	    # Can do better than working with the DOS character encoding returned by locale	    
-	    $deduced_filename_encoding = "unicode";
-	}
-	else {
-	    # Unix of some form or other
-
-	    # See if we can determine the file system encoding through locale
-	    $deduced_filename_encoding = $self->locale_encoding();
-	    
-	}
-		print STDERR "guessing filesystem encoding is $deduced_filename_encoding\n";
-		return $deduced_filename_encoding;
-}
-
-
-
-sub deduce_filename_encoding
-{
-    my $self = shift (@_);  
-    my ($file,$metadata,$plugin_filename_encoding) = @_;
-
-    my $gs_filename_encoding = $metadata->{"gs.filenameEncoding"};
-    my $deduced_filename_encoding = undef;
-    
-    # Start by looking for manually assigned metadata
-    if (defined $gs_filename_encoding) {
-	if (ref ($gs_filename_encoding) eq "ARRAY") {
-	    my $outhandle = $self->{'outhandle'};
-	    
-	    $deduced_filename_encoding = $gs_filename_encoding->[0];
-	    
-	    my $num_vals = scalar(@$gs_filename_encoding);
-	    if ($num_vals>1) {
-		print $outhandle "Warning: gs.filenameEncoding multiply defined for $file\n";
-		print $outhandle "         Selecting first value: $deduced_filename_encoding\n";
-	    }
-	} 
-	else {
-	    $deduced_filename_encoding = $gs_filename_encoding;
-	}
-    }
-    
-    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-	# Look to see if plugin specifies this value
-
-	if (defined $plugin_filename_encoding) {
-	    # First look to see if we're using any of the "older" (i.e. deprecated auto-... plugin options)
-	    if ($plugin_filename_encoding =~ m/^auto-.*$/) {
-		my $outhandle = $self->{'outhandle'};
-		print $outhandle "Warning: $plugin_filename_encoding is no longer supported\n";
-		print $outhandle "         default to 'auto'\n";
-		$self->{'filename_encoding'} = $plugin_filename_encoding = "auto";
-	    }
-	    
-	    if ($plugin_filename_encoding ne "auto") {
-		# We've been given a specific filenamne encoding
-		# => so use it!
-		$deduced_filename_encoding = $plugin_filename_encoding;
-	    }
-	}
-    }
-    
-    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-
-	# Look to file system to provide a character encoding
-
-	# If Windows NTFS, then -- assuming we work with long file names got through
-	# Win32::GetLongFilePath() -- then the underlying file system is UTF16
-
-	if (($ENV{'GSDLOS'} =~ m/^windows$/i) && ($^O ne "cygwin")) {
-	    # Can do better than working with the DOS character encoding returned by locale	    
-	    $deduced_filename_encoding = "unicode";
-	}
-	else {
-	    # Unix of some form or other
-
-	    # See if we can determine the file system encoding through locale
-	    $deduced_filename_encoding = $self->locale_encoding();
-	    
-	    # if locale shows us filesystem is utf8, check to see filename is consistent
-	    # => if not, then we have an "alien" filename on our hands
-
-	    if (defined $deduced_filename_encoding && $deduced_filename_encoding =~ m/^utf-?8$/i) {
-		if (!&unicode::check_is_utf8($file)) {
-		    # "alien" filename, so revert
-		    $deduced_filename_encoding = undef;
-		}
-	    }
-	}
-    }
-    
-#    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-#		# Last chance, apply textcat to deduce filename encoding
-#		$deduced_filename_encoding = $self->textcat_encoding($file);
-#    }
-
-    if ($self->{'verbosity'}>3) {
-	my $outhandle = $self->{'outhandle'};
-
-	if (defined $deduced_filename_encoding) {
-	    print $outhandle "  Deduced filename encoding as: $deduced_filename_encoding\n";
-	}
-	else {
-	    print $outhandle "  No filename encoding deduced\n";
-	}
-    }
-    
-    return $deduced_filename_encoding;
-}
-
 
 
@@ -1258,19 +815,4 @@
 }
 
-# write_file -- used by ConvertToPlug, for example in post processing
-#
-# where should this go, is here the best place??
-sub utf8_write_file {
-    my $self = shift (@_);
-    my ($textref, $filename) = @_;
-    
-    if (!open (FILE, ">:utf8", $filename)) {
-	gsprintf(STDERR, "ConvertToPlug::write_file {ConvertToPlug.could_not_open_for_writing} ($!)\n", $filename);
-	die "\n";
-    }
-    print FILE $$textref;
-    
-    close FILE;
-}
 
 
