###########################################################################
#
# EMAILPlug.pm - a plugin for parsing email files
#
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999-2001 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################



# EMAILPlug
#
# by Gordon Paynter (gwp@cs.waikato.ac.nz)
#
# Email plug reads email files.  These are named with a simple
# number (i.e. as they appear in maildir folders) or with the 
# extension .mbx (for mbox mail file format)
#
# Document text:
#   The document text consists of all the text 
#   after the first blank line in the document.
#
# Metadata (not Dublin Core!):
#   $Headers      All the header content
#   $Subject      Subject: header
#   $To           To: header
#   $From         From: header
#   $FromName     Name of sender (where available)
#   $FromAddr     E-mail address of sender
#   $DateText     Date: header
#   $Date         Date: header in GSDL format (eg: 19990924)
#
#   $Title	  made up of Subject, Date and Sender (for default formatting)
#
#
# John McPherson - June/July 2001
# added (basic) MIME support and quoted-printable and base64 decodings.
# Minor fixes for names that are actually email addresses (ie <...> was lost)
#
# See:  * RFC 822  - ARPA Internet Text Messages
#       * RFC 2045 - Multipurpose Internet Mail Extensions (MIME) -part1
#       * RFC 2046 - MIME (part 2)  Media Types (and multipart messages)
#       * RFC 2047 - MIME (part 3)  Message Header Extensions
#       * RFC 1806 - Content Dispositions (ie inline/attachment)
package EMAILPlug;

use SplitPlug;

use unicode;

use sorttools;
use util;


# EMAILPlug is a sub-class of SplitPlug.

sub BEGIN { 
    @ISA = ('SplitPlug');
}

# Create a new EMAILPlug object with which to parse a file.
# Accomplished by creating a new BasPlug and using bless to 
# turn it into an EMAILPlug.

sub new {
    my ($class) = @_;
    my $self = new BasPlug ("EMAILPlug", @_);
    # this might not actually be true at read-time, but after processing
    # it should all be utf8.
    $self->{'input_encoding'}="utf8";
    return bless $self, $class;
}

sub get_default_process_exp {
    my $self = shift (@_);
    # mbx/email for mailbox file format, \d+ for maildir (each message is
    # in a separate file, with a unique number for filename)
    # mozilla and IE will save individual mbx format files with a ".eml" ext.
    return q@([\\/]\d+|\.(mbx|email|eml))$@;
}

# This plugin splits the mbox mail files at lines starting with From<sp>
# It is supposed to be "\n\nFrom ", but this isn't always used.
sub get_default_split_exp {
    return q^\nFrom .*\n^;
}


# do plugin specific processing of doc_obj
sub process {

    my $self = shift (@_);
    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
    my $outhandle = $self->{'outhandle'};

    # Check that we're dealing with a valid mail file
    # mbox message files start with "From "
    # maildir messages usually start with Return-Path and Delivered-To
    # mh is very similar to maildir
    my $startoffile=substr($$textref,0,256);
    if (($startoffile !~ /^(From )/) &&
	($startoffile !~ /^(From|To|Envelope.*|Received|Return-Path|Date|Subject|Content\-.*|MIME-Version|Forwarded):/im)) {
	return undef;
    }


    print $outhandle "EMAILPlug: processing $file\n" 
	if $self->{'verbosity'} > 1;

    my $cursection = $doc_obj->get_top_section();

    #
    # Parse the document's text and extract metadata
    #

    # Protect backslashes
    $$textref =~ s@\\@\\\\@g;

    # Separate header from body of message
    my $Headers = $$textref;
    $Headers =~ s/\r?\n\r?\n(.*)$//s;
    $$textref = $1;
    
    # Unfold headers - see rfc822
    $Headers =~ s/\r?\n[\t\ ]+/ /gs;
    # Extract basic metadata from header
    my @headers = ("From", "To", "Subject", "Date");
    my %raw;
    foreach my $name (@headers) {
	$raw{$name} = "No $name value";
    }

    # Get a default encoding for the header - RFC says should be ascii...
    my $default_heading_encoding="iso_8859_1";

    # We don't know what character set is the user's default...
    # We could use textcat to guess... for now we'll look at mime content-type
#    if ($Headers =~ /([[:^ascii:]])/) {
#    }
    if ($Headers =~ /^Content\-type:.*charset=\"?([a-z0-9\-_]+)/mi) {
	$default_header_encoding=$1;
	$default_header_encoding =~ s@\-@_@g;
	$default_header_encoding =~ tr/A-Z/a-z/;
    }


    # Examine each line of the headers
    my ($line, $name, $value);
    my @parts;
    foreach $line (split(/\n/, $Headers)) {
	
	# Ignore lines with no content or which begin with whitespace
	next unless ($line =~ /:/);
	next if ($line =~ /^\s/);

	# Find out what metadata is on this line
	@parts = split(/:/, $line);
	$name = shift @parts;
        # get fieldname in canonical form - first cap, then lower case.
	$name =~ tr/A-Z/a-z/;
        # uppercase the first character according to the current locale
	$name=~s/(.+)/\u$1/;
	next unless $name;
	next unless ($raw{$name});

	# Find the value of that metadata
	$value = join(":", @parts);
	$value =~ s/^\s+//;
	$value =~ s/\s+$//;

	# decode headers if stored using =?<charset>?[BQ]?<data>?= (rfc2047)
	if ($value =~ /=\?.*\?[BbQq]\?.*\?=/) {
	    my $original_value=$value;
	    my $encoded=$value;
	    $value="";
	    # this isn't quite right yet regarding spaces between encoded-texts
	    # (see examples, section 8. of rfc).
	    while ($encoded =~ s/(.*?)=\?([^\?]*)\?([bq])\?([^\?]+)\?=\s*//i) {
		my ($charset, $encoding, $data)=($2,$3,$4);
		my ($decoded_data);
		$value.="$1"; # any leading chars
		$data=~s/^\s*//; $data=~s/\s*$//; # strip whitespace from ends
		chomp $data;
		$encoding =~ tr/BQ/bq/;
		if ($encoding eq "q") { # quoted printable
		    $data =~ s/_/\ /g;  # from rfc2047 (sec 4.2.2)
		    $decoded_data=qp_decode($data);
		} else { # base 64
		    $decoded_data=base64_decode($data);
		}
		$self->convert2unicode($charset, \$decoded_data);
		$value .= $decoded_data;
	      } # end of while loop
	    
	    # get any trailing characters
	    $self->convert2unicode($default_header_encoding, \$encoded);
	    $value.=$encoded;

	    if ($value =~ /^\s*$/) { # we couldn't extract anything...
	      $self->convert2unicode($default_header_encoding,
				     \$original_value);
	      $value=original_value;
	    }
        } # end of if =?...?=

        # In the absense of other charset information, assume the
        # header is the default (usually "iso_8859_1") and convert it to unicode.
        else {
	    $self->convert2unicode($default_header_encoding, \$value);
	}
	
	# Store the metadata
	$raw{$name} = $value;
    }

    # Extract the name and e-mail address from the From metadata
    $frommeta = $raw{"From"};
    my $fromnamemeta;
    my $fromaddrmeta;

    $frommeta =~ s/\s*$//;  # Remove trailing space, if any

    if ($frommeta =~ m/(.+)\s*<(.+)>/) {
	$fromnamemeta=$1;
	$fromaddrmeta=$2;
    } elsif ($frommeta =~ m/(.+@.+)\s+\((.*)\)/) {
	$fromnamemeta=$2;
	$fromaddrmeta=$1;
    } 
    if (!defined($fromaddrmeta)) {
	$fromaddrmeta=$frommeta;
    }
    $fromaddrmeta=~s/<//; $fromaddrmeta=~s/>//;
    # minor attempt to prevent spam-bots from harvesting addresses...
    $fromaddrmeta=~s/@/&#64;/;
    $doc_obj->add_utf8_metadata ($cursection, "FromAddr", $fromaddrmeta);

    if (defined($fromnamemeta)) {
	$fromnamemeta =~ s/\"//g;
    }
    else {
	$fromnamemeta = $fromaddrmeta;
    }
    # if name is an address
    $fromnamemeta =~ s/<//g; $fromnamemeta =~ s/>//g;
    $fromnamemeta=~s/@/&#64\;/;
    $doc_obj->add_utf8_metadata ($cursection, "FromName", $fromnamemeta);

    $raw{"From"}=$frommeta;

    # Process Date information
    if ($raw{"Date"} !~ /No Date/) {
	$raw{"DateText"} = $raw{"Date"};
	
	# Convert the date text to internal date format
	$value = $raw{"Date"};
	my ($day, $month, $year) = $value =~ /(\d?\d)\s([A-Z][a-z][a-z])\s(\d\d\d?\d?)/;
	# make some assumptions about the year formatting...
	# some (old) software thinks 2001 is 101, some think 2001 is 01
	if ($year < 20) { $year += 2000; } # assume not really 1920...
	elsif ($year < 150) { $year += 1900; } # assume not really 2150...
	$raw{"Date"} = &sorttools::format_date($day, $month, $year);
	
    } else {
	# We have not extracted a date
	$raw{"DateText"} = "Unknown.";
	$raw{"Date"} = "19000000";
    }

    # Add extracted metadata to document object
    foreach my $name (keys %raw) {
	$value = $raw{$name};
	if ($value) {
	    # assume subject, etc headers have no special HTML meaning.
	    $value = &text_into_html($value);
	    # escape [] so it isn't re-interpreted as metadata
	    $value =~ s/\[/&#91;/g; $value =~ s/\]/&#93;/g;
	} else {
	    $value = "No $name field";
	}
	$doc_obj->add_utf8_metadata ($cursection, $name, $value);
    }

    my $mimetype="text/plain";
    my $mimeinfo="";
    my $charset = $default_header_encoding;
    # Do MIME and encoding stuff. Allow \s in mimeinfo in case there is
    # more than one parameter given to Content-type.
    # eg: Content-type: text/plain; charset="us-ascii"; format="flowed"
    if ($Headers =~ /^content\-type:\s*([\w\/\-]+)\s*(\;\s*.*)\s*$/mi)
	{
	    $mimetype=$1;
	    $mimetype =~ tr/[A-Z]/[a-z]/;

	    if ($mimetype eq "text") { # for pre-RFC2045 messages (c. 1996)
		$mimetype = "text/plain";
	    }

	    $mimeinfo=$2;
	    if (!defined $mimeinfo) {
		$mimeinfo="";
	    } else { # strip leading and trailing stuff
		$mimeinfo =~ s/^\;\s*//;
		$mimeinfo =~ s/\s*$//;
	    }
	    if ($mimeinfo =~ /charset=\"([^\"]+)\"/i) {
	      $charset = $1;
	    }
	}

    my $transfer_encoding="7bit";
    if ($Headers =~ /^content-transfer-encoding:\s*([^\s]+)\s*$/mi) {
	$transfer_encoding=$1;
    }
    if ($mimetype eq "text/html") {
	$$textref= $self->text_from_part("$Headers\n$$textref");
    } elsif ($mimetype ne "text/plain") {
	$$textref= $self->text_from_mime_message($mimetype,$mimeinfo,$$textref,
						 $outhandle);
    } elsif ($transfer_encoding =~ /quoted\-printable/) {
	$$textref=qp_decode($$textref);
	$self->convert2unicode($charset, $textref);
    } elsif ($transfer_encoding =~ /base64/) {
	$$textref=base64_decode($$textref);
	$self->convert2unicode($charset, $textref);
    } else {
	$self->convert2unicode($charset, $textref);
    }
    

    # Add "All headers" metadata
    $Headers = &text_into_html($Headers);

    $Headers = "No headers" unless ($Headers =~ /\w/);
    $Headers =~ s/@/&#64\;/g;
    # escape [] so it isn't re-interpreted as metadata
    $Headers =~ s/\[/&#91;/g; $Headers =~ s/\]/&#93;/g;
    $self->convert2unicode($charset, \$Headers);
    $doc_obj->add_utf8_metadata ($cursection, "Headers", $Headers);


    # Add Title metadata
    my $Title = text_into_html($raw{'Subject'});
    $Title .= "<br>From: " . text_into_html($raw{'From'});
    $Title .= "<br>Date: " . text_into_html($raw{'DateText'});
    $Title =~ s/\[/&#91;/g; $Title =~ s/\]/&#93;/g;

    $doc_obj->add_utf8_metadata ($cursection, "Title", $Title);
	

    # Add text to document object
    if ($mimetype eq "text/plain") {
	$$textref = &text_into_html($$textref);
    }
    $$textref = "No message" unless ($$textref =~ /\w/);
    $doc_obj->add_utf8_text($cursection, $$textref);

    return 1;
}


# Convert a text string into HTML.
#
# The HTML is going to be inserted into a GML file, so 
# we have to be careful not to use symbols like ">",
# which ocurs frequently in email messages (and use
# &gt instead.
#
# This function also turns links and email addresses into hyperlinks,
# and replaces carriage returns with <BR> tags (and multiple carriage
# returns with <P> tags).


sub text_into_html {
    my ($text) = @_;

    # Convert problem characters into HTML symbols
    $text =~ s/&/&amp;/g;
    $text =~ s/</&lt;/g;
    $text =~ s/>/&gt;/g;
    $text =~ s/\"/&quot;/g;

    # convert email addresses and URIs into links
# don't markup email addresses for now
#    $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;

    # try to munge email addresses a little bit...
    $text =~ s/@/&#64;/;
    # assume hostnames are \.\w\- only, then might have a trailing '/.*'
    # assume URI doesn't finish with a '.'
    $text =~ s@((http|ftp|https)://[\w\-]+(\.[\w\-]+)*/?((&amp;|\.)?[\w\?\=\-_/~]+)*)@<a href=\"$1\">$1<\/a>@g;


    # Clean up whitespace and convert \n charaters to <BR> or <P>
    $text =~ s/ +/ /g;
    $text =~ s/\s*$//g; 
    $text =~ s/^\s*//g;
    $text =~ s/\n/\n<br>/g;
    $text =~ s/<br>\s*<br>/<p>/gi;

    return $text;
}




#Process a MIME message.
# the textref we are given DOES NOT include the header.
sub text_from_mime_message {
    my $self = shift(@_);
    my ($mimetype,$mimeinfo,$text,$outhandle)=(@_);

    # Check for multiparts - $mimeinfo will be a boundary
    if ($mimetype =~ /multipart/) {
	$boundary="";
	if ($mimeinfo =~ m@boundary=(\"[^\"]+\"|[^\s]+)\s*$@im) {
	    $boundary=$1;
	    if ($boundary =~ m@^\"@) {
		$boundary =~ s@^\"@@; $boundary =~ s@\"$@@;
	    }
	} else {
	    print $outhandle "EMAILPlug: (warning) couldn't parse MIME boundary\n";
	}
	# parts start with "--$boundary"
	# message ends with "--$boundary--"
	# RFC says boundary is <70 chars, [A-Za-z'()+_,-./:=?], so escape any
	# that perl might want to interpolate. Also allows spaces...
	$boundary=~s/\\/\\\\/g;
	$boundary=~s/([\?\+\.\(\)\:\/\'])/\\$1/g;
	my @message_parts = split("\r?\n\-\-$boundary", "\n$text");
	# remove first "part" and last "part" (final --)
	shift @message_parts;
	my $last=pop @message_parts;
	# if our boundaries are a bit dodgy and we only found 1 part...
	if (!defined($last)) {$last="";}
	# make sure it is only -- and whitespace
	if ($last !~ /^\-\-\s*$/ms) {
	    print $outhandle "EMAILPlug: (warning) last part of MIME message isn't empty\n";
	}
	foreach my $message_part (@message_parts) {
	    # remove the leading newline left from split.
	    $message_part=~s/^\r?\n//;
	}
	if ($mimetype eq "multipart/alternative") {
	    # check for an HTML version first, then TEXT, otherwise use first.
	    my $part_text="";
	    foreach my $message_part (@message_parts) {
		if ($message_part =~ m@\s*content\-type:\s*text/html@mis)
		{
		    # Use the HTML version
		    $part_text= $self->text_from_part($message_part);
		    $mimetype="text/html";
		    last;
		}
	    }
	    if ($part_text eq "") { # try getting a text part instead
		foreach my $message_part (@message_parts) {
		    if ($message_part =~ m@^content\-type:\s*text/plain@mis)
		    {
			# Use the plain version
			$part_text= $self->text_from_part($message_part);
			if ($part_text =~/[^\s]/) {
			    $part_text="<pre>".$part_text."</pre>";
			}
			$mimetype="text/plain";
			last;
		    }
		}
	    }
	    if ($part_text eq "") { # use first part
		$part_text= $self->text_from_part(shift @message_parts);
	    }
	    if ($part_text eq "") { # we couldn't get anything!!!
		# or it was an empty message...
		# do nothing...
		print $outhandle "EMAILPlug: no text - empty body?\n";
	    } else {
		$text=$part_text;
	    }
	} elsif ($mimetype =~ m@multipart/(mixed|digest|related)@) {
	    $text="";
	    foreach my $message_part (@message_parts) {
		my $part_header=$message_part;
		my $part_body;
		if ($message_part=~ /^\s*\n/) {
		    # no header... use defaults
		    $part_body=$message_part;
		    $part_header="Content-type: text/plain; charset=us-ascii";
		} elsif ($part_header=~s/\r?\n\r?\n(.*)$//sg) {
		    $part_body=$1;
		} else {
		    # something's gone wrong...
		    $part_header="";
		    $part_body=$message_part;
		}
		
		$part_header =~ s/\r?\n[\t\ ]+/ /gs; #unfold
		my $part_content_type="";
		my $part_content_info="";
		if ($mimetype eq "multipart/digest") {
		    # default type - RTFRFC!!
		    $part_content_type="message/rfc822";
		}
		if ($part_header =~ m@^content\-type:\s*([\w+/\-]+)\s*\;?\s*(.*?)\s*$@mi) {
		    $part_content_type=$1; $part_content_type =~ tr/A-Z/a-z/;
		    $part_content_info=$2;
		}
		my $filename="";
		if ($part_header =~ m@name=\"?([\w\.\-\\/]+)\"?@mis) {
		    $filename=$1;
		}

		# disposition - either inline or attachment.
		# NOT CURRENTLY USED - we display all text types instead...
		# $part_header =~ /^content\-disposition:\s*([\w+])/mis;

		# add <<attachment>> to each part except the first...
		if ($text ne "") {
		    $text.="\n<p><hr><strong>&lt;&lt;attachment&gt;&gt;</>";
		    # add part info header
		    $text.="<br>Type: $part_content_type<br>\n";
		    if ($filename ne "") {
			$text.="Filename: $filename\n";
		    }
		    $text.="</strong></p>\n";
		}

		if ($part_content_type =~ m@text/@)
		{
		    my $part_text= $self->text_from_part($message_part);
		    if ($part_content_type !~ m@text/(ht|x)ml@) {
			$part_text=text_into_html($part_text);
		    }
		    if ($part_text eq "") {
			$part_text='&lt;&lt;empty message&gt;&gt;';
		    }
		    $text.=$part_text;
		} elsif ($part_content_type =~ m@message/rfc822@) {
		    # This is a forwarded message
		    my $message_part_headers=$part_body;
		    $message_part_headers =~ s/\r?\n[\t\ ]+/ /gs; #unfold
		    $message_part_headers=~s/\r?\n\r?\n(.*)$//s;
		    my $message_part_body=$1;

		    my $rfc822_formatted_body=""; # put result in here
		    if ($message_part_headers =~
			/^content\-type:\s*([\w\/\-]+)\s*\;?\s*(.*?)\s*$/ims)
		    {
			# The message header uses MIME flags
			my $message_content_type=$1;
			my $message_content_info=$2;
			if (!defined($message_content_info)) {
			    $message_content_info="";
			}
			$message_content_type =~ tr/A-Z/a-z/;
			if ($message_content_type =~ /multipart/) {
			    $rfc822_formatted_body=
			      $self->text_from_mime_message($message_content_type,
							    $message_content_info,
							    $message_part_body,
							    $outhandle);
			} else {
			    $message_part_body= $self->text_from_part($part_body);
			    $rfc822_formatted_body=text_into_html($message_part_body);
			}
		    } else {
			# message doesn't use MIME flags
			$rfc822_formatted_body=text_into_html($message_part_body);
		    }
		    # Add the returned text to the output
		    # don't put all the headers...
		    $message_part_headers =~ s/^(X\-.*|received|message\-id|return\-path):.*\n//img;
		    $text.=text_into_html($message_part_headers);
		    $text.="<p>\n";
		    $text.=$rfc822_formatted_body;
		    # end of message/rfc822
		} elsif ($part_content_type =~ /multipart/) { 
		    # recurse again

		    $tmptext= $self->text_from_mime_message($part_content_type,
							    $part_content_info,
							    $part_body,
							    $outhandle);
		    $text.=$tmptext;
		} elsif ($text eq "") {
		    # we can't do anything with this part, but if it's the first
		    # part then make sure it is mentioned..
		    
		    $text.="\n<p><hr><strong>&lt;&lt;attachment&gt;&gt;</>";
		    # add part info header
		    $text.="<br>Type: $part_content_type<br>\n";
		    if ($filename ne "") {
			$text.="Filename: $filename\n";
		    }
		    $text.="</strong></p>\n";
		}
	    } # foreach message part.
	} else {
	    # we can't handle this multipart type (not mixed or alternative)
	    # the RFC also mentions "parallel".
	}
    } # end of ($mimetype !~ multipart)
    elsif ($mimetype =~ m@message/rfc822@) {
	my $msg_header = $text;
	$msg_header =~ s/\r?\n\r?\n(.*)$//s;
	$text = $1;

	if ($msg_header =~ /^content\-type:\s*([\w\/\-]+)\s*\;?\s*(.+?)\s*$/mi)
	{
	    $mimetype=$1;
	    $mimetype =~ tr/[A-Z]/[a-z]/;
	    $mimeinfo=$2;
	    if ($mimeinfo =~ /charset=\"([^\"]+)\"/) {
		$charset = $1;
	    }
	    my $msg_text;
	    if ($mimetype =~ m@multipart/@) {
		$msg_text = text_from_mime_message($self, $mimetype, $mimeinfo,
						   $text, $outhandle);
	    } else {$msg_text=$text;}

	    my $brief_header="";
	    if ($msg_header =~ /^(From:.*)$/im) {$brief_header.="$1<br>";}
	    if ($msg_header =~ /^(To:.*)$/im) {$brief_header.="$1<br>";}
	    if ($msg_header =~ /^(Cc:.*)$/im) {$brief_header.="$1<br>";}
	    if ($msg_header =~ /^(Subject:.*)$/im) {$brief_header.="$1<br>";}
	    if ($msg_header =~ /^(Date:.*)$/im) {$brief_header.="$1<br>";}
	    $text= "\n<b>&lt;&lt;attached message&gt;&gt;</b><br>";
	    $text.= "<table><tr><td width=\"5%\"> </td>\n";
	    $text.="<td>" . $brief_header . "\n</p>" . $msg_text 
		. "</td></tr></table>";
	}
    } else {
	# we don't do any processing of the content.
    }

    return $text;
}






# Process a MIME part. Return "" if we can't decode it.
sub text_from_part {
    my $self = shift;
    my $text = shift || '';
    my $part_header = $text;

    # check for empty part header (leading blank line)
    if ($text =~ /^\s*\r?\n/) {
	$part_header="Content-type: text/plain; charset=us-ascii";
    } else {
	$part_header =~ s/\r?\n\r?\n(.*)$//s;
	$text=$1; if (!defined($text)) {$text="";}
    }
    $part_header =~ s/\r?\n[\t ]+/ /gs; #unfold
    $part_header =~ /content\-type:\s*([\w\/]+).*?charset=\"?([^\;\"\s]+)\"?/is;
    my $type=$1;
    my $charset=$2;
    if (!defined($type)) {$type="";}
    if (!defined($charset)) {$charset="ascii";}
    my $encoding="";
    if ($part_header =~ /^content\-transfer\-encoding:\s*([^\s]+)/mis) {
	$encoding=$1; $encoding=~tr/A-Z/a-z/;
    }
    # Content-Transfer-Encoding is per-part
    if ($encoding ne "") {
	if ($encoding =~ /quoted\-printable/) {
	    $text=qp_decode($text);
	} elsif ($encoding =~ /base64/) {
	    $text=base64_decode($text);
	} elsif ($encoding !~ /[78]bit/) { # leave 7/8 bit as is.
	    # rfc2045 also allows binary, which we ignore (for now).
	    # maybe this shouldn't go to stderr, but anyway...
	    print STDERR "EMAILPlug: unknown encoding: $encoding\n";
	    return "";
	}
    }
    if ($type eq "text/html") {
	# only get stuff between <body> tags, or <html> tags.
	$text =~ s@^.*<html[^>]*>@@is;
	$text =~ s@</html>.*$@@is;
	$text =~ s/^.*?<body[^>]*>//si;
	$text =~ s/<\/body>.*$//si;
    }
    elsif ($type eq "text/xml") {
	$text=~s/</&lt;/g;$text=~s/>/&gt;/g;
	$text="<pre>\n$text\n</pre>\n";
    }
    # convert to unicode
    $self->convert2unicode($charset, \$text);
    return $text;
}


# decode quoted-printable text
sub qp_decode {
    my $text=shift;

    my @lines=split('\n', $text);

    # if a line ends with "=\s*", it is a soft line break, otherwise
    # keep in any newline characters.
    foreach my $line (@lines) {
	if ($line !~ s/=\s*$//) {$line.="\n";}

	if ($line =~ /=[0-9A-Fa-f]{2}/) { # it contains an escaped char
	    my @hexcode_segments=split('=',$line);
	    shift @hexcode_segments;
	    my @hexcodes;
	    foreach my $hexcode (@hexcode_segments) {
		$hexcode =~ s/^(..).*$/$1/;  # only need first 2 chars
		chomp($hexcode); # just in case...
		my $char=chr (hex "0x$hexcode");
		$line =~ s/=$hexcode/$char/g;
	    }
	}
    }
    $text= join('', @lines);
    return $text;
}

# decode base64 text. This is fairly slow (since it's interpreted perl rather
# than compiled XS stuff like in the ::MIME modules, but this is more portable
# for us at least).
# see rfc2045 for description, but basically, bits 7 and 8 are set to zero;
# 4 bytes of encoded text become 3 bytes of binary - remove 2 highest bits
# from each encoded byte.


sub base64_decode {
    my $enc_text = shift;
# A=>0, B=>1, ..., '+'=>62, '/'=>63
# also '=' is used for padding at the end, but we remove it anyway.
    my $mimechars="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
# map each MIME char into it's value, for more efficient lookup.
    my %index;
    map { $index{$_} = index ($mimechars, $_) } (split ('', $mimechars));
# remove all non-base64 chars. eval to get variable in transliteration...
# also remove '=' - we'll assume (!!) that there are no errors in the encoding
    eval "\$enc_text =~ tr|$mimechars||cd";
    my $decoded="";
    while (length ($enc_text)>3)
    { 
	my $fourchars=substr($enc_text,0,4,"");
	my @chars=(split '',$fourchars);
	$decoded.=chr( $index{$chars[0]}        << 2 | $index{$chars[1]} >> 4);
	$decoded.=chr( ($index{$chars[1]} & 15) << 4 | $index{$chars[2]} >> 2);
	$decoded.=chr( ($index{$chars[2]} & 3 ) << 6 |  $index{$chars[3]});
    } 
# if there are any input chars left, there are either
# 2 encoded bytes (-> 1 raw byte) left or 3 encoded (-> 2 raw) bytes left.
    my @chars=(split '',$enc_text);
    if (length($enc_text)) {
	$decoded.=chr($index{$chars[0]} << 2 | (int $index{$chars[1]} >> 4));
    } 
    if (length($enc_text)==3) {
	$decoded.=chr( ($index{$chars[1]} & 15) << 4 | $index{$chars[2]} >> 2);
    }
    return $decoded;
}

sub convert2unicode {
  my $self = shift(@_);
  my ($charset, $textref) = @_;

  # first get our character encoding name in the right form.
  $charset = "iso_8859_1" unless defined $charset;
  $charset=~tr/A-Z/a-z/;
  $charset=~s/\-/_/g;
  $charset=~s/gb2312/gb/;
  # assumes EUC-KR, not ISO-2022 !?
  $charset=~s/ks_c_5601_1987/korean/;

  if ($charset eq "utf_8") {
      # nothing to do!
      return;
  }

  # It appears that we can't always trust ascii text so we'll treat it
  # as iso-8859-1 (letting characters above 0x80 through without
  # converting them to utf-8 will result in invalid XML documents
  # which can't be parsed at build time).
  $charset = "iso_8859_1" if ($charset eq "us_ascii" || $charset eq "ascii");

  if ($charset eq "iso_8859_1") {
      # test if the mailer lied, and it has win1252 chars in it...
      # 1252 has characters between 0x80 and 0x9f, 8859-1 doesn't
      if ($$textref =~ m/[\x80-\x9f]/) {
	  my $outhandle = $self->{'outhandle'};
	  print $outhandle "EMAILPlug: Headers claim ISO charset but MS ";
	  print $outhandle "codepage 1252 detected.\n";
	  $charset = "windows_1252";
      }
  }
  $$textref=&unicode::unicode2utf8(&unicode::convert2unicode($charset,$textref));
}


# Perl packages have to return true if they are run.
1;