###########################################################################
#
# MediaWikiPlug.pm -- html plugin with extra facilities for wiki page 
#
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################
# This plugin is to process an HTML file where sections are divided by 
# user-defined headings tags. As it is difficult to predict what user's definition
# this plugin allows to detect the user-defined titles up to three levels (level1, level2, level3...)
# as well as allows to get rid of user-defined Table of Content (TOC)...
# format:e.g. level1 (Abstract_title|ChapterTitle|Referencing Heading) level2(SectionHeading)...

package MediaWikiPlug;

use HTMLPlug;
use ImagePlug;
use File::Copy;

#use strict; # every perl program should have this!
#no strict 'refs'; # make an exception so we can use variables as filehandles

sub BEGIN {
    @MediaWikiPlug::ISA = ('HTMLPlug');
}

my $arguments = 
    [          
     { 'name' => "show_toc",
       'desc' => "{MediaWikiPlug.show_toc}",
       'type' => "flag",
       'reqd' => "no"},
     { 'name' => "toc_exp",
       'desc' => "{MediaWikiPlug.toc_exp}",
       'type' => "regexp",
       'reqd' => "no",
       'deft' => "" },   
     { 'name' => "delete_toc",
       'desc' => "{MediaWikiPlug.delete_toc}",
       'type' => "flag",
       'reqd' => "no"},
     { 'name' => "delete_nav",
       'desc' => "{MediaWikiPlug.delete_nav}",
       'type' => "flag",
       'reqd' => "no",
       'deft' => ""},     
     { 'name' => "nav_exp",
       'desc' => "{MediaWikiPlug.nav_exp}",
       'type' => "regexp",
       'reqd' => "no",
       'deft' => "" },
     { 'name' => "tag_sections",
       'desc' => "{MediaWikiPlug.tag_sections}",
       'type' => "flag",
       'reqd' => "no"},
     { 'name' => "description_tags",
       'desc' => "{HTMLPlug.description_tags}",
       'type' => "flag",
       'reqd' => "no"}	      
     ];


my $options = { 'name'     => "MediaWikiPlug",
		'desc'     => "{MediaWikiPlug.desc}",
		'abstract' => "no",
		'inherits' => "yes",
		'args'     => $arguments };


sub new {
    my ($class) = shift (@_);
    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
    push(@$pluginlist, $class);
    
    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
    
    my $self = new HTMLPlug($pluginlist, $inputargs, $hashArgOptLists);    
    return bless $self, $class;
}



sub process {
    my $self = shift (@_);
    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    my $outhandle = $self->{'outhandle'};

    print $outhandle "MediaWikiPlug: processing $file\n" if $self->{'verbosity'} > 1;
          
    my @head_and_body = split(/<body/i,$$textref);
    my $head = shift(@head_and_body);
    my $body_text = join("<body", @head_and_body);  
    
    $head =~ m/<title>(.+)<\/title>/i;
    my $doctitle = $1 if defined $1;
    
    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'}=~ /\S/) {
	my @doc_properties = split(/<xml>/i,$head);
	my $doc_heading = shift(@doc_properties);
	my $rest_doc_properties = join(" ", @doc_properties);
	
	my @extracted_metadata = split(/<\/xml>/i, $rest_doc_properties);
	my $extracted_metadata = shift (@extracted_metadata);
	$self->extract_metadata($extracted_metadata, $metadata, $doc_obj);
    }
    
    # set the title here if we haven't found it yet
    if (!defined $doc_obj->get_metadata_element ($doc_obj->get_top_section(), "Title")) {    
	if (defined $doctitle && $doctitle =~ /\S/) {
		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Title", $doctitle);
	} else {
		$self->title_fallback($doc_obj,$doc_obj->get_top_section(),$file);
	}
    } 
    
    if(defined $base_dir && $base_dir ne ""){ 
	# find and download stylesheet
	my @css_files;
	my $css_file_count = 0;
	# find all the style sheets imported with import statement
	while($head =~ m"<style type=\"text/css\"(.+)import \"(.+)\""ig){
		$css_files[$css_file_count++] = $2 if defined $2;
	}    
	
	# check whether the stylesheet exists
	# if not, download it and copy to the collection's images folder
	for($css_file_count = 0; $css_file_count < scalar(@css_files); $css_file_count++){
		my $css_file = $css_files[$css_file_count];    	    	
		$css_file =~ s/^(.+)gli\/cache\///i;
			
		my $css_file_path = "$base_dir/$css_file";    	
		
		if (-e $css_file_path){ # the file already exists
			next;
		}  
		
		# check the css directory and create one if it's not there
		my @dirs = split(/\//i,$css_file);
		my $path_check = "$base_dir/";
		for(my $i = 0; $i < (scalar(@dirs)-1); $i++){    	  			    	
			$path_check .= $dirs[$i] . "/";	    	
			if(! -d $path_check ){
				mkdir($path_check);	
			}
		}    	
		
		# download
		$css_file = "http://$css_file";    	
		system("wget", "--non-verbose", "$css_file", "--output-document=$css_file_path");
		if ($? != 0) {unlink("$css_file_path");}
		
		# change every style element to #wikispecificstyle ...
		if(open(INPUT, "<$css_file_path")){
			my $css_content;
			while(my $line = <INPUT>){				
				if($line =~ m/^(.+)\{/i){
					$line = "#wikispecificstyle " . $line;				
				}
				$css_content .= $line;
			}
			close(INPUT);			
			open(OUTPUT, ">$css_file_path");
			print OUTPUT $css_content;
			close(OUTPUT);
		}
		
		# copy to images folder
		# do not copy, because collection can only have one specific stylesheet
		# better to add and modify the style sheets manually
		# @dirs = split(/\//i,$base_dir);						
		# my $collection_base_dir;		
		# for(my $i = 0; $i < (scalar(@dirs)-1); $i++){
		#	$collection_base_dir .= $dirs[$i] . "/";			
		# }
		# my $images_folder = $collection_base_dir . "images/";
		# copy($css_file_path, $images_folder) || die "File cannot be copied.";
	}
    }
    
    # add sections around h2 tag
    # wrap each section with <div id=\"wikispecificstyle\"></div> to get the wiki styles
    # add search box with each section      
    if ($self->{'tag_sections'}) {
	my @sections = ($body_text =~ /<h2>(.+)<\/h2>/gi);
	for(my $i=1; $i < scalar(@sections); $i++){
		my $section_title = $sections[$i];    	
		$section_title =~ s/<([^>]*)>//g;
		$section_title =~ s/(^\s|\s$)//g;
		my $section_metadata = "<Section>\n<Description>\n<Metadata name=\"Title\">$section_title</Metadata>\n</Description>\n";
		if($i !=1){
			$section_metadata = "</Section>\n" . $section_metadata;
		}
		$section_metadata = "\n<!--\n" . $section_metadata . "-->\n";
		
		$section_metadata .= "<div id=\"wikispecificstyle\">\n<div id=\"content\">\n";
		$section_metadata = "</div></div>\n" . $section_metadata if $i !=1;
		
		$body_text =~ s/<h2>$sections[$i]<\/h2>/$section_metadata<h2>$sections[$i]<\/h2>/i;
		
		if($i==scalar(@sections)-1) {
			# $body_text =~ s/<div class=\"printfooter\">/<!--\n<\/Section>\n-->\n<div class=\"printfooter\">/i;
			$body_text =~ s/<div class=\"printfooter\">/<\/div>\n<\/div>\n<!--\n<\/Section>\n-->\n<div class=\"printfooter\">/i;
		}
	}
     }    
    
    # If delete_nav is enabled, it means to get rid of navigation contents.
    # if (defined $self->{'delete_nav'} && ($self->{'delete_nav'} == 1)){
	#	if (defined $self->{'nav_exp'}&& $self->{'nav_exp'} =~ /\S/){
	#	    print "it matches nav_exp!!\n" if $body_text =~ /$self->{'nav_exp'}/;
	#	    $body_text =~ s/$self->{'nav_exp'}//isg;	    
	#	}
    #}
    my $searchbox = "";
    if (defined $self->{'delete_nav'} && ($self->{'delete_nav'} == 1)){
	my $nav_match_express;
	if (defined $self->{'nav_exp'}&& $self->{'nav_exp'} =~ /\S/) {
		$nav_match_express = $self->{'nav_exp'} ;
	} else { # default setting for mediawiki
		$nav_match_express = "<div class=\"printfooter\">(.|\n)*secs. -->";
	}
	
	print "it matches nav_exp!!\n" if $body_text =~ /$self->{'nav_exp'}/;
	
	# $body_text =~ m/<div class=\"printfooter\">(.|\n)*secs. -->/isg;    
	$body_text =~ m/$nav_match_express/isg;
	my $navigate = $& if defined $&;  	  
	
	# find the search box and add it to the document page
	if(defined $navigate && $navigate =~ /\S/){	
		$navigate =~ m/<div id="p-search" class="portlet">(.|\n)*<\/form>/;
		$searchbox = $& . "\n<\/div>\n<\/div>";
		$searchbox =~ s/action="([^>]*)"/action="\/gsdl\/cgi-bin\/library"/isg;
		$searchbox =~ s/name="search"/name="q"/isg;
		$searchbox =~ s/name="go"//isg;
		$searchbox =~ s/name="fulltext"//isg;
		my $hidden_params = "<input type=\"hidden\" name=\"a\" value=\"q\"/>\n" 
					."<input type=\"hidden\" name=\"c\" value=\"wikitest\"/>\n"
					."<input type=\"hidden\" name=\"fqf\" value=\"TX\"/>"
					."<input type=\"hidden\" name=\"t\" value=\"1\">";
		$searchbox =~ s/<\/form>/$hidden_params<\/form>/isg;    
		$searchbox = "\n</div>\n</div><div id=\"wikispecificstyle\"><div id=\"column-one\">$searchbox</div></div>";
	}
	
	# $body_text =~ s/<div class=\"printfooter\">(.|\n)*secs. -->/$searchbox/isg;
	$body_text =~ s/$nav_match_express/$searchbox/isg;
    }
    
    if ($self->{'tag_sections'}) {
    	$body_text =~ s/<!--\n<\/Section>/$searchbox\n<!--\n<\/Section>/ig;
    }
    
    # Tidy up extra new lines
    $body_text =~ s/(<p[^>]*><span[^>]*><o:p>&nbsp;<\/o:p><\/span><\/p>)//isg;
    $body_text =~ s/(<p[^>]*><o:p>&nbsp;<\/o:p><\/p>)//isg;
    
    $section_text .= "<!--\n<Section>\n-->\n";
    my $body = "<body".$body_text;
    
    $$textref = $body;
    
    # get the base dir for convert absolute links to relative links
    $$textref =~ m"href=\"(.*?)/cache/(.*?)/"i;
    my $basedir = $2;   
        
    $$textref =~ s/<!\[if !vml\]>/<![if vml]>/g;    
    $$textref =~ s/(&nbsp;)+/&nbsp;/sg;      
    
    # get rid of the [edit] button
    $$textref =~ s/\[<a([^>]*)>edit<\/a>]//g;

    # get rid of the last time edit information at the bottom
    $$textref =~ s/<a href="(.+)edit(.*?)"(.*?)>(\w+)<\/a> \d\d:\d\d,(.*?)(PST)//g;
    
    # get rid of the (Redirected from ...)
    $$textref =~ s/(Redirected from <a ([^>]*)>(\w|\s)*<\/a>)//isg;
    
    # escape macros
    $$textref =~ s/_([^\s]*)_/_<span>$1<\/span>_/isg;
    # may change the links, like Greenstone_Documentation_All.html, then change back
    $$textref =~ s/<a([^>]*)_<span>([^>]*)<\/span>_/<a$1_$2_/isg;	
    
    # convert all the urls to relative url, because current wget 1.10 -k and -E option doesn't work together
    # get rid of the title attribute of a tag
    $$textref =~ s/<a([^>]*)title="(.*?)"/<a$1/isg;
    # find the relative path of current directory         
    if($basedir ne ""){
    	my @dirs=split("\/", $file);
	    my $dirnum = scalar(@dirs);
	    my $replace = "";
	    for(my $i=0; $i<$dirnum-2; $i++){
			$replace .= "../";
	    }
	    # test if the linked relative file exists, if not, link to the internet version
    	$$textref =~ s/(href|src)="([^>]*)$basedir\/([^>]*)"/$1="$replace$3"/gi;    	    	
    	# my @total_links = ($$textref =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/gi);
    	# print $outhandle "\nnumber of total links: " . scalar(@total_links)."\n";
    	# for(my $cur_link_no = 0; $cur_link_no < scalar(@total_links); $cur_link_no++){
    	
    	#while($$textref =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/gi){
    		#$total_links[$cur_link_no] =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/i;
    	#	my $prefix = $1;
    	#	my $link = $&; 
    	#	my $rel_file_name = $3; 		    		
    	#	my $rel_link = "$replace$rel_file_name";
    		# print $outhandle "catched link==> $link\nrelative link==> $rel_link\n";
    	#	if(-e $rel_link){
    	#		$rel_link = "$prefix=\"$rel_link\"";
    	#		$$textref =~ s/$link/$rel_link/i;
    	#	}else{
    	#		my $ext_link = "$prefix=\"http:\/\/$basedir\/$rel_file_name\"";
    			# print $outhandle "external link==> $ext_link\n";
    	#		$$textref =~ s/$link/$ext_link/i; #s/$link/$prefix="http:\/\/$rel_file_name"/i;
    	#	}
    	#}   		
    		
    	
    	# tag the link to new wiki pages as red
    	$$textref =~ s/(href|src)="$replace([^>]*)&amp;action=edit([^>]*)"/$1="http:\/\/$basedir\/$2&amp;action=edit$3"/gi;    	
    	$$textref =~ s/<a([^>]*)class="new"([^>]*)>/<a$1style="color:red"$2)>/gi;
    	
    	# tag the link to external pages as blue
    	$$textref =~ s/<a([^>]*)class='external text'([^>]*)>/<a$1style="color:blue"$2)>/gi;   	
    	
    	#print $outhandle $$textref;
    }
    
	# if 'show_toc' is set, put the table of content on the Wiki Main_Page to the about page of the collection
	# 1. read _content_ macro from about.dm
	# 2. append the toc, change all links to the Greenstone internal format for relative links 
	# 3. write to the extra.dm
	# TODO: currently we suppose the _about:content_ hasn't been specified before
	#	    so needs to add function to handle when the macro is already in the extra.dm	
   	if($self->{'show_toc'}==1 && $file =~ m/Main_Page.(html|htm)$/){
        my $macro_path = $base_dir;
    	$macro_path =~ s/import$/macros/;
    	my $extra_dm;
    	my $extradm_file = "$macro_path/extra.dm";
        if(open(INPUT, "<$extradm_file")){    	
    		while(my $line = <INPUT>){
			    $extra_dm .= $line;
   		}
   			close(INPUT);  		   			  		
    		
   			if($extra_dm =~ m/package about/ && $extra_dm =~ m/_content_(\s)*{/){  
 				print $outhandle "already changed!!!!\n";   			
    		} else {
	    		# read _content_ macro from about.dm file
    			my $about_macro = $ENV{'GSDLHOME'} . "/macros/about.dm";    			
   				my $about_page_content = "";
   				if(open(INPUT, "<$about_macro")){
   					while(my $line=<INPUT>){
   						$about_page_content .= $line;
   					}
   				}else{
   					print $outhandle "can't open file $about_macro\n";
   				}    		
   				close(INPUT);
    		    
   				# extract the _content_ macro
   				$about_page_content =~ m/_content_ {(.|\n)*<\/div>\n\n<\/div>\n}/i;
   				$about_page_content = $&;
    				
   				# extract toc of the Main_Page
    			my $mainpage_content = "";
    			if($self->{'toc_exp'} =~ /\S/){
    				$$textref =~ /$self->{'toc_exp'}/;
    				$mainpage_content = $&;
    			} else {
    				# $mainpage_content =~ s/<!-- start content -->(.|\n)*<!-- end content -->/$1/igs;
    			}
    			# print $outhandle "---------\n$$textref\n--------\n\n";	    		
    			# print $outhandle "==========\n$mainpage_content\n==========\n\n";
    				
    			# add toc to the _content_ macro
    			$about_page_content =~ m/{(.|\n)*<\/div>\n\n/;
    			$extra_dm .= "package about\n_content_$&\n\n<div class=\"section\">\n$mainpage_content\n</div>\n</div>\n}";
    				
    			# change all links to the internal Greenstone relative link format
    			$extra_dm =~ s/<a href="([^>]*)"/<a href="_httpquery_&a=extlink&rl=1&href=http:\/\/$basedir$1"/isg;
   				$extra_dm =~ s/(\.\.\/)+/\//isg;
   				# print $outhandle "to add---------\n$extra_dm\n--------\n";
    				   
   				# write to the extra.dm file of the collection
   				open(OUTPUT, ">$extradm_file");
   				print OUTPUT $extra_dm;
   				close(OUTPUT);    		
   			}	
    	} else { 
   			print $outhandle "can't open file $extradm_file\n";
   		}
	}
	
	# If delete_toc is enabled, it means to get rid of toc and tof contents.
    # get rid of TOC and TOF sections and their title
    if (defined $self->{'delete_toc'} && ($self->{'delete_toc'} == 1)){
		if (defined $self->{'toc_exp'} && $self->{'toc_exp'} =~ /\S/){
	    	# $body_text =~ s/<p class=(($self->{'toc_exp'})[^>]*)>(.+?)<\/p>//isg;	    
	    	# print "it matches toc_exp!!\n" if $body_text =~ /$self->{'toc_exp'}/;	    
	    	# $body_text =~ s/$self->{'toc_exp'}//i;
	    	print "it matches toc_exp!!\n" if $$textref =~ /$self->{'toc_exp'}/;	
	    	$$textref =~ s/$self->{'toc_exp'}//i;
		}	
    }
    
    # To add a layer on top of the wiki page
    # so as to keep the wiki style inside the wiki page
    # and keep the Greenstone style at the same time    
    $$textref =~ s/<body([^>]*)>/$&\n<div id="wikispecificstyle">\n/is;    
    $$textref =~ s/<\/body>/<\/div><\/body>/is; 
    
    # tag with sections    
    $$textref =~ s/<body([^>]*)>/$&\n<!--\n<Section>\n<Description>\n<Metadata name=\"Title\">$doctitle<\/Metadata>\n<\/Description>\n-->\n/is;
    $$textref =~ s/<\/body>/\n<!--\n<\/Section>\n-->\n/is;
    
    #print $outhandle "\n\n$$textref\n\n";
    
    # use description tags    
    if ($self->{'description_tags'}) {
    	my $cursection = $doc_obj->get_top_section();
		# remove the html header - note that doing this here means any
		# sections defined within the header will be lost (so all <Section>
		# tags must appear within the body of the HTML)
		my ($head_keep) = ($$textref =~ m/^(.*?)<body[^>]*>/is);

		$$textref =~ s/^.*?<body[^>]*>//is;
		$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;

		my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
		my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';

		my $lt = '(?:<|&lt;)';
		my $gt = '(?:>|&gt;)';
		my $quot = '(?:"|&quot;|&rdquo;|&ldquo;)';

		# my $dont_strip = '';
		# if ($self->{'no_strip_metadata_html'}) {
		#    ($dont_strip = $self->{'no_strip_metadata_html'}) =~ s{,}{|}g;
		# }

		my $found_something = 0; 
		my $top = 1;
		while ($$textref =~ s/^(.*?)$opencom(.*?)$closecom//s) {
	    	my $text = $1;
	   	 	my $comment = $2;
	    	if (defined $text) {
				# text before a comment - note that getting to here
				# doesn't necessarily mean there are Section tags in
				# the document
				# print $outhandle "section text:\n$text\n";
				$self->process_section(\$text, $base_dir, $file, $doc_obj, $cursection);
	    	}
	    	while ($comment =~ s/$lt(.*?)$gt//s) {
				my $tag = $1;
				if ($tag eq "Section") {
		    		$found_something = 1;
		    		$cursection = $doc_obj->insert_section($doc_obj->get_end_child($cursection)) unless $top;
		    		$top = 0;
				} elsif ($tag eq "/Section") {
		    		$found_something = 1;
		    		$cursection = $doc_obj->get_parent_section ($cursection);
				} elsif ($tag =~ /^Metadata name=$quot(.*?)$quot/s) {
		    		my $metaname = $1;
		    		my $accumulate = $tag =~ /mode=${quot}accumulate${quot}/ ? 1 : 0;
		   			$comment =~ s/^(.*?)$lt\/Metadata$gt//s;
				    my $metavalue = $1;
				    $metavalue =~ s/^\s+//;
				    $metavalue =~ s/\s+$//;
                    # assume that no metadata value intentionally includes
                    # carriage returns or HTML tags (if they're there they
                    # were probably introduced when converting to HTML from
                    # some other format).
				    # actually some people want to have html tags in their
				    # metadata.
				    $metavalue =~ s/[\cJ\cM]/ /sg;
				    # $metavalue =~ s/<[^>]+>//sg unless $dont_strip && ($dont_strip eq 'all' || $metaname =~ /^($dont_strip)$/);
				    $metavalue =~ s/\s+/ /sg;
				    # print $outhandle "metaname = $metaname\nmetavalue = $metavalue\n";
				    if ($accumulate) {
						$doc_obj->add_utf8_metadata($cursection, $metaname, $metavalue);
		    		} else {
						$doc_obj->set_utf8_metadata_element($cursection, $metaname, $metavalue);	
		    		}
				} elsif ($tag eq "Description" || $tag eq "/Description") {
		    		# do nothing with containing Description tags
				} else {
				    # simple HTML tag (probably created by the conversion
				    # to HTML from some other format) - we'll ignore it and
				    # hope for the best ;-)
				}
	    	}
		}# end while

		if ($cursection ne "") {
		    print $outhandle "HTMLPlug: WARNING: $file contains unmatched <Section></Section> tags\n";
		}

		$$textref =~ s/^.*?<body[^>]*>//is;
		$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
		if ($$textref =~ /\S/) {
		    if (!$found_something) {
				if ($self->{'verbosity'} > 2) {
				    print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags so\n";
				    print $outhandle "          will be processed as a single section document\n";
				}
	
				# go ahead and process single-section document
				$self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);

		    } else {
				print $outhandle "HTMLPlug: WARNING: $file contains the following text outside\n";
				print $outhandle "          of the final closing </Section> tag. This text will\n";
				print $outhandle "          be ignored.";

				my ($text);
				if (length($$textref) > 30) {
				    $text = substr($$textref, 0, 30) . "...";
				} else {
				    $text = $$textref;
				}
				$text =~ s/\n/ /isg;
				print $outhandle " ($text)\n";
		    }
		} elsif (!$found_something) {
		    if ($self->{'verbosity'} > 2) {
			# may get to here if document contained no valid Section
			# tags but did contain some comments. The text will have
			# been processed already but we should print the warning
			# as above and extract metadata
			print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags and\n";
			print $outhandle "          is blank or empty.  Metadata will be assigned if present.\n";
		    }
		}
    } # if $self->{'description_tags'}
	else {
		# remove header and footer
		# if (!$self->{'keep_head'}) {
		#    $$textref =~ s/^.*?<body[^>]*>//is;
		#    $$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
		# }

		# single section document
		# $self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);		
		
		# Important: to get the relative links to work,
		# 1: use the below statement instead of the above one
		# 2. cannot have process_section method. 
		# why?????
		$self->SUPER::process(@_);
    }
	return 1;
	       
    #$self->SUPER::process(@_);
}



# note that process_section may be called multiple times for a single
# section (relying on the fact that add_utf8_text appends the text to any
# that may exist already).
# sub process_section {
#    my $self = shift (@_);
#    my ($textref, $base_dir, $file, $doc_obj, $cursection) = @_;

    # trap links
    # if (!$self->{'nolinks'}) {
	# usemap="./#index" not handled correctly => change to "#index"
	# $$textref =~ s/(<img[^>]*?usemap\s*=\s*[\"\']?)([^\"\'>\s]+)([\"\']?[^>]*>)/
	    #$self->replace_usemap_links($1, $2, $3)/isge;

	#$$textref =~ s/(<(?:a|area|frame|link|script)\s+[^>]*?\s*(?:href|src)\s*=\s*[\"\']?)([^\"\'>\s]+)([\"\']?[^>]*>)/
	    #$self->replace_href_links ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
    #}

    # trap images

    # allow spaces if inside quotes - jrm21
    #$$textref =~ s/(<(?:img|embed|table|tr|td)[^>]*?(?:src|background)\s*=\s*)([\"\'][^\"\']+[\"\']|[^\s>]+)([^>]*>)/
	#$self->replace_images ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;

    # add text to document object
    # turn \ into \\ so that the rest of greenstone doesn't think there
    # is an escape code following. (Macro parsing loses them...)
#    $$textref =~ s/\\/\\\\/go;
    
#    $doc_obj->add_utf8_text($cursection, $$textref);
#}


sub extract_metadata 
{
    my $self = shift (@_);
    my ($textref, $metadata, $doc_obj) = @_;
    my $outhandle = $self->{'outhandle'};
    
    return if (!defined $textref);

    # metadata fields to extract/save. 'key' is the (lowercase) name of the
    # html meta, 'value' is the metadata name for greenstone to use
    my %find_fields = ();
    my ($tag,$value);

    my $orig_field = "";
    foreach my $field (split /,/, $self->{'metadata_fields'}) {
	# support tag<tagname>
	if ($field =~ /^(.*?)<(.*?)>$/) {
	    # "$2" is the user's preferred gs metadata name
	    $find_fields{lc($1)}=$2; # lc = lowercase
	    $orig_field = $1;
	} else { # no <tagname> for mapping
	    # "$field" is the user's preferred gs metadata name
	    $find_fields{lc($field)}=$field; # lc = lowercase
	    $orig_field = $field;
	}

	if ($textref =~ m/<o:$orig_field>(.*)<\/o:$orig_field>/i){
	    $tag = $orig_field;
	    $value = $1;
	    if (!defined $value || !defined $tag){
		#print $outhandle "StructuredHTMLPlug: can't find VALUE in \"$tag\"\n";
		next;
	    } else {
		# clean up and add
		chomp($value); # remove trailing \n, if any
		$tag = $find_fields{lc($tag)};
		#print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
		#    if ($self->{'verbosity'} > 2);
		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $value);
	    }
	}
    }
}

1;