Index: /trunk/gsdl/perllib/plugins/MediaWikiPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/MediaWikiPlug.pm	(revision 14108)
+++ /trunk/gsdl/perllib/plugins/MediaWikiPlug.pm	(revision 14108)
@@ -0,0 +1,653 @@
+###########################################################################
+#
+# MediaWikiPlug.pm -- html plugin with extra facilities for wiki page 
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+# This plugin is to process an HTML file where sections are divided by 
+# user-defined headings tags. As it is difficult to predict what user's definition
+# this plugin allows to detect the user-defined titles up to three levels (level1, level2, level3...)
+# as well as allows to get rid of user-defined Table of Content (TOC)...
+# format:e.g. level1 (Abstract_title|ChapterTitle|Referencing Heading) level2(SectionHeading)...
+
+package MediaWikiPlug;
+
+use HTMLPlug;
+use ImagePlug;
+use File::Copy;
+
+#use strict; # every perl program should have this!
+#no strict 'refs'; # make an exception so we can use variables as filehandles
+
+sub BEGIN {
+    @MediaWikiPlug::ISA = ('HTMLPlug');
+}
+
+my $arguments = 
+    [          
+     { 'name' => "show_toc",
+       'desc' => "{MediaWikiPlug.show_toc}",
+       'type' => "flag",
+       'reqd' => "no"},
+     { 'name' => "toc_exp",
+       'desc' => "{MediaWikiPlug.toc_exp}",
+       'type' => "regexp",
+       'reqd' => "no",
+       'deft' => "" },   
+     { 'name' => "delete_toc",
+       'desc' => "{MediaWikiPlug.delete_toc}",
+       'type' => "flag",
+       'reqd' => "no"},
+     { 'name' => "delete_nav",
+       'desc' => "{MediaWikiPlug.delete_nav}",
+       'type' => "flag",
+       'reqd' => "no",
+       'deft' => ""},     
+     { 'name' => "nav_exp",
+       'desc' => "{MediaWikiPlug.nav_exp}",
+       'type' => "regexp",
+       'reqd' => "no",
+       'deft' => "" },
+     { 'name' => "tag_sections",
+       'desc' => "{MediaWikiPlug.tag_sections}",
+       'type' => "flag",
+       'reqd' => "no"},
+     { 'name' => "description_tags",
+       'desc' => "{HTMLPlug.description_tags}",
+       'type' => "flag",
+       'reqd' => "no"}	      
+     ];
+
+
+my $options = { 'name'     => "MediaWikiPlug",
+		'desc'     => "{MediaWikiPlug.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+    
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+    
+    my $self = new HTMLPlug($pluginlist, $inputargs, $hashArgOptLists);    
+    return bless $self, $class;
+}
+
+
+
+sub process {
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    print $outhandle "MediaWikiPlug: processing $file\n" if $self->{'verbosity'} > 1;
+          
+    my @head_and_body = split(/<body/i,$$textref);
+    my $head = shift(@head_and_body);
+    my $body_text = join("<body", @head_and_body);  
+    
+    $head =~ m/<title>(.+)<\/title>/i;
+    my $doctitle = $1 if defined $1;
+    
+    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'}=~ /\S/) {
+	my @doc_properties = split(/<xml>/i,$head);
+	my $doc_heading = shift(@doc_properties);
+	my $rest_doc_properties = join(" ", @doc_properties);
+	
+	my @extracted_metadata = split(/<\/xml>/i, $rest_doc_properties);
+	my $extracted_metadata = shift (@extracted_metadata);
+	$self->extract_metadata($extracted_metadata, $metadata, $doc_obj);
+    }
+    
+    # set the title here if we haven't found it yet
+    if (!defined $doc_obj->get_metadata_element ($doc_obj->get_top_section(), "Title")) {    
+	if (defined $doctitle && $doctitle =~ /\S/) {
+		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Title", $doctitle);
+	} else {
+		$self->title_fallback($doc_obj,$doc_obj->get_top_section(),$file);
+	}
+    } 
+    
+    if(defined $base_dir && $base_dir ne ""){ 
+	# find and download stylesheet
+	my @css_files;
+	my $css_file_count = 0;
+	# find all the style sheets imported with import statement
+	while($head =~ m"<style type=\"text/css\"(.+)import \"(.+)\""ig){
+		$css_files[$css_file_count++] = $2 if defined $2;
+	}    
+	
+	# check whether the stylesheet exists
+	# if not, download it and copy to the collection's images folder
+	for($css_file_count = 0; $css_file_count < scalar(@css_files); $css_file_count++){
+		my $css_file = $css_files[$css_file_count];    	    	
+		$css_file =~ s/^(.+)gli\/cache\///i;
+			
+		my $css_file_path = "$base_dir/$css_file";    	
+		
+		if (-e $css_file_path){ # the file already exists
+			next;
+		}  
+		
+		# check the css directory and create one if it's not there
+		my @dirs = split(/\//i,$css_file);
+		my $path_check = "$base_dir/";
+		for(my $i = 0; $i < (scalar(@dirs)-1); $i++){    	  			    	
+			$path_check .= $dirs[$i] . "/";	    	
+			if(! -d $path_check ){
+				mkdir($path_check);	
+			}
+		}    	
+		
+		# download
+		$css_file = "http://$css_file";    	
+		system("wget", "--non-verbose", "$css_file", "--output-document=$css_file_path");
+		if ($? != 0) {unlink("$css_file_path");}
+		
+		# change every style element to #wikispecificstyle ...
+		if(open(INPUT, "<$css_file_path")){
+			my $css_content;
+			while(my $line = <INPUT>){				
+				if($line =~ m/^(.+)\{/i){
+					$line = "#wikispecificstyle " . $line;				
+				}
+				$css_content .= $line;
+			}
+			close(INPUT);			
+			open(OUTPUT, ">$css_file_path");
+			print OUTPUT $css_content;
+			close(OUTPUT);
+		}
+		
+		# copy to images folder
+		# do not copy, because collection can only have one specific stylesheet
+		# better to add and modify the style sheets manually
+		# @dirs = split(/\//i,$base_dir);						
+		# my $collection_base_dir;		
+		# for(my $i = 0; $i < (scalar(@dirs)-1); $i++){
+		#	$collection_base_dir .= $dirs[$i] . "/";			
+		# }
+		# my $images_folder = $collection_base_dir . "images/";
+		# copy($css_file_path, $images_folder) || die "File cannot be copied.";
+	}
+    }
+    
+    # add sections around h2 tag
+    # wrap each section with <div id=\"wikispecificstyle\"></div> to get the wiki styles
+    # add search box with each section      
+    if ($self->{'tag_sections'}) {
+	my @sections = ($body_text =~ /<h2>(.+)<\/h2>/gi);
+	for(my $i=1; $i < scalar(@sections); $i++){
+		my $section_title = $sections[$i];    	
+		$section_title =~ s/<([^>]*)>//g;
+		$section_title =~ s/(^\s|\s$)//g;
+		my $section_metadata = "<Section>\n<Description>\n<Metadata name=\"Title\">$section_title</Metadata>\n</Description>\n";
+		if($i !=1){
+			$section_metadata = "</Section>\n" . $section_metadata;
+		}
+		$section_metadata = "\n<!--\n" . $section_metadata . "-->\n";
+		
+		$section_metadata .= "<div id=\"wikispecificstyle\">\n<div id=\"content\">\n";
+		$section_metadata = "</div></div>\n" . $section_metadata if $i !=1;
+		
+		$body_text =~ s/<h2>$sections[$i]<\/h2>/$section_metadata<h2>$sections[$i]<\/h2>/i;
+		
+		if($i==scalar(@sections)-1) {
+			# $body_text =~ s/<div class=\"printfooter\">/<!--\n<\/Section>\n-->\n<div class=\"printfooter\">/i;
+			$body_text =~ s/<div class=\"printfooter\">/<\/div>\n<\/div>\n<!--\n<\/Section>\n-->\n<div class=\"printfooter\">/i;
+		}
+	}
+     }    
+    
+    # If delete_nav is enabled, it means to get rid of navigation contents.
+    # if (defined $self->{'delete_nav'} && ($self->{'delete_nav'} == 1)){
+	#	if (defined $self->{'nav_exp'}&& $self->{'nav_exp'} =~ /\S/){
+	#	    print "it matches nav_exp!!\n" if $body_text =~ /$self->{'nav_exp'}/;
+	#	    $body_text =~ s/$self->{'nav_exp'}//isg;	    
+	#	}
+    #}
+    my $searchbox = "";
+    if (defined $self->{'delete_nav'} && ($self->{'delete_nav'} == 1)){
+	my $nav_match_express;
+	if (defined $self->{'nav_exp'}&& $self->{'nav_exp'} =~ /\S/) {
+		$nav_match_express = $self->{'nav_exp'} ;
+	} else { # default setting for mediawiki
+		$nav_match_express = "<div class=\"printfooter\">(.|\n)*secs. -->";
+	}
+	
+	print "it matches nav_exp!!\n" if $body_text =~ /$self->{'nav_exp'}/;
+	
+	# $body_text =~ m/<div class=\"printfooter\">(.|\n)*secs. -->/isg;    
+	$body_text =~ m/$nav_match_express/isg;
+	my $navigate = $& if defined $&;  	  
+	
+	# find the search box and add it to the document page
+	if(defined $navigate && $navigate =~ /\S/){	
+		$navigate =~ m/<div id="p-search" class="portlet">(.|\n)*<\/form>/;
+		$searchbox = $& . "\n<\/div>\n<\/div>";
+		$searchbox =~ s/action="([^>]*)"/action="\/gsdl\/cgi-bin\/library"/isg;
+		$searchbox =~ s/name="search"/name="q"/isg;
+		$searchbox =~ s/name="go"//isg;
+		$searchbox =~ s/name="fulltext"//isg;
+		my $hidden_params = "<input type=\"hidden\" name=\"a\" value=\"q\"/>\n" 
+					."<input type=\"hidden\" name=\"c\" value=\"wikitest\"/>\n"
+					."<input type=\"hidden\" name=\"fqf\" value=\"TX\"/>"
+					."<input type=\"hidden\" name=\"t\" value=\"1\">";
+		$searchbox =~ s/<\/form>/$hidden_params<\/form>/isg;    
+		$searchbox = "\n</div>\n</div><div id=\"wikispecificstyle\"><div id=\"column-one\">$searchbox</div></div>";
+	}
+	
+	# $body_text =~ s/<div class=\"printfooter\">(.|\n)*secs. -->/$searchbox/isg;
+	$body_text =~ s/$nav_match_express/$searchbox/isg;
+    }
+    
+    if ($self->{'tag_sections'}) {
+    	$body_text =~ s/<!--\n<\/Section>/$searchbox\n<!--\n<\/Section>/ig;
+    }
+    
+    # Tidy up extra new lines
+    $body_text =~ s/(<p[^>]*><span[^>]*><o:p>&nbsp;<\/o:p><\/span><\/p>)//isg;
+    $body_text =~ s/(<p[^>]*><o:p>&nbsp;<\/o:p><\/p>)//isg;
+    
+    $section_text .= "<!--\n<Section>\n-->\n";
+    my $body = "<body".$body_text;
+    
+    $$textref = $body;
+    
+    # get the base dir for convert absolute links to relative links
+    $$textref =~ m"href=\"(.*?)/cache/(.*?)/"i;
+    my $basedir = $2;   
+        
+    $$textref =~ s/<!\[if !vml\]>/<![if vml]>/g;    
+    $$textref =~ s/(&nbsp;)+/&nbsp;/sg;      
+    
+    # get rid of the [edit] button
+    $$textref =~ s/\[<a([^>]*)>edit<\/a>]//g;
+
+    # get rid of the last time edit information at the bottom
+    $$textref =~ s/<a href="(.+)edit(.*?)"(.*?)>(\w+)<\/a> \d\d:\d\d,(.*?)(PST)//g;
+    
+    # get rid of the (Redirected from ...)
+    $$textref =~ s/(Redirected from <a ([^>]*)>(\w|\s)*<\/a>)//isg;
+    
+    # escape macros
+    $$textref =~ s/_([^\s]*)_/_<span>$1<\/span>_/isg;
+    # may change the links, like Greenstone_Documentation_All.html, then change back
+    $$textref =~ s/<a([^>]*)_<span>([^>]*)<\/span>_/<a$1_$2_/isg;	
+    
+    # convert all the urls to relative url, because current wget 1.10 -k and -E option doesn't work together
+    # get rid of the title attribute of a tag
+    $$textref =~ s/<a([^>]*)title="(.*?)"/<a$1/isg;
+    # find the relative path of current directory         
+    if($basedir ne ""){
+    	my @dirs=split("\/", $file);
+	    my $dirnum = scalar(@dirs);
+	    my $replace = "";
+	    for(my $i=0; $i<$dirnum-2; $i++){
+			$replace .= "../";
+	    }
+	    # test if the linked relative file exists, if not, link to the internet version
+    	$$textref =~ s/(href|src)="([^>]*)$basedir\/([^>]*)"/$1="$replace$3"/gi;    	    	
+    	# my @total_links = ($$textref =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/gi);
+    	# print $outhandle "\nnumber of total links: " . scalar(@total_links)."\n";
+    	# for(my $cur_link_no = 0; $cur_link_no < scalar(@total_links); $cur_link_no++){
+    	
+    	#while($$textref =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/gi){
+    		#$total_links[$cur_link_no] =~ m/(href|src)="([^>]*)$basedir\/([^>]*)"/i;
+    	#	my $prefix = $1;
+    	#	my $link = $&; 
+    	#	my $rel_file_name = $3; 		    		
+    	#	my $rel_link = "$replace$rel_file_name";
+    		# print $outhandle "catched link==> $link\nrelative link==> $rel_link\n";
+    	#	if(-e $rel_link){
+    	#		$rel_link = "$prefix=\"$rel_link\"";
+    	#		$$textref =~ s/$link/$rel_link/i;
+    	#	}else{
+    	#		my $ext_link = "$prefix=\"http:\/\/$basedir\/$rel_file_name\"";
+    			# print $outhandle "external link==> $ext_link\n";
+    	#		$$textref =~ s/$link/$ext_link/i; #s/$link/$prefix="http:\/\/$rel_file_name"/i;
+    	#	}
+    	#}   		
+    		
+    	
+    	# tag the link to new wiki pages as red
+    	$$textref =~ s/(href|src)="$replace([^>]*)&amp;action=edit([^>]*)"/$1="http:\/\/$basedir\/$2&amp;action=edit$3"/gi;    	
+    	$$textref =~ s/<a([^>]*)class="new"([^>]*)>/<a$1style="color:red"$2)>/gi;
+    	
+    	# tag the link to external pages as blue
+    	$$textref =~ s/<a([^>]*)class='external text'([^>]*)>/<a$1style="color:blue"$2)>/gi;   	
+    	
+    	#print $outhandle $$textref;
+    }
+    
+	# if 'show_toc' is set, put the table of content on the Wiki Main_Page to the about page of the collection
+	# 1. read _content_ macro from about.dm
+	# 2. append the toc, change all links to the Greenstone internal format for relative links 
+	# 3. write to the extra.dm
+	# TODO: currently we suppose the _about:content_ hasn't been specified before
+	#	    so needs to add function to handle when the macro is already in the extra.dm	
+   	if($self->{'show_toc'}==1 && $file =~ m/Main_Page.(html|htm)$/){
+        my $macro_path = $base_dir;
+    	$macro_path =~ s/import$/macros/;
+    	my $extra_dm;
+    	my $extradm_file = "$macro_path/extra.dm";
+        if(open(INPUT, "<$extradm_file")){    	
+    		while(my $line = <INPUT>){
+			    $extra_dm .= $line;
+   		}
+   			close(INPUT);  		   			  		
+    		
+   			if($extra_dm =~ m/package about/ && $extra_dm =~ m/_content_(\s)*{/){  
+ 				print $outhandle "already changed!!!!\n";   			
+    		} else {
+	    		# read _content_ macro from about.dm file
+    			my $about_macro = $ENV{'GSDLHOME'} . "/macros/about.dm";    			
+   				my $about_page_content = "";
+   				if(open(INPUT, "<$about_macro")){
+   					while(my $line=<INPUT>){
+   						$about_page_content .= $line;
+   					}
+   				}else{
+   					print $outhandle "can't open file $about_macro\n";
+   				}    		
+   				close(INPUT);
+    		    
+   				# extract the _content_ macro
+   				$about_page_content =~ m/_content_ {(.|\n)*<\/div>\n\n<\/div>\n}/i;
+   				$about_page_content = $&;
+    				
+   				# extract toc of the Main_Page
+    			my $mainpage_content = "";
+    			if($self->{'toc_exp'} =~ /\S/){
+    				$$textref =~ /$self->{'toc_exp'}/;
+    				$mainpage_content = $&;
+    			} else {
+    				# $mainpage_content =~ s/<!-- start content -->(.|\n)*<!-- end content -->/$1/igs;
+    			}
+    			# print $outhandle "---------\n$$textref\n--------\n\n";	    		
+    			# print $outhandle "==========\n$mainpage_content\n==========\n\n";
+    				
+    			# add toc to the _content_ macro
+    			$about_page_content =~ m/{(.|\n)*<\/div>\n\n/;
+    			$extra_dm .= "package about\n_content_$&\n\n<div class=\"section\">\n$mainpage_content\n</div>\n</div>\n}";
+    				
+    			# change all links to the internal Greenstone relative link format
+    			$extra_dm =~ s/<a href="([^>]*)"/<a href="_httpquery_&a=extlink&rl=1&href=http:\/\/$basedir$1"/isg;
+   				$extra_dm =~ s/(\.\.\/)+/\//isg;
+   				# print $outhandle "to add---------\n$extra_dm\n--------\n";
+    				   
+   				# write to the extra.dm file of the collection
+   				open(OUTPUT, ">$extradm_file");
+   				print OUTPUT $extra_dm;
+   				close(OUTPUT);    		
+   			}	
+    	} else { 
+   			print $outhandle "can't open file $extradm_file\n";
+   		}
+	}
+	
+	# If delete_toc is enabled, it means to get rid of toc and tof contents.
+    # get rid of TOC and TOF sections and their title
+    if (defined $self->{'delete_toc'} && ($self->{'delete_toc'} == 1)){
+		if (defined $self->{'toc_exp'} && $self->{'toc_exp'} =~ /\S/){
+	    	# $body_text =~ s/<p class=(($self->{'toc_exp'})[^>]*)>(.+?)<\/p>//isg;	    
+	    	# print "it matches toc_exp!!\n" if $body_text =~ /$self->{'toc_exp'}/;	    
+	    	# $body_text =~ s/$self->{'toc_exp'}//i;
+	    	print "it matches toc_exp!!\n" if $$textref =~ /$self->{'toc_exp'}/;	
+	    	$$textref =~ s/$self->{'toc_exp'}//i;
+		}	
+    }
+    
+    # To add a layer on top of the wiki page
+    # so as to keep the wiki style inside the wiki page
+    # and keep the Greenstone style at the same time    
+    $$textref =~ s/<body([^>]*)>/$&\n<div id="wikispecificstyle">\n/is;    
+    $$textref =~ s/<\/body>/<\/div><\/body>/is; 
+    
+    # tag with sections    
+    $$textref =~ s/<body([^>]*)>/$&\n<!--\n<Section>\n<Description>\n<Metadata name=\"Title\">$doctitle<\/Metadata>\n<\/Description>\n-->\n/is;
+    $$textref =~ s/<\/body>/\n<!--\n<\/Section>\n-->\n/is;
+    
+    #print $outhandle "\n\n$$textref\n\n";
+    
+    # use description tags    
+    if ($self->{'description_tags'}) {
+    	my $cursection = $doc_obj->get_top_section();
+		# remove the html header - note that doing this here means any
+		# sections defined within the header will be lost (so all <Section>
+		# tags must appear within the body of the HTML)
+		my ($head_keep) = ($$textref =~ m/^(.*?)<body[^>]*>/is);
+
+		$$textref =~ s/^.*?<body[^>]*>//is;
+		$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+
+		my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
+		my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
+
+		my $lt = '(?:<|&lt;)';
+		my $gt = '(?:>|&gt;)';
+		my $quot = '(?:"|&quot;|&rdquo;|&ldquo;)';
+
+		# my $dont_strip = '';
+		# if ($self->{'no_strip_metadata_html'}) {
+		#    ($dont_strip = $self->{'no_strip_metadata_html'}) =~ s{,}{|}g;
+		# }
+
+		my $found_something = 0; 
+		my $top = 1;
+		while ($$textref =~ s/^(.*?)$opencom(.*?)$closecom//s) {
+	    	my $text = $1;
+	   	 	my $comment = $2;
+	    	if (defined $text) {
+				# text before a comment - note that getting to here
+				# doesn't necessarily mean there are Section tags in
+				# the document
+				# print $outhandle "section text:\n$text\n";
+				$self->process_section(\$text, $base_dir, $file, $doc_obj, $cursection);
+	    	}
+	    	while ($comment =~ s/$lt(.*?)$gt//s) {
+				my $tag = $1;
+				if ($tag eq "Section") {
+		    		$found_something = 1;
+		    		$cursection = $doc_obj->insert_section($doc_obj->get_end_child($cursection)) unless $top;
+		    		$top = 0;
+				} elsif ($tag eq "/Section") {
+		    		$found_something = 1;
+		    		$cursection = $doc_obj->get_parent_section ($cursection);
+				} elsif ($tag =~ /^Metadata name=$quot(.*?)$quot/s) {
+		    		my $metaname = $1;
+		    		my $accumulate = $tag =~ /mode=${quot}accumulate${quot}/ ? 1 : 0;
+		   			$comment =~ s/^(.*?)$lt\/Metadata$gt//s;
+				    my $metavalue = $1;
+				    $metavalue =~ s/^\s+//;
+				    $metavalue =~ s/\s+$//;
+                    # assume that no metadata value intentionally includes
+                    # carriage returns or HTML tags (if they're there they
+                    # were probably introduced when converting to HTML from
+                    # some other format).
+				    # actually some people want to have html tags in their
+				    # metadata.
+				    $metavalue =~ s/[\cJ\cM]/ /sg;
+				    # $metavalue =~ s/<[^>]+>//sg unless $dont_strip && ($dont_strip eq 'all' || $metaname =~ /^($dont_strip)$/);
+				    $metavalue =~ s/\s+/ /sg;
+				    # print $outhandle "metaname = $metaname\nmetavalue = $metavalue\n";
+				    if ($accumulate) {
+						$doc_obj->add_utf8_metadata($cursection, $metaname, $metavalue);
+		    		} else {
+						$doc_obj->set_utf8_metadata_element($cursection, $metaname, $metavalue);	
+		    		}
+				} elsif ($tag eq "Description" || $tag eq "/Description") {
+		    		# do nothing with containing Description tags
+				} else {
+				    # simple HTML tag (probably created by the conversion
+				    # to HTML from some other format) - we'll ignore it and
+				    # hope for the best ;-)
+				}
+	    	}
+		}# end while
+
+		if ($cursection ne "") {
+		    print $outhandle "HTMLPlug: WARNING: $file contains unmatched <Section></Section> tags\n";
+		}
+
+		$$textref =~ s/^.*?<body[^>]*>//is;
+		$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+		if ($$textref =~ /\S/) {
+		    if (!$found_something) {
+				if ($self->{'verbosity'} > 2) {
+				    print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags so\n";
+				    print $outhandle "          will be processed as a single section document\n";
+				}
+	
+				# go ahead and process single-section document
+				$self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
+
+		    } else {
+				print $outhandle "HTMLPlug: WARNING: $file contains the following text outside\n";
+				print $outhandle "          of the final closing </Section> tag. This text will\n";
+				print $outhandle "          be ignored.";
+
+				my ($text);
+				if (length($$textref) > 30) {
+				    $text = substr($$textref, 0, 30) . "...";
+				} else {
+				    $text = $$textref;
+				}
+				$text =~ s/\n/ /isg;
+				print $outhandle " ($text)\n";
+		    }
+		} elsif (!$found_something) {
+		    if ($self->{'verbosity'} > 2) {
+			# may get to here if document contained no valid Section
+			# tags but did contain some comments. The text will have
+			# been processed already but we should print the warning
+			# as above and extract metadata
+			print $outhandle "HTMLPlug: WARNING: $file appears to contain no Section tags and\n";
+			print $outhandle "          is blank or empty.  Metadata will be assigned if present.\n";
+		    }
+		}
+    } # if $self->{'description_tags'}
+	else {
+		# remove header and footer
+		# if (!$self->{'keep_head'}) {
+		#    $$textref =~ s/^.*?<body[^>]*>//is;
+		#    $$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
+		# }
+
+		# single section document
+		# $self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);		
+		
+		# Important: to get the relative links to work,
+		# 1: use the below statement instead of the above one
+		# 2. cannot have process_section method. 
+		# why?????
+		$self->SUPER::process(@_);
+    }
+	return 1;
+	       
+    #$self->SUPER::process(@_);
+}
+
+
+
+# note that process_section may be called multiple times for a single
+# section (relying on the fact that add_utf8_text appends the text to any
+# that may exist already).
+# sub process_section {
+#    my $self = shift (@_);
+#    my ($textref, $base_dir, $file, $doc_obj, $cursection) = @_;
+
+    # trap links
+    # if (!$self->{'nolinks'}) {
+	# usemap="./#index" not handled correctly => change to "#index"
+	# $$textref =~ s/(<img[^>]*?usemap\s*=\s*[\"\']?)([^\"\'>\s]+)([\"\']?[^>]*>)/
+	    #$self->replace_usemap_links($1, $2, $3)/isge;
+
+	#$$textref =~ s/(<(?:a|area|frame|link|script)\s+[^>]*?\s*(?:href|src)\s*=\s*[\"\']?)([^\"\'>\s]+)([\"\']?[^>]*>)/
+	    #$self->replace_href_links ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
+    #}
+
+    # trap images
+
+    # allow spaces if inside quotes - jrm21
+    #$$textref =~ s/(<(?:img|embed|table|tr|td)[^>]*?(?:src|background)\s*=\s*)([\"\'][^\"\']+[\"\']|[^\s>]+)([^>]*>)/
+	#$self->replace_images ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
+
+    # add text to document object
+    # turn \ into \\ so that the rest of greenstone doesn't think there
+    # is an escape code following. (Macro parsing loses them...)
+#    $$textref =~ s/\\/\\\\/go;
+    
+#    $doc_obj->add_utf8_text($cursection, $$textref);
+#}
+
+
+sub extract_metadata 
+{
+    my $self = shift (@_);
+    my ($textref, $metadata, $doc_obj) = @_;
+    my $outhandle = $self->{'outhandle'};
+    
+    return if (!defined $textref);
+
+    # metadata fields to extract/save. 'key' is the (lowercase) name of the
+    # html meta, 'value' is the metadata name for greenstone to use
+    my %find_fields = ();
+    my ($tag,$value);
+
+    my $orig_field = "";
+    foreach my $field (split /,/, $self->{'metadata_fields'}) {
+	# support tag<tagname>
+	if ($field =~ /^(.*?)<(.*?)>$/) {
+	    # "$2" is the user's preferred gs metadata name
+	    $find_fields{lc($1)}=$2; # lc = lowercase
+	    $orig_field = $1;
+	} else { # no <tagname> for mapping
+	    # "$field" is the user's preferred gs metadata name
+	    $find_fields{lc($field)}=$field; # lc = lowercase
+	    $orig_field = $field;
+	}
+
+	if ($textref =~ m/<o:$orig_field>(.*)<\/o:$orig_field>/i){
+	    $tag = $orig_field;
+	    $value = $1;
+	    if (!defined $value || !defined $tag){
+		#print $outhandle "StructuredHTMLPlug: can't find VALUE in \"$tag\"\n";
+		next;
+	    } else {
+		# clean up and add
+		chomp($value); # remove trailing \n, if any
+		$tag = $find_fields{lc($tag)};
+		#print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
+		#    if ($self->{'verbosity'} > 2);
+		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $value);
+	    }
+	}
+    }
+}
+
+1;
