Index: /trunk/gsdl/perllib/doc.pm
===================================================================
--- /trunk/gsdl/perllib/doc.pm	(revision 2809)
+++ /trunk/gsdl/perllib/doc.pm	(revision 2810)
@@ -155,122 +155,42 @@
 }
 
-
-sub buffer_section_gml {
-    my $self = shift (@_);
-    my ($section, $suppress_subject_info) = @_;
-
-    $suppress_subject_info = 0 unless defined $suppress_subject_info;
-    my ($all_text,$data, $subsection);
-    
+sub buffer_section_xml {
+    my $self = shift (@_);
+    my ($section) = @_;
+
     my $section_ptr = $self->_lookup_section ($section);
-    my ($section_num) = $section =~ /(\d+)$/;
-    
     return "" unless defined $section_ptr;
 
-    # output the section header (including the section number
-    # and metadata)
-
-    $all_text = "<gsdlsection";
-    $all_text .= " gsdlnum=\"$section_num\"" if defined $section_num;
-    foreach $data (@{$section_ptr->{'metadata'}}) {
-	$all_text .= " $data->[0]=\"" . &_escape_text($data->[1]) . "\"" 
-	    unless $suppress_subject_info && $data->[0] eq "Subject";
-    }
-    $all_text .= ">";
+    my $all_text = "<Section>\n";
+    $all_text .= "  <Description>\n";
+    
+    # output metadata
+    foreach my $data (@{$section_ptr->{'metadata'}}) {
+	my $escaped_value = &_escape_text($data->[1]);
+	$all_text .= '    <Metadata name="' . $data->[0] . '">' . $escaped_value . "</Metadata>\n";
+    }
+
+    $all_text .= "  </Description>\n";
 
     # output the text
+    $all_text .= "  <Content>\n";
     $all_text .= &_escape_text($section_ptr->{'text'});
-
+    $all_text .= "  </Content>\n";
+    
     # output all the subsections
-    foreach $subsection (@{$section_ptr->{'subsection_order'}}) {
-	$all_text .= $self->buffer_section_gml("$section.$subsection", 
-					       $suppress_subject_info);
-    }
-    
-    # output the closing tag
-    $all_text .=  "</gsdlsection>\n";
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
+	$all_text .= $self->buffer_section_xml("$section.$subsection");
+    }
+    
+    $all_text .=  "</Section>\n";
 
     return $all_text;
 }
 
-
-sub buffer_section_xml {
-    my $self = shift (@_);
-    my ($section, $dtd_metadata, $suppress_subject_info) = @_;
-
-    $suppress_subject_info = 0 unless defined $suppress_subject_info;
-    my ($all_text, $data, $subsection);
-    
-    my $section_ptr = $self->_lookup_section ($section);
-    my ($section_num) = $section =~ /(\d+)$/;
-    
-    return "" unless defined $section_ptr;
-
-    # output the section header (including the section number
-    # and metadata)
-
-    $all_text .= "<gsdlsection";
-    $all_text .= " gsdlnum=\"$section_num\"" if defined $section_num;
-    $all_text .= ">\n";
-    
-    $all_text .= "  <metadata>\n";
-    
-    # output metadata
-    foreach $data (@{$section_ptr->{'metadata'}}) {
-	my $tag_name = $data->[0];
-
-	# a tagname beginning with '/' (like </srclink>) will cause problems
-	# so we'll escape any leading '/'
-	$tag_name =~ s/^\//&\#47;/; 
-
-	my $tag_value = &_escape_text($data->[1]);
-
-	unless ($suppress_subject_info && $tag_name eq "Subject")
-	{
-	    if (defined $dtd_metadata)
-	    {
-		$dtd_metadata->{$tag_name}++;
-	    }
-
-	    $all_text .= "    <$tag_name>$tag_value</$tag_name>\n";
-	}
-    }
-
-    $all_text .= "  </metadata>\n";
-
-    # output the text
-    $all_text .= &_escape_text($section_ptr->{'text'});
-
-    # output all the subsections
-    foreach $subsection (@{$section_ptr->{'subsection_order'}}) {
-	$all_text .= $self->buffer_section_xml("$section.$subsection", 
-					       $dtd_metadata,
-					       $suppress_subject_info);
-    }
-    
-    # output the closing tag
-    $all_text .=  "</gsdlsection>\n";
-
-    return $all_text;
-}
-
 sub output_section {
     my $self = shift (@_);
-    my ($handle, $section, $colname, $dtd_metadata, 
-	$suppress_subject_info) = @_;
-
-    my $all_text = $self->buffer_section_xml($section, $dtd_metadata, 
-					     $suppress_subject_info);
-
-    # xml header    
-    if (defined $collection)
-    {
-	my $xml_head
-	    = '<? xml version="1.0" standalone="no" encoding="UTF-8" ?>';
-	$xml_head .= "\n<!DOCTYPE gsdl:$colname SYSTEM \"$colname.dtd\">\n";
-	$all_text = $xml_head.$all_text;
-    }
-
-    print $handle $all_text;
+    my ($handle, $section) = @_;
+
+    print $handle $self->buffer_section_xml($section);
 }
 
Index: /trunk/gsdl/perllib/docsave.pm
===================================================================
--- /trunk/gsdl/perllib/docsave.pm	(revision 2809)
+++ /trunk/gsdl/perllib/docsave.pm	(revision 2810)
@@ -25,5 +25,5 @@
 
 # This document processor saves a document in the 
-# archives directory of a collection
+# archives directory of a collection (as xml)
 
 
@@ -53,8 +53,4 @@
     $self->{'groupsize'} = $groupsize;
     $self->{'gs_count'} = 0;
-
-    # keep an associate array of all metavalues used by collection to
-    # help generate the XML DTD 
-    $self->{'dtd_metadata'} = {}; 
 
     $self->{'outhandle'} = STDERR;
@@ -93,5 +89,5 @@
 	
     } else {
-	# groupsize is 1 (i.e. one document per GML file) so sortmeta 
+	# groupsize is 1 (i.e. one document per XML file) so sortmeta 
 	# may be used
 	
@@ -107,6 +103,6 @@
 	
 	my $doc_file 
-	    = &util::filename_cat ($self->{'archive_dir'}, $doc_dir, "doc.gml");
-	my $short_doc_file = &util::filename_cat ($doc_dir, "doc.gml");
+	    = &util::filename_cat ($self->{'archive_dir'}, $doc_dir, "doc.xml");
+	my $short_doc_file = &util::filename_cat ($doc_dir, "doc.xml");
 	
 	if (!open (OUTDOC, ">$doc_file")) {
@@ -116,9 +112,7 @@
 
 	# save this document
-	$doc_obj->output_section('docsave::OUTDOC', 
-				 $doc_obj->get_top_section(),
-				 $self->{'collection'},
-				 $self->{'dtd_metadata'},0);
-
+	$self->output_xml_header('docsave::OUTDOC');
+	$doc_obj->output_section('docsave::OUTDOC', $doc_obj->get_top_section());
+	$self->output_xml_footer('docsave::OUTDOC');
 	close OUTDOC;
 
@@ -172,6 +166,6 @@
 	    # only if opening new file
 	    my $doc_file 
-		= &util::filename_cat ($self->{'archive_dir'}, $doc_dir, "doc.gml");
-	    my $short_doc_file = &util::filename_cat ($doc_dir, "doc.gml");
+		= &util::filename_cat ($self->{'archive_dir'}, $doc_dir, "doc.xml");
+	    my $short_doc_file = &util::filename_cat ($doc_dir, "doc.xml");
 	    
 	    if ($gs_count>0)
@@ -187,12 +181,11 @@
 	    $self->{'gs_short_filename'} = $short_doc_file;
 	    $self->{'gs_OID'} = $OID;
+
+	    $self->output_xml_header('docsave::OUTDOC');
 	}
     }
 
     # save this document
-    $doc_obj->output_section('docsave::OUTDOC', 
-			     $doc_obj->get_top_section(),
-			     $self->{'collection'},
-			     $self->{'dtd_metadata'},0);
+    $doc_obj->output_section('docsave::OUTDOC', $doc_obj->get_top_section());
 
     $self->{'gs_count'}++;
@@ -210,5 +203,5 @@
 	# same one.
 	$doc_dir = $doc_info->[0];
-	$doc_dir =~ s/\/?doc\.gml(\.gz)?$//;
+	$doc_dir =~ s/\/?doc\.xml(\.gz)?$//;
     } else {
 	# have to get a new document directory
@@ -264,5 +257,6 @@
 {
     my ($self) = @_;
-    
+
+    $self->output_xml_footer('docsave::OUTDOC');    
     close OUTDOC;
 
@@ -288,3 +282,20 @@
 }
 
+sub output_xml_header {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    print $handle '<?xml version="1.0" encoding="UTF-8" standalone="no"?>' . "\n";
+    print $handle '<!DOCTYPE GreenstoneArchive SYSTEM ' .
+	'"http://greenstone.org/dtd/GreenstoneArchive/1.0/GreenstoneArchive.dtd">' . "\n";
+    print $handle '<GreenstoneArchive>' . "\n";
+}
+
+sub output_xml_footer {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    print $handle '</GreenstoneArchive>' . "\n";
+}
+
 1;
Index: /trunk/gsdl/perllib/plugins/GAPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/GAPlug.pm	(revision 2810)
+++ /trunk/gsdl/perllib/plugins/GAPlug.pm	(revision 2810)
@@ -0,0 +1,155 @@
+###########################################################################
+#
+# GAPlug.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Processes GreenstoneArchive XML documents. Note that this plugin does no
+# syntax checking (though the XML::Parser module tests for
+# well-formedness). It's assumed that the GreenstoneArchive files conform
+# to their DTD.
+
+package GAPlug;
+
+use XMLPlug;
+
+sub BEGIN {
+    @ISA = ('XMLPlug');
+}
+
+sub new {
+    my $class = shift (@_);
+    my $self = new XMLPlug ($class, @_);
+
+    $self->{'section'} = "";
+    $self->{'section_level'} = 0;
+    $self->{'metadata_name'} = "";
+    $self->{'metadata_value'} = "";
+    $self->{'content'} = "";
+
+    return bless $self, $class;
+}
+
+sub xml_start_document {
+}
+
+sub xml_end_document {
+}
+
+sub xml_doctype {
+    my $self = shift(@_);
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+
+    die if ($name ne "GreenstoneArchive");
+    my $outhandle = $self->{'outhandle'};
+    print $outhandle "GAPLug: processing $self->{'file'}\n";
+}
+
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    $self->{'element'} = $element;
+    if ($element eq "Section") {
+	if ($self->{'section_level'} == 0) {
+	    $self->open_document();
+	} else {
+	    my $doc_obj = $self->{'doc_obj'};
+	    $self->{'section'} = 
+		$doc_obj->insert_section($doc_obj->get_end_child($self->{'section'}));
+	}
+	$self->{'section_level'} ++;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'metadata_name'} = $_{'name'};
+    }
+}
+
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    if ($element eq "Section") {
+	$self->{'section_level'} --;
+	$self->{'section'} = $self->{'doc_obj'}->get_parent_section ($self->{'section'});
+	$self->close_document() if $self->{'section_level'} == 0;
+    }
+    elsif ($element eq "Metadata") {
+	$self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, $self->{'metadata_name'},
+					      $self->{'metadata_value'});
+	$self->{'metadata_name'} = "";
+	$self->{'metadata_value'} = "";
+    }
+    elsif ($element eq "Content" && $self->{'content'} ne "") {
+	$self->{'doc_obj'}->add_utf8_text($self->{'section'}, $self->{'content'});
+	$self->{'content'} = "";
+    }
+    
+    $self->{'element'} = "";
+}
+
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    if ($self->{'element'} eq "Metadata") {
+	$self->{'metadata_value'} .= $_;
+    }
+    elsif ($self->{'element'} eq "Content") {
+	$self->{'content'} .= $_;
+    }
+}
+
+sub open_document {
+    my $self = shift(@_);
+
+    # create a new document
+    $self->{'doc_obj'} = new doc ();
+    $self->{'section'} = "";
+}
+
+sub close_document {
+    my $self = shift(@_);
+    
+    # add the associated files
+    my $assoc_files = 
+	$self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    my $parent_dir = $self->{'filename'};
+    $parent_dir =~ s/[^\\\/]*$//;
+
+    foreach my $assoc_file_info (@$assoc_files) {
+	my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
+	my $real_dir = &util::filename_cat($parent_dir, $assoc_file),
+	my $assoc_dir = (defined $dir && $dir ne "") 
+	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
+	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
+    }
+    $self->{'doc_obj'}->delete_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
+
+    # process the document
+    $self->{'processor'}->process($self->{'doc_obj'}, $self->{'file'});
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/XMLPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/XMLPlug.pm	(revision 2810)
+++ /trunk/gsdl/perllib/plugins/XMLPlug.pm	(revision 2810)
@@ -0,0 +1,224 @@
+###########################################################################
+#
+# XMLPlug.pm -- base class for XML plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package XMLPlug;
+
+use BasPlug;
+use doc;
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+}
+
+use XML::Parser;
+use XML::Parser::Expat;
+
+my ($self);
+sub new {
+    my $class = shift (@_);
+
+    # $self is global for use within subroutines called by XML::Parser
+    $self = new BasPlug ($class, @_);
+
+    my $parser = new XML::Parser('Style' => 'Stream',
+				 'Handlers' => {'Char' => \&Char,
+						'XMLDecl' => \&XMLDecl,
+						'Doctype' => \&Doctype,
+						'Default' => \&Default
+						}
+				 );
+    $self->{'parser'} = $parser;
+
+    return bless $self, $class;
+}
+
+
+sub read {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return 0;
+    }
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	return undef;
+    }
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+    $self->{'file'} = $file;
+    $self->{'filename'} = $filename;
+    $self->{'processor'} = $processor;
+    $self->{'metadata'} = $metadata;
+
+    eval {
+	$self->{'parser'}->parsefile($filename);
+    };
+    if ($@) {
+
+	# parsefile may either croak somewhere in XML::Parser (e.g. because
+	# the document is not well formed) or die somewhere in XMLPlug or a
+	# derived plugin (e.g. because we're attempting to process a
+	# document whose DOCTYPE is not meant for this plugin). For the
+	# first case we'll print a warning and continue, for the second
+	# we'll just continue quietly
+
+	my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
+	if (defined $msg) {	
+	    my $outhandle = $self->{'outhandle'};
+	    my $plugin_name = ref ($self);
+	    print $outhandle "$plugin_name failed to process $file ($msg)\n";
+	}
+
+	print $@;
+
+	return undef;
+    }
+
+    return 1; # processed the file
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.xml$^;
+}
+
+sub StartDocument {$self->xml_start_document(@_);}
+sub XMLDecl {$self->xml_xmldecl(@_);}
+sub Doctype {$self->xml_doctype(@_);}
+sub StartTag {$self->xml_start_tag(@_);}
+sub EndTag {$self->xml_end_tag(@_);}
+sub Text {$self->xml_text(@_);}
+sub PI {$self->xml_pi(@_);}
+sub EndDocument {$self->xml_end_document(@_);}
+sub Default {$self->xml_default(@_);}
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {
+  $_[0]->{'Text'} .= $_[1];
+  return undef;
+}
+
+# Called at the beginning of the XML document.
+sub xml_start_document {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    $self->open_document();
+}
+
+# Called for XML declarations
+sub xml_xmldecl {
+    my $self = shift(@_);
+    my ($expat, $version, $encoding, $standalone) = @_;
+}
+
+# Called for DOCTYPE declarations - use die to bail out if this doctype
+# is not meant for this plugin
+sub xml_doctype {
+    my $self = shift(@_);
+    my ($expat, $name, $sysid, $pubid, $internal) = @_;
+    die "XMLPlug Cannot process XML document with DOCTYPE of $name";
+}
+
+# Called for every start tag. The $_ variable will contain a copy of the
+# tag and the %_ variable will contain the element's attributes.
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+}
+
+# Called for every end tag. The $_ variable will contain a copy of the tag.
+sub xml_end_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+}
+
+# Called just before start or end tags with accumulated non-markup text in
+# the $_ variable.
+sub xml_text {
+    my $self = shift(@_);
+    my ($expat) = @_;
+}
+
+# Called for processing instructions. The $_ variable will contain a copy
+# of the pi.
+sub xml_pi {
+    my $self = shift(@_);
+    my ($expat, $target, $data) = @_;
+}
+
+# Called at the end of the XML document.
+sub xml_end_document {
+    my $self = shift(@_);
+    my ($expat) = @_;
+
+    $self->close_document();
+}
+
+# Called for any characters not handled by the above functions.
+sub xml_default {
+    my $self = shift(@_);
+    my ($expat, $text) = @_;
+}
+
+sub open_document {
+    my $self = shift(@_);
+
+    # create a new document
+    $self->{'doc_obj'} = new doc ($self->{'filename'}, "indexed_doc");
+    $self->{'doc_obj'}->set_OIDtype ($self->{'processor'}->{'OIDtype'});
+}
+
+sub close_document {
+    my $self = shift(@_);
+
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($self->{'doc_obj'}, 
+			   $self->{'doc_obj'}->get_top_section(), 
+			   $self->{'metadata'});
+   
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($self->{'doc_obj'});
+   
+    # add an OID
+    $self->{'doc_obj'}->set_OID();
+    
+    # process the document
+    $self->{'processor'}->process($self->{'doc_obj'});
+    
+    $self->{'num_processed'} ++;
+}
+
+1;
+
