Index: /gsdl/trunk/perllib/marcmapping.pm
===================================================================
--- /gsdl/trunk/perllib/marcmapping.pm	(revision 16692)
+++ /gsdl/trunk/perllib/marcmapping.pm	(revision 16692)
@@ -0,0 +1,146 @@
+###########################################################################
+#
+# marcmapping.pm -- code to read in the marc mapping files
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2008 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package marcmapping;
+
+sub parse_marc_metadata_mapping
+{
+    my ($mm_file_or_files, $outhandle) = @_;
+
+    my $metadata_mapping = {};
+
+    if (ref ($mm_file_or_files) eq 'SCALAR') {
+	my $mm_file = $mm_file_or_files;
+	&_parse_marc_metadata_mapping($mm_file,$metadata_mapping, $outhandle);
+    }
+    else {
+	my $mm_files = $mm_file_or_files;
+	
+	# Need to process files in reverse order.  This is so in the
+	# case where we have both a "collect" and "main" version,
+	# the "collect" one tops up the main one
+
+	my $mm_file;
+	while ($mm_file = pop(@$mm_files)) {
+	    &_parse_marc_metadata_mapping($mm_file,$metadata_mapping, $outhandle);
+	}
+    }
+
+    return $metadata_mapping;
+}
+
+sub _parse_marc_metadata_mapping
+{
+    my ($mm_file,$metadata_mapping, $outhandle) = @_;
+
+    if (open(MMIN, "<$mm_file"))
+    {
+	my $l=0;
+	my $line;
+	while (defined($line=<MMIN>))
+	{
+	    $l++;
+	    chomp $line;
+	    $line =~ s/#.*$//; # strip out any comments, including end of line
+	    next if ($line =~ m/^\s*$/); 
+	    $line =~ s/\s+$//; # remove any white space at end of line
+
+	    my $parse_error_count = 0;
+	    if ($line =~ m/^-(\d+)\s*$/) {
+		# special "remove" rule syntax
+		my $marc_info = $1;
+		if (defined $metadata_mapping->{$marc_info}) {
+		    delete $metadata_mapping->{$marc_info};
+		}
+		else {
+		    print $outhandle "Parse Warning: Did not find pre-existing rule $marc_info to remove";
+		    print $outhandle " on line $l of $mm_file:\n";
+		    print $outhandle "  $line\n";
+		}
+	    }
+	    elsif ($line =~ m/^(.*?)->\s*([\w\^]+)$/)
+	    {
+		my $lhs = $1;
+		my $gsdl_info = $2;
+
+		my @fields = split(/,\s*/,$lhs);
+		my $f;
+		while ($f  = shift (@fields)) {
+		    $f =~ s/\s+$//; # remove any white space at end of line
+
+		    if ($f =~ m/^(\d+)\-(\d+)$/) {
+			# number range => generate number in range and
+			# push on to array
+			push(@fields,$1..$2);
+			next;
+		    }
+
+		    if ($f =~ m/^(\d+)((?:(?:\$|\^)\w)*)\s*$/) {
+
+			my $marc_info = $1;
+			my $opt_sub_fields = $2;
+
+			if ($opt_sub_fields ne "") {			
+			    my @sub_fields = split(/\$|\^/,$opt_sub_fields);
+			    shift @sub_fields; # skip first entry, which is blank
+
+			    foreach my $sub_field (@sub_fields) {
+				$metadata_mapping->{$marc_info."\$".$sub_field} = $gsdl_info;
+			    }
+			}
+			else {
+			    # no subfields to worry about
+			    $marc_info =~ s/\^/\$/;
+			    $metadata_mapping->{$marc_info} = $gsdl_info;
+			}
+		    }
+		    else {
+			$parse_error_count++;
+		    }
+		}
+	    }
+	    else
+	    {
+		$parse_error_count++;
+	    }
+
+	    if ($parse_error_count>0) {
+		
+		print $outhandle "Parse Error: $parse_error_count syntax error(s) on line $l of $mm_file:\n";
+		print $outhandle "  $line\n";
+	    }
+	}
+	close(MMIN);
+    }
+    else
+    {
+	print $outhandle "Unable to open $mm_file: $!\n";
+    }
+}
+
+
+
+1;
Index: /gsdl/trunk/perllib/plugins/MARCPlugin.pm
===================================================================
--- /gsdl/trunk/perllib/plugins/MARCPlugin.pm	(revision 16691)
+++ /gsdl/trunk/perllib/plugins/MARCPlugin.pm	(revision 16692)
@@ -31,4 +31,5 @@
 use unicode;
 use util;
+use marcmapping;
 
 use strict;
@@ -42,7 +43,7 @@
 my $arguments = 
     [ { 'name' => "metadata_mapping",
-	'desc' => "{MARCPlugin.metadata_mapping}",
+	'desc' => "{common.deprecated} {MARCPlugin.metadata_mapping}",
 	'type' => "string",
-	'deft' => "marctodc.txt",
+	'deft' => "",
 	'hiddengli' => "yes", # deprecated in favour of 'metadata_mapping_file'
 	'reqd' => "no" },
@@ -50,5 +51,5 @@
 	'desc' => "{MARCXMLPlugin.metadata_mapping_file}",
 	'type' => "string",
-	'deft' => "",
+	'deft' => "marctodc.txt",
 	'reqd' => "no" },
       { 'name' => "process_exp",
@@ -99,10 +100,10 @@
     # preference
 
-    if ($self->{'metadata_mapping_file'} eq "") {
-	# If nothing set in the new version, use the old version
-	# that defaults to 'marctodc.txt'
+    if ($self->{'metadata_mapping'} ne "") {
+	print STDERR "MARCPlugin WARNING:: the metadata_mapping option is set but has been deprecated. Please use metadata_mapping_file option instead\n";
+	# If the old version is set, use it. 
 	$self->{'metadata_mapping_file'} = $self->{'metadata_mapping'};
     }
-
+    $self->{'metadata_mapping'} = undef;
     $self->{'type'} = "";
     
@@ -114,15 +115,17 @@
     my ($verbosity, $outhandle, $failhandle) = @_;
 
-    my @metadata_mapping = ();
-
-    # read in the metadata mapping file
-
-    my $mm_file = &util::locate_config_file($self->{'metadata_mapping_file'}); 
-
-    if (!defined $mm_file)
-    {
-
+    ## the mapping file has already been loaded
+    if (defined $self->{'metadata_mapping'} ){ 
+	$self->SUPER::init(@_);
+	return;
+    }
+
+    # read in the metadata mapping files
+    my $mm_files = &util::locate_config_files($self->{'metadata_mapping_file'});
+
+    if (scalar(@$mm_files)==0)
+    {
 	my $msg = "MARCPlugin ERROR: Can't locate mapping file \"" .
-	    $self->{'metadata_mapping_file'} . "\".\n" .
+	    $self->{'metadata_mapping_file'} . "\".\n " .
 		"    No marc files can be processed.\n";
 
@@ -133,40 +136,14 @@
 	# If we exit here, then pluginfo.pl will exit too!
     }
-    elsif (open(MMIN, "<$mm_file"))
-    {
-	my $l=1;
-	my $line;
-	while (defined($line=<MMIN>))
-	{
-	    chomp $line;
-	    if ($line =~ m/^(\d+)\s*->\s*([\w\^]+)$/)
-	    {
-		my $marc_info = $1;
-		my $gsdl_info = $2;
-		my $mapping = { 'marc' => $marc_info, 'gsdl' => $gsdl_info };
-		push(@metadata_mapping,$mapping);
-	    }
-	    elsif ($line !~ m/^\#/       # allow comments (# in first column)
-		   && $line !~ m/^\s*$/) # allow blank lines
-	    {
-		print $outhandle "Parse error on line $l of $mm_file:\n";
-		print $outhandle "  \"$line\"\n";
-	    }
-	    $l++
-	}
-	close(MMIN);
-
-	$self->{'metadata_mapping'} = \@metadata_mapping;
-    }
-    else
-    {
-	print STDERR "Unable to open $mm_file: $!\n";
-    }
-
-
+    else {
+	$self->{'metadata_mapping'} = &marcmapping::parse_marc_metadata_mapping($mm_files, $outhandle);
+    }
+
+    ##map { print STDERR $_."=>".$self->{'metadata_mapping'}->{$_}."\n"; } keys %{$self->{'metadata_mapping'}};
 
     $self->SUPER::init(@_);
 }
-    
+
+
 
 sub get_default_process_exp {
@@ -340,7 +317,8 @@
 
 
-sub extract_metadata 
+sub extract_metadata
 {
     my $self = shift (@_);
+ 
     my ($marc, $metadata, $encoding, $doc_obj, $section) = @_;
     my $outhandle = $self->{'outhandle'};
@@ -350,29 +328,48 @@
     }
 
-    my $metadata_mapping = $self->{'metadata_mapping'};
-    my $mm;
-
-    foreach $mm ( @$metadata_mapping )
-    {
-	my $marc_field = $mm->{'marc'};
-
-	my @metavalues = $marc->field($marc_field);
-
-	if (scalar(@metavalues)>0)
-	{
-	    my $metaname = $mm->{'gsdl'};
-	    my $metavalue;
-	    foreach $metavalue ( @metavalues )
+    my $metadata_mapping = $self->{'metadata_mapping'};;
+
+    foreach my $marc_field ( keys %$metadata_mapping )
+    {
+	my $gsdl_field = $metadata_mapping->{$marc_field};
+	my $meta_value = undef;
+	
+	# have we got a subfield?
+	my $subfield = undef;
+	if ($marc_field =~ /(\d\d\d)(?:\$|\^)?(\w)/){
+	    $marc_field = $1;
+	    $subfield = $2;
+	}
+	if (defined $subfield) {
+	    $meta_value = $marc->subfield($marc_field, $subfield);
+	    if (defined $meta_value) {
+		## escape [ and ]
+		$meta_value =~ s/\[/\\\[/g;
+		$meta_value =~ s/\]/\\\]/g;
+		my $metavalue_str = $self->to_utf8($encoding, $meta_value);
+		$doc_obj->add_utf8_metadata ($section, $gsdl_field, $metavalue_str);
+	    }
+	} else {
+	    
+	    my @metavalues = $marc->field($marc_field);
+	    
+	    if (scalar(@metavalues)>0)
 	    {
-		my $metavalue_str = $self->to_utf8($encoding,$metavalue->as_string());
-		$doc_obj->add_utf8_metadata ($section, $metaname, $metavalue_str);
-	    }
-	}
-    }
-}
+		my $metavalue = undef;
+		foreach $metavalue ( @metavalues )
+		{
+		    my $metavalue_str = $self->to_utf8($encoding,$metavalue->as_string());
+		    $doc_obj->add_utf8_metadata ($section, $gsdl_field, $metavalue_str);
+		}
+	    }
+	}
+    }
+}
+
 
 sub extract_ascii_metadata 
 {
     my $self = shift (@_);
+
     my ($text, $metadata,$doc_obj, $section) = @_;
     my $outhandle = $self->{'outhandle'};
@@ -386,5 +383,5 @@
 	    $field =~ /^(\d\d\d)\s/;
 	    my $code = $1;
-	    $field = $';
+	    $field = $'; #'
 	    ##get subfields
 	    my @subfields = split(/\$/,$field);
@@ -403,5 +400,5 @@
 		     ##print STDERR "$1=>$'\n";
 		     push(@{$marc_mapping->{$code}},$1);
-                     push(@{$marc_mapping->{$code}},$'); 
+                     push(@{$marc_mapping->{$code}},$'); #'
 		 }
 	    }
@@ -410,7 +407,6 @@
 
 
-     foreach my $mm ( @$metadata_mapping )
-    {
-	my $marc_field = $mm->{'marc'};
+     foreach my $marc_field ( keys %$metadata_mapping )
+    {
 		
 	my $matched_field = $marc_mapping->{$marc_field};
@@ -422,5 +418,5 @@
 		$subfield = $1;
 	    }
-	    my $metaname = $mm->{'gsdl'};
+	    my $metaname = $metadata_mapping->{$marc_field};
  
 	    my $metavalue;
Index: /gsdl/trunk/perllib/plugins/MARCXMLPlugin.pm
===================================================================
--- /gsdl/trunk/perllib/plugins/MARCXMLPlugin.pm	(revision 16691)
+++ /gsdl/trunk/perllib/plugins/MARCXMLPlugin.pm	(revision 16692)
@@ -31,4 +31,5 @@
 
 use ReadXMLFile;
+use marcmapping;
 
 use strict;
@@ -85,125 +86,4 @@
 
 
-sub _parse_marc_metadata_mapping
-{
-    my $self = shift(@_);
-    my ($mm_file,$metadata_mapping) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    if (open(MMIN, "<$mm_file"))
-    {
-	my $l=0;
-	my $line;
-	while (defined($line=<MMIN>))
-	{
-	    $l++;
-	    chomp $line;
-	    $line =~ s/#.*$//; # strip out any comments, including end of line
-	    next if ($line =~ m/^\s*$/); 
-	    $line =~ s/\s+$//; # remove any white space at end of line
-
-	    my $parse_error_count = 0;
-	    if ($line =~ m/^-(\d+)\s*$/) {
-		# special "remove" rule syntax
-		my $marc_info = $1;
-		if (defined $metadata_mapping->{$marc_info}) {
-		    delete $metadata_mapping->{$marc_info};
-		}
-		else {
-		    print $outhandle "Parse Warning: Did not file pre-existing rule $marc_info to remove";
-		    print $outhandle " on line $l of $mm_file:\n";
-		    print $outhandle "  $line\n";
-		}
-	    }
-	    elsif ($line =~ m/^(.*?)->\s*([\w\^]+)$/)
-	    {
-		my $lhs = $1;
-		my $gsdl_info = $2;
-
-		my @fields = split(/,\s*/,$lhs);
-		my $f;
-		while ($f  = shift (@fields)) {
-		    $f =~ s/\s+$//; # remove any white space at end of line
-
-		    if ($f =~ m/^(\d+)\-(\d+)$/) {
-			# number range => genrate number in range and
-			# push on to array
-			push(@fields,$1..$2);
-			next;
-		    }
-
-		    if ($f =~ m/^(\d+)((?:(?:\$|\^)\w)*)\s*$/) {
-
-			my $marc_info = $1;
-			my $opt_sub_fields = $2;
-
-			if ($opt_sub_fields ne "") {			
-			    my @sub_fields = split(/\$|\^/,$opt_sub_fields);
-			    shift @sub_fields; # skip first entry, which is blank
-
-			    foreach my $sub_field (@sub_fields) {
-				$metadata_mapping->{$marc_info."\$".$sub_field} = $gsdl_info;
-			    }
-			}
-			else {
-			    # no subfields to worry about
-			    $marc_info =~ s/\^/\$/;
-			    $metadata_mapping->{$marc_info} = $gsdl_info;
-			}
-		    }
-		    else {
-			$parse_error_count++;
-		    }
-		}
-	    }
-	    else
-	    {
-		$parse_error_count++;
-	    }
-
-	    if ($parse_error_count>0) {
-		
-		print $outhandle "Parse Error: $parse_error_count syntax error(s) on line $l of $mm_file:\n";
-		print $outhandle "  $line\n";
-	    }
-	}
-	close(MMIN);
-    }
-    else
-    {
-	print STDERR "Unable to open $mm_file: $!\n";
-    }
-}
-
-
-sub parse_marc_metadata_mapping
-{
-    my $self = shift(@_);
-    my ($mm_file_or_files) = @_;
-
-    my $metadata_mapping = {};
-
-    if (ref ($mm_file_or_files) eq 'SCALAR') {
-	my $mm_file = $mm_file_or_files;
-	$self->_parse_marc_metadata_mapping($mm_file,$metadata_mapping);
-    }
-    else {
-	my $mm_files = $mm_file_or_files;
-	
-	# Need to process files in reverse order.  This is so in the
-	# case where we have both a "collect" and "main" version,
-	# the "collect" one tops up the main one
-
-	my $mm_file;
-	while ($mm_file = pop(@$mm_files)) {
-	    $self->_parse_marc_metadata_mapping($mm_file,$metadata_mapping);
-	}
-    }
-
-    return $metadata_mapping;
-}
-
-
 sub init {
     my $self = shift (@_);
@@ -216,5 +96,5 @@
     }
 
-    # read in the metadata mapping file
+    # read in the metadata mapping files
     my $mm_files = &util::locate_config_files($self->{'metadata_mapping_file'}); 
 
@@ -233,5 +113,5 @@
     }
     else {
-	$self->{'metadata_mapping'} = $self->parse_marc_metadata_mapping($mm_files);
+	$self->{'metadata_mapping'} = &marcmapping::parse_marc_metadata_mapping($mm_files, $outhandle);
     }
 
@@ -316,5 +196,5 @@
 	}
 	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "XML");
+	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "MARCXML");
 
 	my $outhandle = $self->{'outhandle'};
