Index: /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 24289)
+++ /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 24290)
@@ -95,4 +95,6 @@
       { 'name' => "hash",
         'desc' => "{import.OIDtype.hash}" },
+      { 'name' => "hash_on_ga_xml",
+        'desc' => "{import.OIDtype.hash_on_ga_xml}" },
       { 'name' => "assigned",
         'desc' => "{import.OIDtype.assigned}" },
Index: /main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm	(revision 24289)
+++ /main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm	(revision 24290)
@@ -289,5 +289,5 @@
     print STDERR "calling cmd $cmd\n";
     $output_type = `$cmd`;
-
+	
     # remove symbolic link to original file
     &util::rm($tmp_filename);
@@ -368,5 +368,5 @@
 	}	
     }
-
+	
     my $secondary_plugins =  $self->{'secondary_plugins'};
     my $num_secondary_plugins = scalar(keys %$secondary_plugins);
@@ -488,6 +488,6 @@
      my $tmp_dir = $self->{'tmp_dir'};
      if (defined $tmp_dir && -d $tmp_dir) {
-##	 print STDERR "**** Supressing clean up of tmp dir\n";
-	 &util::rm_r($tmp_dir);
+	 print STDERR "**** Supressing clean up of tmp dir\n";
+	 ##&util::rm_r($tmp_dir);
 	 $self->{'tmp_dir'} = undef;
      }
Index: /main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm	(revision 24289)
+++ /main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm	(revision 24290)
@@ -39,32 +39,48 @@
 sub BEGIN
 {
-  @EmbeddedMetadataPlugin::ISA = ('BasePlugin');
-  binmode(STDERR, ":utf8");
-}
-
-my $encoding_plus_auto_list = 
-    [ { 'name' => "auto",
-	'desc' => "{ReadTextFile.input_encoding.auto}" } ];
+	@EmbeddedMetadataPlugin::ISA = ('BasePlugin');
+	binmode(STDERR, ":utf8");
+}
+
+my $encoding_plus_auto_list = [{ 
+		'name' => "auto",
+		'desc' => "{ReadTextFile.input_encoding.auto}" }];
 push(@{$encoding_plus_auto_list},@{$BasePlugin::encoding_list});
 
-my $arguments =
-    [ { 'name' => "metadata_field_separator",
+my $arguments = [{ 
+	'name' => "metadata_field_separator",
 	'desc' => "{HTMLPlugin.metadata_field_separator}",
 	'type' => "string",
-	'deft' => "" },
-	  { 'name' => "input_encoding",
+	'deft' => "" 
+	},{ 
+	'name' => "input_encoding",
 	'desc' => "{ReadTextFile.input_encoding}",
 	'type' => "enum",
 	'list' => $encoding_plus_auto_list,
 	'reqd' => "no",
-	'deft' => "auto" }
-];
-
-
-my $options = { 'name'     => "EmbeddedMetadataPlugin",
-		'desc'     => "{EmbeddedMetadataPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
+	'deft' => "auto" 
+	},{
+	'name' => "join_before_split",
+	'desc' => "{EmbeddedMetadataPlugin.join_before_split}",
+	'type' => "flag"
+	},{
+	'name' => "join_character",
+	'desc' => "{EmbeddedMetadataPlugin.join_character}",
+	'type' => "string",
+	'deft' => " "
+	},{
+	'name' => "trim_whitespace",
+	'desc' => "{EmbeddedMetadataPlugin.trim_whitespace}",
+	'type' => "enum",
+	'list' => [{'name' => "true", 'desc' => "{common.true}"}, {'name' => "false", 'desc' => "{common.false}"}],
+	'deft' => "true"
+	}];
+
+my $options = { 
+	'name'     => "EmbeddedMetadataPlugin",
+	'desc'     => "{EmbeddedMetadataPlugin.desc}",
+	'abstract' => "no",
+	'inherits' => "yes",
+	'args'     => $arguments };
 
 sub new()
@@ -79,13 +95,11 @@
     my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
 
-
     # Create a new Image::ExifTool object
     my $exifTool = new Image::ExifTool;
     $exifTool->Options(Duplicates => 0);
-    $exifTool->Options(PrintConv => 0);
+	$exifTool->Options(PrintConv => 0);
     $exifTool->Options(Unknown => 1);
     $exifTool->Options('Verbose');
     $self->{'exiftool'} = $exifTool;
-
 
     return bless $self, $class;
@@ -122,88 +136,118 @@
     my $separator = $self->{'metadata_field_separator'};
     if ($separator eq "") {
-	undef $separator;
+		undef $separator;
     }
 
     my @group_list = Image::ExifTool::GetAllGroups(0);
-    foreach my $group (@group_list)
-      {
-##	  print STDERR "**** group = $group\n";
-
-        # Extract meta information from an image
-        $self->{'exiftool'}->Options(Group0 => [$group]);
-        $self->{'exiftool'}->ExtractInfo($filename);
-
-        # Get list of tags in the order they were found in the file
-        my @tag_list = $self->{'exiftool'}->GetFoundTags('File');
-        foreach my $tag (@tag_list)
-          {
-###	      print STDERR "**** tag = $tag\n";
+    foreach my $group (@group_list) {
+##	print STDERR "**** group = $group\n";
+
+		# Extract meta information from an image
+		$self->{'exiftool'}->Options(Group0 => [$group]);
+		$self->{'exiftool'}->ExtractInfo($filename);
+
+		# Get list of tags in the order they were found in the file
+		my @tag_list = $self->{'exiftool'}->GetFoundTags('File');
+		foreach my $tag (@tag_list) {
 
             # Strip any numbering suffix
-            $tag =~ s/^([^\s]+)\s.*$/$1/i;
-            my $value = $self->{'exiftool'}->GetValue($tag);
-            if (defined $value && $value =~ /[a-z0-9]+/i) {
-
-		my $field = "ex.$group.$tag";
+			$tag =~ s/^([^\s]+)\s.*$/$1/i;
+			my $value = $self->{'exiftool'}->GetValue($tag);
+			if (defined $value && $value =~ /[a-z0-9]+/i) {
+				my $field = "ex.$group.$tag";
 		
-			my $encoding = $self->{'input_encoding'};
-			if($encoding eq "auto")
-			{
-				$encoding = "utf8"
+				my $encoding = $self->{'input_encoding'};
+				if($encoding eq "auto")
+				{
+					$encoding = "utf8"
+				}
+
+				if (!defined $exif_metadata{$field})
+				{
+					$exif_metadata{$field} = [];
+				}
+
+				$field = Encode::decode($encoding,$field);
+				my $metadata_done = 0;
+				if (ref $value eq 'SCALAR') {
+					if ($$value =~ /^Binary data/) {
+						$value = "($$value)";
+					}
+                    else {
+						my $len = length($$value);
+						$value = "(Binary data $len bytes)";
+					}
+				}
+				elsif (ref $value eq 'ARRAY') {
+					$metadata_done = 1;
+					
+					my $allvals = "";
+					foreach my $v (@$value) {
+						$v = Encode::decode($encoding,$v);
+						
+						if(!$self->{'join_before_split'}){
+							if (defined $separator) {
+								my @vs = split($separator, $v);
+								foreach my $val (@vs) {
+									if ($val =~ /\S/) {
+										push (@{$exif_metadata{$field}}, $self->gsSafe($val));
+										++$metadata_count;
+									}
+								}
+							}
+							else
+							{
+								push (@{$exif_metadata{$field}}, $self->gsSafe($v));
+								++$metadata_count;
+							}
+						}
+						else{
+							if($allvals ne ""){
+								$allvals = $allvals . $self->{'join_character'};
+							}
+							$allvals = $allvals . $v;
+						}
+					}
+					
+					if($self->{'join_before_split'}){
+						if (defined $separator) {
+							my @vs = split($separator, $allvals);
+							foreach my $val (@vs) {
+								if ($val =~ /\S/) {
+									push (@{$exif_metadata{$field}}, $self->gsSafe($val));
+									++$metadata_count;
+								}
+							}
+						}
+						else
+						{
+							push (@{$exif_metadata{$field}}, $self->gsSafe($allvals));
+							++$metadata_count;
+						}
+					}
+				}
+				else {
+					$value = Encode::decode($encoding,$value);
+					if (defined $separator) {
+						my @vs = split($separator, $value);
+						$metadata_done = 1;
+						foreach my $v (@vs) {
+							if ($v =~ /\S/) {
+								push (@{$exif_metadata{$field}}, $self->gsSafe($v));
+								++$metadata_count; 
+							}
+						}
+					}
+				}
+				if (!$metadata_done) {
+					push (@{$exif_metadata{$field}}, $self->gsSafe($value));
+					++$metadata_count;
+				}
 			}
-		
-			
-
-		if (!defined $exif_metadata{$field})
-		{
-		    $exif_metadata{$field} = [];
 		}
-
-		$field = Encode::decode($encoding,$field);
-		my $metadata_done = 0;
-		if (ref $value eq 'SCALAR') {
-		    
-                    if ($$value =~ /^Binary data/) {
-			
-                        $value = "($$value)";
-		    }
-                    else {
-			
-                        my $len = length($$value);
-                        $value = "(Binary data $len bytes)";
-		    }
-		}
-		elsif (ref $value eq 'ARRAY') {
-		    $metadata_done = 1;
-		    foreach my $v (@$value) {
-			$v = Encode::decode($encoding,$v);
-			push (@{$exif_metadata{$field}}, $self->gsSafe($v));
-			++$metadata_count;
-		    }
-		}
-		else {
-			$value = Encode::decode($encoding,$value);
-			if (defined $separator) {
-			my @vs = split($separator, $value);
-			$metadata_done = 1;
-			foreach my $v (@vs) {
-			    if ($v =~ /\S/) {
-				push (@{$exif_metadata{$field}}, $self->gsSafe($v));
-				++$metadata_count; 
-			    }
-			}
-		    }
-		}
-		if (!$metadata_done) {
-		    push (@{$exif_metadata{$field}}, $self->gsSafe($value));
-		    ++$metadata_count;
-		}
-	    }
-          }
-      }
-
+	}
 
     if ($metadata_count > 0) {
-	print $outhandle " Extracted $metadata_count pieces of metadata from $filename EXIF block\n";
+		print $outhandle " Extracted $metadata_count pieces of metadata from $filename EXIF block\n";
     }
 
@@ -221,24 +265,22 @@
 sub metadata_read()
 {
-  my $self = shift (@_);
-  my ($pluginfo, $base_dir, $file, $block_hash, 
-      $extrametakeys, $extrametadata, $extrametafile,
-      $processor, $gli, $aux) = @_;
+	my $self = shift (@_);
+	my ($pluginfo, $base_dir, $file, $block_hash, 
+		$extrametakeys, $extrametadata, $extrametafile,
+		$processor, $gli, $aux) = @_;
   
-  my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+	my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
   
-  # we don't want to process directories
-  if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
-      return undef;
-  }
-  print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli);
-  print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
-
-
-  $self->extractEmbeddedMetadata($filename_no_path,$filename_full_path,
-				 $extrametadata,$extrametakeys);
-
-    
-  return undef;
+	# we don't want to process directories
+	if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
+		return undef;
+	}
+	print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli);
+	print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
+
+	$self->extractEmbeddedMetadata($filename_no_path,$filename_full_path,
+		$extrametadata,$extrametakeys);
+
+	return undef;
 }
 
@@ -250,19 +292,27 @@
 }
 
-sub gsSafe()
-  {
-    my $self = shift(@_);
-    my ($text) = @_;
-    # Replace dangerous characters
-    $text =~ s/\(/&#40;/g;
-    $text =~ s/\)/&#41;/g;
-    $text =~ s/,/&#44;/g;
-    $text =~ s/\</&#60;/g;
-    $text =~ s/\</&#62;/g;
-    $text =~ s/\[/&#91;/g;
-    $text =~ s/\]/&#93;/g;
-    # Done
-    return $text;
-  }
+sub gsSafe() {
+	my $self = shift(@_);
+	my ($text) = @_;
+	
+	# Replace potentially problematic characters
+	$text =~ s/\(/&#40;/g;
+	$text =~ s/\)/&#41;/g;
+	$text =~ s/,/&#44;/g;
+	$text =~ s/\</&#60;/g;
+	$text =~ s/\>/&#62;/g;
+	$text =~ s/\[/&#91;/g;
+	$text =~ s/\]/&#93;/g;
+	$text =~ s/\{/&#123;/g;
+	$text =~ s/\}/&#125;/g;
+	# Done
+	
+	if ($self->{'trim_whitespace'} eq "true"){
+		$text =~ s/^\s+//;
+		$text =~ s/\s+$//;
+	}
+	
+	return $text;
+}
 
 1;
Index: /main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm	(revision 24289)
+++ /main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm	(revision 24290)
@@ -173,5 +173,5 @@
     if ($secondary_plugin_name eq "HTMLPlugin") {
 	# pdftohtml always produces utf8 - What about pdfbox???
-	push(@$specific_options, "-input_encoding", "utf8");
+	# push(@$specific_options, "-input_encoding", "utf8");
 	push(@$specific_options, "-extract_language") if $self->{'extract_language'};
 	push(@$specific_options, "-processing_tmp_files");
@@ -238,5 +238,23 @@
 }
   
-
+# By setting hashing to be on ga xml this ensures that two 
+# PDF files that are identical except for the metadata
+# to hash to different values. Without this, when each PDF
+# file is converted to HTML there is a chance that they 
+# will both be *identical* if the conversion utility does
+# not embed the metadata in the generated HTML. This is
+# certainly the case when PDFBOX is being used. 
+
+# This change makes this convert to based plugin more 
+# consistent with the original vision that the same document 
+# with different metadata should
+# be seen as different.
+
+sub get_oid_hash_type {
+    my $self = shift (@_);
+    return "hash_on_ga_xml";
+}
+  
+  
 sub tmp_area_convert_file {
 
