Index: /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/etc/collectionConfig.xml
===================================================================
--- /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/etc/collectionConfig.xml	(revision 35990)
+++ /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/etc/collectionConfig.xml	(revision 35991)
@@ -131,6 +131,9 @@
     <index name="Year">
       <displayItem name="name" key="Year.buttonname" dictionary="collectionConfig"/>
-    </index>      o
-
+    </index>
+    <index name="lyrics">
+      <displayItem name="name" key="Lyrics.buttonname" dictionary="collectionConfig"/>
+    </index>
+    
     <sort name="rank"/>
     <sort name="dc.Title,Title">
@@ -258,4 +261,9 @@
 	<option name="process_exp" value="metadata([-_].*)?\.json$" />
       </plugin>
+      <plugin name="MetadataCSVPlugin">
+	<option name="process_exp" value="metadata([-_].*)?\.csv$" />
+	<option name="filename_formed_from" value="to_country,year" />
+      </plugin>
+
       <plugin name="ArchivesInfPlugin"/>
       <plugin name="DirectoryPlugin"/>
@@ -656,4 +664,5 @@
 	<gsf:metadata name="Country"/>
 	<gsf:metadata name="Year"/>
+	<gsf:metadata name="lyrics"/>
 
 	<gsf:metadata name="Abstract_en"/>
Index: /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/MetadataCSVPlugin.pm
===================================================================
--- /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/MetadataCSVPlugin.pm	(revision 35991)
+++ /main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/MetadataCSVPlugin.pm	(revision 35991)
@@ -0,0 +1,364 @@
+###########################################################################
+#
+# MetadataCSVPlugin.pm -- A plugin for metadata in comma-separated value format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package MetadataCSVPlugin;
+
+
+use BaseImporter;
+use MetadataRead;
+use CSVFieldSeparator;
+
+use strict;
+no strict 'refs';
+
+use extrametautil;
+use multiread;
+use util;
+
+use Encode;
+use Text::CSV;
+
+# methods with identical signatures take precedence in the order given in the ISA list.
+sub BEGIN {
+    @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter', 'CSVFieldSeparator');
+}
+
+
+
+my $arguments = [
+      { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+    { 'name' => "filename_formed_from",
+	  'desc' => "Specify in a comma separated list the metadata fields that get combined together to form the filename",
+	  'type' => "string",
+	  'reqd' => "no"
+    },
+    { 'name' => "filename_formed_with_no_spaces",
+	  'desc' => "Set this flag to remove an spaces in the metadata values used to form the filename",
+	  'type' => "flag",
+	  'reqd' => "no"
+    },
+    { 'name' => "filename_formed_from_ext",
+	  'desc' => "The filename extension to add on to the end of the filename_formed_from metadata", 
+	  'type' => "string",
+	  'reqd' => "no",
+	  'deft' => ".nul"
+    }
+];
+
+
+my $options = { 'name'     => "MetadataCSVPlugin",
+		'desc'     => "{MetadataCSVPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    new CSVFieldSeparator($pluginlist, $inputargs, $hashArgOptLists);
+    my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
+
+    return bless $self, $class;
+}
+
+
+sub get_default_process_exp
+{
+    return q^(?i)\.csv$^;
+}
+
+sub file_block_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
+
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
+	return undef; # can't recognise
+    }
+
+    # set this so we know this is a metadata file - needed for incremental 
+    # build
+    # if this file changes, then we need to reimport everything
+    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
+
+    return 1;
+}
+
+sub metadata_read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+	$extrametakeys, $extrametadata, $extrametafile,
+	$processor, $gli, $aux) = @_;
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
+    
+    print STDERR "\n<Processing n='$file' p='MetadataCSVPlugin'>\n" if ($gli);
+    print STDERR "MetadataCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
+
+    my $outhandle = $self->{'outhandle'};
+    my $failhandle = $self->{'failhandle'};
+
+    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
+    $self->block_raw_filename($block_hash,$filename_full_path);
+
+
+    # Read the CSV file to get the metadata
+    my $csv_file_content;
+    open(CSV_FILE, "$filename_full_path");
+    my $csv_file_reader = new multiread();
+    $csv_file_reader->set_handle('MetadataCSVPlugin::CSV_FILE');
+    $csv_file_reader->read_file(\$csv_file_content);
+
+    # Would be nice if MetadataCSVPlugin was extended to support a minus
+    # option to choose the character encoding the CSV file is in 
+    # For now we will assume it is always in UTF8
+    $csv_file_content = decode("utf8",$csv_file_content);
+
+    close(CSV_FILE);
+
+    # Split the file into lines and read the first line (contains the metadata names)
+    $csv_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
+    $csv_file_content =~ s/\n+/\n/g;
+
+    my $separate_char = $self->{'csv_field_separator'};
+
+    my $md_val_sep = $self->{'metadata_value_separator'};
+    undef $md_val_sep if ($md_val_sep eq "");
+
+    my @csv_file_lines = split(/\n/, $csv_file_content);
+    my $csv_file_field_line = shift(@csv_file_lines);
+
+    if ($separate_char =~ m/^auto$/i) {
+	$separate_char = $self->resolve_auto($csv_file_field_line,$self->{'plugin_type'});
+    }
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+ 
+    my @csv_file_fields = undef;
+    if ($csv->parse($csv_file_field_line)) {
+	@csv_file_fields = $csv->fields;
+    }
+    else {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Error: Badly formatted CSV header line: $csv_file_field_line");
+	return -1;
+    }
+
+    my $found_filename_field = 0;
+    for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
+	# Remove any spaces from the field names, and surrounding quotes too
+	$csv_file_fields[$i] =~ s/ //g;
+	$csv_file_fields[$i] =~ s/^"//;
+	$csv_file_fields[$i] =~ s/"$//;
+
+	if ($csv_file_fields[$i] eq "Filename") {
+	    $found_filename_field = 1;
+	}
+    }
+
+    if (!defined $self->{'filename_formed_from'}) {
+	if (!$found_filename_field) {
+	    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename field in CSV file");
+	    return -1; # error
+	}
+    }
+    
+    # Read each line of the file and assign the metadata appropriately
+    foreach my $csv_line (@csv_file_lines) {
+	# Ignore lines containing only whitespace
+	next if ($csv_line =~ /^\s*$/);
+	my $orig_csv_line = $csv_line;
+
+	# Build a hash of metadata name to metadata value for this line
+	my %csv_line_metadata;
+
+	if ($csv->parse($csv_line)) {
+	    my @md_vals = $csv->fields;
+	    my $md_vals_len = scalar(@md_vals);
+	    
+	    for (my $i=0; $i<$md_vals_len; $i++) {
+		my $md_val = $md_vals[$i];
+		# Only bother with non-empty values
+		if ($md_val ne "" && defined($csv_file_fields[$i])) {
+
+		    my $md_name = $csv_file_fields[$i];
+
+		    if (!defined $md_name) {
+			$csv_line_metadata{$md_name} = [];
+		    }
+
+		    if (defined $md_val_sep) {			
+			
+			my @within_md_vals = split(/${md_val_sep}/,$md_val);
+			#push (@{$csv_line_metadata{$md_name}}, @within_md_vals);
+
+			# protect square brackets in metadata values by hex entity encoding them
+			# As unescaped square bracket chars in metadata
+			# have special meaning in GS' Java runtime code
+			my @escaped_within_md_vals = ();
+			for my $meta_value (@within_md_vals) {
+			    $meta_value =~ s/\[/&\#091;/g;
+			    $meta_value =~ s/\]/&\#093;/g;
+			    push(@escaped_within_md_vals, $meta_value);
+			}
+			push (@{$csv_line_metadata{$md_name}}, @escaped_within_md_vals);
+
+#			foreach my $within_md_val (@within_md_vals) {
+#			    push (@{$csv_line_metadata{$md_name}}, $within_md_val);
+#			}
+		    }
+		    else {
+			#push (@{$csv_line_metadata{$md_name}}, $md_val);
+			# protect square brackets in metadata values by hex entity encoding them
+			my $escaped_metadata_value = $md_val;
+			$escaped_metadata_value =~ s/\[/&\#091;/g;
+			$escaped_metadata_value =~ s/\]/&\#093;/g;
+			push (@{$csv_line_metadata{$md_name}}, $escaped_metadata_value);
+		    }
+		}
+	    }
+	}
+	else {
+	    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Badly formatted CSV line: $csv_line");
+	    last;
+	}
+
+	# We can't associate any metadata without knowing the file to associate it with
+	if (defined $self->{'filename_formed_from'}) {
+	    my @filename_formed_from_metanames = split(",",$self->{'filename_formed_from'});
+
+	    my $filename_metavalue = ""; 
+	    foreach my $metaname (@filename_formed_from_metanames) {
+		my $metavalue = shift(@{$csv_line_metadata{$metaname}});
+		if (defined $self->{'filename_formed_with_no_spaces'}) {
+		    $metavalue =~ s/\s+//g;
+		}
+
+		
+		$filename_metavalue .= $metavalue if defined $metavalue;
+	    }
+
+	    if (defined $self->{'filename_formed_from_ext'}) {
+		$filename_metavalue .= $self->{'filename_formed_from_ext'};
+	    }
+	    
+	    if ($filename_metavalue ne "") {
+		# print STDERR "**** Setting up Metadata Filename match on: $filename_metavalue\n";
+		
+		$csv_line_metadata{"Filename"} = [ $filename_metavalue ];
+	    }
+	}
+	
+	my $csv_line_filename_array = $csv_line_metadata{"Filename"};
+	if (!defined $csv_line_filename_array) {
+	    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename metadata in CSV line: $orig_csv_line");
+	    next;
+	}
+	my $csv_line_filename = shift(@$csv_line_filename_array);
+	delete $csv_line_metadata{"Filename"};
+
+	my $csv_line_section_array = $csv_line_metadata{"Section"};
+	my $section_suffix = "";
+	if (defined $csv_line_section_array) {
+		my $section_value = shift(@$csv_line_section_array);
+		if ($section_value =~ /[\d.]+/m){
+			my $section_suffix = "///Section/" . $section_value;
+			foreach my $metaname (keys %csv_line_metadata) {
+				my $new_name = $metaname . $section_suffix;
+				$csv_line_metadata{$new_name} = delete $csv_line_metadata{$metaname};
+			}
+		} else{
+			unshift(@$csv_line_section_array, $section_value);
+		}
+	}
+
+	# For CSV files to include line-breaks these are typically stored as '\n'
+	# Unescape them here so they present more nicely in the browser
+	#
+	# foreach my $metaname (keys %csv_line_metadata) {
+	#     my $csv_line_metavalue_array = $csv_line_metadata{$metaname};
+        #
+	#     map { $_ =~ s/\\n/\n/g } @$csv_line_metavalue_array;
+	# }
+	
+	
+ 	# Associate the metadata now
+	# Indexing into the extrameta data structures requires the filename's style of slashes to be in URL format
+	# Then need to convert the filename to a regex, no longer to protect windows directory chars \, but for
+	# protecting special characters like brackets in the filepath such as "C:\Program Files (x86)\Greenstone".
+	$csv_line_filename = &util::filepath_to_url_format($csv_line_filename);
+	$csv_line_filename = &util::filename_to_regex($csv_line_filename);
+
+	if (defined &extrametautil::getmetadata($extrametadata, $csv_line_filename)) { # merge with existing meta    
+
+	    my $file_metadata_table = &extrametautil::getmetadata($extrametadata, $csv_line_filename);
+	    
+	    foreach my $metaname (keys %csv_line_metadata) {
+		# will create new entry if one does not already exist
+		push(@{$file_metadata_table->{$metaname}}, @{$csv_line_metadata{$metaname}});	    
+	    }
+	    
+	    # no need to push $file on to $extrametakeys as it is already in the list
+	} else { # add as new meta
+	    
+	    &extrametautil::setmetadata($extrametadata, $csv_line_filename, \%csv_line_metadata);
+	    &extrametautil::addmetakey($extrametakeys, $csv_line_filename);
+	}
+	# record which file the metadata came from 
+	if (!defined &extrametautil::getmetafile($extrametafile, $csv_line_filename)) {
+	    &extrametautil::setmetafile($extrametafile, $csv_line_filename, {});
+	}
+	# maps the file to full path
+	&extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename_full_path);
+    }
+}
+
+sub print_error
+{
+
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "MetadataCSVPlugin Error: $file: $error\n";
+    print $failhandle "MetadataCSVPlugin Error: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+}
+1;
