Index: /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/CSVPlugin.pm
===================================================================
--- /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/CSVPlugin.pm	(revision 32810)
+++ /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/CSVPlugin.pm	(revision 32810)
@@ -0,0 +1,236 @@
+###########################################################################
+#
+# CSVPlugin.pm -- A plugin for files in comma-separated value format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package CSVPlugin;
+
+use SplitTextFile;
+use MetadataRead;
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use Text::CSV;
+
+# CSVPlugin is a sub-class of SplitTextFile.
+sub BEGIN {
+    @CSVPlugin::ISA = ('MetadataRead', 'SplitTextFile');
+}
+
+
+my $arguments = 
+    [ 
+      { 'name' => "separate_char",
+	'desc' => "{CSVPlugin.separate_char}",
+	'type' => "string",
+	'deft' => "auto",
+	'reqd' => "no" },
+      { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "split_exp",
+	'desc' => "{SplitTextFile.split_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_split_exp(),
+        'hiddengli' => "yes" }
+      ];
+
+
+my $options = { 'name'     => "CSVPlugin",
+		'desc'     => "{CSVPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'explodes' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes files with the suffix ".csv"
+sub get_default_process_exp {
+    return q^(?i)(\.csv)$^;
+}
+
+    
+# This plugin splits the input text by line
+sub get_default_split_exp {
+    return q^\r?\n^;
+}
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}}, $options);
+
+    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
+
+    return bless $self, $class;
+}
+
+
+sub read_file
+{
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    # Read in file the usual ReadTextFile way
+    # This ensure that $textref is a unicode aware string
+    $self->SUPER::read_file(@_);
+
+    #
+    # Now top-up the processing of the text with what this plugin
+    # needs
+    #
+
+    # Remove any blank lines so the data is split and processed properly
+    $$textref =~ s/\n(\s*)\n/\n/g;
+
+    # The first line contains the metadata element names
+    $$textref =~ s/^(.*?)\r?\n//;
+    my @csv_file_fields = ();
+    my $csv_file_field_line = $1;
+
+    my $separate_char = $self->{'separate_char'};
+    if ($separate_char =~ m/^auto$/i) {
+	$separate_char = CSVSeparateChar::resolve_auto($csv_file_field_line,$self->{'plugin_type'},$self->{'outhandle'},$self->{'verbosity'});
+	# Replace the 'auto' setting the resolved value (for use later on)
+	$self->{'separate_char'} = $separate_char;
+    }
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+ 
+    if ($csv->parse($csv_file_field_line)) {
+	@csv_file_fields = $csv->fields;
+    }
+    else {
+	    print STDERR "Error: Badly formatted CSV field line: $csv_file_field_line.\n";
+    }
+
+    # $csv_file_field_line .= $separate_char;  # To make the regular expressions simpler # ****
+
+    # while ($csv_file_field_line ne "") {
+    # 	# Handle quoted values
+    # 	if ($csv_file_field_line =~ s/^\"(.*?)\"${separate_char}//) { # ****
+    # 	    my $csv_file_field = $1;
+    # 	    $csv_file_field =~ s/ //g;  # Remove any spaces from the field names
+    # 	    push(@csv_file_fields, $csv_file_field);
+    # 	}
+    # 	# Normal comma-separated case
+    # 	elsif ($csv_file_field_line =~ s/^(.*?)${separate_char}//) { # ****
+    # 	    my $csv_file_field = $1;
+    # 	    $csv_file_field =~ s/ //g;  # Remove any spaces from the field names
+    # 	    push(@csv_file_fields, $csv_file_field);
+    # 	}
+    # 	# The line must be formatted incorrectly
+    # 	else {
+    # 	    print STDERR "Error: Badly formatted CSV field line: $csv_file_field_line.\n";
+    # 	    last;
+    # 	}
+    # }
+
+    $self->{'csv_file_fields'} = \@csv_file_fields;
+
+    print STDERR "****** csv file fields joined = ", join(" ||| ", @{$self->{'csv_file_fields'}}), "\n";
+}
+
+
+sub process
+{
+    my $self = shift (@_);
+    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $section = $doc_obj->get_top_section();
+    my $csv_line = $$textref;
+    my @csv_file_fields = @{$self->{'csv_file_fields'}};
+
+    # Add the raw line as the document text
+    $doc_obj->add_utf8_text($section, $csv_line);
+
+    my $separate_char = $self->{'separate_char'};
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+ 
+    # Build a hash of metadata name to metadata value for this line
+    if ($csv->parse($csv_line)) {
+	my @md_vals = $csv->fields;
+	my $md_vals_len = scalar(@md_vals);
+
+	for (my $i=0; $i<$md_vals_len; $i++) {
+	    my $md_val = $md_vals[$i];
+	    # Only bother with non-empty values
+	    if ($md_val ne "" && defined($csv_file_fields[$i])) {
+		$doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $md_val);
+	    }
+	}
+    }
+    else {
+	print STDERR "Error: Badly formatted CSV line: $csv_line.\n";
+    }
+
+
+
+    # # Build a hash of metadata name to metadata value for this line
+    # my $i = 0;
+    # $csv_line .= $separate_char;  # To make the regular expressions simpler # ****
+    # while ($csv_line ne "") {
+    # 	# Metadata values containing commas are quoted
+    # 	if ($csv_line =~ s/^\"(.*?)\"${separate_char}//) { # ****
+    # 	    # Only bother with non-empty values
+    # 	    my $match = $1;
+    # 	    if ($match ne "" && defined($csv_file_fields[$i])) {
+    # 		$doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $match);
+    # 	    }
+    # 	}
+    # 	# Normal comma-separated case
+    # 	elsif ($csv_line =~ s/^(.*?)${separate_char}//) { # ****
+    # 	    # Only bother with non-empty values
+    # 	    my $match = $1;
+    # 	    if ($match ne "" && defined($csv_file_fields[$i])) {
+    # 		$doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $match);
+    # 	    }
+    # 	}
+    # 	# The line must be formatted incorrectly
+    # 	else {
+    # 	    print STDERR "Error: Badly formatted CSV line: $csv_line.\n";
+    # 	    last;
+    # 	}
+
+    # 	$i++;
+    # }
+
+    # Record was processed successfully
+    return 1;
+}
+
+
+1;
Index: /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/MetadataCSVPlugin.pm
===================================================================
--- /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/MetadataCSVPlugin.pm	(revision 32810)
+++ /main/trunk/model-sites-dev/heritage-nz/collect/reports-2019/perllib/plugins/MetadataCSVPlugin.pm	(revision 32810)
@@ -0,0 +1,320 @@
+###########################################################################
+#
+# MetadataCSVPlugin.pm -- A plugin for metadata in comma-separated value format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package MetadataCSVPlugin;
+
+
+use BaseImporter;
+use MetadataRead;
+
+use strict;
+no strict 'refs';
+
+use extrametautil;
+use multiread;
+use util;
+
+use CSVSeparateChar; 
+
+use Encode;
+
+# methods with identical signatures take precedence in the order given in the ISA list.
+sub BEGIN {
+    @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter');
+}
+
+
+
+my $arguments = [
+      { 'name' => "separate_char",
+	'desc' => "{CSVPlugin.separate_char}",
+	'type' => "string",
+	'deft' => "auto",
+	'reqd' => "no" },
+      { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() }
+
+];
+
+
+my $options = { 'name'     => "MetadataCSVPlugin",
+		'desc'     => "{MetadataCSVPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
+
+    return bless $self, $class;
+}
+
+
+sub get_default_process_exp
+{
+    return q^(?i)\.csv$^;
+}
+
+sub file_block_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
+
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
+	return undef; # can't recognise
+    }
+
+    # set this so we know this is a metadata file - needed for incremental 
+    # build
+    # if this file changes, then we need to reimport everything
+    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
+
+    return 1;
+}
+
+sub metadata_read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+	$extrametakeys, $extrametadata, $extrametafile,
+	$processor, $gli, $aux) = @_;
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
+    
+    print STDERR "\n<Processing n='$file' p='MetadataCSVPlugin'>\n" if ($gli);
+    print STDERR "MetadataCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
+
+    my $outhandle = $self->{'outhandle'};
+    my $failhandle = $self->{'failhandle'};
+
+    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
+    $self->block_raw_filename($block_hash,$filename_full_path);
+
+
+    # Read the CSV file to get the metadata
+    my $csv_file_content;
+    open(CSV_FILE, "$filename_full_path");
+    my $csv_file_reader = new multiread();
+    $csv_file_reader->set_handle('MetadataCSVPlugin::CSV_FILE');
+    $csv_file_reader->read_file(\$csv_file_content);
+
+    # Would be nice if MetadataCSVPlugin was extended to support a minus
+    # option to choose the character encoding the CSV file is in 
+    # For now we will assume it is always in UTF8
+    $csv_file_content = decode("utf8",$csv_file_content);
+
+    close(CSV_FILE);
+
+    # Split the file into lines and read the first line (contains the metadata names)
+    $csv_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
+    $csv_file_content =~ s/\n+/\n/g;
+
+    my $separate_char = $self->{'separate_char'};
+
+    my @csv_file_lines = split(/\n/, $csv_file_content);
+    my $csv_file_field_line = shift(@csv_file_lines);
+
+    if ($separate_char =~ m/^auto$/i) {
+	$separate_char = CSVSeparateChar::resolve_auto($csv_file_field_line,$self->{'plugin_type'},$outhandle,$self->{'verbosity'});
+    }
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+ 
+    my @csv_file_fields = undef;
+    if ($csv->parse($csv_file_field_line)) {
+	@csv_file_fields = $csv->fields;
+    }
+    else {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Error: Badly formatted CSV header line: $csv_file_field_line");
+	return -1;
+    }
+
+##    my @csv_file_fields = split(/${separate_char}/, $csv_file_field_line); # ****
+    my $found_filename_field = 0;
+    for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
+	# Remove any spaces from the field names, and surrounding quotes too
+	$csv_file_fields[$i] =~ s/ //g;
+	$csv_file_fields[$i] =~ s/^"//;
+	$csv_file_fields[$i] =~ s/"$//;
+
+	if ($csv_file_fields[$i] eq "Filename") {
+	    $found_filename_field = 1;
+	}
+    }
+
+    if (!$found_filename_field) {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename field in CSV file");
+	return -1; # error
+    }
+    # Read each line of the file and assign the metadata appropriately
+    foreach my $csv_line (@csv_file_lines) {
+	# Ignore lines containing only whitespace
+	next if ($csv_line =~ /^\s*$/);
+	my $orig_csv_line = $csv_line;
+
+	# Build a hash of metadata name to metadata value for this line
+	my %csv_line_metadata;
+
+	if ($csv->parse($csv_line)) {
+	    my @md_vals = $csv->fields;
+	    my $md_vals_len = scalar(@md_vals);
+	    
+	    for (my $i=0; $i<$md_vals_len; $i++) {
+		my $md_val = $md_vals[$i];
+		# Only bother with non-empty values
+		if ($md_val ne "" && defined($csv_file_fields[$i])) {
+		    if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
+			$csv_line_metadata{$csv_file_fields[$i]} = [];
+		    }
+		    push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $md_val);
+		}
+	    }
+	}
+	else {
+	    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Badly formatted CSV line: $csv_line");
+	    last;
+	}
+
+	# my $i = 0;
+	# $csv_line .= "$separate_char";  # To make the regular expressions simpler
+	# while ($csv_line ne "") {
+	#     # Metadata values containing commas are quoted
+	#     if ($csv_line =~ s/^\"(.*?)\"${separate_char}//) { # ****
+	# 	# Only bother with non-empty values
+	# 	if ($1 ne "" && defined($csv_file_fields[$i])) {
+	# 	    if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
+	# 		$csv_line_metadata{$csv_file_fields[$i]} = [];
+	# 	    }
+	# 	    push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $1);
+	# 	}
+	#     }
+	#     # Normal comma-separated case
+	#     elsif ($csv_line =~ s/^(.*?)${separate_char}//) { # ****
+	# 	# Only bother with non-empty values
+	# 	if ($1 ne "" && defined($csv_file_fields[$i])) {
+	# 	    if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
+	# 		$csv_line_metadata{$csv_file_fields[$i]} = [];
+	# 	    }
+	# 	    # remove any surrounding quotes. (When exporting to CSV, some spreadsheet
+	# 	    # programs add quotes even around field values that don't contain commas.)
+	# 	    my $value = $1;
+	# 	    $value =~ s/^"//;
+	# 	    $value =~ s/"$//;
+	# 	    push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $value);
+	# 	}
+	#     }
+	#     # The line must be formatted incorrectly
+	#     else {
+	# 	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Badly formatted CSV line: $csv_line");
+	# 	last;
+	#     }
+
+	#     $i++;
+	# }
+
+	# We can't associate any metadata without knowing the file to associate it with
+	my $csv_line_filename_array = $csv_line_metadata{"Filename"};
+	if (!defined $csv_line_filename_array) {
+	    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename metadata in CSV line: $orig_csv_line");
+	    next;
+	}
+	my $csv_line_filename = shift(@$csv_line_filename_array);
+	delete $csv_line_metadata{"Filename"};
+
+	my $csv_line_section_array = $csv_line_metadata{"Section"};
+	my $section_suffix = "";
+	if (defined $csv_line_section_array) {
+		my $section_value = shift(@$csv_line_section_array);
+		if ($section_value =~ /[\d.]+/m){
+			my $section_suffix = "///Section/" . $section_value;
+			foreach my $metaname (keys %csv_line_metadata) {
+				my $new_name = $metaname . $section_suffix;
+				$csv_line_metadata{$new_name} = delete $csv_line_metadata{$metaname};
+			}
+		} else{
+			unshift(@$csv_line_section_array, $section_value);
+		}
+	}
+
+
+ 	# Associate the metadata now
+	# Indexing into the extrameta data structures requires the filename's style of slashes to be in URL format
+	# Then need to convert the filename to a regex, no longer to protect windows directory chars \, but for
+	# protecting special characters like brackets in the filepath such as "C:\Program Files (x86)\Greenstone".
+	$csv_line_filename = &util::filepath_to_url_format($csv_line_filename);
+	$csv_line_filename = &util::filename_to_regex($csv_line_filename);
+
+	if (defined &extrametautil::getmetadata($extrametadata, $csv_line_filename)) { # merge with existing meta    
+
+	    my $file_metadata_table = &extrametautil::getmetadata($extrametadata, $csv_line_filename);
+	    
+	    foreach my $metaname (keys %csv_line_metadata) {
+		# will create new entry if one does not already exist
+		push(@{$file_metadata_table->{$metaname}}, @{$csv_line_metadata{$metaname}});	    
+	    }
+	    
+	    # no need to push $file on to $extrametakeys as it is already in the list
+	} else { # add as new meta
+	    
+	    &extrametautil::setmetadata($extrametadata, $csv_line_filename, \%csv_line_metadata);
+	    &extrametautil::addmetakey($extrametakeys, $csv_line_filename);
+	}
+	# record which file the metadata came from 
+	if (!defined &extrametautil::getmetafile($extrametafile, $csv_line_filename)) {
+	    &extrametautil::setmetafile($extrametafile, $csv_line_filename, {});
+	}
+	# maps the file to full path
+	&extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename_full_path);
+    }
+}
+
+sub print_error
+{
+
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "MetadataCSVPlugin Error: $file: $error\n";
+    print $failhandle "MetadataCSVPlugin Error: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+}
+1;
