Index: /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm	(revision 36480)
+++ /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm	(revision 36480)
@@ -0,0 +1,399 @@
+###########################################################################
+#
+# CSVPlugin.pm -- A plugin for files in comma-separated value format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package NewCSVPlugin;
+
+use extrametautil;
+
+use ReadTextFile;
+use SplitTextFile; # for a couple routines, but we not inheriting
+use MetadataRead;
+use CSVFieldSeparator;
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use Text::CSV;
+
+sub BEGIN {
+    @NewCSVPlugin::ISA = ('MetadataRead', 'ReadTextFile', 'CSVFieldSeparator');
+    binmode(STDERR, ":utf8");
+
+}
+
+
+my $arguments = 
+    [ 
+      { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "filename_field",
+	'desc' => "{CSVPlugin.filename_field}",
+	'type' => "string",
+	'reqd' => "no",
+	'deft' => "Filename" }
+      ];
+
+
+my $options = { 'name'     => "NewCSVPlugin",
+		'desc'     => "{CSVPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'explodes' => "yes",
+		'args'     => $arguments };
+
+
+# This plugin processes files with the suffix ".csv"
+sub get_default_process_exp {
+    return q^(?i)(\.csv)$^;
+}
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}}, $options);
+
+    new CSVFieldSeparator($pluginlist, $inputargs, $hashArgOptLists);
+    my $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
+
+    $self->{'textcat_store'} = {};
+    $self->{'metapass_srcdoc'} = {}; # which segments have valid metadata_srcdoc
+    return bless $self, $class;
+}
+
+# mark the file as a metadata file
+sub file_block_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
+	
+    my $filename_full_path = &FileUtils::filenameConcatenate($base_dir, $file);
+    return undef unless $self->can_process_this_file($filename_full_path);    
+
+    if (($ENV{'GSDLOS'} =~ m/^windows$/) && ($^O ne "cygwin")) {
+	# convert to full name - paths stored in block hash are long filenames
+	$filename_full_path = &util::upgrade_if_dos_filename($filename_full_path);
+    }
+# kjdon - upgrade method converts everyhting to lower case drive letter.
+# so would we need the following stuff???
+#	my $lower_drive = $filename_full_path;
+#	$lower_drive =~ s/^([A-Z]):/\l$1:/i;
+	
+#	my $upper_drive = $filename_full_path;
+#	$upper_drive =~ s/^([A-Z]):/\u$1:/i;
+	
+#	$block_hash->{'metadata_files'}->{$lower_drive} = 1;
+#	$block_hash->{'metadata_files'}->{$upper_drive} = 1;
+		
+#    }
+#    else {
+###	$block_hash->{'metadata_files'}->{$filename_full_path} = 1;
+ #   }
+    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
+    return undef; #1
+}
+
+sub metadata_read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+	$extrametakeys, $extrametadata, $extrametafile,
+	$processor, $gli, $aux) = @_;
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
+    
+    print STDERR "\n<Processing n='$file' p='NewCSVPlugin'>\n" if ($gli);
+    print STDERR "NewCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
+
+    my $outhandle = $self->{'outhandle'};
+    my $failhandle = $self->{'failhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    # don't add to block list, as we may do some processing in read.
+
+    # Do encoding stuff
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    if ($self->{'verbosity'} > 2) {
+	print $outhandle "NewCSVPlugin: reading $file as ($content_encoding,$language)\n";
+    }
+    # store these values for read
+    my $le_rec = { 'language' => $language, 'encoding' => $content_encoding };
+    $self->{'textcat_store'}->{$file} = $le_rec;
+
+    my $metadata_store = {};
+    $self->{'metadata_store'}->{$file} = $metadata_store; # used to record metadata for segments with no src doc
+
+    my $CSV_FILE;
+    open($CSV_FILE, "<:encoding($content_encoding)", "$filename_full_path");
+    my $separate_char = $self->{'csv_field_separator'};
+
+    my $md_val_sep = $self->{'metadata_value_separator'};
+    undef $md_val_sep if ($md_val_sep eq "");
+
+    my $csv_file_field_line;
+    if ($separate_char =~ m/^auto$/i) {
+	
+	$csv_file_field_line = <$CSV_FILE>;
+	$separate_char = $self->resolve_auto($csv_file_field_line,$self->{'plugin_type'});
+	seek $CSV_FILE, 0, 0; # move pointer back to start of file, as we want to read in the fields using csv.
+    }
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+    $csv->binary(1);
+
+    my @csv_file_fields = undef;
+
+    my $first_row = $csv->getline ($CSV_FILE);
+    if (defined $first_row) {
+	@csv_file_fields = @$first_row;
+    }
+    else {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Error: Badly formatted CSV header line: $csv_file_field_line");
+	return -1;
+    }
+
+    my $found_filename_field = 0;
+    my $filename_field = $self->{'filename_field'};
+    print STDERR "looking for $filename_field field\n";
+    for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
+	# Remove any spaces from the field names, and surrounding quotes too
+	$csv_file_fields[$i] =~ s/ //g;
+	$csv_file_fields[$i] =~ s/^"//;
+	$csv_file_fields[$i] =~ s/"$//;
+
+	if ($csv_file_fields[$i] eq  $filename_field) {
+	    $found_filename_field = 1;
+	}
+    }
+
+    
+    if (!$found_filename_field) {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field field in CSV file, metadata cannot be assigned to documents, will use metadata only dummy documents");
+	
+    }
+
+    my $count = 0;
+    while (my $csv_line = $csv->getline($CSV_FILE)) {
+	my @md_vals = @$csv_line;
+	$count++;
+
+	# Build a hash of metadata name to metadata value for this line
+	my %csv_line_metadata;
+
+	my $md_vals_len = scalar(@md_vals);
+	
+	for (my $i=0; $i<$md_vals_len; $i++) {
+	    my $md_val = $md_vals[$i];
+	    print STDERR "$count: md val = $md_val\n";
+	    # Only bother with non-empty values
+	    if ($md_val ne "" && defined($csv_file_fields[$i])) {
+		
+		my $md_name = $csv_file_fields[$i];
+		$csv_line_metadata{$md_name} = [];
+		if (defined $md_val_sep) {			
+		    
+		    my @within_md_vals = split(/${md_val_sep}/,$md_val);
+		    
+		    # protect square brackets in metadata values by hex entity encoding them
+		    # As unescaped square bracket chars in metadata
+		    # have special meaning in GS' Java runtime code
+		    my @escaped_within_md_vals = ();
+		    for my $meta_value (@within_md_vals) {
+			
+			$meta_value =~ s/\[/&\#091;/g;
+			$meta_value =~ s/\]/&\#093;/g;
+			push(@escaped_within_md_vals, $meta_value);
+		    }
+		    push (@{$csv_line_metadata{$md_name}}, @escaped_within_md_vals);
+		}
+		else {
+		    # protect square brackets in metadata values by hex entity encoding them
+		    my $escaped_metadata_value = $md_val;
+		    $escaped_metadata_value =~ s/\[/&\#091;/g;
+		    $escaped_metadata_value =~ s/\]/&\#093;/g;
+		    push (@{$csv_line_metadata{$md_name}}, $escaped_metadata_value);
+		}
+	    }
+	}
+
+	my $csv_line_section_array = $csv_line_metadata{"Section"};
+	my $section_suffix = "";
+	if (defined $csv_line_section_array) {
+	    my $section_value = shift(@$csv_line_section_array);
+	    if ($section_value =~ /[\d.]+/m){
+		my $section_suffix = "///Section/" . $section_value;
+		foreach my $metaname (keys %csv_line_metadata) {
+		    my $new_name = $metaname . $section_suffix;
+		    $csv_line_metadata{$new_name} = delete $csv_line_metadata{$metaname};
+		}
+	    } else{
+		unshift(@$csv_line_section_array, $section_value);
+	    }
+	}
+	
+	# do we have filename field?
+	# We can't associate any metadata without knowing the file to associate it with
+	my $has_srcdoc = 0;
+	my $csv_line_filename="";;
+	if ($found_filename_field) {
+	    # is there a srcdoc mentioned?
+	    my $csv_line_filename_array = $csv_line_metadata{$filename_field};
+	    if (!defined $csv_line_filename_array) {
+		$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field metadata in CSV line num $count");
+	    } else {
+		$csv_line_filename = shift(@$csv_line_filename_array);
+		# TODO - have an option for whether we do this or not
+		if (&FileUtils::fileExists(&FileUtils::filenameConcatenate($base_dir, $csv_line_filename))) {
+		    $has_srcdoc = 1;
+		    
+		    delete $csv_line_metadata{$filename_field};
+		}
+	    }
+	    
+	}
+	if ($has_srcdoc) {
+	    print STDERR "storing meta in extra meta for $csv_line_filename\n";
+	    $self->store_meta_in_extrametadata($csv_line_filename, \%csv_line_metadata, $file, $filename_full_path, $extrametakeys, $extrametadata, $extrametafile);
+	} else {
+	    print STDERR "storing meta for $count, $csv_line_filename\n";
+	    $metadata_store->{$count} = \%csv_line_metadata;
+	}
+    } # while csv_line = csv->getline
+    close ($CSV_FILE);
+}
+
+#adapted from read in splittextfile
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file($filename_full_path);
+
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    my $le_rec = $self->{'textcat_store'}->{$file};
+    if (!defined $le_rec) {
+	# means no text was found;
+	return 0; # not processed but no point in passing it on
+    }
+
+    print STDERR "<Processing n='$file' p='$self->{'plugin_type'}'>\n" if ($gli);
+    print $outhandle "$self->{'plugin_type'} processing $file\n"
+	    if $self->{'verbosity'} > 1;    
+
+    my $language = $le_rec->{'language'};
+    my $encoding = $le_rec->{'encoding'};
+    $self->{'textcat_store'}->{$file} = undef;
+
+    my $metadata_store = $self->{'metadata_store'}->{$file}; # a hash of seg num to metadata hash
+
+
+    # Process each segment in turn
+    my $segment = 0; #which segment/record number we have
+    my $count = 0; # num doc objs produced
+
+    my ($filemeta) = $file =~ /([^\\\/]+)$/; #why?
+    my $plugin_filename_encoding = $self->{'filename_encoding'};
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
+
+    my $id;
+    print STDERR "num keys  = ".scalar(keys (%$metadata_store))."\n";
+    foreach $segment (sort keys (%$metadata_store)) {
+	print $outhandle "processing segment $segment\n"
+	    if $self->{'verbosity'} > 1;    
+	$count++;
+	# create a new document
+	my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
+
+	$self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
+
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$segment");
+	if ($self->{'cover_image'}) {
+	    $self->associate_cover_image($doc_obj, $filename_full_path);
+	}
+	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+
+	# include any metadata passed in from previous plugins 
+	# note that this metadata is associated with the top level section
+	$self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+
+	# add our stored metadata from metadata_read pass
+	my $segment_metadata = $metadata_store->{$segment};
+	$self->extra_metadata($doc_obj, $doc_obj->get_top_section(), $segment_metadata);
+
+	# do any automatic metadata extraction - does this make sense??
+	#$self->auto_extract_metadata ($doc_obj);
+
+	# Calculate a "base" document ID.
+	if (!defined $id) {
+	    $id = &SplitTextFile::get_base_OID($self,$doc_obj);
+	}
+	
+	# add an OID
+	&SplitTextFile::add_segment_OID($self, $doc_obj, $id, $segment);
+
+	# process the document
+	$processor->process($doc_obj);
+
+	$self->{'num_processed'} ++;
+	if ($maxdocs != -1 && $self->{'num_processed'} >= $maxdocs) {
+	    last;
+	}
+    }
+
+    delete $self->{'metadata_store'}->{$file};
+
+    # Return number of document objects produced
+    return $count; 
+}
+
+sub print_error
+{
+
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "NewCSVPlugin Error: $file: $error\n";
+    print $failhandle "NewCSVPlugin Error: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+}
+
+
+1;
