Index: /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm	(revision 36481)
+++ /main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm	(revision 36482)
@@ -57,6 +57,24 @@
 	'type' => "string",
 	'reqd' => "no",
-	'deft' => "Filename" }
-      ];
+	'deft' => "Filename" },
+      { 'name' => "store_raw_values_as_text",
+	'desc' => "{CSVPlugin.store_raw_values_as_text}",
+	'type' => "flag",
+	'reqd' => "no"},
+       { 'name' => "no_document_if_source_unspecified",
+	'desc' => "{CSVPlugin.no_document_if_source_unspecified}",
+	'type' => "flag",
+	'reqd' => "no"},
+       { 'name' => "no_document_if_source_missing",
+	'desc' => "{CSVPlugin.no_document_if_source_missing}",
+	'type' => "flag",
+	'reqd' => "no"},
+       { 'name' => "store_field_values_as_document_text",
+	'desc' => "{CSVPlugin.store_field_values_as_document_text}",
+	'type' => "flag",
+	'reqd' => "no"},
+
+
+     ];
 
 
@@ -186,5 +204,4 @@
     my $found_filename_field = 0;
     my $filename_field = $self->{'filename_field'};
-    print STDERR "looking for $filename_field field\n";
     for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
 	# Remove any spaces from the field names, and surrounding quotes too
@@ -200,5 +217,5 @@
     
     if (!$found_filename_field) {
-	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field field in CSV file, metadata cannot be assigned to documents, will use metadata only dummy documents");
+	$self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field field in CSV file, metadata cannot be assigned to documents, will use metadata only dummy documents");
 	
     }
@@ -216,5 +233,4 @@
 	for (my $i=0; $i<$md_vals_len; $i++) {
 	    my $md_val = $md_vals[$i];
-	    print STDERR "$count: md val = $md_val\n";
 	    # Only bother with non-empty values
 	    if ($md_val ne "" && defined($csv_file_fields[$i])) {
@@ -266,4 +282,5 @@
 	# We can't associate any metadata without knowing the file to associate it with
 	my $has_srcdoc = 0;
+	my $missing_srcdoc = 0;
 	my $csv_line_filename="";;
 	if ($found_filename_field) {
@@ -271,5 +288,5 @@
 	    my $csv_line_filename_array = $csv_line_metadata{$filename_field};
 	    if (!defined $csv_line_filename_array) {
-		$self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field metadata in CSV line num $count");
+		$self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field metadata in CSV line num $count");
 	    } else {
 		$csv_line_filename = shift(@$csv_line_filename_array);
@@ -279,4 +296,7 @@
 		    
 		    delete $csv_line_metadata{$filename_field};
+		} else {
+		    $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "$csv_line_filename in $filename_field metadata in CSV line num $count is not found");
+		    $missing_srcdoc = 1; # there was one mentioned but its not found
 		}
 	    }
@@ -284,9 +304,20 @@
 	}
 	if ($has_srcdoc) {
-	    print STDERR "storing meta in extra meta for $csv_line_filename\n";
+	    print $outhandle "Storing metadata, segment $count, for document $csv_line_filename\n" if ($verbosity > 2);
 	    $self->store_meta_in_extrametadata($csv_line_filename, \%csv_line_metadata, $file, $filename_full_path, $extrametakeys, $extrametadata, $extrametafile);
 	} else {
-	    print STDERR "storing meta for $count, $csv_line_filename\n";
-	    $metadata_store->{$count} = \%csv_line_metadata;
+	    my $store_for_dummy = 1;
+	    if ($missing_srcdoc && $self->{'no_document_if_source_missing'}) {
+		$self->print_warning($outhandle, $failhandle, $gli, $filename_full_path,"Not storing metadata for line $count as source doc is missing");
+		$store_for_dummy = 0;
+	    } elsif(!$missing_srcdoc && $self->{'no_document_if_source_unspecified'}) {
+		$self->print_warning($outhandle, $failhandle, $gli, $filename_full_path,"Not storing metadata for line $count as source doc is unspecified");
+		$store_for_dummy = 0;
+	    } 
+	    if ($store_for_dummy) {
+		
+		print $outhandle "Storing metadata for dummy document, segment $count\n" if ($verbosity > 2);
+		$metadata_store->{$count} = \%csv_line_metadata;
+	    }
 	}
     } # while csv_line = csv->getline
@@ -333,30 +364,39 @@
 
     my $id;
-    print STDERR "num keys  = ".scalar(keys (%$metadata_store))."\n";
-    foreach $segment (sort keys (%$metadata_store)) {
-	print $outhandle "processing segment $segment\n"
+
+    foreach $segment (sort { $a <=> $b } keys (%$metadata_store)) {
+	print $outhandle "processing segment $segment as its own document\n"
 	    if $self->{'verbosity'} > 1;    
 	$count++;
 	# create a new document
 	my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
+	my $cursection = $doc_obj->get_top_section();
+	$doc_obj->add_utf8_metadata($cursection, "Language", $language);
+	$doc_obj->add_utf8_metadata($cursection, "Encoding", $encoding);
 
 	$self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
 
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$segment");
+	$doc_obj->add_utf8_metadata($cursection, "SourceSegment", "$segment");
 	if ($self->{'cover_image'}) {
 	    $self->associate_cover_image($doc_obj, $filename_full_path);
 	}
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+	$doc_obj->add_utf8_metadata($cursection, "Plugin", "$self->{'plugin_type'}");
 
 	# include any metadata passed in from previous plugins 
 	# note that this metadata is associated with the top level section
-	$self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+	$self->extra_metadata ($doc_obj, $cursection, $metadata);
 
 	# add our stored metadata from metadata_read pass
 	my $segment_metadata = $metadata_store->{$segment};
-	$self->extra_metadata($doc_obj, $doc_obj->get_top_section(), $segment_metadata);
-
+	$self->extra_metadata($doc_obj, $cursection, $segment_metadata);
+	if ($self->{'store_field_values_as_document_text'}) {
+	    my $new_text = "";
+	    foreach my $f (keys %$segment_metadata) {
+		my $values = $segment_metadata->{$f};
+		$new_text .= join (", ", @$values).", ";
+	    }
+
+	    $doc_obj->add_utf8_text($cursection, $new_text);
+	}
 	# do any automatic metadata extraction - does this make sense??
 	#$self->auto_extract_metadata ($doc_obj);
@@ -385,4 +425,13 @@
 }
 
+sub print_warning {
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "CSVPlugin Warning: $file: $error\n";
+    print $failhandle "CSVPlugin Warning: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+
+}
 sub print_error
 {
Index: /main/trunk/greenstone2/perllib/strings.properties
===================================================================
--- /main/trunk/greenstone2/perllib/strings.properties	(revision 36481)
+++ /main/trunk/greenstone2/perllib/strings.properties	(revision 36482)
@@ -884,4 +884,12 @@
 CSVPlugin.desc:A plugin for files in comma-separated value format. Metadata can be assigned to source documents (specified in the Filename field), or new documents created for each line of the file.
 
+CSVPlugin.filename_field:Which field in the CSV file to use for specifying source documents.
+
+CSVPlugin.store_field_values_as_document_text:Store all the metadata values as the text of the document. Only applies if there is no source document specified. Useful for searching.
+
+CSVPlugin.no_document_if_source_unspecified:IF there is no source document specified, don't create a dummy document.
+
+CSVPlugin.no_document_if_source_missing:If there is a specified source document, but it is not there, don't create a dummy document.
+
 CSVDeprecatedPlugin.desc:An old plugin for files in comma-separated value format. A new document will be created for each line of the file.
 
