Changeset 36482 for main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm

Timestamp:

2022-08-25T11:23:53+12:00 (20 months ago)

Author:

kjdon

Message:

added some new options for CSVPlugin

File:

• main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm (modified) (10 diffs)

main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm

@@ -57 +57 @@
     'type' => "string",
     'reqd' => "no",
-    'deft' => "Filename" }
-      ];
+    'deft' => "Filename" },
+      { 'name' => "store_raw_values_as_text",
+    'desc' => "{CSVPlugin.store_raw_values_as_text}",
+    'type' => "flag",
+    'reqd' => "no"},
+       { 'name' => "no_document_if_source_unspecified",
+    'desc' => "{CSVPlugin.no_document_if_source_unspecified}",
+    'type' => "flag",
+    'reqd' => "no"},
+       { 'name' => "no_document_if_source_missing",
+    'desc' => "{CSVPlugin.no_document_if_source_missing}",
+    'type' => "flag",
+    'reqd' => "no"},
+       { 'name' => "store_field_values_as_document_text",
+    'desc' => "{CSVPlugin.store_field_values_as_document_text}",
+    'type' => "flag",
+    'reqd' => "no"},
+
+
+     ];
 
 
@@ -186 +204 @@
     my $found_filename_field = 0;
     my $filename_field = $self->{'filename_field'};
-    print STDERR "looking for $filename_field field\n";
     for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
     # Remove any spaces from the field names, and surrounding quotes too
@@ -200 +217 @@
     
     if (!$found_filename_field) {
-    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field field in CSV file, metadata cannot be assigned to documents, will use metadata only dummy documents");
+    $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field field in CSV file, metadata cannot be assigned to documents, will use metadata only dummy documents");
     
     }
@@ -216 +233 @@
     for (my $i=0; $i<$md_vals_len; $i++) {
         my $md_val = $md_vals[$i];
-        print STDERR "$count: md val = $md_val\n";
         # Only bother with non-empty values
         if ($md_val ne "" && defined($csv_file_fields[$i])) {
@@ -266 +282 @@
     # We can't associate any metadata without knowing the file to associate it with
     my $has_srcdoc = 0;
+    my $missing_srcdoc = 0;
     my $csv_line_filename="";;
     if ($found_filename_field) {
@@ -271 +288 @@
         my $csv_line_filename_array = $csv_line_metadata{$filename_field};
         if (!defined $csv_line_filename_array) {
-        $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field metadata in CSV line num $count");
+        $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "No $filename_field metadata in CSV line num $count");
         } else {
         $csv_line_filename = shift(@$csv_line_filename_array);
@@ -279 +296 @@
             
             delete $csv_line_metadata{$filename_field};
+        } else {
+            $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path, "$csv_line_filename in $filename_field metadata in CSV line num $count is not found");
+            $missing_srcdoc = 1; # there was one mentioned but its not found
         }
         }
@@ -284 +304 @@
     }
     if ($has_srcdoc) {
-        print STDERR "storing meta in extra meta for $csv_line_filename\n";
+        print $outhandle "Storing metadata, segment $count, for document $csv_line_filename\n" if ($verbosity > 2);
         $self->store_meta_in_extrametadata($csv_line_filename, \%csv_line_metadata, $file, $filename_full_path, $extrametakeys, $extrametadata, $extrametafile);
     } else {
-        print STDERR "storing meta for $count, $csv_line_filename\n";
-        $metadata_store->{$count} = \%csv_line_metadata;
+        my $store_for_dummy = 1;
+        if ($missing_srcdoc && $self->{'no_document_if_source_missing'}) {
+        $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path,"Not storing metadata for line $count as source doc is missing");
+        $store_for_dummy = 0;
+        } elsif(!$missing_srcdoc && $self->{'no_document_if_source_unspecified'}) {
+        $self->print_warning($outhandle, $failhandle, $gli, $filename_full_path,"Not storing metadata for line $count as source doc is unspecified");
+        $store_for_dummy = 0;
+        } 
+        if ($store_for_dummy) {
+        
+        print $outhandle "Storing metadata for dummy document, segment $count\n" if ($verbosity > 2);
+        $metadata_store->{$count} = \%csv_line_metadata;
+        }
     }
     } # while csv_line = csv->getline
@@ -333 +364 @@
 
     my $id;
-    print STDERR "num keys  = ".scalar(keys (%$metadata_store))."\n";
-    foreach $segment (sort keys (%$metadata_store)) {
-    print $outhandle "processing segment $segment\n"
+
+    foreach $segment (sort { $a <=> $b } keys (%$metadata_store)) {
+    print $outhandle "processing segment $segment as its own document\n"
         if $self->{'verbosity'} > 1;    
     $count++;
     # create a new document
     my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
+    my $cursection = $doc_obj->get_top_section();
+    $doc_obj->add_utf8_metadata($cursection, "Language", $language);
+    $doc_obj->add_utf8_metadata($cursection, "Encoding", $encoding);
 
     $self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
 
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$segment");
+    $doc_obj->add_utf8_metadata($cursection, "SourceSegment", "$segment");
     if ($self->{'cover_image'}) {
         $self->associate_cover_image($doc_obj, $filename_full_path);
     }
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
+    $doc_obj->add_utf8_metadata($cursection, "Plugin", "$self->{'plugin_type'}");
 
     # include any metadata passed in from previous plugins 
     # note that this metadata is associated with the top level section
-    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    $self->extra_metadata ($doc_obj, $cursection, $metadata);
 
     # add our stored metadata from metadata_read pass
     my $segment_metadata = $metadata_store->{$segment};
-    $self->extra_metadata($doc_obj, $doc_obj->get_top_section(), $segment_metadata);
-
+    $self->extra_metadata($doc_obj, $cursection, $segment_metadata);
+    if ($self->{'store_field_values_as_document_text'}) {
+        my $new_text = "";
+        foreach my $f (keys %$segment_metadata) {
+        my $values = $segment_metadata->{$f};
+        $new_text .= join (", ", @$values).", ";
+        }
+
+        $doc_obj->add_utf8_text($cursection, $new_text);
+    }
     # do any automatic metadata extraction - does this make sense??
     #$self->auto_extract_metadata ($doc_obj);
@@ -385 +425 @@
 }
 
+sub print_warning {
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "CSVPlugin Warning: $file: $error\n";
+    print $failhandle "CSVPlugin Warning: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+
+}
 sub print_error
 {