Changeset 34249 for main/trunk/greenstone2/perllib/plugins/MetadataCSVPlugin.pm

Timestamp:

2020-07-09T09:38:42+12:00 (4 years ago)

Author:

ak19

Message:

Dr Bainbridge in his commit 32810 had expressed that he intended to commit his MetadataCSVPlugin related work for dlheritage to the main GS after the then upcoming GS3 release. His plugin changes support multiple values for a metadata field work and these changes for me in the GS3tutorials collection that uses a metadata.csv file. Like dlheritage, I also use the pipe symbol to separate multiple meta values for a meta field/column. Kathy had made a bugfix to MetadataCSVPlugin since Dr Bainbridge's branched the code off for dlheritage. I will incorporate her bugfix into Dr Bainbridge's work and test things still work and will commit that separately next. Committing from uni machine, as something weird about WMTB VM where I tested these plugin changes and additions: svn committing hasn't been working for a few days now but freezes trying to transmit data.

File:

• main/trunk/greenstone2/perllib/plugins/MetadataCSVPlugin.pm (modified) (10 diffs)

main/trunk/greenstone2/perllib/plugins/MetadataCSVPlugin.pm

@@ -30 +30 @@
 use BaseImporter;
 use MetadataRead;
+use CSVFieldSeparator;
 
 use strict;
@@ -39 +40 @@
 
 use Encode;
+use Text::CSV;
 
 # methods with identical signatures take precedence in the order given in the ISA list.
 sub BEGIN {
-    @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter');
-}
+    @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter', 'CSVFieldSeparator');
+}
+
 
 
@@ -72 +75 @@
     push(@{$hashArgOptLists->{"OptList"}},$options);
 
+    new CSVFieldSeparator($pluginlist, $inputargs, $hashArgOptLists);
     my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
 
@@ -108 +112 @@
     $processor, $gli, $aux) = @_;
 
-    # Read metadata from CSV files
-    my $filename = &util::filename_cat($base_dir, $file);
-    if ($filename !~ /\.csv$/ || !-f $filename) {
-    return undef;
-    }
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
+    
     print STDERR "\n<Processing n='$file' p='MetadataCSVPlugin'>\n" if ($gli);
     print STDERR "MetadataCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
@@ -120 +123 @@
 
     # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
-    $self->block_raw_filename($block_hash,$filename);
+    $self->block_raw_filename($block_hash,$filename_full_path);
 
 
     # Read the CSV file to get the metadata
     my $csv_file_content;
-    open(CSV_FILE, "$filename");
+    open(CSV_FILE, "$filename_full_path");
     my $csv_file_reader = new multiread();
     $csv_file_reader->set_handle('MetadataCSVPlugin::CSV_FILE');
@@ -140 +143 @@
     $csv_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
     $csv_file_content =~ s/\n+/\n/g;
+
+    my $separate_char = $self->{'csv_field_separator'};
+
+    my $md_val_sep = $self->{'metadata_value_separator'};
+    undef $md_val_sep if ($md_val_sep eq "");
+
     my @csv_file_lines = split(/\n/, $csv_file_content);
     my $csv_file_field_line = shift(@csv_file_lines);
-    my @csv_file_fields = split(/\,/, $csv_file_field_line);
+
+    if ($separate_char =~ m/^auto$/i) {
+    $separate_char = $self->resolve_auto($csv_file_field_line,$self->{'plugin_type'});
+    }
+
+    my $csv = Text::CSV->new();
+    $csv->sep_char($separate_char);
+ 
+    my @csv_file_fields = undef;
+    if ($csv->parse($csv_file_field_line)) {
+    @csv_file_fields = $csv->fields;
+    }
+    else {
+    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Error: Badly formatted CSV header line: $csv_file_field_line");
+    return -1;
+    }
+
     my $found_filename_field = 0;
     for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
@@ -156 +181 @@
 
     if (!$found_filename_field) {
-    $self->print_error($outhandle, $failhandle, $gli, $filename, "No Filename field in CSV file");
+    $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename field in CSV file");
     return -1; # error
     }
@@ -164 +189 @@
     next if ($csv_line =~ /^\s*$/);
     my $orig_csv_line = $csv_line;
+
     # Build a hash of metadata name to metadata value for this line
     my %csv_line_metadata;
-    my $i = 0;
-    $csv_line .= ",";  # To make the regular expressions simpler
-    while ($csv_line ne "") {
-        # Metadata values containing commas are quoted
-        if ($csv_line =~ s/^\"(.*?)\"\,//) {
+
+    if ($csv->parse($csv_line)) {
+        my @md_vals = $csv->fields;
+        my $md_vals_len = scalar(@md_vals);
+        
+        for (my $i=0; $i<$md_vals_len; $i++) {
+        my $md_val = $md_vals[$i];
         # Only bother with non-empty values
-        if ($1 ne "" && defined($csv_file_fields[$i])) {
-            if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
-            $csv_line_metadata{$csv_file_fields[$i]} = [];
+        if ($md_val ne "" && defined($csv_file_fields[$i])) {
+
+            my $md_name = $csv_file_fields[$i];
+
+            if (!defined $md_name) {
+            $csv_line_metadata{$md_name} = [];
             }
-            push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $1);
+
+            if (defined $md_val_sep) {          
+            
+            my @within_md_vals = split(/${md_val_sep}/,$md_val);
+            #push (@{$csv_line_metadata{$md_name}}, @within_md_vals);
+
+            # protect square brackets in metadata values by hex entity encoding them
+            # As unescaped square bracket chars in metadata
+            # have special meaning in GS' Java runtime code
+            my @escaped_within_md_vals = ();
+            for my $meta_value (@within_md_vals) {
+                $meta_value =~ s/\[/&\#091;/g;
+                $meta_value =~ s/\]/&\#093;/g;
+                push(@escaped_within_md_vals, $meta_value);
+            }
+            push (@{$csv_line_metadata{$md_name}}, @escaped_within_md_vals);
+
+#           foreach my $within_md_val (@within_md_vals) {
+#               push (@{$csv_line_metadata{$md_name}}, $within_md_val);
+#           }
+            }
+            else {
+            #push (@{$csv_line_metadata{$md_name}}, $md_val);
+            # protect square brackets in metadata values by hex entity encoding them
+            my $escaped_metadata_value = $md_val;
+            $escaped_metadata_value =~ s/\[/&\#091;/g;
+            $escaped_metadata_value =~ s/\]/&\#093;/g;
+            push (@{$csv_line_metadata{$md_name}}, $escaped_metadata_value);
+            }
         }
         }
-        # Normal comma-separated case
-        elsif ($csv_line =~ s/^(.*?)\,//) {
-        # Only bother with non-empty values
-        if ($1 ne "" && defined($csv_file_fields[$i])) {
-            if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
-            $csv_line_metadata{$csv_file_fields[$i]} = [];
-            }
-            # remove any surrounding quotes. (When exporting to CSV, some spreadsheet
-            # programs add quotes even around field values that don't contain commas.)
-            my $value = $1;
-            $value =~ s/^"//;
-            $value =~ s/"$//;
-            push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $value);
-        }
-        }
-        # The line must be formatted incorrectly
-        else {
-        $self->print_error($outhandle, $failhandle, $gli, $filename, "Badly formatted CSV line: $csv_line");
-        last;
-        }
-
-        $i++;
+    }
+    else {
+        $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Badly formatted CSV line: $csv_line");
+        last;
     }
 
@@ -206 +247 @@
     my $csv_line_filename_array = $csv_line_metadata{"Filename"};
     if (!defined $csv_line_filename_array) {
-        $self->print_error($outhandle, $failhandle, $gli, $filename, "No Filename metadata in CSV line: $orig_csv_line");
+        $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename metadata in CSV line: $orig_csv_line");
         next;
     }
@@ -255 +296 @@
     }
     # maps the file to full path
-    &extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename);
+    &extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename_full_path);
     }
 }