- Timestamp:
- 2020-07-09T09:38:42+12:00 (4 years ago)
- Location:
- main/trunk/greenstone2/perllib/plugins
- Files:
-
- 1 added
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
main/trunk/greenstone2/perllib/plugins/CSVPlugin.pm
r33389 r34249 27 27 package CSVPlugin; 28 28 29 30 29 use SplitTextFile; 31 30 use MetadataRead; 31 use CSVFieldSeparator; 32 32 33 use strict; 33 34 no strict 'refs'; # allow filehandles to be variables and viceversa 34 35 36 use Text::CSV; 35 37 36 38 # CSVPlugin is a sub-class of SplitTextFile. 37 39 sub BEGIN { 38 @CSVPlugin::ISA = ('MetadataRead', 'SplitTextFile' );40 @CSVPlugin::ISA = ('MetadataRead', 'SplitTextFile', 'CSVFieldSeparator'); 39 41 } 40 42 41 43 42 44 my $arguments = 43 [ { 'name' => "process_exp", 45 [ 46 { 'name' => "process_exp", 44 47 'desc' => "{BaseImporter.process_exp}", 45 48 'type' => "regexp", … … 84 87 push(@{$hashArgOptLists->{"OptList"}}, $options); 85 88 89 new CSVFieldSeparator($pluginlist, $inputargs, $hashArgOptLists); 86 90 my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists); 87 91 … … 110 114 $$textref =~ s/^(.*?)\r?\n//; 111 115 my @csv_file_fields = (); 112 my $csv_file_field_line = $1 . ","; # To make the regular expressions simpler 113 while ($csv_file_field_line ne "") { 114 # Handle quoted values 115 if ($csv_file_field_line =~ s/^\"(.*?)\"\,//) { 116 my $csv_file_field = $1; 117 $csv_file_field =~ s/ //g; # Remove any spaces from the field names 118 push(@csv_file_fields, $csv_file_field); 119 } 120 # Normal comma-separated case 121 elsif ($csv_file_field_line =~ s/^(.*?)\,//) { 122 my $csv_file_field = $1; 123 $csv_file_field =~ s/ //g; # Remove any spaces from the field names 124 push(@csv_file_fields, $csv_file_field); 125 } 126 # The line must be formatted incorrectly 127 else { 128 print STDERR "Error: Badly formatted CSV field line: $csv_file_field_line.\n"; 129 last; 130 } 116 my $csv_file_field_line = $1; 117 118 my $separate_char = $self->{'csv_field_separator'}; 119 if ($separate_char =~ m/^auto$/i) { 120 $separate_char = $self->resolve_auto($csv_file_field_line,$self->{'plugin_type'}); 121 # Replace the 'auto' setting the resolved value (for use later on) 122 $self->{'separate_char'} = $separate_char; 131 123 } 132 124 133 $self->{'csv_file_fields'}->{$filename} = \@csv_file_fields; 134 ###print STDERR "**** CSV file fields joined ($filename) = ", join(" ||| ", @{$self->{'csv_file_fields'}->{$filename}}), "\n"; 125 my $csv = Text::CSV->new(); 126 $csv->sep_char($separate_char); 127 128 if ($csv->parse($csv_file_field_line)) { 129 @csv_file_fields = $csv->fields; 130 } 131 else { 132 print STDERR "Error: Badly formatted CSV field line: $csv_file_field_line.\n"; 133 } 134 135 $self->{'csv_file_fields'} = \@csv_file_fields; 136 137 # print STDERR "**** CSV file fields joined = ", join(" ||| ", @{$self->{'csv_file_fields'}}), "\n"; 135 138 } 136 139 … … 144 147 my $section = $doc_obj->get_top_section(); 145 148 my $csv_line = $$textref; 146 147 my $filename_full_path = &FileUtils::filenameConcatenate($base_dir,$file); 148 my @csv_file_fields = @{$self->{'csv_file_fields'}->{$filename_full_path}}; 149 150 ###print STDERR "**** CSV file fields joined = ", join(" ||| ", @csv_file_fields), "\n"; 151 152 # Add the raw line as the document text 149 my @csv_file_fields = @{$self->{'csv_file_fields'}}; 150 151 # Add the raw line as the document text 153 152 $doc_obj->add_utf8_text($section, $csv_line); 154 153 154 my $separate_char = $self->{'separate_char'}; 155 156 my $md_val_sep = $self->{'metadata_value_separator'}; 157 undef $md_val_sep if ($md_val_sep eq ""); 158 159 my $csv = Text::CSV->new(); 160 $csv->sep_char($separate_char); 161 155 162 # Build a hash of metadata name to metadata value for this line 156 my $i = 0; 157 $csv_line .= ","; # To make the regular expressions simpler 158 while ($csv_line ne "") { 159 # Metadata values containing commas are quoted 160 if ($csv_line =~ s/^\"(.*?)\"\,//) { 163 if ($csv->parse($csv_line)) { 164 my @md_vals = $csv->fields; 165 my $md_vals_len = scalar(@md_vals); 166 167 for (my $i=0; $i<$md_vals_len; $i++) { 168 my $md_val = $md_vals[$i]; 161 169 # Only bother with non-empty values 162 if ($1 ne "" && defined($csv_file_fields[$i])) { 163 $doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $1); 170 if ($md_val ne "" && defined($csv_file_fields[$i])) { 171 if (defined $md_val_sep) { 172 my $md_name = $csv_file_fields[$i]; 173 174 my @within_md_vals = split(/${md_val_sep}/,$md_val); 175 foreach my $within_md_val (@within_md_vals) { 176 $doc_obj->add_utf8_metadata($section, $md_name, $within_md_val); 177 } 178 } 179 else { 180 $doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $md_val); 181 } 164 182 } 165 183 } 166 # Normal comma-separated case 167 elsif ($csv_line =~ s/^(.*?)\,//) { 168 # Only bother with non-empty values 169 if ($1 ne "" && defined($csv_file_fields[$i])) { 170 $doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $1); 171 } 172 } 173 # The line must be formatted incorrectly 174 else { 175 print STDERR "Error: Badly formatted CSV line: $csv_line.\n"; 176 last; 177 } 178 179 $i++; 184 } 185 else { 186 print STDERR "Error: Badly formatted CSV line: $csv_line.\n"; 180 187 } 181 188 -
main/trunk/greenstone2/perllib/plugins/MetadataCSVPlugin.pm
r32501 r34249 30 30 use BaseImporter; 31 31 use MetadataRead; 32 use CSVFieldSeparator; 32 33 33 34 use strict; … … 39 40 40 41 use Encode; 42 use Text::CSV; 41 43 42 44 # methods with identical signatures take precedence in the order given in the ISA list. 43 45 sub BEGIN { 44 @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter'); 45 } 46 @MetadataCSVPlugin::ISA = ('MetadataRead', 'BaseImporter', 'CSVFieldSeparator'); 47 } 48 46 49 47 50 … … 72 75 push(@{$hashArgOptLists->{"OptList"}},$options); 73 76 77 new CSVFieldSeparator($pluginlist, $inputargs, $hashArgOptLists); 74 78 my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists); 75 79 … … 108 112 $processor, $gli, $aux) = @_; 109 113 110 # Read metadata from CSV files 111 my $filename = &util::filename_cat($base_dir, $file); 112 if ($filename !~ /\.csv$/ || !-f $filename) { 113 return undef; 114 } 114 # can we process this file?? 115 my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file); 116 return undef unless $self->can_process_this_file_for_metadata($filename_full_path); 117 115 118 print STDERR "\n<Processing n='$file' p='MetadataCSVPlugin'>\n" if ($gli); 116 119 print STDERR "MetadataCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1; … … 120 123 121 124 # add the file to the block list so that it won't be processed in read, as we will do all we can with it here 122 $self->block_raw_filename($block_hash,$filename );125 $self->block_raw_filename($block_hash,$filename_full_path); 123 126 124 127 125 128 # Read the CSV file to get the metadata 126 129 my $csv_file_content; 127 open(CSV_FILE, "$filename ");130 open(CSV_FILE, "$filename_full_path"); 128 131 my $csv_file_reader = new multiread(); 129 132 $csv_file_reader->set_handle('MetadataCSVPlugin::CSV_FILE'); … … 140 143 $csv_file_content =~ s/\r/\n/g; # Handle non-Unix line endings 141 144 $csv_file_content =~ s/\n+/\n/g; 145 146 my $separate_char = $self->{'csv_field_separator'}; 147 148 my $md_val_sep = $self->{'metadata_value_separator'}; 149 undef $md_val_sep if ($md_val_sep eq ""); 150 142 151 my @csv_file_lines = split(/\n/, $csv_file_content); 143 152 my $csv_file_field_line = shift(@csv_file_lines); 144 my @csv_file_fields = split(/\,/, $csv_file_field_line); 153 154 if ($separate_char =~ m/^auto$/i) { 155 $separate_char = $self->resolve_auto($csv_file_field_line,$self->{'plugin_type'}); 156 } 157 158 my $csv = Text::CSV->new(); 159 $csv->sep_char($separate_char); 160 161 my @csv_file_fields = undef; 162 if ($csv->parse($csv_file_field_line)) { 163 @csv_file_fields = $csv->fields; 164 } 165 else { 166 $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Error: Badly formatted CSV header line: $csv_file_field_line"); 167 return -1; 168 } 169 145 170 my $found_filename_field = 0; 146 171 for (my $i = 0; $i < scalar(@csv_file_fields); $i++) { … … 156 181 157 182 if (!$found_filename_field) { 158 $self->print_error($outhandle, $failhandle, $gli, $filename , "No Filename field in CSV file");183 $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename field in CSV file"); 159 184 return -1; # error 160 185 } … … 164 189 next if ($csv_line =~ /^\s*$/); 165 190 my $orig_csv_line = $csv_line; 191 166 192 # Build a hash of metadata name to metadata value for this line 167 193 my %csv_line_metadata; 168 my $i = 0; 169 $csv_line .= ","; # To make the regular expressions simpler 170 while ($csv_line ne "") { 171 # Metadata values containing commas are quoted 172 if ($csv_line =~ s/^\"(.*?)\"\,//) { 194 195 if ($csv->parse($csv_line)) { 196 my @md_vals = $csv->fields; 197 my $md_vals_len = scalar(@md_vals); 198 199 for (my $i=0; $i<$md_vals_len; $i++) { 200 my $md_val = $md_vals[$i]; 173 201 # Only bother with non-empty values 174 if ($1 ne "" && defined($csv_file_fields[$i])) { 175 if (!defined $csv_line_metadata{$csv_file_fields[$i]}) { 176 $csv_line_metadata{$csv_file_fields[$i]} = []; 202 if ($md_val ne "" && defined($csv_file_fields[$i])) { 203 204 my $md_name = $csv_file_fields[$i]; 205 206 if (!defined $md_name) { 207 $csv_line_metadata{$md_name} = []; 177 208 } 178 push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $1); 209 210 if (defined $md_val_sep) { 211 212 my @within_md_vals = split(/${md_val_sep}/,$md_val); 213 #push (@{$csv_line_metadata{$md_name}}, @within_md_vals); 214 215 # protect square brackets in metadata values by hex entity encoding them 216 # As unescaped square bracket chars in metadata 217 # have special meaning in GS' Java runtime code 218 my @escaped_within_md_vals = (); 219 for my $meta_value (@within_md_vals) { 220 $meta_value =~ s/\[/&\#091;/g; 221 $meta_value =~ s/\]/&\#093;/g; 222 push(@escaped_within_md_vals, $meta_value); 223 } 224 push (@{$csv_line_metadata{$md_name}}, @escaped_within_md_vals); 225 226 # foreach my $within_md_val (@within_md_vals) { 227 # push (@{$csv_line_metadata{$md_name}}, $within_md_val); 228 # } 229 } 230 else { 231 #push (@{$csv_line_metadata{$md_name}}, $md_val); 232 # protect square brackets in metadata values by hex entity encoding them 233 my $escaped_metadata_value = $md_val; 234 $escaped_metadata_value =~ s/\[/&\#091;/g; 235 $escaped_metadata_value =~ s/\]/&\#093;/g; 236 push (@{$csv_line_metadata{$md_name}}, $escaped_metadata_value); 237 } 179 238 } 180 239 } 181 # Normal comma-separated case 182 elsif ($csv_line =~ s/^(.*?)\,//) { 183 # Only bother with non-empty values 184 if ($1 ne "" && defined($csv_file_fields[$i])) { 185 if (!defined $csv_line_metadata{$csv_file_fields[$i]}) { 186 $csv_line_metadata{$csv_file_fields[$i]} = []; 187 } 188 # remove any surrounding quotes. (When exporting to CSV, some spreadsheet 189 # programs add quotes even around field values that don't contain commas.) 190 my $value = $1; 191 $value =~ s/^"//; 192 $value =~ s/"$//; 193 push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $value); 194 } 195 } 196 # The line must be formatted incorrectly 197 else { 198 $self->print_error($outhandle, $failhandle, $gli, $filename, "Badly formatted CSV line: $csv_line"); 199 last; 200 } 201 202 $i++; 240 } 241 else { 242 $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "Badly formatted CSV line: $csv_line"); 243 last; 203 244 } 204 245 … … 206 247 my $csv_line_filename_array = $csv_line_metadata{"Filename"}; 207 248 if (!defined $csv_line_filename_array) { 208 $self->print_error($outhandle, $failhandle, $gli, $filename , "No Filename metadata in CSV line: $orig_csv_line");249 $self->print_error($outhandle, $failhandle, $gli, $filename_full_path, "No Filename metadata in CSV line: $orig_csv_line"); 209 250 next; 210 251 } … … 255 296 } 256 297 # maps the file to full path 257 &extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename );298 &extrametautil::setmetafile_for_named_file($extrametafile, $csv_line_filename, $file, $filename_full_path); 258 299 } 259 300 }
Note:
See TracChangeset
for help on using the changeset viewer.