Changeset 38742 for main/trunk/greenstone2/perllib/plugins/BaseImporter.pm

Timestamp:

2024-02-13T17:15:40+13:00 (4 months ago)

Author:

davidb

Message:

New mapping_metadata_file rule-table functionalilty

File:

• main/trunk/greenstone2/perllib/plugins/BaseImporter.pm (modified) (6 diffs)

main/trunk/greenstone2/perllib/plugins/BaseImporter.pm

@@ -99 +99 @@
     'type' => "string",
     'reqd' => "no" },
+      { 'name' => "metadata_mapping_file",
+    'desc' => "{BaseImporter.metadata_mapping_file}",
+    'type' => "string",
+    #'deft' => "metadata_mapping_rules.csv", # commenting out, explicitly giving this as the default means every plugin automatically activates it
+    'deft' => "",
+    'reqd' => "no" },      
       { 'name' => "OIDtype",
     'desc' => "{import.OIDtype}",
@@ -155 +161 @@
         'args'     => $arguments };
 
+sub setup_metadata_mapping_rules
+{
+    my $self = shift (@_);  
+    my ($metadata_mapping_full_filename) = @_;
+
+    my $csv_file = &util::read_utf8_textfile($metadata_mapping_full_filename);
+
+    my @csv_lines = split('\n',$csv_file);
+
+    my $line_num = 0;
+
+    my $metadata_mapping_rules = [];
+    
+    foreach my $csv_line (@csv_lines) {
+    chomp($csv_line);
+
+    $line_num++;
+    
+    next if ($csv_line =~ m/^\s*$/);
+    next if ($csv_line =~ m/^#/);
+
+    # my @tsv_entries = split('\t',$tsv_line);
+    my @csv_entries = split(/\s*,\s*/,$csv_line);
+
+    my $num_csv_entries = scalar(@csv_entries);
+
+    # print STDERR join(",",@tsv_entries), "\n\n";
+    
+    if (($num_csv_entries == 5) || ($num_csv_entries >= 6 && $csv_entries[5] =~ m/^#/)) {
+        my $src_metadata_name  = $csv_entries[0];
+        (my $src_metadata_regex = $csv_entries[1]) =~ s/(^\/)|(\/$)//g; #assigns as a /qr (quoted regex) and then strips off leading and trailing slash
+        (my $dst_metadata_regex = $csv_entries[2]) =~ s/(^\/)|(\/$)//g; #as line above
+        (my $regex_modifiers    = $csv_entries[3]) =~ s/(^')|('$)//g;   #similar to above but strips out single quotes
+        my $dst_metadata_name  = $csv_entries[4];
+
+        # If the last entry has a '# comments' after it, needs a bit more string manipulation
+        # to tidy it up
+        $dst_metadata_name =~ s/#.*$//;
+        $dst_metadata_name =~ s/\s*$//;
+        
+        my $regex_sub = "s/$src_metadata_regex/$dst_metadata_regex/$regex_modifiers";
+
+        my $metadata_mapping_rule_rec = {
+        "src_metadata_name" => $src_metadata_name,
+        "regex_sub"         => $regex_sub,
+        "dst_metadata_name" => $dst_metadata_name           
+        };
+            
+        push(@$metadata_mapping_rules,$metadata_mapping_rule_rec);
+    }
+    else {
+        print STDERR "Warning: syntax error in $metadata_mapping_full_filename, line $line_num\n";
+        print STDERR "  $csv_line\n";
+        print STDERR "Did not contain 5 tab-delimited entries\n";
+    }
+        
+    }
+
+    if (scalar(@$metadata_mapping_rules)>0) {
+    $self->{'metadata_mapping_rules'} = $metadata_mapping_rules;
+    }
+}
+
 sub new {
 
@@ -165 +234 @@
 
     my $self = new CommonUtil($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
-    
+
     if ($self->{'info_only'}) {
         # don't worry about any options etc
@@ -205 +274 @@
     }
 
-    return bless $self, $class;
+    my $blessed_self = bless $self, $class;
+    
+    if ($blessed_self->{'metadata_mapping_file'}) {
+    my $metadata_mapping_file = $blessed_self->{'metadata_mapping_file'};
+    my $mmf_full_filename;  
+    $mmf_full_filename = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'},"etc", $metadata_mapping_file);
+    if (!-e $mmf_full_filename) {
+        my $mmf_full_col_filename = $mmf_full_filename;
+        $mmf_full_filename = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"etc", $metadata_mapping_file);
+
+        if (!-e $mmf_full_filename) {
+        my $outhandle = $blessed_self->{'outhandle'};
+        print STDERR "\nBaseImporter Error: Can't locate metadata_mapping_file '$metadata_mapping_file'\n";
+        print STDERR "This file should be in $mmf_full_col_filename or $mmf_full_filename\n";
+        $blessed_self->print_txt_usage("");  # Use default resource bundle
+        print STDERR "\nBaseImporter Error: Can't locate metadata_mapping_file '$metadata_mapping_file'\n";
+        print STDERR "This file should be in $mmf_full_col_filename or $mmf_full_filename\n";
+        die "\n";
+        }
+    }
+
+    $blessed_self->{'metadata_mapping_full_filename'} = $mmf_full_filename;
+    $blessed_self->setup_metadata_mapping_rules($mmf_full_filename);
+    }
+
+    
+    return $blessed_self;
 
 }
@@ -688 +783 @@
 }
 
+
+sub apply_metadata_mapping_file {
+    my $self = shift (@_);  
+    my ($doc_obj) = @_;
+
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+    
+    my $top_section = $doc_obj->get_top_section();
+
+    my $metadata_mapping_rules = $self->{'metadata_mapping_rules'};
+
+    my $transient_metadata_names = {};
+    my $new_metadata_names = {};
+
+    print $outhandle "  Applying metadata_mapping_file to document\n";
+    
+    foreach my $metadata_mapping_rule_rec (@$metadata_mapping_rules) {
+        
+    my $src_metadata_name = $metadata_mapping_rule_rec->{"src_metadata_name"};
+    my $dst_metadata_name = $metadata_mapping_rule_rec->{"dst_metadata_name"};
+
+    my $regex_sub = $metadata_mapping_rule_rec->{"regex_sub"};
+
+    my $metadata_vals = $doc_obj->get_metadata($top_section,$src_metadata_name);
+
+    foreach my $metadata_val (@$metadata_vals) {
+
+        my $store_metadata_val = $metadata_val;
+        
+        if ($verbosity >= 4) {
+        print $outhandle "    Testing for match with: \$metadata_val =~ $regex_sub\n"
+        }
+        eval ( "\$metadata_val =~ $regex_sub" );
+        if ($@) {
+        warn "$@";
+        }
+        else {
+        # print STDERR "**** after metadata_val = $metadata_val\n\n";
+        if ($verbosity >=2) {
+            if ($metadata_val ne $store_metadata_val) {
+
+            if ($verbosity >= 3) {
+                print $outhandle "      Transformed metadata to $dst_metadata_name: '$store_metadata_val' -> '$metadata_val'\n"
+            }
+            if ($verbosity >= 2 && ($dst_metadata_name !~ m/^_transient/)) {
+                print $outhandle "    Added new metadata $dst_metadata_name: '$metadata_val'\n"
+            }
+            }
+        }
+        $doc_obj->add_utf8_metadata($top_section,$dst_metadata_name,$metadata_val);
+
+        if ($dst_metadata_name =~ m/^_transient/) {
+            $transient_metadata_names->{$dst_metadata_name} = 1;
+        }
+        else {
+            if ($metadata_val ne $store_metadata_val) {
+            $new_metadata_names->{$dst_metadata_name} = 1;
+            }
+        }       
+        }
+    }
+    
+    }
+
+    if ($verbosity >=1) {
+    print $outhandle "  Number of metadata_mapping_file transformations for this document: ", scalar(keys %$new_metadata_names), "\n";
+
+    if ($verbosity >=2) {
+        print $outhandle "  Generated metadata names: ", join(", ",keys %$new_metadata_names), "\n";
+    }
+    }
+    
+    # Now remove any of dst_metadata_name which began _transient
+    foreach my $transient_metadata_name (keys %$transient_metadata_names) {
+    #print STDERR "transient_metadata_name = $transient_metadata_name\n";   
+    $doc_obj->delete_metadata($top_section,$transient_metadata_name);
+    }
+}
+
 sub post_process_doc_obj {
     my $self = shift (@_);  
     my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
 
+    if ($self->{'metadata_mapping_rules'}) {
+    $self->apply_metadata_mapping_file($doc_obj);
+    }
+    
     return 1;
 }
@@ -820 +999 @@
     return undef unless $self->can_process_this_file($filename_full_path);
     
-    #print STDERR "**** BEFORE READ INTO DOC OBJ: $file\n";
+    #print STDERR "**** BEFORE READ INTO DOC OBJ: $file\n";
     my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
     #print STDERR "**** AFTER READ INTO DOC OBJ: $file\n";