Changeset 38742 for main/trunk

Timestamp:

2024-02-13T17:15:40+13:00 (3 months ago)

Author:

davidb

Message:

New mapping_metadata_file rule-table functionalilty

Location:

main/trunk/greenstone2/perllib

Files:

• plugins/BaseImporter.pm (modified) (6 diffs)
• strings.properties (modified) (1 diff)

main/trunk/greenstone2/perllib/plugins/BaseImporter.pm

@@ -99 +99 @@
     'type' => "string",
     'reqd' => "no" },
+      { 'name' => "metadata_mapping_file",
+    'desc' => "{BaseImporter.metadata_mapping_file}",
+    'type' => "string",
+    #'deft' => "metadata_mapping_rules.csv", # commenting out, explicitly giving this as the default means every plugin automatically activates it
+    'deft' => "",
+    'reqd' => "no" },      
       { 'name' => "OIDtype",
     'desc' => "{import.OIDtype}",
@@ -155 +161 @@
         'args'     => $arguments };
 
+sub setup_metadata_mapping_rules
+{
+    my $self = shift (@_);  
+    my ($metadata_mapping_full_filename) = @_;
+
+    my $csv_file = &util::read_utf8_textfile($metadata_mapping_full_filename);
+
+    my @csv_lines = split('\n',$csv_file);
+
+    my $line_num = 0;
+
+    my $metadata_mapping_rules = [];
+    
+    foreach my $csv_line (@csv_lines) {
+    chomp($csv_line);
+
+    $line_num++;
+    
+    next if ($csv_line =~ m/^\s*$/);
+    next if ($csv_line =~ m/^#/);
+
+    # my @tsv_entries = split('\t',$tsv_line);
+    my @csv_entries = split(/\s*,\s*/,$csv_line);
+
+    my $num_csv_entries = scalar(@csv_entries);
+
+    # print STDERR join(",",@tsv_entries), "\n\n";
+    
+    if (($num_csv_entries == 5) || ($num_csv_entries >= 6 && $csv_entries[5] =~ m/^#/)) {
+        my $src_metadata_name  = $csv_entries[0];
+        (my $src_metadata_regex = $csv_entries[1]) =~ s/(^\/)|(\/$)//g; #assigns as a /qr (quoted regex) and then strips off leading and trailing slash
+        (my $dst_metadata_regex = $csv_entries[2]) =~ s/(^\/)|(\/$)//g; #as line above
+        (my $regex_modifiers    = $csv_entries[3]) =~ s/(^')|('$)//g;   #similar to above but strips out single quotes
+        my $dst_metadata_name  = $csv_entries[4];
+
+        # If the last entry has a '# comments' after it, needs a bit more string manipulation
+        # to tidy it up
+        $dst_metadata_name =~ s/#.*$//;
+        $dst_metadata_name =~ s/\s*$//;
+        
+        my $regex_sub = "s/$src_metadata_regex/$dst_metadata_regex/$regex_modifiers";
+
+        my $metadata_mapping_rule_rec = {
+        "src_metadata_name" => $src_metadata_name,
+        "regex_sub"         => $regex_sub,
+        "dst_metadata_name" => $dst_metadata_name           
+        };
+            
+        push(@$metadata_mapping_rules,$metadata_mapping_rule_rec);
+    }
+    else {
+        print STDERR "Warning: syntax error in $metadata_mapping_full_filename, line $line_num\n";
+        print STDERR "  $csv_line\n";
+        print STDERR "Did not contain 5 tab-delimited entries\n";
+    }
+        
+    }
+
+    if (scalar(@$metadata_mapping_rules)>0) {
+    $self->{'metadata_mapping_rules'} = $metadata_mapping_rules;
+    }
+}
+
 sub new {
 
@@ -165 +234 @@
 
     my $self = new CommonUtil($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
-    
+
     if ($self->{'info_only'}) {
         # don't worry about any options etc
@@ -205 +274 @@
     }
 
-    return bless $self, $class;
+    my $blessed_self = bless $self, $class;
+    
+    if ($blessed_self->{'metadata_mapping_file'}) {
+    my $metadata_mapping_file = $blessed_self->{'metadata_mapping_file'};
+    my $mmf_full_filename;  
+    $mmf_full_filename = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'},"etc", $metadata_mapping_file);
+    if (!-e $mmf_full_filename) {
+        my $mmf_full_col_filename = $mmf_full_filename;
+        $mmf_full_filename = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"etc", $metadata_mapping_file);
+
+        if (!-e $mmf_full_filename) {
+        my $outhandle = $blessed_self->{'outhandle'};
+        print STDERR "\nBaseImporter Error: Can't locate metadata_mapping_file '$metadata_mapping_file'\n";
+        print STDERR "This file should be in $mmf_full_col_filename or $mmf_full_filename\n";
+        $blessed_self->print_txt_usage("");  # Use default resource bundle
+        print STDERR "\nBaseImporter Error: Can't locate metadata_mapping_file '$metadata_mapping_file'\n";
+        print STDERR "This file should be in $mmf_full_col_filename or $mmf_full_filename\n";
+        die "\n";
+        }
+    }
+
+    $blessed_self->{'metadata_mapping_full_filename'} = $mmf_full_filename;
+    $blessed_self->setup_metadata_mapping_rules($mmf_full_filename);
+    }
+
+    
+    return $blessed_self;
 
 }
@@ -688 +783 @@
 }
 
+
+sub apply_metadata_mapping_file {
+    my $self = shift (@_);  
+    my ($doc_obj) = @_;
+
+    my $verbosity = $self->{'verbosity'};
+    my $outhandle = $self->{'outhandle'};
+    
+    my $top_section = $doc_obj->get_top_section();
+
+    my $metadata_mapping_rules = $self->{'metadata_mapping_rules'};
+
+    my $transient_metadata_names = {};
+    my $new_metadata_names = {};
+
+    print $outhandle "  Applying metadata_mapping_file to document\n";
+    
+    foreach my $metadata_mapping_rule_rec (@$metadata_mapping_rules) {
+        
+    my $src_metadata_name = $metadata_mapping_rule_rec->{"src_metadata_name"};
+    my $dst_metadata_name = $metadata_mapping_rule_rec->{"dst_metadata_name"};
+
+    my $regex_sub = $metadata_mapping_rule_rec->{"regex_sub"};
+
+    my $metadata_vals = $doc_obj->get_metadata($top_section,$src_metadata_name);
+
+    foreach my $metadata_val (@$metadata_vals) {
+
+        my $store_metadata_val = $metadata_val;
+        
+        if ($verbosity >= 4) {
+        print $outhandle "    Testing for match with: \$metadata_val =~ $regex_sub\n"
+        }
+        eval ( "\$metadata_val =~ $regex_sub" );
+        if ($@) {
+        warn "$@";
+        }
+        else {
+        # print STDERR "**** after metadata_val = $metadata_val\n\n";
+        if ($verbosity >=2) {
+            if ($metadata_val ne $store_metadata_val) {
+
+            if ($verbosity >= 3) {
+                print $outhandle "      Transformed metadata to $dst_metadata_name: '$store_metadata_val' -> '$metadata_val'\n"
+            }
+            if ($verbosity >= 2 && ($dst_metadata_name !~ m/^_transient/)) {
+                print $outhandle "    Added new metadata $dst_metadata_name: '$metadata_val'\n"
+            }
+            }
+        }
+        $doc_obj->add_utf8_metadata($top_section,$dst_metadata_name,$metadata_val);
+
+        if ($dst_metadata_name =~ m/^_transient/) {
+            $transient_metadata_names->{$dst_metadata_name} = 1;
+        }
+        else {
+            if ($metadata_val ne $store_metadata_val) {
+            $new_metadata_names->{$dst_metadata_name} = 1;
+            }
+        }       
+        }
+    }
+    
+    }
+
+    if ($verbosity >=1) {
+    print $outhandle "  Number of metadata_mapping_file transformations for this document: ", scalar(keys %$new_metadata_names), "\n";
+
+    if ($verbosity >=2) {
+        print $outhandle "  Generated metadata names: ", join(", ",keys %$new_metadata_names), "\n";
+    }
+    }
+    
+    # Now remove any of dst_metadata_name which began _transient
+    foreach my $transient_metadata_name (keys %$transient_metadata_names) {
+    #print STDERR "transient_metadata_name = $transient_metadata_name\n";   
+    $doc_obj->delete_metadata($top_section,$transient_metadata_name);
+    }
+}
+
 sub post_process_doc_obj {
     my $self = shift (@_);  
     my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
 
+    if ($self->{'metadata_mapping_rules'}) {
+    $self->apply_metadata_mapping_file($doc_obj);
+    }
+    
     return 1;
 }
@@ -820 +999 @@
     return undef unless $self->can_process_this_file($filename_full_path);
     
-    #print STDERR "**** BEFORE READ INTO DOC OBJ: $file\n";
+    #print STDERR "**** BEFORE READ INTO DOC OBJ: $file\n";
     my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
     #print STDERR "**** AFTER READ INTO DOC OBJ: $file\n";

main/trunk/greenstone2/perllib/strings.properties

@@ -809 +809 @@
 BaseImporter.no_cover_image:Do not look for a prefix.jpg file (where prefix is the same prefix as the file being processed) to associate as a cover image.
 
+BaseImporter.metadata_mapping_file:Use the specified metadata mapping file to generate additional metadata for a document.  The specified comma-separated value file (csv) needs to be encoded as UTF8, and consists of a series of rules, with 5 entries per line.  The first entry in the line specifyies a source metadata value to select from the doucment being process, and the second entry is a regular expression the metadata must match for the rule to be applied (Note: the syntax used is Perl's regular expression substitution, where use of parentheses form capture groups).  If it does match, then the third element is what the matching metadata value is transformed into (groups formed with brackets from the source metadata matching term can be referenced as $1, $2 and so on). The fourth entry specifies any modifiers for the substitution, such as 'g' for global and 'i' for case-insensitive.  The fifth entry specifies the metadata name that is set with the newly created value.  The rules are applied in the order they are provided in the comma-separated value file, so it is permissible for metadata set by one of the earlier rules to then be used in a later matching rule. Destination metadata names that start '_transient' are not stored in the final document.  For an example of a metadata_mapping_file, refer to the one provided in GSDLHOME/etc/metadta_mapping_rules.csv
+
 BaseImporter.OIDtype.auto:Use OIDtype set in import.pl