Changeset 34823

Timestamp:

2021-02-11T18:13:01+13:00 (3 years ago)

Author:

davidb

Message:

Merging of records introduced

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/JSONSPARQLResultPlugin.pm (modified) (8 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/JSONSPARQLResultPlugin.pm

@@ -32 +32 @@
 package JSONSPARQLResultPlugin;
 
+
+
 use strict;
 no strict 'refs'; # allow filehandles to be variables and viceversa
@@ -52 +54 @@
     'type' => "regexp",
     'reqd' => "no",
-    'deft' => &get_default_process_exp() }
+    'deft' => &get_default_process_exp() },
+      { 'name' => "decode_json_with_backslash_u",
+    'desc' => "{JSONSPARQLResultPlugin.decode_json_with_backslash_u}",
+    'type' => "flag",
+    'reqd' => "no" },
+      { 'name' => "metadata_merge_on_concat_fields",
+    'desc' => "{JSONSPARQLResultPlugin.metadata_merge_on_concat_fields}",
+    'type' => "string",
+    'reqd' => "no" }
       ];
 
@@ -83 +93 @@
 
 
+sub sparqlresult_items_equal
+{
+    my $self = shift (@_);
+    my ($lhs_entry,$rhs_entry) = @_;
+
+    my $is_equal = 0;
+    
+    my $lhs_type = $lhs_entry->{'type'};
+    my $rhs_type = $rhs_entry->{'type'};
+
+    if ($lhs_type eq $rhs_type) {
+##  print STDERR " checking $lhs_entry->{'value'} == $rhs_entry->{'value'}\n";
+
+    $is_equal = ($lhs_entry->{'value'} eq $rhs_entry->{'value'});
+    }
+
+    return $is_equal;
+}
+
+    
 sub split_text_into_segments {
     my $self = shift (@_);
@@ -90 +120 @@
 
     # Convert textref (a json-string) into nested object
-    
-    my $results_hashmap = decode_json($$textref);
+
+    my $results_hashmap;
+    
+    if ($self->{'decode_json_with_backslash_u'}) {
+    $results_hashmap = decode_json($$textref);
+    }
+    else {
+    $results_hashmap = from_json($$textref);
+    }
     
     # To get each result record returned:
@@ -98 +135 @@
     my $results_array = $results_hashmap->{'results'}->{'bindings'};
 
+    my $merge_on = $self->{'metadata_merge_on_concat_fields'};
+
+    ## print STDERR "**** merge_on = '$merge_on'\n";
+
+    if ($merge_on ne "") {
+    # build hashmap on merged metadata fields, and then regenerate @results_array
+
+    my $merged_hashmap = {};
+    
+    my @merge_fields = split(/\s*,\s*/,$merge_on);
+    
+    foreach my $result_entry (@$results_array) {
+        my $merged_key = "";
+        foreach my $field (@merge_fields) {
+        $merged_key .= $result_entry->{$field}->{'value'};
+        }
+
+        if (!defined $merged_hashmap->{$merged_key}) {
+        
+        $merged_hashmap->{$merged_key} = {};
+        foreach my $md_key (keys %$result_entry) {          
+            my $md_val = $result_entry->{$md_key};
+            $merged_hashmap->{$merged_key}->{$md_key} = [ $md_val ];
+        }
+        
+        }
+        else {
+        foreach my $md_key (keys %$result_entry) {
+            my $existing_md_vals = \@{$merged_hashmap->{$merged_key}->{$md_key}};
+            my $md_val = $result_entry->{$md_key};
+            
+            # append only if not already present
+            my $found_val = 0;
+            foreach my $existing_md_val (@$existing_md_vals) {
+
+##          print STDERR "$merged_key: ";
+            my $is_match = $self->sparqlresult_items_equal($md_val,$existing_md_val);
+            
+            if ($is_match) {
+                $found_val = 1;
+                last;
+            }
+            }
+
+            if (!$found_val) {
+            push(@$existing_md_vals, $md_val);
+            }
+        }
+        
+        }
+    }
+
+    # Iterate through merged_hashmap to egenerate result_array with newly merge record entires
+    
+    $results_array = [];
+    
+    foreach my $merged_key (keys %$merged_hashmap) {
+        push(@$results_array,$merged_hashmap->{$merged_key});
+    }
+    }
+
+    
     foreach my $result_entry (@$results_array) {
-    my $result_entry_stringified = encode_json($result_entry);
+    my $result_entry_stringified;
+    if ($self->{'decode_json_with_backslash_u'}) {  
+        $result_entry_stringified = encode_json($result_entry);
+    }
+    else {
+        $result_entry_stringified = to_json($result_entry);
+    }
     push(@segments,$result_entry_stringified);
     
     #print STDERR "*** result_entry = $result_entry_stringified\n";
     }
-
+    
     return \@segments;
 }
@@ -116 +221 @@
 
     my $section = $doc_obj->get_top_section();
-    
+
+    my $result_entry;
     #my $result_entry = decode_json($$textref);
-    # $textref is UTF8 byte encoded, not "Unicode-aware", so need to specify this in optional param to from_json()
-    my $result_entry = from_json($$textref, {utf8 => 1} ); 
+    if ($self->{'decode_json_with_backslash_u'}) {  
+    # $textref is UTF8 byte encoded, not "Unicode-aware", so need to specify this in optional param to from_json()
+    $result_entry = from_json($$textref, {utf8 => 1} );
+    }
+    else {
+    $result_entry = from_json($$textref);
+    }
 
     # In JSON syntax, it seems permissible to represent Unicode characters in the form \uNNNN
@@ -132 +243 @@
 
     # Set the "text document" to be a pre-formatted, HTML friendly version of the JSON syntax
-    $self->text_to_html(\$text_utf8); 
-    $doc_obj->add_utf8_text($section, $text_utf8);
+    #
+    # **** Now done later on, as pretty-print text, pp_txt
+    #$self->text_to_html(\$text_utf8); 
+    #$doc_obj->add_utf8_text($section, $text_utf8);
 
     # entrant{value,type=uri}
@@ -139 +252 @@
     # country{value,xml:lang=en,type=literal}
 
+    my $pp_text = "";
+
+    print STDERR "[";
+    
     foreach my $md_name (keys %$result_entry)
     {
-    print STDERR "*** md_name = $md_name\n";
-    my $md_name_lod_rec = $result_entry->{$md_name};
-    if ($md_name_lod_rec->{'type'} eq "literal") {
-        my $md_val = $md_name_lod_rec->{'value'};
-        $doc_obj->add_utf8_metadata($section, $md_name, $md_val);
-    }
-    elsif ($md_name_lod_rec->{'type'} eq "uri") {
-        my $md_val = $md_name_lod_rec->{'value'};
-        $doc_obj->add_utf8_metadata($section, $md_name."_uri", $md_val);
-    }
-    
-    }
+    print STDERR " $md_name ";
+
+    my $md_name_lod_generic_entry = $result_entry->{$md_name};
+    my $md_name_lod_array;
+    
+    if (ref($md_name_lod_generic_entry) eq "ARRAY") {
+        $md_name_lod_array = $md_name_lod_generic_entry;
+    }
+    else {
+        $md_name_lod_array = [ $md_name_lod_generic_entry ];
+    }
+
+    foreach my $md_name_lod_rec (@$md_name_lod_array) {
+
+        ## the original line to pick up on record
+        # my $md_name_lod_rec = $result_entry->{$md_name};
+        
+        if ($md_name_lod_rec->{'type'} eq "literal") {
+        my $md_val = $md_name_lod_rec->{'value'};
+        
+        # Deal with any %XX encodings
+        $md_val =~ s/%([A-Fa-f\d]{2})/chr hex $1/eg;
+        
+        $doc_obj->add_utf8_metadata($section, $md_name, $md_val);
+        $pp_text .= "$md_name: $md_val\n";
+        }
+        elsif ($md_name_lod_rec->{'type'} eq "uri") {
+        my $md_val = $md_name_lod_rec->{'value'};
+        $doc_obj->add_utf8_metadata($section, $md_name."_uri", $md_val);
+        }
+    }
+    
+    }
+    print STDERR "]\n";
+    
+    $doc_obj->add_utf8_text($section, $pp_text);
     
     return 1;