Changeset 34695

Timestamp:

2021-01-24T12:44:31+13:00 (3 years ago)

Author:

davidb

Message:

More careful treatment of UTF8 and Unicode-aware strings while beig processed as JSON. Addresses an issue with JSON syntax using \uNNNN format (literal ASCII chars) for unicode chars

File:

• main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/JSONSPARQLResultPlugin.pm (modified) (2 diffs)

main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/JSONSPARQLResultPlugin.pm

@@ -117 +117 @@
     my $section = $doc_obj->get_top_section();
     
-    # Add the raw line as the document text
-    $doc_obj->add_utf8_text($section, $$textref);
+    #my $result_entry = decode_json($$textref);
+    # $textref is UTF8 byte encoded, not "Unicode-aware", so need to specify this in optional param to from_json()
+    my $result_entry = from_json($$textref, {utf8 => 1} ); 
 
-    my $result_entry = decode_json($$textref);
+    # In JSON syntax, it seems permissible to represent Unicode characters in the form \uNNNN
+    # These are not automatically converted into UTF8 chars by reading in the file as text
+    # (it is tempting to think about putting such decoding into the read-text-file subroutine,
+    #  but holding off on that idea for now, as could have wider unforseen implications)
+    #
+    # For now: The \uNNNN is dealt with by having the to_json() result re-encoded
+    my $text_utf8 = to_json($result_entry, { pretty => 1} );
 
-    # entrant{value,type=url}
+    $doc_obj->add_utf8_metadata($section, "JSON", $text_utf8);
+
+    # Set the "text document" to be a pre-formatted, HTML friendly version of the JSON syntax
+    $self->text_to_html(\$text_utf8); 
+    $doc_obj->add_utf8_text($section, $text_utf8);
+
+    # entrant{value,type=uri}
     # song{value,type=uri}
     # country{value,xml:lang=en,type=literal}
@@ -134 +147 @@
         $doc_obj->add_utf8_metadata($section, $md_name, $md_val);
     }
+    elsif ($md_name_lod_rec->{'type'} eq "uri") {
+        my $md_val = $md_name_lod_rec->{'value'};
+        $doc_obj->add_utf8_metadata($section, $md_name."_uri", $md_val);
+    }
     
     }