Index: main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/EssentiaMetadataJSONPlugin.pm
===================================================================
--- main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/EssentiaMetadataJSONPlugin.pm	(revision 35837)
+++ main/trunk/model-sites-dev/eurovision-lod/collect/eurovision/perllib/plugins/EssentiaMetadataJSONPlugin.pm	(revision 35837)
@@ -0,0 +1,264 @@
+###########################################################################
+#
+# EssentiaMetadataJSONPlugin.pm -- A plugin for JSON files resulting
+# from running essentia_stream_music_extract tool from UPF
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright 2016 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package EssentiaMetadataJSONPlugin;
+
+
+use BaseImporter;
+use MetadataRead;
+
+use strict;
+no strict 'refs';
+use multiread;
+
+use Encode;
+use JSON;
+
+# methods with identical signatures take precedence in the order given in the ISA list.
+sub BEGIN {
+    @EssentiaMetadataJSONPlugin::ISA = ('MetadataRead', 'BaseImporter');
+}
+
+
+my $arguments = [
+      { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'reqd' => "no",
+	'deft' => &get_default_process_exp() },
+      { 'name' => "metadata_name_joinchar",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "char",
+	'reqd' => "no",
+	'deft' => "_" },
+#     { 'name' => "associcate_ext",
+#	'desc' => "{BaseImporter.associate_ext}",
+#	'type' => "string",
+#	'reqd' => "no",
+#	'deft' => "nul" },
+      { 'name' => "metadata_bind_to_filename",
+	'desc' => "{EssentiaMetadataJSONPlugin.metadata_bind_to_filename}",
+	'type' => "regexp",
+	'reqd' => "yes" },
+    
+
+];
+
+
+my $options = { 'name'     => "EssentiaMetadataJSONPlugin",
+		'desc'     => "{EssentiaMetadataJSONPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new
+{
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
+
+    return bless $self, $class;
+}
+
+
+sub get_default_process_exp
+{
+    return q^(?i)\.json$^;
+}
+
+sub file_block_read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
+
+    my $ret_status = $self->SUPER::file_block_read(@_);
+
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+#    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
+#	return undef; # can't recognise
+#    }
+
+    if ((defined $ret_status) && ($ret_status == 1)) {
+	# set this so we know this is a metadata file - needed for incremental 
+	# build
+	# if this file changes, then we need to reimport everything
+	$block_hash->{'metadata_files'}->{$filename_full_path} = 1;
+
+	$ret_status = 1;
+    }
+    
+    return $ret_status;
+}
+
+sub nested_json_to_flat_json
+{
+    my $self = shift (@_);
+    my ($md_prefix,$md_joinchar,$json_metadata_node,$json_metadata_flat) = @_;
+
+	 
+    # Set up append items from json_metadata_node into json_metadat_flat
+    # If item is itself a nested-node, then recurse
+    
+    
+    foreach my $md_name (keys %$json_metadata_node) {
+	my $full_md_name = ($md_prefix eq "essentia")
+	    ? join(".",$md_prefix,$md_name)
+	    : join($md_joinchar,$md_prefix,$md_name);
+
+	my $md_value = $json_metadata_node->{$md_name};
+	
+	if (ref($md_value) eq "HASH") {
+	    $self->nested_json_to_flat_json($full_md_name,$md_joinchar,$md_value,$json_metadata_flat);
+	}
+	elsif (ref($md_value) eq "ARRAY") {
+	    # Essentia array are typically numeric features, so no real benefit representing them
+	    # "natively" as a Greenstone array of single values (as we would want to do with multiple
+	    # author metadata values)
+	    # => turn into a JSON string representing the array, and assume a client retrieving
+	    #    this, will then turn it into something useful at that point
+
+	    $json_metadata_flat->{$full_md_name} = [ encode_json($md_value) ];
+	}
+	else {
+	    # scalar
+	    $json_metadata_flat->{$full_md_name} = [ $json_metadata_node->{$md_name} ];
+	}
+	
+	# delete $json_metadata_node->{$md_name};
+    }
+}
+
+
+sub metadata_read
+{
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+	$extrametakeys, $extrametadata, $extrametafile,
+	$processor, $gli, $aux) = @_;
+
+    # Read metadata from JSON files
+    #my $filename = &util::filename_cat($base_dir, $file);
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    
+    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
+	return undef;
+    }
+    
+    print STDERR "\n<Processing n='$file' p='EssentiaMetadataJSONPlugin'>\n" if ($gli);
+    print STDERR "EssentiaMetadataJSONPlugin: metadata_read() processing $file\n" if ($self->{'verbosity'}) > 1;
+    
+    my $outhandle = $self->{'outhandle'};
+    my $failhandle = $self->{'failhandle'};
+
+    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
+    $self->block_filename($block_hash,$filename_full_path);
+
+
+    # Read the JSON file to get the metadata
+    my $json_file_content;
+    open(JSON_FILE, "$filename_full_path");
+    my $json_file_reader = new multiread();
+    $json_file_reader->set_handle('EssentiaMetadataJSONPlugin::JSON_FILE');
+    $json_file_reader->read_file(\$json_file_content);
+
+    # Would be nice if EssentiaMetadataJSONPlugin was extended to support a minus
+    # option to choose the character encoding the JSON file is in 
+    # For now we will assume it is always in UTF8
+
+    my $json_file_content_bytes = encode('UTF-8', $json_file_content);
+
+    $json_file_content = decode("utf8",$json_file_content_bytes);
+
+    close(JSON_FILE);
+
+    # print STDERR "**** json_metadata = $json_file_content\n";
+
+    # my $json_metadata = decode_json $json_file_content;
+    my $json_metadata_nested = JSON->new->utf8->decode($json_file_content);
+
+
+    my $metadata_bind_to_filename_regex = $self->{'metadata_bind_to_filename'};
+    # print STDERR "**** metadata_bind_to_filename_regex = $metadata_bind_to_filename_regex\n";
+    my $bind_to_filename = $filename_full_path;    
+    eval ( "\$bind_to_filename =~ $metadata_bind_to_filename_regex"  );
+
+    my ($bind_to_file) = ($bind_to_filename =~ m/^.*(?:\/|\\)(.*?)$/);
+    
+    if (! -f $bind_to_filename) {
+	$self->print_error($outhandle, $failhandle, $gli, $filename_full_path,
+			   "Unable to bind JSON file '$file' to '$bind_to_file' through regular expression '$metadata_bind_to_filename_regex' as file does not exist");
+	return -1;
+    }
+
+    my $metadata_name_joinchar = $self->{'metadata_name_joinchar'};
+    
+    my $json_metadata_flat = {};
+    $self->nested_json_to_flat_json("essentia",$metadata_name_joinchar,$json_metadata_nested,$json_metadata_flat);
+    
+				    
+#    # Set up metadata table based on JSON file read in
+#    foreach my $k (keys %$json_metadata) {
+#	$json_metadata->{'essentia.'.$k} = [ $json_metadata->{$k} ];
+#	delete $json_metadata->{$k};
+#    }
+
+    my $bind_to_file_re = &util::filename_to_regex($bind_to_file);
+
+
+    # Set up mapping from bind_to_file_re to the created JSON metadata table 
+    # so it 'sticks' to the file
+
+    $extrametadata->{$bind_to_file_re} = $json_metadata_flat;
+    push(@$extrametakeys, $bind_to_file_re);
+    # record which file the metadata came from 
+    if (!defined $extrametafile->{$bind_to_file_re}) {
+	$extrametafile->{$bind_to_file_re} = {};
+    }
+
+    # maps the file to full path
+    $extrametafile->{$bind_to_file_re}->{$file} = $filename_full_path;
+
+    return 1; 
+}
+
+sub print_error
+{
+    my $self = shift(@_);
+    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
+
+    print $outhandle "EssentiaMetadataJSONPlugin Error: $file: $error\n";
+    print $failhandle "EssentiaMetadataJSONPlugin Error: $file: $error\n";
+    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
+}
+
+1;
