root/gs3-extensions/mars-src/trunk/perllib/plugins/AMCMetadataJSONPlugin.pm @ 34356

Revision 34356, 6.3 KB (checked in by davidb, 7 weeks ago)

Some initial work computing essensia audio features when the collection is built

Line 
1###########################################################################
2#
3# AMCMetadataJSONPlugin.pm -- A plugin for JSON files resulting from page
4#                             scrape of artists on the Australian Music Centre
5#                             website
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright 2016 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29package AMCMetadataJSONPlugin;
30
31
32use BaseImporter;
33use MetadataRead;
34
35use strict;
36no strict 'refs';
37use multiread;
38
39use Encode;
40use JSON;
41
42# methods with identical signatures take precedence in the order given in the ISA list.
43sub BEGIN {
44    @AMCMetadataJSONPlugin::ISA = ('MetadataRead', 'BaseImporter');
45}
46
47
48my $arguments = [
49      { 'name' => "process_exp",
50    'desc' => "{BaseImporter.process_exp}",
51    'type' => "regexp",
52    'reqd' => "no",
53    'deft' => &get_default_process_exp() }
54
55];
56
57
58my $options = { 'name'     => "AMCMetadataJSONPlugin",
59        'desc'     => "{AMCMetadataJSONPlugin.desc}",
60        'abstract' => "no",
61        'inherits' => "yes",
62        'args'     => $arguments };
63
64
65sub new
66{
67    my ($class) = shift (@_);
68    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
69    push(@$pluginlist, $class);
70
71    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
72    push(@{$hashArgOptLists->{"OptList"}},$options);
73
74    my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
75
76    return bless $self, $class;
77}
78
79
80sub get_default_process_exp
81{
82    return q^(?i)\.json$^;
83}
84
85sub file_block_read {
86    my $self = shift (@_);
87    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
88
89    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
90
91    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
92    return undef; # can't recognise
93    }
94
95    # set this so we know this is a metadata file - needed for incremental
96    # build
97    # if this file changes, then we need to reimport everything
98    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
99
100    return 1;
101}
102
103sub metadata_read
104{
105    my $self = shift (@_);
106    my ($pluginfo, $base_dir, $file, $block_hash,
107    $extrametakeys, $extrametadata, $extrametafile,
108    $processor, $gli, $aux) = @_;
109
110    # Read metadata from JSON files
111    my $filename = &util::filename_cat($base_dir, $file);
112
113    if (!-f $filename || !$self->can_process_this_file($filename)) {
114
115    return undef;
116    }
117    print STDERR "\n<Processing n='$file' p='AMCMetadataJSONPlugin'>\n" if ($gli);
118    print STDERR "AMCMetadataJSONPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
119
120    my $outhandle = $self->{'outhandle'};
121    my $failhandle = $self->{'failhandle'};
122
123    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
124    $self->block_filename($block_hash,$filename);
125
126
127    # Read the JSON file to get the metadata
128    my $json_file_content;
129    open(JSON_FILE, "$filename");
130    my $json_file_reader = new multiread();
131    $json_file_reader->set_handle('AMCMetadataJSONPlugin::JSON_FILE');
132    $json_file_reader->read_file(\$json_file_content);
133
134    # Would be nice if AMCMetadataJSONPlugin was extended to support a minus
135    # option to choose the character encoding the JSON file is in
136    # For now we will assume it is always in UTF8
137
138    my $json_file_content_bytes = encode('UTF-8', $json_file_content);
139
140    $json_file_content = decode("utf8",$json_file_content_bytes);
141
142    close(JSON_FILE);
143
144    # Split the file into lines and read the first line (contains the metadata names)
145#    $json_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
146#    $json_file_content =~ s/\n+/\n/g;
147#    my @json_file_lines = split(/;/s, $json_file_content);
148
149#    my $matching_file = $file;
150#    $matching_filename =~ s/\.js/\.{ogg,mp3,wav}/;
151
152
153#    my $json_metadata = {};
154
155    print STDERR "**** json_metadata = $json_file_content\n";
156    # my $json_metadata = decode_json $json_file_content;
157    my $json_metadata = JSON->new->utf8->decode($json_file_content);
158
159    # We can't associate any metadata without knowing the file to associate it with
160    my $audio_url = $json_metadata->{"audio_url"};
161    if (!defined $audio_url) {
162    $self->print_error($outhandle, $failhandle, $gli, $filename, "No audio_url metadata in JSON file: $file");
163   
164    return -1;
165    }
166
167    my $local_audio_file = $audio_url;
168    $local_audio_file =~ s/^http:\/\/.*\///;
169   
170    print STDERR "***** local audio file = $local_audio_file\n";
171 
172    foreach my $k (keys %$json_metadata) {
173    $json_metadata->{'amc.'.$k} = [ $json_metadata->{$k} ];
174    delete $json_metadata->{$k};
175    }
176
177
178#    my $json_filename = shift(@$json_trackurl_array);
179#    $json_filename =~ s/\"//g; # strip off quotes
180#    delete $json_metadata->{"track_url"};
181   
182    # Associate the metadata now
183    $local_audio_file = &util::filename_to_regex($local_audio_file);
184
185###  print STDERR "**** setting up mapping for: $json_filename\n";
186
187    $extrametadata->{$local_audio_file} = $json_metadata;
188    push(@$extrametakeys, $local_audio_file);
189    # record which file the metadata came from
190    if (!defined $extrametafile->{$local_audio_file}) {
191    $extrametafile->{$local_audio_file} = {};
192    }
193    # maps the file to full path
194    $extrametafile->{$local_audio_file}->{$file} = $filename;
195
196    return 1;
197}
198
199sub print_error
200{
201
202    my $self = shift(@_);
203    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
204
205    print $outhandle "AMCMetadataJSONPlugin Error: $file: $error\n";
206    print $failhandle "AMCMetadataJSONPlugin Error: $file: $error\n";
207    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
208}
2091;
Note: See TracBrowser for help on using the browser.