root/main/trunk/model-sites-dev/atea/collect/digital-nz/perllib/plugins/DNZMetadataJSON.pm @ 33166

Revision 33166, 6.3 KB (checked in by davidb, 2 months ago)

Collection config files and initial programming work for atea collections

Line 
1###########################################################################
2#
3# DNZMetadataJSONPlugin.pm -- A plugin for JSON files resulting from downloadin
4#                             records using the Digital NZ API
5#
6
7# NEVER BEEN TESTED
8# EVEN THE NAME SHOULD PERHAPS BE CHANGED TO MetadataDNZJSONPlugin
9
10
11# A component of the Greenstone digital library software
12# from the New Zealand Digital Library Project at the
13# University of Waikato, New Zealand.
14#
15# Copyright 2016 New Zealand Digital Library Project
16#
17# This program is free software; you can redistribute it and/or modify
18# it under the terms of the GNU General Public License as published by
19# the Free Software Foundation; either version 2 of the License, or
20# (at your option) any later version.
21#
22# This program is distributed in the hope that it will be useful,
23# but WITHOUT ANY WARRANTY; without even the implied warranty of
24# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
25# GNU General Public License for more details.
26#
27# You should have received a copy of the GNU General Public License
28# along with this program; if not, write to the Free Software
29# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
30#
31###########################################################################
32
33package DNZMetadataJSONPlugin;
34
35
36use BasePlugin;
37use MetadataRead;
38
39use strict;
40no strict 'refs';
41use multiread;
42
43use Encode;
44use JSON;
45
46# methods with identical signatures take precedence in the order given in the ISA list.
47sub BEGIN {
48    @DNZJSONPlugin::ISA = ('MetadataRead', 'BaseImporter');
49}
50
51
52my $arguments = [
53      { 'name' => "process_exp",
54    'desc' => "{BasePlugin.process_exp}",
55    'type' => "regexp",
56    'reqd' => "no",
57    'deft' => &get_default_process_exp() }
58
59];
60
61
62my $options = { 'name'     => "DNZJSONPlugin",
63        'desc'     => "{DNZJSONPlugin.desc}",
64        'abstract' => "no",
65        'inherits' => "yes",
66        'args'     => $arguments };
67
68
69sub new
70{
71    my ($class) = shift (@_);
72    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
73    push(@$pluginlist, $class);
74
75    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
76    push(@{$hashArgOptLists->{"OptList"}},$options);
77
78    my $self = new BaseImporter($pluginlist, $inputargs, $hashArgOptLists);
79
80    return bless $self, $class;
81}
82
83
84sub get_default_process_exp
85{
86    return q^(?i)\.json$^;
87}
88
89sub file_block_read {
90    my $self = shift (@_);
91    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
92
93    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
94
95    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
96    return undef; # can't recognise
97    }
98
99    # set this so we know this is a metadata file - needed for incremental
100    # build
101    # if this file changes, then we need to reimport everything
102    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
103
104    return 1;
105}
106
107
108
109sub metadata_read
110{
111    my $self = shift (@_);
112    my ($pluginfo, $base_dir, $file, $block_hash,
113    $extrametakeys, $extrametadata, $extrametafile,
114    $processor, $gli, $aux) = @_;
115
116    # Read metadata from JSON files
117    my $filename = &util::filename_cat($base_dir, $file);
118
119    if (!-f $filename || !$self->can_process_this_file($filename)) {
120
121    return undef;
122    }
123    print STDERR "\n<Processing n='$file' p='DNZJSONPlugin'>\n" if ($gli);
124    print STDERR "DNZJSONPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
125
126    my $outhandle = $self->{'outhandle'};
127    my $failhandle = $self->{'failhandle'};
128
129    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
130    &util::block_filename($block_hash,$filename);
131
132
133    # Read the JSON file to get the metadata
134    my $json_file_content;
135    open(JSON_FILE, "$filename");
136    my $json_file_reader = new multiread();
137    $json_file_reader->set_handle('DNZJSONPlugin::JSON_FILE');
138    $json_file_reader->read_file(\$json_file_content);
139
140    # Would be nice if DNZJSONPlugin was extended to support a minus
141    # option to choose the character encoding the JSON file is in
142    # For now we will assume it is always in UTF8
143
144    my $json_file_content_bytes = encode('UTF-8', $json_file_content);
145
146    $json_file_content = decode("utf8",$json_file_content_bytes);
147
148    close(JSON_FILE);
149
150    # Split the file into lines and read the first line (contains the metadata names)
151#    $json_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
152#    $json_file_content =~ s/\n+/\n/g;
153#    my @json_file_lines = split(/;/s, $json_file_content);
154
155#    my $matching_file = $file;
156#    $matching_filename =~ s/\.js/\.{ogg,mp3,wav}/;
157
158
159#    my $json_metadata = {};
160
161    print STDERR "**** json_metadata = $json_file_content\n";
162    # my $json_metadata = decode_json $json_file_content;
163    my $json_metadata = JSON->new->utf8->decode($json_file_content);
164
165    # We can't associate any metadata without knowing the file to associate it with
166    my $audio_url = $json_metadata->{"audio_url"};
167    if (!defined $audio_url) {
168    $self->print_error($outhandle, $failhandle, $gli, $filename, "No audio_url metadata in JSON file: $file");
169   
170    return -1;
171    }
172
173    my $local_audio_file = $audio_url;
174    $local_audio_file =~ s/^http:\/\/.*\///;
175   
176    print STDERR "***** local audio file = $local_audio_file\n";
177 
178    foreach my $k (keys %$json_metadata) {
179    $json_metadata->{'amc.'.$k} = [ $json_metadata->{$k} ];
180    delete $json_metadata->{$k};
181    }
182
183
184#    my $json_filename = shift(@$json_trackurl_array);
185#    $json_filename =~ s/\"//g; # strip off quotes
186#    delete $json_metadata->{"track_url"};
187   
188    # Associate the metadata now
189    $local_audio_file = &util::filename_to_regex($local_audio_file);
190
191###  print STDERR "**** setting up mapping for: $json_filename\n";
192
193    $extrametadata->{$local_audio_file} = $json_metadata;
194    push(@$extrametakeys, $local_audio_file);
195    # record which file the metadata came from
196    if (!defined $extrametafile->{$local_audio_file}) {
197    $extrametafile->{$local_audio_file} = {};
198    }
199    # maps the file to full path
200    $extrametafile->{$local_audio_file}->{$file} = $filename;
201
202    return 1;
203}
204
205sub print_error
206{
207
208    my $self = shift(@_);
209    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
210
211    print $outhandle "DNZJSONPlugin Error: $file: $error\n";
212    print $failhandle "DNZJSONPlugin Error: $file: $error\n";
213    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
214}
215
Note: See TracBrowser for help on using the browser.