source: gs2-extensions/music-ir-src/trunk/perllib/plugins/jSongMinerExtractor.pm@ 30436

Last change on this file since 30436 was 30436, checked in by davidb, 8 years ago

Tweak to allow %xx%xx encodings expressing UTF8 to be correctly handled

  • Property svn:executable set to *
File size: 12.0 KB
Line 
1###########################################################################
2#
3# jSongMinerExtractor - helper plugin that identifies audio through
4# external web services based on either an audio
5# computed fingerprint or ID3 title and album
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2010 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28package jSongMinerExtractor;
29
30use BaseMediaConverter;
31
32use Cwd;
33use URI::Escape;
34
35use strict;
36no strict 'refs'; # allow filehandles to be variables and viceversa
37
38
39BEGIN {
40 @jSongMinerExtractor::ISA = ('BaseMediaConverter');
41}
42
43
44my $arguments = [
45 { 'name' => "track_identification",
46 'desc' => "{jSongMinerExtractor.track_identification}",
47 'type' => "enum",
48 'list' => [{'name' => "Fingerprint then ID3 tags", 'desc' => "{jSongMinerExtractor.fingerprint_first}"},
49 {'name' => "ID3 tags only", 'desc' => "{jSongMinerExtractor.only_ids}"},
50 {'name' => "Disabled", 'desc' => "{jSongMinerExtractor.off}"} ],
51 'deft' => 'Fingerprint then ID3 tags',
52 'reqd' => "no" }
53 ];
54
55my $options = { 'name' => "jSongMinerExtractor",
56 'desc' => "{jSongMinerExtractor.desc}",
57 'abstract' => "yes",
58 'inherits' => "yes",
59 'args' => $arguments };
60
61sub new {
62 my ($class) = shift (@_);
63 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
64 push(@$pluginlist, $class);
65
66 push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
67 push(@{$hashArgOptLists->{"OptList"}},$options);
68
69 my $self = new BaseMediaConverter($pluginlist, $inputargs, $hashArgOptLists, 1);
70
71 # Set controlling variables
72 my $gsdl_home = $ENV{'GSDLHOME'};
73 my $music_ir_home = $ENV{'GEXT_MUSICIR'};
74
75 $self->{'jmir_directory'} = &util::filename_cat($music_ir_home,"lib","java"); # Set the directory holding the jMIR .jar files
76
77 return bless $self, $class;
78}
79
80sub urlEncode{
81 # ARG 1: $to_encode is the string to URL encode
82 my ($to_encode) = @_;
83 return uri_escape($to_encode);
84}
85
86# URL Decode the given string
87sub urlDecode{
88 # ARG 1: $to_decode is the string to URL decode
89 my ($to_decode) = @_;
90 # really want to use 'uri_unescape_utf8' in the following, but
91 # it doesn't exist in the Perl module. Follow will suffice
92 # my $decoded= uri_unescape_utf8($to_decode);
93 my $decoded = Encode::decode_utf8(uri_unescape($to_decode));
94
95 return $decoded;
96}
97
98sub map_id3v1_genre_num
99{
100 my ($genre_num) = @_;
101
102 $genre_num =~ s/(/\(/g;
103 $genre_num =~ s/)/\)/g;
104 $genre_num =~ s/^[\(\[\{](.*)[\)\]\}]$/$1/;
105
106 if (($genre_num =~ m/^\d+$/) && ($genre_num >= 148)) {
107 return "Unknown";
108 }
109
110 my $lookup_id3v1_genre = {
111 "0" => "Blues",
112 "1" => "Classic Rock",
113 "2" => "Country",
114 "3" => "Dance",
115 "4" => "Disco",
116 "5" => "Funk",
117 "6" => "Grunge",
118 "7" => "Hip-Hop",
119 "8" => "Jazz",
120 "9" => "Metal",
121 "10" => "New Age",
122 "11" => "Oldies",
123 "12" => "Other",
124 "13" => "Pop",
125 "14" => "R&B",
126 "15" => "Rap",
127 "16" => "Reggae",
128 "17" => "Rock",
129 "18" => "Techno",
130 "19" => "Industrial",
131 "20" => "Alternative",
132 "21" => "Ska",
133 "22" => "Death Metal",
134 "23" => "Pranks",
135 "24" => "Soundtrack",
136 "25" => "Euro-Techno",
137 "26" => "Ambient",
138 "27" => "Trip-Hop",
139 "28" => "Vocal",
140 "29" => "Jazz+Funk",
141 "30" => "Fusion",
142 "31" => "Trance",
143 "32" => "Classical",
144 "33" => "Instrumental",
145 "34" => "Acid",
146 "35" => "House",
147 "36" => "Game",
148 "37" => "Sound Clip",
149 "38" => "Gospel",
150 "39" => "Noise",
151 "40" => "Alternative Rock",
152 "41" => "Bass",
153 "42" => "Soul",
154 "43" => "Punk",
155 "44" => "Space",
156 "45" => "Meditative",
157 "46" => "Instrumental Pop",
158 "47" => "Instrumental Rock",
159 "48" => "Ethnic",
160 "49" => "Gothic",
161 "50" => "Darkwave",
162 "51" => "Techno-Industrial",
163 "52" => "Electronic",
164 "53" => "Pop-Folk",
165 "54" => "Eurodance",
166 "55" => "Dream",
167 "56" => "Southern Rock",
168 "57" => "Comedy",
169 "58" => "Cult",
170 "59" => "Gangsta",
171 "60" => "Top 40",
172 "61" => "Christian Rap",
173 "62" => "Pop/Funk",
174 "63" => "Jungle",
175 "64" => "Native US",
176 "65" => "Cabaret",
177 "66" => "New Wave",
178 "67" => "Psychadelic",
179 "68" => "Rave",
180 "69" => "Showtunes",
181 "70" => "Trailer",
182 "71" => "Lo-Fi",
183 "72" => "Tribal",
184 "73" => "Acid Punk",
185 "74" => "Acid Jazz",
186 "75" => "Polka",
187 "76" => "Retro",
188 "77" => "Musical",
189 "78" => "Rock & Roll",
190 "79" => "Hard Rock",
191 "80" => "Folk",
192 "81" => "Folk-Rock",
193 "82" => "National Folk",
194 "83" => "Swing",
195 "84" => "Fast Fusion",
196 "85" => "Bebob",
197 "86" => "Latin",
198 "87" => "Revival",
199 "88" => "Celtic",
200 "89" => "Bluegrass",
201 "90" => "Avantgarde",
202 "91" => "Gothic Rock",
203 "92" => "Progressive Rock",
204 "93" => "Psychedelic Rock",
205 "94" => "Symphonic Rock",
206 "95" => "Slow Rock",
207 "96" => "Big Band",
208 "97" => "Chorus",
209 "98" => "Easy Listening",
210 "99" => "Acoustic",
211 "100" => "Humour",
212 "101" => "Speech",
213 "102" => "Chanson",
214 "103" => "Opera",
215 "104" => "Chamber Music",
216 "105" => "Sonata",
217 "106" => "Symphony",
218 "107" => "Booty Bass",
219 "108" => "Primus",
220 "109" => "Porn Groove",
221 "110" => "Satire",
222 "111" => "Slow Jam",
223 "112" => "Club",
224 "113" => "Tango",
225 "114" => "Samba",
226 "115" => "Folklore",
227 "116" => "Ballad",
228 "117" => "Power Ballad",
229 "118" => "Rhythmic Soul",
230 "119" => "Freestyle",
231 "120" => "Duet",
232 "121" => "Punk Rock",
233 "122" => "Drum Solo",
234 "123" => "Acapella",
235 "124" => "Euro-House",
236 "125" => "Dance Hall",
237 "126" => "Goa",
238 "127" => "Drum & Bass",
239 "128" => "Club - House",
240 "129" => "Hardcore",
241 "130" => "Terror",
242 "131" => "Indie",
243 "132" => "BritPop",
244 "133" => "Negerpunk",
245 "134" => "Polsk Punk",
246 "135" => "Beat",
247 "136" => "Christian Gangsta Rap",
248 "137" => "Heavy Metal",
249 "138" => "Black Metal",
250 "139" => "Crossover",
251 "140" => "Contemporary Christian",
252 "141" => "Christian Rock",
253 "142" => "Merengue",
254 "143" => "Salsa",
255 "144" => "Thrash Metal",
256 "145" => "Anime",
257 "146" => "JPop",
258 "147" => "Synthpop"
259 };
260
261 my $mapped_genre;
262
263 if (defined $lookup_id3v1_genre->{$genre_num}) {
264# print STDERR "*** changing $genre_num -> ", $lookup_id3v1_genre->{$genre_num}, "\n";
265 $mapped_genre = $lookup_id3v1_genre->{$genre_num};
266 }
267 else {
268 $mapped_genre = $genre_num;
269 }
270
271 return $mapped_genre;
272}
273
274sub check_for_existing_id3_genre
275{
276 my $self = shift @_;
277 my ($doc_obj) = @_;
278
279 my $top_section=$doc_obj->get_top_section();
280
281 # Look for ex.ID3.Genre as well, as special case
282 my $genre_md_list = $doc_obj->get_metadata($top_section,"ex.ID3.Genre");
283 my @new_genre_md_list = ();
284
285 foreach my $gv (@$genre_md_list) {
286# print STDERR "*** got match on ex.ID3.Genre -> '$gv'\n";
287 my $new_gv = map_id3v1_genre_num($gv);
288
289 push(@new_genre_md_list,$new_gv);
290 }
291
292 $doc_obj->delete_metadata($top_section,"ex.ID3.Genre");
293 foreach my $gv (@new_genre_md_list) {
294 $doc_obj->add_utf8_metadata($top_section,"ex.ID3.Genre",$gv);
295 }
296}
297
298
299sub parse_txt_metadata
300{
301 my $self = shift @_;
302 my ($doc_obj,$target_txt_file_path) = @_;
303
304 if (open(MIN,"<$target_txt_file_path")) {
305
306 my ($md_name, $md_value);
307
308 while (defined($md_name=<MIN>) && defined($md_value=<MIN>)) {
309
310 chomp $md_name;
311 chomp $md_value;
312
313 my $top_section=$doc_obj->get_top_section();
314
315 $md_name =~ s/\+//g;
316 $md_value =~ s/\+/ /g;
317
318 $md_name = urlDecode($md_name);
319 $md_value = urlDecode($md_value);
320
321 # $md_name =~ s/\s+/ /sg;
322 $md_name =~ s/\(.*?\)$//s; # can stretch over multiple lines
323 $md_name =~ s/Last\.FM/LastFM/g;
324 $md_name =~ s/:/^/g;
325 $md_name =~ s/(API)?\^/./;
326
327# print STDERR "*** md_name = '$md_name'\n";
328
329 if ($md_name =~ m/genre$/i) {
330# print STDERR "*** got match on $md_name -> $md_value\n";
331 $md_value = map_id3v1_genre_num($md_value);
332 }
333
334 $doc_obj->add_utf8_metadata($top_section,$md_name,$md_value);
335 }
336
337 close(MIN);
338
339 $self->check_for_existing_id3_genre($doc_obj);
340
341 }
342 else {
343 print STDERR "Error: Failed to open $target_txt_file_path\n";
344 print STDERR " !$\n";
345 }
346}
347
348
349sub retrieve_metadata
350{
351 my $self = shift(@_);
352 my ($source_file_path,$id3_title,$id3_artist,$convert_options) = @_;
353
354 $convert_options = "" if (!defined $convert_options);
355
356 my $outhandle = $self->{'outhandle'};
357 my $verbosity = $self->{'verbosity'};
358
359 my $source_file_no_path = &File::Basename::basename($source_file_path);
360
361 $self->init_cache_for_file($source_file_path);
362
363 my $target_txt_file_path;
364 my $target_acexml_file_path;
365
366 if ($self->{'enable_cache'}) {
367 my $cached_dir = $self->{'cached_dir'};
368 my $file_root = $self->{'cached_file_root'};
369
370 my $target_txt_file = "${file_root}_metadata.txt";
371 my $target_acexml_file = "${file_root}.xml";
372
373 $target_txt_file_path = &util::filename_cat($cached_dir,$target_txt_file);
374 $target_acexml_file_path = &util::filename_cat($cached_dir,$target_acexml_file);
375 }
376 else {
377 $target_txt_file_path = &util::get_tmp_filename("_metadata.txt");
378 $target_acexml_file_path = &util::get_tmp_filename(".xml");
379 }
380
381 my $jmir_directory = $self->{'jmir_directory'};
382
383 my $store_cwd = cwd();
384
385 if (!-d $jmir_directory) {
386 print STDERR "Error: Unable able to find directory '$jmir_directory'\n";
387 print STDERR " Cannot run jAudio\n";
388 }
389 elsif (chdir($jmir_directory)) {
390
391 my $source_file_path_os = $source_file_path;
392 if ($^O eq "cygwin") {
393 $source_file_path_os = `cygpath -w "$source_file_path"`;
394 $source_file_path_os =~ s/\s+$//;
395 }
396 my $target_txt_file_path_os = $target_txt_file_path;
397 if ($^O eq "cygwin") {
398 $target_txt_file_path_os = `cygpath -w "$target_txt_file_path"`;
399 $target_txt_file_path_os =~ s/\s+$//;
400 }
401 my $target_acexml_file_path_os = $target_acexml_file_path;
402 if ($^O eq "cygwin") {
403 $target_acexml_file_path_os = `cygpath -w "$target_acexml_file_path"`;
404 $target_acexml_file_path_os =~ s/\s+$//;
405 }
406
407
408 my $jsongminer_cmd = "java -Xmx1024M -jar \"" . $jmir_directory . "/jSongMiner.jar\" $convert_options";
409 $jsongminer_cmd .= " -title \"$id3_title\"" if defined $id3_title;
410 $jsongminer_cmd .= " -artist \"$id3_artist\"" if defined $id3_artist;
411 $jsongminer_cmd .= " -audio \"$source_file_path_os\"";
412 $jsongminer_cmd .= " -savetxtfile \"$target_txt_file_path_os\"";
413 $jsongminer_cmd .= " -saveacexmlfile \"$target_acexml_file_path_os\"";
414
415 if ($verbosity>2) {
416 print $outhandle "jSongMinerExtractor: Running ...\n";
417 print $outhandle "jSongMinerExtractor: $jsongminer_cmd\n";
418 }
419
420 my $print_info = { 'message_prefix' => "jSongMiner",
421 'message' => "jSongMinerExtractor: Retrieving audio metadata for $source_file_no_path" };
422
423 my ($regenerated,$result,$had_error)
424 = $self->autorun_general_cmd($jsongminer_cmd,$source_file_path,$target_txt_file_path,$print_info);
425
426 if ($verbosity>2) {
427 print $outhandle "jSongMinerExtractor: ...done\n";
428 }
429 }
430 else {
431 print STDERR "Error: failed to change directory to '$jmir_directory'\n";
432 print STDERR " Cannot run jAudio\n";
433 }
434
435 chdir($store_cwd);
436
437 return ($target_acexml_file_path,$target_txt_file_path);
438}
439
440
441
442
4431;
Note: See TracBrowser for help on using the repository browser.