source: gs3-extensions/structured-image/trunk/perllib/plugins/GoogleVisionAPIConverter.pm@ 37046

Last change on this file since 37046 was 37046, checked in by davidb, 9 months ago

Code extended to now generate Open Annotation (JSON format) of the OCR'd blocks of text; some refinement of the existing Google Vision perl code
File size: 20.2 KB
Line 
1######################################################################
2#
3# GoogleVisionAPIConverter.pm -- helper plugin that allows other plugins
4# (such as ImagePlugin and PagedImagePlugin) to extend their
5# processing capability through sub-classing inheritence (such as
6# GoogleVisionImagePlugin and GoogleVisionPagedImagePlugin) to
7# expand the image processing capabilities at ingest time to
8# include the Google Vision API allowing for: metadata labelling
9# of objects within a scene; and OCR text recognition.
10#
11# A component of the Greenstone digital library software
12# from the New Zealand Digital Library Project at the
13# University of Waikato, New Zealand.
14#
15# Copyright (C) 1999 New Zealand Digital Library Project
16#
17# This program is free software; you can redistribute it and/or modify
18# it under the terms of the GNU General Public License as published by
19# the Free Software Foundation; either version 2 of the License, or
20# (at your option) any later version.
21#
22# This program is distributed in the hope that it will be useful,
23# but WITHOUT ANY WARRANTY; without even the implied warranty of
24# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25# GNU General Public License for more details.
26#
27# You should have received a copy of the GNU General Public License
28# along with this program; if not, write to the Free Software
29# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
30#
31###########################################################################
32
33package GoogleVisionAPIConverter;
34
35use strict;
36no strict 'refs'; # allow filehandles to be variables and viceversa
37no strict 'subs';
38
39use gsprintf;
40use FileUtils;
41
42##use ImagePlugin;
43use BaseMediaConverter;
44
45use utf8;
46use JSON; # qw( from_json, encode_json );
47
48sub BEGIN {
49 @GoogleVisionAPIConverter::ISA = ('BaseMediaConverter');
50}
51
52my $arguments = [
53 { 'name' => "google_application_credentials",
54 'desc' => "{GoogleVisionAPIConverter.google_applicatio_credentials}",
55 'type' => "string",
56 'reqd' => "no",
57 'deft' => "google-sa-credentials-key.json"
58 },
59 { 'name' => "enable_image_labelling",
60 'desc' => "{GoogleVisionAPIConverter.enable_image_labelling}",
61 'type' => "flag",
62 'reqd' => "no" },
63 { 'name' => "enable_image_ocr",
64 'desc' => "{GoogleVisionAPIConverter.enable_image_ocr}",
65 'type' => "flag",
66 'reqd' => "no" },
67 { 'name' => "enable_document_ocr",
68 'desc' => "{GoogleVisionAPIConverter.enable_document_ocr}",
69 'type' => "flag",
70 'reqd' => "no" }
71];
72
73my $options = { 'name' => "GoogleVisionAPIConverter",
74 'desc' => "{GoogleVisionAPIConverter.desc}",
75 'abstract' => "no",
76 'inherits' => "yes",
77 'args' => $arguments };
78
79sub new {
80 my ($class) = shift (@_);
81 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
82 push(@$pluginlist, $class);
83
84 push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
85 push(@{$hashArgOptLists->{"OptList"}},$options);
86
87 my $self = new BaseMediaConverter($pluginlist, $inputargs, $hashArgOptLists, 1);
88
89 return bless $self, $class;
90}
91
92sub begin {
93 my $self = shift (@_);
94 my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
95
96 if ($self->{'enable_image_ocr'} && $self->{'enable_document_ocr'}) {
97 print STDERR "Please use the following command syntax for vision types: (--enable_image_ocr | --enable_document_ocr) [--enable_image_labelling]\n";
98 print STDERR "\t\t --enable_image_ocr : optical character recognition for text within images\n";
99 print STDERR "\t\t --enable_document_ocr : optical character recognition for text within documents\n";
100 print STDERR "\t\t --enable_image_labelling : annotation labeling for objects within images\n";
101 exit(2);
102 }
103
104 $self->SUPER::begin(@_);
105}
106
107sub vision_monitor_line {
108 my ($line) = @_;
109
110 my $had_error = 0;
111 my $generate_dot = 0;
112
113 if ($line =~ m/^.*$/)
114 {
115 $generate_dot = 1;
116 }
117
118 return ($had_error,$generate_dot);
119}
120
121sub run_gv_convert {
122 my $self = shift (@_);
123 my ($filename,$file,$doc_obj,$opt_section) = @_;
124
125 my $section = (defined $opt_section) ? $opt_section : $doc_obj->get_top_section();
126
127 my $verbosity = $self->{'verbosity'};
128 my $outhandle = $self->{'outhandle'};
129 print $outhandle "----- GoogleVisionAPIConveter run_gv_convert -----\n";
130
131 my @vision_types = (); # array containing target ocr / labelling type(s)
132
133 if ($self->{'enable_image_labelling'}) { push(@vision_types, "enable_image_labelling"); }
134 if ($self->{'enable_image_ocr'}) { push(@vision_types, "enable_image_ocr"); }
135 if ($self->{'enable_document_ocr'}) { push(@vision_types, "enable_document_ocr"); }
136
137 my $vision_types_length = scalar(@vision_types);
138
139 if ($vision_types_length != 0) {
140
141 $self->init_cache_for_file($filename);
142 my $cached_image_dir = $self->{'cached_dir'};
143 # my $cached_image_root = $self->{'cached_file_root'};
144
145 # my $filename_no_path = &File::Basename::basename($filename);
146
147 my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
148 my $credentials_filename = &FileUtils::filenameConcatenate($collect_dir, "etc", $self->{'google_application_credentials'});
149
150 for my $vision_type (@vision_types) {
151
152 my $ofile = "${vision_type}-google-vision-output.json";
153 my $ofilename = &FileUtils::filenameConcatenate($cached_image_dir,$ofile);
154
155 my $vision_cmd = "vision.py --$vision_type --credentials \"$credentials_filename\" \"$filename\" \"$ofilename\"";
156
157 $self->run_vision($file, $filename, $ofile, $ofilename, $vision_cmd, $vision_type, $doc_obj,$section);
158
159 if ($vision_type eq "enable_document_ocr") {
160 my $gv_dococr_rec = { 'filename' => $ofilename, 'section' => $section };
161
162 push(@{$self->{'gv-dococr-json-filename-recs'}}, $gv_dococr_rec);
163 }
164 }
165 }
166
167 return "json";
168}
169
170sub gv_ocr_bounding_box_rect
171{
172 my $self = shift (@_);
173 my ($gv_block,) = @_;
174
175 my $bbox_rect = undef;
176
177 my $gv_boundingBox = $gv_block->{'boundingBox'};
178
179 my $gv_vertices = $gv_boundingBox->{'vertices'};
180 my $gv_num_vertices = scalar(@$gv_vertices);
181
182 if ($gv_num_vertices > 0) {
183 # print STDERR "**** gs_vertices[0] = ", JSON::encode_json($gv_vertices->[0]), "\n";
184
185 # Discovered that sometimes the 'x' value in the 'vertices' structure is not defined
186 # So can't rely on picking up $gv_vertices->[0 for 'x' and 'y'
187 # start off with 'undef' and test for !defined in for-loop
188
189 my $min_x = undef;
190 my $min_y = undef;
191 my $max_x = undef;
192 my $max_y = undef;
193
194
195 for (my $v=0; $v<$gv_num_vertices; $v++) {
196 my $x = $gv_vertices->[$v]->{'x'};
197 my $y = $gv_vertices->[$v]->{'y'};
198
199 if (defined $x) {
200 $min_x = $x if (!defined $min_x || ($x < $min_x));
201 $max_x = $x if (!defined $max_x || ($x > $max_x));
202 }
203
204 if (defined $y) {
205 $min_y = $y if (!defined $min_y || ($y < $min_y));
206 $max_y = $y if (!defined $max_y || ($y > $max_y));
207 }
208 }
209
210 my $x_org = $min_x;
211 my $y_org = $min_y;
212 my $x_dim = $max_x - $min_x +1;
213 my $y_dim = $max_y - $min_y +1;
214
215 $bbox_rect = { "x_org" => $x_org, "y_org" => $y_org, "x_dim" => $x_dim, "y_dim" => $y_dim};
216 }
217
218 return $bbox_rect;
219}
220
221sub run_vision
222{
223 my $self = shift (@_);
224 my ($file, $filename, $ofile, $ofilename, $vision_cmd, $vision_type, $doc_obj,$section) = @_;
225
226 my $vision_regenerated;
227 my $vision_result;
228 my $vision_error;
229
230 my $print_info = {
231 'message_prefix' => "GoogleVisionAPI",
232 'message' => "Sending $file to GoogleVisionAPI using vision.py"
233 };
234
235 ($vision_regenerated,$vision_result,$vision_error)
236 = $self->run_cached_general_cmd($vision_cmd,$filename,$ofilename,$print_info);
237
238 # Need to work a bit harder in setting up the associated JSON file
239 # => strip of 'enable_' in favour of 'gv_'
240 # => add in section number as part of the file name to avoid clashes
241
242 my $section_file_suffix = $section;
243 $section_file_suffix =~ s/\./_/g;
244
245 my $assoc_ofile = $ofile;
246 $assoc_ofile =~ s/^enable_/gv_/;
247 $assoc_ofile =~ s/-google-vision//;
248 $assoc_ofile =~ s/\.(.*?)$/$section_file_suffix.$1/;
249
250 $doc_obj->associate_file($ofilename,$assoc_ofile,"application/json",$section);
251
252 my $json_text = do { # read in json file
253 open(my $json_fh, "<:encoding(UTF-8)", $ofilename)
254 or die("Can't open \"$ofilename\": $!\n");
255 local $/;
256 <$json_fh>
257 };
258
259 my $decoded_json = JSON::from_json($json_text);
260
261 my $ocr_text;
262 if ($vision_type eq "enable_document_ocr" || $vision_type eq "enable_image_ocr") {
263 $ocr_text = $decoded_json->{'textAnnotations'}[0]{'description'}; # access full ocr content
264 $doc_obj->add_utf8_text($section, $ocr_text); # append text to section
265
266 my $blocks = $decoded_json->{'fullTextAnnotation'}{'pages'}[0]{'blocks'};
267 my %text_and_language;
268
269 foreach my $block (@{$blocks}) {
270 foreach my $paragraph (@{$block->{'paragraphs'}}) {
271 foreach my $word (@{$paragraph->{'words'}}) {
272 my $detected_language = $word->{'property'}{'detectedLanguages'}[0]{'languageCode'} || "no_lang";
273 my $word_text = "";
274 foreach my $letter (@{$word->{'symbols'}}) {
275 $word_text .= $letter->{'text'};
276 }
277 $text_and_language{$detected_language} .= $word_text . " ";
278 }
279 }
280 }
281
282 for (keys %text_and_language) {
283 $doc_obj->add_utf8_metadata($section, "z_" . $_, $text_and_language{$_});
284 }
285
286
287 my $assoc_json_metaname = "HasGoogleVision";
288
289 if ($vision_type eq "enable_document_ocr") {
290 $assoc_json_metaname .= "DocumentOCRJSON";
291
292 $doc_obj->add_utf8_metadata($section, "GVDocumentOCRJSON",$assoc_ofile);
293 }
294 else {
295 # $vision_type eq "enable_image_ocr"
296 $assoc_json_metaname .= "ImageOCRJSON";
297
298 $doc_obj->add_utf8_metadata($section, "GVImageOCRJSON",$assoc_ofile);
299 }
300
301 $doc_obj->add_utf8_metadata($section, $assoc_json_metaname, 1);
302 }
303 elsif ($vision_type eq "enable_image_labelling") {
304 $ocr_text = $decoded_json->{'labelAnnotations'};
305 foreach my $label (@{$ocr_text}) {
306 # write to metadata : 'description'='Book' 'score'='0.9' 'topicality'='0.9' 'mid'='/m/0123'
307 $doc_obj->add_utf8_metadata($section, "description", $label->{'description'});
308 $doc_obj->add_utf8_metadata($section, "score", $label->{'score'});
309 $doc_obj->add_utf8_metadata($section, "topicality", $label->{'topicality'});
310 $doc_obj->add_utf8_metadata($section, "mid", $label->{'mid'});
311
312 # write to metadata : 'descriptions'='Book' 'Book_score'='0.9' 'Book_topicality'='0.9' 'Book_mid'='/m/0123'
313 $doc_obj->add_utf8_metadata($section, "descriptions", $label->{'description'});
314 $doc_obj->add_utf8_metadata($section, $label->{'description'} . "_score", $label->{'score'});
315 $doc_obj->add_utf8_metadata($section, $label->{'description'} . "_topicality", $label->{'topicality'});
316 $doc_obj->add_utf8_metadata($section, $label->{'description'} . "_mid", $label->{'mid'});
317
318 }
319
320 $doc_obj->add_utf8_metadata($section, "HasGoogleVisionImageLabellingJSON", 1);
321 $doc_obj->add_utf8_metadata($section, "GVImageLabellingJSON",$assoc_ofile);
322
323 }
324}
325
326sub start_openannotation_list
327{
328 my $self = shift (@_);
329 my ($doc_obj) = @_;
330
331 my $OID = $doc_obj->get_OID();
332
333 my $openannotation_list = {
334 "\@context" => "http://www.shared-canvas.org/ns/context.json",
335 # "\@id" => "https://iiif.harvardartmuseums.org/manifests/object/299843/list/47174896",
336 "\@id" => "${OID}/openannotation-list.json", # #### **** make full URL to be unique? or greenstone3:site:collect:OID ??
337 "\@type" => "sc:AnnotationList",
338 "resources" => []
339 };
340
341 $self->{'openannotation-list'} = $openannotation_list;
342}
343
344
345
346sub convert_gvocr_to_openannotation_resource
347{
348 my $self = shift (@_);
349 my ($gv_blocks, $doc_obj, $section) = @_;
350
351 my $OID = $doc_obj->get_OID();
352 my $OID_with_section = ($section ne "") ? "${OID}_$section" : $OID;
353 $section = 1 if ($section eq ""); # occurs when the document is a single image
354
355
356 # Example Open Annotation resource (for single annotation):
357# {
358# "@context": "http://iiif.io/api/presentation/2/context.json",
359# "@id": "https://iiif.harvardartmuseums.org/annotations/9641482",
360# "@type": "oa:Annotation",
361# "motivation": [
362# "oa:commenting"
363# ],
364# "on": {
365# "@type": "oa:SpecificResource",
366# "full": "https://iiif.harvardartmuseums.org/manifests/object/299843/canvas/canvas-47174896",
367# "selector": {
368# "@type": "oa:FragmentSelector",
369# "value": "xywh=622,591,642,940"
370# },
371# "within": {
372# "@id": "https://iiif.harvardartmuseums.org/manifests/object/299843",
373# "@type": "sc:Manifest"
374# }
375# },
376# "resource": [
377# {
378# "@type": "dctypes:Text",
379# "chars": "<p>age: 35-52<br/>gender: Female(66.337677%)<br/>CALM: 55.438412%<br/>CONFUSED: 3.949288%<br/>SURPRISED: 2.33092%<br/>DISGUSTED:
380# 0.545727%<br/>HAPPY: 1.549943%<br/>ANGRY: 2.082294%<br/>SAD: 34.103416%<br/></p><p>Generated by AWS Rekognition</p>",
381# "format": "text/html"
382# }
383# ]
384# },
385
386 my $self_openannotation_resources = $self->{'openannotation-list'}->{'resources'};
387
388 my $block_i = 0;
389
390 foreach my $block (@{$gv_blocks}) {
391 $block_i++;
392
393 my $openannotation_resource = {
394 "\@context" => "http://iiif.io/api/presentation/2/context.json",
395 # "\@id" => "https://iiif.harvardartmuseums.org/annotations/9641482",
396 "\@id" => "${OID_with_section}/annotation/gv-block-$block_i", # #### **** make full URL to be unique? or greenstone3:site:collect:OID ??
397 "\@type" => "oa:Annotation",
398 "motivation" => [ "oa:commenting" ]
399 };
400
401
402 my $bbox_rect = $self->gv_ocr_bounding_box_rect($block);
403 my $bb_x_org = $bbox_rect->{'x_org'};
404 my $bb_y_org = $bbox_rect->{'y_org'};
405 my $bb_x_dim = $bbox_rect->{'x_dim'};
406 my $bb_y_dim = $bbox_rect->{'y_dim'};
407
408 my $openannotation_on = {
409 "\@type" => "oa:SpecificResource",
410 # "full" => "https://iiif.harvardartmuseums.org/manifests/object/299843/canvas/canvas-47174896",
411 "full" => "${OID}/canvas/$section", # doc id + /canvas + page-i/sect # #### **** make full URL to be unique? or greenstone3:site:collect:OID ??
412 "selector" => {
413 "\@type" => "oa:FragmentSelector",
414 "value" => "xywh=${bb_x_org},${bb_y_org},${bb_x_dim},${bb_y_dim}"
415 },
416 "within" => {
417 #"\@id" => "https://iiif.harvardartmuseums.org/manifests/object/299843",
418 "\@id" => "${OID_with_section}/manifest", # #### **** make full URL to be unique? or greenstone3:site:collect:OID... ??
419 "\@type" => "sc:Manifest"
420 }
421 };
422 $openannotation_resource->{'on'} = $openannotation_on;
423
424
425 my $block_text_html = "";
426
427 foreach my $paragraph (@{$block->{'paragraphs'}}) {
428 my $para_text = "";
429
430 foreach my $word (@{$paragraph->{'words'}}) {
431 my $word_text = "";
432
433 foreach my $letter (@{$word->{'symbols'}}) {
434 $word_text .= $letter->{'text'};
435 }
436
437 $para_text .= " " if $para_text ne "";
438 $para_text .= $word_text;
439 }
440
441 $block_text_html .= "<p>\n$para_text\n</p>\n\n";
442 }
443
444 my $openannotation_inner_resource = [{
445 "\@type" => "dctypes:Text",
446 "chars" => "$block_text_html",
447 "format" => "text/html"
448 }];
449
450 $openannotation_resource->{'resource'} = $openannotation_inner_resource;
451
452 push(@$self_openannotation_resources,$openannotation_resource);
453 }
454}
455
456
457sub convert_and_append_openannotation_resources
458{
459 my $self = shift (@_);
460 my ($gv_dococr_json_filename, $doc_obj, $section) = @_;
461
462
463 # Read in JSON file
464 my $json_text = do {
465 open(my $json_fh, "<:encoding(UTF-8)", $gv_dococr_json_filename)
466 or die("Can't open \"$gv_dococr_json_filename\": $!\n");
467 local $/;
468 <$json_fh>
469 };
470
471 my $decoded_json = JSON::from_json($json_text);
472
473 my $gv_blocks = $decoded_json->{'fullTextAnnotation'}->{'pages'}->[0]->{'blocks'};
474
475 $self->convert_gvocr_to_openannotation_resource($gv_blocks, $doc_obj, $section);
476}
477
478
479
480sub end_openannotation_list
481{
482 my $self = shift (@_);
483 my ($doc_obj,$json_ofilename) = @_;
484
485 my $ret_status = 1;
486
487 if (!open(JOUT, "> $json_ofilename")) {
488 print STDERR "Error: Failed save Open Annotation List JSON to \"$json_ofilename\":\n $!\n";
489 $ret_status = 0;
490 }
491 else {
492 binmode(JOUT, ":utf8");
493
494 my $openannotation_list = $self->{'openannotation-list'};
495 my $openannotation_list_json_text = JSON::encode_json($openannotation_list);
496
497 print JOUT $openannotation_list_json_text;
498 close JOUT;
499
500 }
501
502 $self->{'openannotation-list'} = undef;
503
504 return $ret_status;
505}
506
507sub openannotation_list_associate_json
508{
509 my $self = shift (@_);
510 my ($doc_obj, $gv_dococr_json_filename_recs) = @_;
511
512 my $outhandle = $self->{'outhandle'};
513
514 # Guaranteed to have at least one value in gv_dococr_json_filename_recs
515 #
516 # Legacy code used to have a '\d+' just before the '.json' reflecting page/section number
517 # Keep this in the regular expression, just in case,
518 #
519 my ($gv_dococr_filename_root) = ($gv_dococr_json_filename_recs->[0]->{'filename'} =~ m/^(.+)(?:\d+)?\.json$/);
520
521 # slight of hand so new directory spot in cache_dir picked out is where we want it!
522 $gv_dococr_filename_root .= "/";
523
524 my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
525 my $toplevel_cached_dir = &FileUtils::filenameConcatenate($collect_dir,"cached");
526
527 $self->init_cache_for_file($gv_dococr_filename_root);
528 my $cached_dir = $self->{'cached_dir'};
529
530 my $assoc_openannotation_json_ofile = "openannotation-list.json";
531 my $cached_openannotation_json_ofilename = &FileUtils::filenameConcatenate($cached_dir,$assoc_openannotation_json_ofile);
532
533 my $needs_json_regen = 0;
534
535 if (!-f $cached_openannotation_json_ofilename) {
536 $needs_json_regen = 1;
537 }
538 else {
539 for my $gv_json_filename_rec (@$gv_dococr_json_filename_recs) {
540 my $gv_json_filename = $gv_json_filename_rec->{'filename'};
541 if (-M $gv_json_filename > -M $cached_openannotation_json_ofilename) {
542 $needs_json_regen = 1;
543 last;
544 }
545 }
546 }
547
548 my $saved_ok = 1;
549
550 if ($needs_json_regen) {
551
552 print $outhandle " OpenAnnotation-List: Generating $cached_openannotation_json_ofilename\n";
553
554 $self->start_openannotation_list($doc_obj);
555
556 for my $gv_json_filename_rec (@$gv_dococr_json_filename_recs) {
557 my $gv_json_filename = $gv_json_filename_rec->{'filename'};
558 my $section = $gv_json_filename_rec->{'section'};
559 $self->convert_and_append_openannotation_resources($gv_json_filename, $doc_obj,$section);
560 }
561
562 $saved_ok = $self->end_openannotation_list($doc_obj,$cached_openannotation_json_ofilename);
563 }
564
565 if ($saved_ok) {
566 print $outhandle " OpenAnnotation-List: Cached file $cached_openannotation_json_ofilename already exists\n";
567
568 my $top_section = $doc_obj->get_top_section();
569 $doc_obj->associate_file($cached_openannotation_json_ofilename,$assoc_openannotation_json_ofile,"application/json",$top_section);
570 }
571
572 return $saved_ok;
573}
574
575sub opt_run_gen_openannotation
576{
577 my $self = shift (@_);
578 my ($doc_obj) = @_;
579
580 my $gv_dococr_json_filename_recs = $self->{'gv-dococr-json-filename-recs'};
581 my $num_gv_dococr_json_filename_recs = scalar(@$gv_dococr_json_filename_recs);
582
583 my $ret_val_ok = 1;
584
585 if ($num_gv_dococr_json_filename_recs > 0) {
586 $ret_val_ok = $self->openannotation_list_associate_json($doc_obj,$gv_dococr_json_filename_recs);
587 }
588
589 return $ret_val_ok;
590}
591
5921;
593
Note: See TracBrowser for help on using the repository browser.