########################################################################### # # PagedImgPlug.pm -- plugin for sets of images and OCR text that # make up a document # A component of the Greenstone digital library software # from the New Zealand Digital Library Project at the # University of Waikato, New Zealand. # # Copyright (C) 1999 New Zealand Digital Library Project # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # ########################################################################### # PagedImgPlug # processes sequences of images, with optional OCR text # # This plugin takes *.item files, which contain metadata and lists of image # files, and produces a document containing sections, one for each page. # The files should be named something.item, then you can have more than one # book in a directory. You will need to create these files, one for each # document/book. # # The format of the xxx.item file is as follows: # The first lines contain any metadata for the whole document # metadata-value # eg. # Snail farming # <Date>19230102 # Then comes a list of pages, one page per line, each line has the format # # pagenum:imagefile:textfile:r # # page num and imagefile are required. pagenum is used for the Title # of the section, and in the display is shown as page <pagenum>. # imagefile is the image for the page. textfile is an optional text # file containing the OCR (or any) text for the page - this gets added # as the text for the section. r is optional, and signals that the image # should be rotated 180deg. Eg use this if the image has been made upside down. # So an example item file looks like: # <Title>Snail farming # <Date>19960403 # 1:p1.gif:p1.txt: # 2:p2.gif:: # 3:p3.gif:p3.txt: # 3b:p3b.gif:p3b.txt:r # The second page has no text, the fourth page is a back page, and # should be rotated. # # All the supplemetary image amd text files should be in the same folder as # the .item file. # # To display the images instead of the document text, you can use [srcicon] # in the DocumentText format statement. # For example, # # format DocumentText "<center><table width=_pagewidth_><tr><td>[srcicon]</td></tr></table></center>" # # To have it create thumbnail size images, use the '-thumbnail' option. # To have it create medium size images for display, use the '-screenview' # option. As usual, running # 'perl -S pluginfo.pl PagedImgPlug' will list all the options. # If you want the resulting documents to be presented with a table of # contents, use '-documenttype hierarchy', otherwise they will have # next and previous arrows, and a goto page X box. # If you have used -screenview, you can also use [screenicon] in the format # statement to display the smaller image. Here is an example that switches # between the two: # # format DocumentText "<center><table width=_pagewidth_><tr><td>_If_(\"_cgiargp_\" eq \"full\",<a href='_httpdocument_&d=_cgiargd_&p=small'>Switch to small version.</a>,<a href='_httpdocument_&d=_cgiargd_&p=full'>Switch to fullsize version</a>)</td></tr><tr><td>_If_(\"_cgiargp_\" eq \"full\",<a href='_httpdocument_&d=_cgiargd_&p=small' title='Switch to small version'>[srcicon]</a>,<a href='_httpdocument_&d=_cgiargd_&p=full' title='Switch to fullsize version'>[screenicon]</a>)</td></tr></table></center>" # # Additional metadata can be added into the .item files, alternatively you can # use normal metadata.xml files, with the name of the xxx.item file as the # FileName. package PagedImgPlug; use BasPlug; sub BEGIN { @ISA = ('BasPlug'); } my $type_list = [ { 'name' => "paged", 'desc' => "{PagedImgPlug.documenttype.paged}" }, { 'name' => "hierarchy", 'desc' => "{PagedImgPlug.documenttype.hierarchy}" } ]; my $arguments = [ { 'name' => "process_exp", 'desc' => "{BasPlug.process_exp}", 'type' => "string", 'deft' => &get_default_process_exp(), 'reqd' => "no" }, { 'name' => "block_exp", 'desc' => "{BasPlug.block_exp}", 'type' => "string", 'deft' => &get_default_block_exp(), 'reqd' => "no" }, { 'name' => "noscaleup", 'desc' => "{ImagePlug.noscaleup}", 'type' => "flag", 'reqd' => "no" }, { 'name' => "thumbnail", 'desc' => "{PagedImgPlug.thumbnail}", 'type' => "flag", 'reqd' => "no" }, { 'name' => "thumbnailsize", 'desc' => "{ImagePlug.thumbnailsize}", 'type' => "int", 'deft' => "100", 'range' => "1,", 'reqd' => "no" }, { 'name' => "thumbnailtype", 'desc' => "{ImagePlug.thumbnailtype}", 'type' => "string", 'deft' => "gif", 'reqd' => "no" }, { 'name' => "screenview", 'desc' => "{PagedImgPlug.screenview}", 'type' => "flag", 'reqd' => "no" }, { 'name' => "screenviewsize", 'desc' => "{PagedImgPlug.screenviewsize}", 'type' => "int", 'deft' => "500", 'range' => "1,", 'reqd' => "no" }, { 'name' => "screenviewtype", 'desc' => "{PagedImgPlug.screenviewtype}", 'type' => "string", 'deft' => "jpg", 'reqd' => "no" }, { 'name' => "converttotype", 'desc' => "{ImagePlug.converttotype}", 'type' => "string", 'deft' => "", 'reqd' => "no" }, { 'name' => "minimumsize", 'desc' => "{ImagePlug.minimumsize}", 'type' => "int", 'deft' => "100", 'range' => "1,", 'reqd' => "no" }, { 'name' => "headerpage", 'desc' => "{PagedImgPlug.headerpage}", 'type' => "flag", 'reqd' => "no" }, { 'name' => "documenttype", 'desc' => "{PagedImgPlug.documenttype}", 'type' => "enum", 'list' => $type_list, 'deft' => "paged", 'reqd' => "no" } ]; my $options = { 'name' => "PagedImgPlug", 'desc' => "{PagedImgPlug.desc}", 'inherits' => "yes", 'args' => $arguments }; sub new { my ($class) = @_; my $plugin_name = shift (@_); my $self = new BasPlug ("PagedImgPlug", @_); my $option_list = $self->{'option_list'}; push( @{$option_list}, $options ); if (!parsargv::parse(\@_, q^noscaleup^, \$self->{'noscaleup'}, q^converttotype/.*/^, \$self->{'converttotype'}, q^minimumsize/[0-9]*/100^, \$self->{'minimumsize'}, q^thumbnailsize/[0-9]*/100^, \$self->{'thumbnailsize'}, q^thumbnailtype/.*/gif^, \$self->{'thumbnailtype'}, q^screenviewsize/[0-9]*/0^, \$self->{'screenviewsize'}, q^screenviewtype/.*/jpg^, \$self->{'screenviewtype'}, q^thumbnail^, \$self->{'thumbnail'}, q^screenview^, \$self->{'screenview'}, q^headerpage^, \$self->{'headerpage'}, 'documenttype/^(paged|hierarchy)$/paged', \$self->{'doctype'}, "allow_extra_options")) { print STDERR "\nPagedImgPlug uses an incorrect option.\n"; print STDERR "Check your collect.cfg configuration file.\n"; $self->print_txt_usage(""); # Use default resource bundle die "\n"; } return bless $self, $class; } sub get_default_process_exp { my $self = shift (@_); return q^\.item$^; } # want to block everything except the .item ones # but instead we will block images and txt files sub get_default_block_exp { my $self = shift (@_); return q^(?i)(\.jpe?g|\.gif|\.png|\.tif?f|\.te?xt|~)$^ } # Create the thumbnail and screenview images, and discover the Image's # size, width, and height using the convert utility. sub process_image { my $self = shift (@_); my $filename = shift (@_); # filename with full path my $srcfile = shift (@_); # filename without path my $doc_obj = shift (@_); my $section = shift (@_); #the current section my $rotate = shift (@_); # whether to rotate the image or not my $top=0; if ($section eq $doc_obj->get_top_section()) { $top=1; } my $verbosity = $self->{'verbosity'}; my $outhandle = $self->{'outhandle'}; # check the filename is okay return 0 if ($srcfile eq "" || $filename eq ""); my $minimumsize = $self->{'minimumsize'}; if (defined $minimumsize && (-s $filename < $minimumsize)) { print $outhandle "PagedImgPlug: \"$filename\" too small, skipping\n" if ($verbosity > 1); } # Convert the image to a new type (if required), and rotate if required. my $converttotype = $self->{'converttotype'}; my $originalfilename = ""; # only set if we do a conversion my $type = "unknown"; my $converted = 0; my $rotated=0; if ($converttotype ne "" && $filename !~ /$converttotype$/) { $converted=1; $originalfilename = $filename; my $filehead = &util::get_tmp_filename(); $filename = $filehead . ".$converttotype"; $n = 1; while (-e $filename) { $filename = "$filehead$n\.$converttotype"; $n++; } $self->{'tmp_filename1'} = $filename; my $rotate_option = ""; if ($rotate eq "r") { $rotate_option = "-rotate 180 "; } my $command = "convert -verbose \"$originalfilename\" $rotate_option \"$filename\""; print $outhandle "CONVERT: $command\n" if ($verbosity > 2); my $result = ''; $result = `$command`; print $outhandle "CONVERT RESULT = $result\n" if ($verbosity > 2); $type = $converttotype; } elsif ($rotate eq "r") { $rotated=1; $originalfilename = $filename; $filename = &util::get_tmp_filename(); my $command = "convert \"$originalfilename\" -rotate 180 \"$filename\""; print $outhandle "ROTATE: $command\n" if ($verbosity > 2); my $result = ''; $result = `$command`; print $outhandle "ROTATE RESULT = $result\n" if ($verbosity > 2); } # Add the image metadata my $file; # the new file name my $id = $srcfile; $id =~ s/\.([^\.]*)$//; # the new file name without an extension if ($converted) { # we have converted the image # add on the new extension $file .= "$id.$converttotype"; } else { $file = $srcfile; } my $url =$file; # the new file name prepared for a url my $srcurl = $srcfile; $url =~ s/ /%20/g; $srcurl =~ s/ /%20/g; $doc_obj->add_metadata ($section, "Image", $url); # Also want to set filename as 'Source' metadata to be # consistent with other plugins $doc_obj->add_metadata ($section, "Source", $srcurl); my ($image_type, $image_width, $image_height, $image_size) = &identify($filename, $outhandle, $verbosity); $doc_obj->add_metadata ($section, "ImageType", $image_type); $doc_obj->add_metadata ($section, "ImageWidth", $image_width); $doc_obj->add_metadata ($section, "ImageHeight", $image_height); $doc_obj->add_metadata ($section, "ImageSize", $image_size); $doc_obj->add_metadata ($section, "FileFormat", "PagedImg"); if ($type eq "unknown" && $image_type) { $type = $image_type; } if ($top) { $doc_obj->add_metadata ($section, "srclink", "<a href=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\">"); $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Image]\">"); } else { $doc_obj->add_metadata ($section, "srclink", "<a href=\"_httpcollection_/index/assoc/[parent:assocfilepath]/[Image]\">"); $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpcollection_/index/assoc/[parent:assocfilepath]/[Image]\">"); } $doc_obj->add_metadata ($section, "/srclink", "</a>"); # Add the image as an associated file $doc_obj->associate_file($filename,$file,"image/$type",$section); print $outhandle "associating file $filename as name $file\n" if ($verbosity > 2); if ($self->{'thumbnail'}) { # Make the thumbnail image my $thumbnailsize = $self->{'thumbnailsize'} || 100; my $thumbnailtype = $self->{'thumbnailtype'} || 'gif'; my $filehead = &util::get_tmp_filename(); my $thumbnailfile = $filehead . ".$thumbnailtype"; my $n=1; while (-e $thumbnailfile) { $thumbnailfile = $filehead . $n . ".$thumbnailtype"; $n++; } $self->{'tmp_filename2'} = $thumbnailfile; # Generate the thumbnail with convert my $command = "convert -verbose -geometry $thumbnailsize" . "x$thumbnailsize \"$filename\" \"$thumbnailfile\""; print $outhandle "THUMBNAIL: $command\n" if ($verbosity > 2); my $result = ''; $result = `$command 2>&1` ; print $outhandle "THUMB RESULT: $result\n" if ($verbosity > 2); # Add the thumbnail as an associated file ... if (-e "$thumbnailfile") { $doc_obj->associate_file("$thumbnailfile", $id."thumb.$thumbnailtype", "image/$thumbnailtype",$section); $doc_obj->add_metadata ($section, "ThumbType", $thumbnailtype); $doc_obj->add_metadata ($section, "Thumb", $id."thumb.$thumbnailtype"); $doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpcollection_/index/assoc/[parent:assocfilepath]/[Thumb]\" width=[ThumbWidth] height=[ThumbHeight]>"); } # Extract Thumnail metadata from convert output if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) { $doc_obj->add_metadata ($section, "ThumbWidth", $1); $doc_obj->add_metadata ($section, "ThumbHeight", $2); } } # Make a screen-sized version of the picture if requested if ($self->{'screenview'}) { # To do: if the actual image is smaller than the screenview size, # we should use the original ! my $screenviewsize = $self->{'screenviewsize'} || 500; my $screenviewtype = $self->{'screenviewtype'} || 'jpeg'; my $filehead = &util::get_tmp_filename(); my $screenviewfilename = $filehead . ".$screenviewtype"; my $n=1; while (-e $screenviewfilename) { $screenviewfilename = "$filehead$n\.$screenviewtype"; $n++; } $self->{'tmp_filename3'} = $screenviewfilename; # make the screenview image my $command = "convert -verbose -geometry $screenviewsize" . "x$screenviewsize \"$filename\" \"$screenviewfilename\""; print $outhandle "SCREENVIEW: $command\n" if ($verbosity > 2); my $result = ""; $result = `$command 2>&1` ; print $outhandle "SCREENVIEW RESULT: $result\n" if ($verbosity > 3); # get screenview dimensions, size and type if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) { $doc_obj->add_metadata ($section, "ScreenWidth", $1); $doc_obj->add_metadata ($section, "ScreenHeight", $2); }elsif ($result =~ m/([0-9]+)x([0-9]+)/) { #if the image hasn't changed size, the previous regex doesn't match $doc_obj->add_metadata ($section, "ScreenWidth", $1); $doc_obj->add_metadata ($section, "ScreenHeight", $2); } #add the screenview as an associated file ... if (-e "$screenviewfilename") { $doc_obj->associate_file("$screenviewfilename", $id."sv.$screenviewtype", "image/$screenviewtype",$section); print $outhandle "associating screen file $screenviewfilename as name $id sv.$screenviewtype\n" if ($verbosity > 2); $doc_obj->add_metadata ($section, "ScreenType", $screenviewtype); $doc_obj->add_metadata ($section, "Screen", $id."sv.$screenviewtype"); if ($top) { $doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpcollection_/index/assoc/[assocfilepath]/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>"); } else { $doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpcollection_/index/assoc/{If}{[parent:assocfilepath],[parent:assocfilepath],[assocfilepath]}/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>"); } } else { print $outhandle "PagedImgPlug: couldn't find \"$screenviewfilename\"\n"; } } return $type; } # Discover the characteristics of an image file with the ImageMagick # "identify" command. sub identify { my ($image, $outhandle, $verbosity) = @_; # Use the ImageMagick "identify" command to get the file specs my $command = "identify \"$image\" 2>&1"; print $outhandle "$command\n" if ($verbosity > 2); my $result = ''; $result = `$command`; print $outhandle "$result\n" if ($verbosity > 3); # Read the type, width, and height my $type = 'unknown'; my $width = 'unknown'; my $height = 'unknown'; my $image_safe = quotemeta $image; if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) { $type = $1; $width = $2; $height = $3; } # Read the size my $size = "unknown"; if ($result =~ m/^.* ([0-9]+)b/) { $size = $1; } elsif ($result =~ m/^.* ([0-9]+)kb/) { $size = 1024 * $1; } print $outhandle "file: $image:\t $type, $width, $height, $size\n" if ($verbosity > 3); # Return the specs return ($type, $width, $height, $size); } # The PagedImgPlug read() function. This function does all the right things # to make general options work for a given plugin. It calls the process() # function which does all the work specific to a plugin (like the old # read functions used to do). Most plugins should define their own # process() function and let this read() function keep control. # # PagedImgPlug overrides read() because there is no need to read the actual # text of the file in, because the contents of the file is not text... # # Return number of files processed, undef if can't process # Note that $base_dir might be "" and that $file might # include directories sub read { my $self = shift (@_); my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_; my $outhandle = $self->{'outhandle'}; my $smart_block = $self->{'smart_block'}; my $filename = &util::filename_cat($base_dir, $file); if ($self->associate_with($file,$filename,$metadata)) { # a form of smart block $self->{'num_blocked'} ++; return 0; # blocked } if ($smart_block) { if (defined $self->{'file_blocks'}->{$filename} && $self->{'file_blocks'}->{$filename} == 1){ $self->{'num_blocked'} ++; return 0; # blocked } } elsif ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) { $self->{'num_blocked'} ++; return 0; # blocked } if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) { return undef; } print $outhandle "PagedImgPlug processing \"$filename\"\n" if $self->{'verbosity'} > 1; my ($dir); ($dir, $file) = $filename =~ /^(.*?)([^\/\\]*)$/; #process the .item file my $doc_obj = $self->process_item($filename, $dir, $file, $processor); if ($self->{'cover_image'}) { $self->associate_cover_image($doc_obj, $filename); } # include any metadata passed in from previous plugins # note that this metadata is associated with the top level section my $section = $doc_obj->get_top_section(); $self->extra_metadata ($doc_obj, $section, $metadata); # do plugin specific processing of doc_obj return -1 unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj)); # do any automatic metadata extraction $self->auto_extract_metadata ($doc_obj); # process the document $processor->process($doc_obj); # clean up temporary files - we do this here instead of in # process_image becuase associated files aren't actually copied # until after process has been run. if (defined $self->{'tmp_filename1'} && -e $self->{'tmp_filename1'}) { &util::rm($self->{'tmp_filename1'}) } if (defined $self->{'tmp_filename2'} && -e $self->{'tmp_filename2'}) { &util::rm($self->{'tmp_filename2'}) } if (defined $self->{'tmp_filename3'} && -e $self->{'tmp_filename3'}) { &util::rm($self->{'tmp_filename3'}) } $self->{'num_processed'}++; return 1; } sub process_item { my $self = shift (@_); my ($filename, $dir, $file, $processor) = @_; my $doc_obj = new doc ($file, "indexed_doc"); my $topsection = $doc_obj->get_top_section(); if ($self->{'doctype'} eq 'paged') { # set the gsdlthistype metadata to Paged - this ensures this document will # be treated as a Paged doc, even if Titles are not numeric $doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Paged"); } else { $doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Hierarchy"); } open (ITEMFILE, $filename) || die "couldn't open $filename\n"; my $line = ""; my $num = 0; while (defined ($line = <ITEMFILE>)) { next unless $line =~ /\w/; chomp $line; if ($line =~ /^<([^>]*)>(.*?)\s*$/) { $doc_obj->set_utf8_metadata_element ($topsection, $1, $2); $meta->{$1} = $2; } else { $num++; # line should be like page:imagefilename:textfilename:r - the r is optional -> means rotate the image 180 deg $line =~ s/^\s+//; #remove space at the front $line =~ s/\s+$//; #remove space at the end my ($pagenum, $imgname, $txtname, $rotate) = split /:/, $line; # create a new section for each image file my $cursection = $doc_obj->insert_section($doc_obj->get_end_child($topsection)); # the page number becomes the Title $doc_obj->set_utf8_metadata_element($cursection, 'Title', $pagenum); # process the image for this page my $result = $self->process_image($dir.$imgname, $imgname, $doc_obj, $cursection, $rotate); if (!defined $result) { print "PagedImgPlug: couldn't process image \"$dir.$imgname\" for item \"$filename\"\n"; } # process the text file if one is there if (defined $txtname && $txtname ne "") { $result = undef; $result = $self->process_text ($dir.$txtname, $txtname, $doc_obj, $cursection); if (!defined $result) { print "PagedImgPlug: couldn't process text file \"$dir.$txtname\" for item \"$filename\"\n"; } } else { # otherwise add in some dummy text $doc_obj->add_text($cursection, &gsprintf::lookup_string("{BasPlug.dummy_text}")); } } } close ITEMFILE; # if we want a header page, we need to add some text into the top section, otherwise this section will become invisible if ($self->{'headerpage'}) { $doc_obj->add_text($topsection, &gsprintf::lookup_string("{BasPlug.dummy_text}")); } $file =~ s/\.item//i; $doc_obj->set_OID (); # add numpages metadata $doc_obj->set_utf8_metadata_element ($topsection, 'NumPages', "$num"); return $doc_obj; } sub process_text { my $self = shift (@_); my ($fullpath, $file, $doc_obj, $cursection) = @_; # Do encoding stuff my ($language, $encoding) = $self->textcat_get_language_encoding ($fullpath); my $text=""; &BasPlug::read_file($self, $fullpath, $encoding, $language, \$text); if (!length ($text)) { my $plugin_name = ref ($self); print "PagedImgPlug: ERROR: $fullpath contains no text\n" if $self->{'verbosity'}; return 0; } # we need to escape the escape character, or else mg will convert into # eg literal newlines, instead of leaving the text as '\n' $text =~ s/\\/\\\\/g; # macro language $text =~ s/_/\\_/g; # macro language $text =~ s/</</g; $text =~ s/>/>/g; # insert preformat tags and add text to document object $doc_obj->add_utf8_text($cursection, "<pre>\n$text\n</pre>"); return 1; } # do plugin specific processing of doc_obj sub process { my $self = shift (@_); my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_; my $outhandle = $self->{'outhandle'}; return 1; } 1;