source: trunk/gsdl/perllib/plugins/ConvertToPlug.pm@ 10592

Last change on this file since 10592 was 10592, checked in by kjdon, 18 years ago

in read, call title_fallback to make sure that we have a title - pdf paged_img format doesn't get a title

  • Property svn:keywords set to Author Date Id Revision
File size: 15.2 KB
Line 
1###########################################################################
2#
3# ConvertToPlug.pm -- plugin that inherits from BasPlug
4#
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright (C) 1999 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26
27# This plugin is inherited by such plugins as WordPlug, PPTPlug, PSPlug, RTFPlug
28# and PDFPlug. It facilitates the conversion of these document types to either
29# HTML, Text or auto (allow user to choose which format to convert to).
30# It works by dynamically inheriting BasPlug and base on the plugin type in
31# secondary_plugins to devide which format to 'convert_to'. If the argument is
32# not present, the default is to inherit auto.
33package ConvertToPlug;
34
35use BasPlug;
36use ghtml;
37use HTMLPlug;
38use TEXTPlug;
39use PagedImgPlug;
40
41use strict;
42no strict 'refs'; # allow filehandles to be variables and viceversa
43
44sub BEGIN {
45 @ConvertToPlug::ISA = ('BasPlug');
46}
47
48my $convert_to_list =
49 [ { 'name' => "auto",
50 'desc' => "{ConvertToPlug.convert_to.auto}" },
51 { 'name' => "html",
52 'desc' => "{ConvertToPlug.convert_to.html}" },
53 { 'name' => "text",
54 'desc' => "{ConvertToPlug.convert_to.text}" }
55 ];
56
57my $arguments =
58 [ { 'name' => "convert_to",
59 'desc' => "{ConvertToPlug.convert_to}",
60 'type' => "enum",
61 'reqd' => "yes",
62 'list' => $convert_to_list,
63 'deft' => "html" },
64 { 'name' => "title_sub",
65 'desc' => "{HTMLPlug.title_sub}",
66 'type' => "string",
67 #'type' => "regexp",
68 'deft' => "" },
69 { 'name' => "use_strings",
70 'desc' => "{ConvertToPlug.use_strings}",
71 'type' => "flag",
72 'reqd' => "no" },
73 { 'name' => "extract_keyphrases",
74 'desc' => "{BasPlug.extract_keyphrases}",
75 'type' => "flag",
76 'reqd' => "no",
77 'hiddengli' => "yes" },
78 { 'name' => "extract_keyphrase_options",
79 'desc' => "{BasPlug.extract_keyphrase_options}",
80 'type' => "string",
81 'reqd' => "no",
82 'hiddengli' => "yes" } ];
83
84my $options = { 'name' => "ConvertToPlug",
85 'desc' => "{ConvertToPlug.desc}",
86 'abstract' => "yes",
87 'inherits' => "yes",
88 'args' => $arguments };
89
90sub findType
91{
92 my ($inputargs) = @_;
93 my $convert_to_type = "html";
94 my $windows_scripting = "false";
95 for(my $intCounter = 0; $intCounter < scalar(@{$inputargs}) ; $intCounter++)
96 {
97 if($inputargs->[$intCounter] eq "-convert_to")
98 {
99 if($inputargs->[$intCounter+1] eq "auto" || $inputargs->[$intCounter+1] =~ /pagedimg.*/i || $inputargs->[$intCounter+1] eq "text" || $inputargs->[$intCounter+1] eq "html")
100 {
101 $convert_to_type = $inputargs->[$intCounter+1];
102 }
103 }
104 if ($inputargs->[$intCounter] eq "-windows_scripting")
105 {
106 $windows_scripting = "true";
107 }
108 }
109 return ($convert_to_type,$windows_scripting);
110}
111
112sub load_secondary_plugins
113{
114 my $self = shift (@_);
115 my ($class,$input_args,$hashArgOptLists) = @_;
116
117 my @convert_to_list = split(",",$self->{'convert_to'});
118 my $secondary_plugins = {};
119
120 foreach my $convert_to (@convert_to_list) {
121 # load in "convert_to" plugin package
122 my $plugin_class = $convert_to."Plug";
123 my $plugin_package = $plugin_class.".pm";
124
125 require $plugin_package;
126
127 # call its constructor with extra options that we've worked out!
128 my $arglist = $input_args->{$plugin_class};
129 my $secondary_plugin = new $plugin_class([],$arglist, $hashArgOptLists);
130 $secondary_plugins->{$plugin_class} = $secondary_plugin;
131 }
132 $self->{'secondary_plugins'} = $secondary_plugins;
133}
134
135sub new {
136 my ($class) = shift (@_);
137 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
138 push(@$pluginlist, $class);
139 my $classPluginName = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
140 my ($strConvertTo,$blnWindowsScripting) = findType($inputargs);
141
142 if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
143 if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
144
145 my $self = (defined $hashArgOptLists)? new BasPlug($pluginlist,$inputargs,$hashArgOptLists): new BasPlug($pluginlist,$inputargs);
146
147
148 if ($classPluginName eq "PDFPlug") {
149 if ($strConvertTo eq "text" &&
150 $ENV{'GSDLOS'} =~ /^windows$/i) {
151 print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
152 $strConvertTo = "html";
153 }
154 } elsif ($classPluginName eq "WordPlug") {
155 if ($blnWindowsScripting eq "true" && $ENV{'GSDLOS'} =~ /^windows$/i && $strConvertTo =~ /^(html|auto)$/) {
156 # we use structured HTML, not normal html
157 $strConvertTo = "structuredhtml";
158 }
159 } elsif ($classPluginName eq "PPTPlug") {
160 if ($blnWindowsScripting eq "true" && $ENV{'GSDLOS'} =~ /^windows$/i && $strConvertTo eq "auto") {
161 # we use paged img
162 $strConvertTo = "pagedimg_jpg";
163 }
164 } elsif ($classPluginName eq "PSPlug") {
165 if ($strConvertTo eq "auto") {
166 # we use text
167 $strConvertTo = "text";
168 }
169 }
170
171 if ($strConvertTo eq "auto") {
172 # choose html for now - should choose a format based on doc type
173 $strConvertTo = "html";
174 }
175
176 if ($strConvertTo eq "html") {
177 $self->{'convert_to'} = "HTML";
178 $self->{'convert_to_ext'} = "html";
179 } elsif ($strConvertTo eq "text") {
180 $self->{'convert_to'} = "TEXT";
181 $self->{'convert_to_ext'} = "txt";
182 } elsif ($strConvertTo eq "structuredhtml") {
183 $self->{'convert_to'} = "StructuredHTML";
184 $self->{'convert_to_ext'} = "html";
185 } elsif ($strConvertTo =~ /^pagedimg/) {
186 $self->{'convert_to'} = "PagedImg";
187 my ($convert_to_ext) = $strConvertTo =~ /pagedimg\_(jpg|gif|png)/i;
188 $convert_to_ext = 'jpg' unless defined $convert_to_ext;
189 $self->{'convert_to_ext'} = $convert_to_ext;
190 }
191
192 return bless $self, $class;
193}
194
195
196sub init {
197 my $self = shift (@_);
198 my ($verbosity, $outhandle, $failhandle) = @_;
199
200 $self->SUPER::init($verbosity,$outhandle,$failhandle);
201
202 my $secondary_plugins = $self->{'secondary_plugins'};
203
204 foreach my $plug_name (keys %$secondary_plugins) {
205 my $plugin = $secondary_plugins->{$plug_name};
206 $plugin->init($verbosity,$outhandle,$failhandle);
207 }
208}
209
210sub deinit {
211 # called only once, after all plugin passes have been done
212
213 my ($self) = @_;
214
215 my $secondary_plugins = $self->{'secondary_plugins'};
216
217 foreach my $plug_name (keys %$secondary_plugins) {
218 my $plugin = $secondary_plugins->{$plug_name};
219 $plugin->deinit();
220 }
221}
222
223sub convert_post_process
224{
225 # by default do no post processing
226 return;
227}
228
229
230# Run conversion utility on the input file.
231#
232# The conversion takes place in a collection specific 'tmp' directory so
233# that we don't accidentally damage the input.
234#
235# The desired output type is indicated by $output_ext. This is usually
236# something like "html" or "word", but can be "best" (or the empty string)
237# to indicate that the conversion utility should do the best it can.
238sub tmp_area_convert_file {
239 my $self = shift (@_);
240 my ($output_ext, $input_filename, $textref) = @_;
241
242 my $outhandle = $self->{'outhandle'};
243 my $convert_to = $self->{'convert_to'};
244 my $failhandle = $self->{'failhandle'};
245 my $convert_to_ext = $self->{'convert_to_ext'};
246
247 # softlink to collection tmp dir
248 my $tmp_dirname
249 = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
250 &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
251
252 # derive tmp filename from input filename
253 my ($tailname, $dirname, $suffix)
254 = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
255
256 # Remove any white space from filename -- no risk of name collision, and
257 # makes later conversion by utils simpler. Leave spaces in path...
258 # tidy up the filename with space, dot, hyphen between
259 $tailname =~ s/\s+//g;
260 $tailname =~ s/\.+//g;
261 $tailname =~ s/\-+//g;
262 $suffix = lc($suffix);
263 my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
264 &util::soft_link($input_filename, $tmp_filename);
265 my $verbosity = $self->{'verbosity'};
266 if ($verbosity > 0) {
267 print $outhandle "Converting $tailname$suffix to $convert_to format\n";
268 }
269
270 my $errlog = &util::filename_cat($tmp_dirname, "err.log");
271
272 # Execute the conversion command and get the type of the result,
273 # making sure the converter gives us the appropriate output type
274 my $output_type="";
275 if ($convert_to =~ m/PagedImg/i) {
276 $output_type = lc($convert_to)."_".lc($convert_to_ext);
277 } else {
278 $output_type = lc($convert_to);
279 }
280
281 my $cmd = "perl -S gsConvert.pl -verbose $verbosity ";
282 if (defined $self->{'convert_options'}) {
283 $cmd .= $self->{'convert_options'} . " ";
284 }
285 if ($self->{'use_strings'}) {
286 $cmd .= "-use_strings ";
287 }
288 $cmd .= "-errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
289
290 $output_type = `$cmd`;
291
292 # remove symbolic link to original file
293 &util::rm($tmp_filename);
294
295 # Check STDERR here
296 chomp $output_type;
297 if ($output_type eq "fail") {
298 print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
299 print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
300 $self->{'num_not_processed'} ++;
301 if (-s "$errlog") {
302 open(ERRLOG, "$errlog");
303 while (<ERRLOG>) {
304 print $outhandle "$_";
305 }
306 print $outhandle "\n";
307 close ERRLOG;
308 }
309 &util::rm("$errlog") if (-e "$errlog");
310 return "";
311 }
312
313 # store the *actual* output type and return the output filename
314 # it's possible we requested conversion to html, but only to text succeeded
315 #$self->{'convert_to_ext'} = $output_type;
316 if ($output_type =~ /html/i) {
317 $self->{'converted_to'} = "HTML";
318 } elsif ($output_type =~ /te?xt/i) {
319 $self->{'converted_to'} = "TEXT";
320 } elsif ($output_type =~ /item/i){
321 $self->{'converted_to'} = "PagedImg";
322 }
323
324 my $output_filename = $tmp_filename;
325 if ($output_type =~ /item/i) {
326 # running under windows
327 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
328 $output_filename = $tmp_dirname . "\\$tailname\\" . $tailname . ".$output_type";
329 } else {
330 $output_filename = $tmp_dirname . "\/$tailname\/" . $tailname . ".$output_type";
331 }
332 } else {
333 $output_filename =~ s/$suffix$/.$output_type/;
334 }
335 return $output_filename;
336}
337
338
339# Override BasPlug read
340# We don't want to get language encoding stuff until after we've converted
341# our file to either TEXT or HTML or PagedImage.
342sub read {
343 my $self = shift (@_);
344 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
345 #if ($self->is_recursive()) {
346 # die "BasPlug::read function must be implemented in sub-class for recursive plugins\n";
347 # }
348
349 my $outhandle = $self->{'outhandle'};
350
351 my ($block_status,$filename) = $self->read_block(@_);
352 return $block_status if ((!defined $block_status) || ($block_status==0));
353 $file = $self->read_tidy_file($file);
354
355 my $output_ext = $self->{'convert_to_ext'};
356 my $conv_filename = "";
357 $conv_filename = $self->tmp_area_convert_file($output_ext, $filename);
358
359 if ("$conv_filename" eq "") {return 0;} # allows continue on errors
360 if (! -e "$conv_filename") {return 0;} # allows continue on errors
361 $self->{'conv_filename'} = $conv_filename;
362 $self->convert_post_process($conv_filename);
363
364 my $secondary_plugins = $self->{'secondary_plugins'};
365 my $num_secondary_plugins = scalar(keys %$secondary_plugins);
366
367 if ($num_secondary_plugins == 0) {
368 print $outhandle "Warning: No secondary plugin to use in conversion. Skipping $file\n";
369 return 0; # effectively block it
370 }
371
372 my @plugin_names = keys %$secondary_plugins;
373 my $plugin_name = shift @plugin_names;
374
375 if ($num_secondary_plugins > 1) {
376 print $outhandle "Warning: Multiple secondary plugins not supported yet! Choosing $plugin_name\n.";
377 }
378
379 my $secondary_plugin = $secondary_plugins->{$plugin_name};
380
381 # note: metadata is not carried on to the next level
382 my ($rv,$doc_obj)
383 = $secondary_plugin->read_into_doc_obj ($pluginfo,"", $conv_filename,
384 $metadata, $processor, $maxdocs, $total_count,
385 $gli);
386
387 if ((!defined $rv) || ($rv<1)) {
388 # wasn't processed
389 return $rv;
390 }
391
392 # Override previous gsdlsourcefilename set by secondary plugin
393 my $collect_file = &util::filename_within_collection($filename);
394 my $collect_conv_file = &util::filename_within_collection($conv_filename);
395 $doc_obj->set_source_filename ($collect_file);
396 $doc_obj->set_converted_filename($collect_conv_file);
397
398 my ($filemeta) = $file =~ /([^\\\/]+)$/;
399 $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
400 $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
401 $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FileSize", (-s $filename));
402
403 if ($self->{'cover_image'}) {
404 $self->associate_cover_image($doc_obj, $filename);
405 }
406
407 # do plugin specific processing of doc_obj
408 unless (defined ($self->process(undef, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
409 print STDERR "<ProcessingError n='$file'>\n" if ($gli);
410 return -1;
411 }
412 # do any automatic metadata extraction
413 $self->auto_extract_metadata ($doc_obj);
414
415 # have we found a Title??
416 $self->title_fallback($doc_obj,$doc_obj->get_top_section(),$filemeta);
417
418 # add an OID
419 $doc_obj->set_OID();
420 # process the document
421 $processor->process($doc_obj);
422
423 $self->{'num_processed'} ++;
424
425 return 1;
426}
427
428
429# do plugin specific processing of doc_obj for doc_ext type
430sub process_type {
431 my $self = shift (@_);
432 my ($doc_ext, $base_dir, $file, $doc_obj) = @_;
433
434 # associate original file with doc object
435 my $cursection = $doc_obj->get_top_section();
436 my $filename = &util::filename_cat($base_dir, $file);
437 $doc_obj->associate_file($filename, "doc.$doc_ext", undef, $cursection);
438
439 my $file_type;
440
441 if ($doc_ext eq "doc") {
442 $file_type = "Word";
443 } elsif ($doc_ext eq "xls") {
444 $file_type = "Excel";
445 } elsif ($doc_ext eq "ppt") {
446 $file_type = "PPT";
447 } elsif ($doc_ext eq "pdf") {
448 $file_type = "PDF";
449 } elsif ($doc_ext eq "rtf") {
450 $file_type = "RTF";
451 } elsif ($doc_ext eq "ps") {
452 $file_type = "PS";
453 }
454
455 my $file_format = $file_type || "unknown";
456
457 # We use set instead of add here because we only want one value
458 $doc_obj->set_utf8_metadata_element($cursection, "FileFormat", $file_format);
459
460 my $doclink = "<a href=\"_httpcollection_/index/assoc/[archivedir]/doc.$doc_ext\">";
461 $doc_obj->add_utf8_metadata ($cursection, "srclink", $doclink);
462 $doc_obj->add_utf8_metadata ($cursection, "srcicon", "_icon".$doc_ext."_");
463 $doc_obj->add_utf8_metadata ($cursection, "/srclink", "</a>");
464
465 return 1;
466}
467
4681;
469
470
471
472
473
474
475
Note: See TracBrowser for help on using the repository browser.