Changeset 3350

Timestamp:

2002-08-13T12:28:15+12:00 (22 years ago)

Author:

sjboddie

Message:

Added -use_strings option to ConvertToPlug. The default behaviour for
plugins derived from ConvertToPlug (WordPlug, PDFPlug etc) is now to
exclude documents that can't be converted correctly. They won't use the
perl strings stuff to extract text unless the -use_strings option is
specified.

Location:

trunk/gsdl

Files:

• bin/script/gsConvert.pl (modified) (5 diffs)
• perllib/plugins/ConvertToPlug.pm (modified) (6 diffs)

trunk/gsdl/bin/script/gsConvert.pl

@@ -57 +57 @@
 my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
 if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
+
+my $use_strings;
 
 sub print_usage
@@ -68 +70 @@
     print STDERR "\t-output\thtml|text\n";
     print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
+    print STDERR "\t-use_strings\t(use strnigs to extract text if conversion fails)\n";
     exit(1);
 }
@@ -85 +88 @@
              'output/(html|text)/', \$output_type,
              'timeout/\d+/0',\$timeout,
-             'verbose/\d+/0',   \$verbose))
+             'verbose/\d+/0',   \$verbose,
+             'use_strings', \$use_strings))
     {
     print_usage();
@@ -469 +473 @@
         close FAILLOG if ($write_to_fail_log);
     }
-    print STDERR "Continuing...\n";
     return 0; # we can try any_to_text
     }
@@ -954 +957 @@
     ($input_filename, $output_filestem) = @_;
 
+    if (!$use_strings) {
+      return 0;
+    }
+
     open(IN, "<$input_filename") || return 0;
     binmode(IN);

trunk/gsdl/perllib/plugins/ConvertToPlug.pm

@@ -61 +61 @@
     print STDERR "   -convert_to (html|text) plugin converts to TEXT or HTML\n";
     print STDERR "                           (default html)\n";
+    print STDERR "   -use_strings            if set a simple strings function\n";
+    print STDERR "                           will be called to extract text\n";
+    print STDERR "                           if the conversion utility fails\n";
 }
 
@@ -71 +74 @@
     $plugin_name =~ s/\.pm$//;
 
-    my $generate_format;
-    my $kea_arg;
+    my $newargs = {};
 
     if (!parsargv::parse($args,  
-             q^extract_keyphrases^, \$kea_arg->{'kea'}, #with extra options 
-             q^extract_keyphrase_options/.*/^, \$kea_arg->{'kea_options'}, #no extra options
-             q^convert_to/(html|text)/html^, \$generate_format,
+             q^extract_keyphrases^, \$newargs->{'kea'}, #with extra options 
+             q^extract_keyphrase_options/.*/^, \$newargs->{'kea_options'}, #no extra options
+             q^convert_to/(html|text)/html^, \$newargs->{'generate_format'},
+             q^use_strings^, \$newargs->{'use_strings'},
              "allow_extra_options")) {
 
@@ -86 +89 @@
     }
    
-    return ($plugin_name,$generate_format, $kea_arg);
+    return ($plugin_name, $newargs);
 }
 
@@ -96 +99 @@
     # of the argument list.
     my @arglist = @_;
-    my ($plugin_name, $generate_format, $kea_arg) = $class->parse_args(\@_);
-
-    if ($class eq "PDFPlug" && $generate_format eq "text" && 
+    my ($plugin_name, $args) = $class->parse_args(\@_);
+
+    if ($class eq "PDFPlug" && $args->{'generate_format'} eq "text" && 
     $ENV{'GSDLOS'} =~ /^windows$/i) {
     print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
-    $generate_format = "html";
-    }
-
-    if ($generate_format eq "text")
+    $args->{'generate_format'} = "html";
+    }
+
+    if ($args->{'generate_format'} eq "text")
     {
     $self = new TEXTPlug ($class, @arglist);
@@ -120 +123 @@
     }
 
-    #if kea data to be extracted...
-    $self->{'kea'} = 1 if($kea_arg->{'kea'});
-    $self->{'kea_options'} = 1 if($kea_arg->{'kea_options'});
+    foreach my $key (keys %$args) {
+      $self->{$key} = $args->{$key};
+    }
  
     return bless $self, $class;
@@ -173 +176 @@
     # making sure the converter gives us the appropriate output type
     my $output_type = lc($convert_to);
-    my $cmd = "perl -S gsConvert.pl -verbose $verbosity -errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
+    my $cmd = "perl -S gsConvert.pl -verbose $verbosity ";
+    if ($self->{'use_strings'}) {
+      $cmd .= "-use_strings ";
+    }
+    $cmd .= "-errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
     $output_type = `$cmd`;