Changeset 1219 for trunk/gsdl/perllib/plugins/BasPlug.pm

Timestamp:

2000-06-21T10:14:14+12:00 (24 years ago)

Author:

sjboddie

Message:

Made BasPlug take options (these options are available to all plugins
derived from BasPlug). Also added a -input_encoding option and related
sub-routines. Existing plugins will need to be caught up with these changes
before they can take advantage of the new option (or any others I may get
around to adding).

File:

• trunk/gsdl/perllib/plugins/BasPlug.pm (modified) (2 diffs)

trunk/gsdl/perllib/plugins/BasPlug.pm

@@ -26 +26 @@
 package BasPlug;
 
+use parsargv;
+use multiread;
+use cnseg;
+use strict;
+
+sub print_usage {
+    print STDERR "\nOne of your plugins uses an incorrect general option (general options are those\n";
+    print STDERR "available to all plugins). Check your collect.cfg configuration file.\n";
+    
+    print STDERR "\n  usage: plugin plugin-name [options]\n\n";
+    print STDERR "  currently supported general options are:\n";
+    print STDERR "   -input_encoding  The encoding of the source documents. Documents will be\n";
+    print STDERR "                    converted from these encodings and stored internally as\n";
+    print STDERR "                    utf8. The default input_encoding is Latin1. Accepted values\n";
+    print STDERR "                    are:\n";
+    print STDERR "                      iso_8859_1 (extended ascii)\n";
+    print STDERR "                      Latin1 (the same as iso-8859-1)\n";
+    print STDERR "                      ascii (7 bit ascii -- may be faster than Latin1 as no\n";
+    print STDERR "                             conversion is neccessary)\n";
+    print STDERR "                      gb (GB or GBK simplified Chinese)\n";
+    print STDERR "                      iso_8859_6 (8 bit Arabic)\n";
+    print STDERR "                      Arabic (the same as iso-8859-6)\n";
+    print STDERR "                      utf8 (either utf8 or unicode -- automatically detected)\n";
+    print STDERR "                      unicode (just unicode -- doesn't currently do endian\n";
+    print STDERR "                               detection)\n\n";
+}
 
 sub new {
-    my ($class) = @_;
+    my $class = shift (@_);
 
-    return bless {}, $class;
+    my $self = {};
+    my $encodings = "^(iso_8859_1|Latin1|ascii|gb|iso_8859_6|Arabic|utf8|unicode)\$";
+
+    # general options available to all plugins
+    if (!parsargv::parse(\@_, "input_encoding/$encodings/Latin1", \$self->{'input_encoding'},
+             "allow_extra_options")) {
+    &print_usage();
+    die "\n";
+    }
+
+    return bless $self, $class;
 }
 
@@ -61 +97 @@
 }
 
-sub extra_metadata
-{
-    my ($self,$doc_obj,$cursection, $metadata) = @_;
+# uses the multiread package to read in the entire file pointed to
+# by filename and loads the resulting text into $$textref. Input text
+# may be in any of the encodings handled by multiread, output text
+# will be in utf8
+sub read_file {
+    my $self = shift (@_);
+    my ($filename, $textref) = @_;
 
-    foreach $field (keys(%$metadata)) {
+    $$textref = "";
+    my $encoding = "";
+    if ($self->{'input_encoding'} =~ /^(Latin1|iso_8859_1)$/) {
+    $encoding = "latin1";
+    } elsif ($self->{'input_encoding'} =~ /^(Arabic|iso_8859_6)$/) {
+    $encoding = "arabic";
+    } else {
+    $encoding = $self->{'input_encoding'};
+    }
+
+    open (FILE, $filename) || die "BasPlug::read_file could not open $filename for reading ($!)\n";
+
+    if ($encoding eq "ascii") {
+    undef $/;
+    $$textref = <FILE>;
+    $/ = "\n";
+    } else {
+    my $reader = new multiread();
+    $reader->set_handle ('BasPlug::FILE');
+    $reader->set_encoding ($encoding);
+    $reader->read_file ($textref);
+
+    if ($encoding eq "gb") {
+        # segment the Chinese words
+        $$textref = &cnseg::segment($$textref);
+    }
+    }
+
+    close FILE;
+}
+
+# add any extra metadata that's been passed around from one
+# plugin to another.
+# extra_metadata uses add_utf8_metadata so it expects metadata values
+# to already be in utf8
+sub extra_metadata {
+    my $self = shift (@_);
+    my ($doc_obj, $cursection, $metadata) = @_;
+
+    foreach my $field (keys(%$metadata)) {
     # $metadata->{$field} may be an array reference
     if (ref ($metadata->{$field}) eq "ARRAY") {
         map { 
-        $doc_obj->add_metadata ($cursection, $field, $_); 
+        $doc_obj->add_utf8_metadata ($cursection, $field, $_); 
         } @{$metadata->{$field}};
     } else {
-        $doc_obj->add_metadata ($cursection, $field, $metadata->{$field}); 
+        $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
     }
     }