source: trunk/gsdl/perllib/plugins/BasPlug.pm@ 1240

###########################################################################
#
# BasPlug.pm -- base class for all the import plugins
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package BasPlug;

use parsargv;
use multiread;
use cnseg;
use strict;

sub print_usage {
    print STDERR "\nOne of your plugins uses an incorrect general option (general options are those\n";
    print STDERR "available to all plugins). Check your collect.cfg configuration file.\n";
    
    print STDERR "\n  usage: plugin plugin-name [options]\n\n";
    print STDERR "  currently supported general options are:\n";
    print STDERR "   -input_encoding  The encoding of the source documents. Documents will be\n";
    print STDERR "                    converted from these encodings and stored internally as\n";
    print STDERR "                    utf8. The default input_encoding is Latin1. Accepted values\n";
    print STDERR "                    are:\n";
    print STDERR "                      iso_8859_1 (extended ascii)\n";
    print STDERR "                      Latin1 (the same as iso-8859-1)\n";
    print STDERR "                      ascii (7 bit ascii -- may be faster than Latin1 as no\n";
    print STDERR "                             conversion is neccessary)\n";
    print STDERR "                      gb (GB or GBK simplified Chinese)\n";
    print STDERR "                      iso_8859_6 (8 bit Arabic)\n";
    print STDERR "                      windows_1256 (Windows codepage 1256 (Arabic))\n";
    print STDERR "                      Arabic (the same as windows_1256)\n";
    print STDERR "                      utf8 (either utf8 or unicode -- automatically detected)\n";
    print STDERR "                      unicode (just unicode -- doesn't currently do endian\n";
    print STDERR "                               detection)\n\n";
}

sub new {
    my $class = shift (@_);

    my $self = {};
    my $encodings = "^(iso_8859_1|Latin1|ascii|gb|iso_8859_6|windows_1256|Arabic|utf8|unicode)\$";

    # general options available to all plugins
    if (!parsargv::parse(\@_, "input_encoding/$encodings/Latin1", \$self->{'input_encoding'},
             "allow_extra_options")) {
    &print_usage();
    die "\n";
    }

    return bless $self, $class;
}

sub begin {
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
}

sub end {
    my ($self) = @_;
}

# return 1 if this class might recurse using $pluginfo
sub is_recursive {
    my $self = shift (@_);

    die "BasPlug::is_recursive function must be implemented in sub classes\n";
}

# return number of files processed, undef if can't process
# Note that $base_dir might be "" and that $file might 
# include directories
sub read {
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;

    die "BasPlug::read function must be implemented in sub classes\n";

    return undef; # will never get here
}

# uses the multiread package to read in the entire file pointed to
# by filename and loads the resulting text into $$textref. Input text
# may be in any of the encodings handled by multiread, output text
# will be in utf8
sub read_file {
    my $self = shift (@_);
    my ($filename, $textref) = @_;

    $$textref = "";
    my $encoding = "";
    if ($self->{'input_encoding'} =~ /^(Latin1|iso_8859_1)$/) {
    $encoding = "iso_8859_1";
    } elsif ($self->{'input_encoding'} =~ /^(Arabic|windows_1256)$/) {
    $encoding = "windows_1256";
    } else {
    $encoding = $self->{'input_encoding'};
    }

    open (FILE, $filename) || die "BasPlug::read_file could not open $filename for reading ($!)\n";

    if ($encoding eq "ascii") {
    undef $/;
    $$textref = <FILE>;
    $/ = "\n";
    } else {
    my $reader = new multiread();
    $reader->set_handle ('BasPlug::FILE');
    $reader->set_encoding ($encoding);
    $reader->read_file ($textref);

    if ($encoding eq "gb") {
        # segment the Chinese words
        $$textref = &cnseg::segment($$textref);
    }
    }

    close FILE;
}

# add any extra metadata that's been passed around from one
# plugin to another.
# extra_metadata uses add_utf8_metadata so it expects metadata values
# to already be in utf8
sub extra_metadata {
    my $self = shift (@_);
    my ($doc_obj, $cursection, $metadata) = @_;

    foreach my $field (keys(%$metadata)) {
    # $metadata->{$field} may be an array reference
    if (ref ($metadata->{$field}) eq "ARRAY") {
        map { 
        $doc_obj->add_utf8_metadata ($cursection, $field, $_); 
        } @{$metadata->{$field}};
    } else {
        $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
    }
    }
}

1;