source: trunk/cic-hcap/perllib/plugins/CICWordPlug.pm@ 12170

###########################################################################
#
# CICWordPlug.pm
#
# Copyright (C) 2006 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package CICWordPlug;


use ConvertToPlug;
use strict;
no strict 'refs';


sub BEGIN {
    @CICWordPlug::ISA = ('ConvertToPlug');
}


my $arguments = [ ];

my $options = { 'name'     => "CICWordPlug",
        'desc'     => "{CICWordPlug.desc}",
        'abstract' => "no",
        'inherits' => "yes",
        'args'     => $arguments };


sub get_default_process_exp
{
    return q^(?i)\.doc$^;
}


sub new {
    my ($class) = shift (@_);
    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
    push(@$pluginlist, $class);

    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};

    my $self = new ConvertToPlug($pluginlist, $inputargs, $hashArgOptLists);

    return bless $self, $class;
}


sub read
{
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;

    $self->{'file'} = $file;
    $self->{'processor'} = $processor;
    $self->{'gli'} = $gli;

    $self->SUPER::read($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli);
}


sub convert_post_process
{
    my $self = shift(@_);
    my $conv_filename = shift(@_);

    my $in_record = 0;
    my $num_records = 0;
    open(HTML_IN, $conv_filename);
    while (my $line = <HTML_IN>) {
    # Is this the end of a record?
    if ($line =~ /^<\/p>/) {
        $in_record = 0;
    }
    # Is this the start of a possible record?
    if ($line =~ /^<p /) {
        $in_record = 1;
    }
    elsif ($in_record == 1) {
        my $line_sans_tags = $line;
        $line_sans_tags =~ s/<.*?>//g;

        # Type 1: No author, title in double quotes
        if ($line =~ /^&ldquo;(.*)&rdquo;/) {
        $num_records++;
        $self->add_reference_document($num_records, $line, $1, "");
        }

        # Type 2: Author, title in italics or quotes
        elsif ($line =~ /^(.*)<i>(.*)<\/i>/ || $line =~ /^(.*)&quot;(.*)&quot;/ || $line =~ /^(.*)&ldquo;(.*)&rdquo;/) {
        $num_records++;
        $self->add_reference_document($num_records, $line, $2, $1);
        }

        # Error: Could not parse record
        elsif ($line_sans_tags =~ /\w/) {
        print "Unknown type: $line";
        }
    }
    }
    close(HTML_IN);
}


sub add_reference_document
{
    my $self = shift(@_);
    my $reference_id = shift(@_);
    my $reference = shift(@_);
    my $reference_title = shift(@_);
    my $reference_author = shift(@_);

    my $reference_doc_obj = new doc($self->{'file'} . "-", "indexed_doc");
    $reference_doc_obj->set_OID("r$reference_id");
    &new_metadata_entry($reference_doc_obj, "DocumentType", "Reference");
    &new_metadata_entry($reference_doc_obj, "Author", $reference_author);
    &new_metadata_entry($reference_doc_obj, "Title", $reference_title);

    $reference_doc_obj->add_utf8_text($reference_doc_obj->get_top_section(), $reference);
    $self->{'processor'}->process($reference_doc_obj);
    $self->{'num_processed'}++;
}


sub new_metadata_entry
{
    my ($doc_obj, $metadata_name, $metadata_value) = (@_);

    # Don't bother with empty metadata
    return if ($metadata_value eq "");

    # Spaces aren't allowed in metadata names
    $metadata_name =~ s/ /_/g;

    # Anything from the database is ISO 8859-1 encoded, so convert to UTF-8
    $metadata_value = &unicode::ascii2utf8(\$metadata_value);

    # Remove '#' characters around links
    if ($metadata_value =~ /\#(.*?)\#/) {
    $metadata_value = $1;
    }

    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $metadata_name, $metadata_value);
}


1;