source: gs2-extensions/apache-jena/trunk/src/perllib/jenaTDBBuildproc.pm@ 28391

##########################################################################
#
# jenaTDBBuildproc.pm -- 
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

# This document processor outputs a document for indexing (should be 
# implemented by subclass) and storing in the database

package jenaTDBBuildproc;

use strict; 
#no strict 'refs'; # allow filehandles to be variables and viceversa

use util;

use extrabuildproc;


BEGIN {
    @jenaTDBBuildproc::ISA = ('extrabuildproc');
}

sub new()
  {
    my $class = shift @_;

    my $self = new extrabuildproc (@_);

    return bless $self, $class;
}


sub textedit {
    my $self = shift (@_);
    my ($doc_obj,$file,$mode) = @_;

    # Code written on the assumption that that jenaTDB does a replace
    # operation when presented with a docid that already extis.
    # => don't need to do anything special to distinguish between
    #    a mode of "add" and "update"

    my $outhandle = $self->{'outhandle'};


    my $source_dir = $self->{'source_dir'}; # typically the archives dir
    my $build_dir  = $self->{'build_dir'};

    # full path to adb database
    my $adb_filename 
    = &util::filename_cat($build_dir, "jenaTDB", "lsh-features.adb");

    # get doc id
    my $doc_oid = $doc_obj->get_OID();

    # map to assoc dir
    my $top_section = $doc_obj->get_top_section();
    my $assoc_file
    = $doc_obj->get_metadata_element ($top_section,"assocfilepath");
    my $assoc_filename = &util::filename_cat($source_dir,$assoc_file);

    my $chr12_filename = &util::filename_cat($assoc_filename,"doc.chr12");
    my $powerlog_filename = &util::filename_cat($assoc_filename,"doc.power");

    print $outhandle "  Inserting tripples for $doc_oid\n";

#    my $cmd = "jenaTDB -d \"$adb_filename\" -I -k \"$doc_oid\" -f \"$chr12_filename\" -w \"$powerlog_filename\"";

#    my $status = system($cmd);
#    if ($status != 0) {
#   print STDERR "Error: failed to run:\n  $cmd\n$!\n";
#    }

}

sub text {
    my $self = shift (@_);
    my ($doc_obj,$file) = @_;

    $self->textedit($doc_obj,$file,"add");
}

sub textreindex
{
    my $self = shift @_;
    my ($doc_obj,$file) = @_;

    $self->textedit($doc_obj,$file,"update");
}

sub textdelete
{
    my $self = shift @_;

    my ($doc_obj,$file) = @_;

    print STDERR "Warning: jenaTDB command-line does not currently support delete operation\n";

    # $self->textedit($doc_obj,$file,"delete");
}






sub MGtext {
    my $self = shift (@_);
    my ($doc_obj) = @_;
    my $handle = $self->{'output_handle'};
    
    # only output this document if it is one to be indexed
    return if ($doc_obj->get_doc_type() ne "indexed_doc");
    
    # see if this document belongs to this subcollection
    my $indexed_doc = $self->is_subcollection_doc($doc_obj);

    # this is another document
    $self->{'num_docs'} += 1;

    # get the parameters for the output
    my ($level, $fields) = split (/:/, $self->{'index'});
    $fields =~ s/\ball\b/Title,Creator,text/;
    $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;

    my $doc_section = 0; # just for this document
    my $text = "";
    my $text_extra = "";

    # get the text for this document
    my $section = $doc_obj->get_top_section();
    while (defined $section) {
    # update a few statistics
    $doc_section++;
    $self->{'num_sections'} += 1;

    my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
    if (($indexed_doc) && ($indexed_section eq "indexed_section" || $indexed_section eq "indexed_doc")) {
        $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
        foreach my $field (split (/,/, $fields)) {
        # only deal with this field if it doesn't start with top or
        # this is the first section
        my $real_field = $field;
        if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
            my $new_text = "";
            if ($level eq "dummy") {
            # a dummy index is a special case used when no
            # indexes are specified (since there must always be
            # at least one index or we can't retrieve the
            # compressed text) - we add a small amount of text
            # to these dummy indexes which will never be seen
            # but will overcome mg's problems with building
            # empty indexes
            $new_text = "this is dummy text to stop mg barfing";
            $self->{'num_processed_bytes'} += length ($new_text);

            } elsif ($real_field eq "text") {
            $new_text = $doc_obj->get_text ($section) if $self->{'store_text'};
            $self->{'num_processed_bytes'} += length ($new_text);
            $new_text =~ s/[\cB\cC]//g;
            $self->find_paragraphs($new_text);
            
            } else {
            my $first = 1;
            $real_field =~ s/^ex\.([^.]+)$/$1/; # remove ex. namespace iff it's the only namespace prefix (will leave ex.dc.* intact)
            my @section_metadata = @{$doc_obj->get_metadata ($section, $real_field)};
            if ($level eq "section" && $section ne $doc_obj->get_top_section() && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
                if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
                push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $real_field)});
                }
            }
            foreach my $meta (@section_metadata) {
                $meta =~ s/[\cB\cC]//g;
                $self->{'num_processed_bytes'} += length ($meta);
                $new_text .= "\cC" unless $first;
                $new_text .= $meta if $self->{'store_text'};
                $first = 0;
            }
            }
            
            # filter the text
            $new_text = $self->filter_text ($field, $new_text);

            $text .= "$new_text\cC";
        }
        }
    }
    
    if ($level eq "document") { $text_extra .= "\cB"; }
    else { $text .= "\cB"; }
    
    $section = $doc_obj->get_next_section($section);
    }

    print $handle "$text$text_extra";
}


1;