###########################################################################
#
# lucenebuildproc.pm -- perl wrapper for building index with Lucene
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package lucenebuildproc;

# This document processor outputs a document
# for lucene to process

# Use same basic XML structure setup by mgppbuilder/mgppbuildproc

use mgppbuildproc; 
use ghtml;

sub BEGIN {
    @lucenebuildproc::ISA = ('mgppbuildproc');
}


sub new {
    my $class = shift @_;
    my $self = new mgppbuildproc (@_);

    return bless $self, $class;
}


sub is_incremental
{
    my $self = shift (@_);

    # Unlike MG and MGPP, Lucene supports incremental building
    return 1;
}


sub preprocess_text {
    my $self = shift (@_);
    my ($text, $strip_html, $para) = @_;

    my ($outtext) = "";
    if ($strip_html) { 
	while ($text =~ /<([^>]*)>/ && $text ne "") {
	    
	    my $tag = $1;
	    $outtext .= $`." "; #add everything before the matched tag
	    $text = $'; #'everything after the matched tag
	    if ($para && $tag =~ /^\s*p\s/i) {
		$outtext .= $para;
	    }
	    elsif ($tag =~ /^pre$/) { # a pre tag
		$text =~ /<\/pre>/; # find the closing pre tag
		my $tmp_text = $`; #everything before the closing pre tag
		$text = $'; #'everything after the </pre>
		$tmp_text =~ s/[<>]//g; # remove all < and >
		$outtext.= $tmp_text . " ";
	    }
	}
    
	$outtext .= $text; # add any remaining text
    } #if strip_html
    else {
	$outtext = $text;
    }
    #if ($para) {
	#$text =~ s/(<p\b)/$para$1/gi;
	#return $text;
    #}

    # remove entities
    $outtext =~ s/&\w{1,10};//g; 

    return $outtext;
}

sub text {
    my $self = shift (@_);
    my ($doc_obj,$file) = @_;
    my $handle = $self->{'output_handle'};
    my $outhandle = $self->{'outhandle'};
    my $indexed_doc = 1;

    # only output this document if it is one to be indexed
    return if ($doc_obj->get_doc_type() ne "indexed_doc");

    # see if this document belongs to this subcollection
    foreach my $indexexp (@{$self->{'indexexparr'}}) {
	$indexed_doc = 0;
	my ($field, $exp, $options) = split /\//, $indexexp;
	if (defined ($field) && defined ($exp)) {
	    my ($bool) = $field =~ /^(.)/;
	    $field =~ s/^.// if $bool eq '!';
	    if ($field =~ /^filename$/i) {
		$field = $doc_obj->get_source_filename();
	    } else {
		$field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
	    }
	    next unless defined $field;
	    if ($bool eq '!') {
		if ($options =~ /^i$/i) {
		    if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
		} else {
		    if ($field !~ /$exp/) {$indexed_doc = 1; last;}
		}
	    } else {
		if ($options =~ /^i$/i) {
		    if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
		} else {
		    if ($field =~ /$exp/) {$indexed_doc = 1; last;}
		}
	    }
	}
    }

    # if this doc is so far in the sub collection, and we have lang info, 
    # now we check the languages to see if it matches
    if($indexed_doc && defined $self->{'lang_meta'}) {
	$indexed_doc = 0;
	my $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'lang_meta'});
	if (defined $field) {
	    foreach my $lang (@{$self->{'langarr'}}) {
		my ($bool) = $lang =~ /^(.)/;
		if ($bool eq '!') {
		    $lang =~ s/^.//;
		    if ($field !~ /$lang/) {
			$indexed_doc = 1; last;
		    }
		} else {
		    if ($field =~ /$lang/) {
			$indexed_doc = 1; last;
		    }
		}
	    }
	} 
    }

    # this is another document
    $self->{'num_docs'} += 1;

    # get the parameters for the output
    # split on : just in case there is subcoll and lang stuff
    my ($fields) = split (/:/, $self->{'index'});

    my $doc_level = $mgppbuildproc::level_map{'document'};
    my $gs2ns = 'xmlns:gs2="http://www.greenstone.org/gs2"';

    my $levels = $self->{'levels'};
    my $ldoc_level = $levels->{'document'};
    my $lsec_level = $levels->{'section'};
    my $lpar_level = $levels->{'paragraph'};

    my $docid="";
    if ($ldoc_level) {
	if ($self->{'gdbm_level'} eq 'document') {
	    my $doc_sec_num = $self->{'num_docs'};	
	    $docid = "gs2:id=\"$doc_sec_num\"";
	} else  {
	    # default is section level
	    my $doc_sec_num = $self->{'num_sections'}+1;	
	    $docid = "gs2:id=\"$doc_sec_num\"";
	}
    }
    my $documenttag = "<$doc_level $gs2ns file=\"$file\" $docid >\n";
    my $documentendtag = "\n</$doc_level>\n";
    
    my ($sectiontag) = "";    
    if ($lsec_level) {
	$sectiontag = $mgppbuildproc::level_map{'section'};
    }
    my ($parastarttag) = "";
    my ($paraendtag) = "";
    if ($self->{'levels'}->{'paragraph'}) {
	if ($self->{'strip_html'}) {
	    $parastarttag = "<".$mgppbuildproc::level_map{'paragraph'}.">";
	    $paraendtag = "</".$mgppbuildproc::level_map{'paragraph'}.">";
	} else {
	    print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
	}
    }
    
    my $doc_section = 0; # just for this document

    my $text = "";
    $text .= $documenttag;
    # get the text for this document
    my $section = $doc_obj->get_top_section();
    while (defined $section) {
	# update a few statistics
	$doc_section++;
	$self->{'num_sections'} += 1;
 
	if ($sectiontag ne "") {
	    my $secid = "gs2:id=\"".$self->{'num_sections'}."\"";	    
	    $text .= "\n<$sectiontag $secid >\n";
	}

	# if we are doing subcollections, then some docs shouldn't be indexed.
	# but we need to put the section tag placeholders in there so the 
	# sections match up with gdbm db
	if ($indexed_doc) {
	    #if ($self->{'indexing_text'}) {
	#	$text .= "$parastarttag"; # only add para tags for indexing
		# note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text
	   # }
	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
	    foreach my $field (split (/,/, $fields)) {
		# only deal with this field if it doesn't start with top or
		# this is the first section
		my $real_field = $field;
		if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
		    my $new_text = ""; 
		    my $tmp_text = "";
		    if ($real_field eq "text") {
			if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed
			    $new_text .= "$parastarttag<TX index=\"1\">\n";
			    $tmp_text .= $doc_obj->get_text ($section);
			    if ($parastarttag =~ "") {
				# we don't want to individually tag each paragraph if not doing para indexing
				$tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "");
			    } else {
				$tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</TX>$paraendtag$parastarttag<TX index=\"1\">");
			    }
			    
			    $new_text .= "$tmp_text</TX>$paraendtag\n";
			    #if (!defined $self->{'indexfields'}->{'TextOnly'}) {
				#$self->{'indexfields'}->{'TextOnly'} = 1;   
			    #}
			}
			else { # leave html stuff in, but escape the tags, and dont add Paragraph tags - never retrieve paras at the moment
			    if ($self->{'store_text'}) {
				$tmp_text .= $doc_obj->get_text ($section);
				&ghtml::htmlsafe($tmp_text);
				$new_text .= $tmp_text;
			    }
			}
		    } else { # metadata field
			if ($real_field eq "allfields") { #ignore
			}
			elsif ($real_field eq "metadata") { # insert all metadata 
			    #except gsdl stuff
			    my $shortname = "";
			    my $metadata = $doc_obj->get_all_metadata ($section);
			    foreach $pair (@$metadata) {
				my ($mfield, $mvalue) = (@$pair);
				# check fields here, maybe others dont want - change to use dontindex!!
				if ($mfield ne "Identifier" 
				    && $mfield !~ /^gsdl/ 
				    && $mfield ne "classifytype"
				    && $mfield ne "assocfilepath"
				    && defined $mvalue && $mvalue ne "") {
				    
				    if (defined $self->{'indexfieldmap'}->{$mfield}) {
					$shortname = $self->{'indexfieldmap'}->{$mfield};
				    }
				    else {
					$shortname = $self->create_shortname($mfield);
					$self->{'indexfieldmap'}->{$mfield} = $shortname;
					$self->{'indexfieldmap'}->{$shortname} = 1;
				    }	   
				    $new_text .= "$parastarttag<$shortname index=\"1\">$mvalue</$shortname>$paraendtag\n";
				    if (!defined $self->{'indexfields'}->{$mfield}) {
					$self->{'indexfields'}->{$mfield} = 1;
				    }				    
				}
			    }
			    
			}
			else { #individual metadata specified
			    my $shortname="";
			    #if (!defined $self->{'indexfields'}->{$real_field}) {
				#$self->{'indexfields'}->{$real_field} = 1;
			    #}
			    if (defined $self->{'indexfieldmap'}->{$real_field}) {
				$shortname = $self->{'indexfieldmap'}->{$real_field};
			    }
			    else {
				$shortname = $self->create_shortname($real_field);
				$self->{'indexfieldmap'}->{$real_field} = $shortname;
				$self->{'indexfieldmap'}->{$shortname} = 1;
			    }
			    foreach $item (@{$doc_obj->get_metadata ($section, $real_field)}) {
				$new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n";
				# remove entities
				$new_text =~ s/&\w{1,10};//g; 
			    }
			}

		    }
		    
		    # filter the text
		    $self->filter_text ($field, $new_text);
		    $self->{'num_processed_bytes'} += length ($new_text);
		    $text .= "$new_text";
		}
	    }
	} # if (indexed_doc)
	
	$text .= "\n</$sectiontag>\n" if ($sectiontag ne "");

	$section = $doc_obj->get_next_section($section);
    } #while defined section
    print $handle "$text\n$documentendtag"; 
}

1;