source: trunk/gsdl/perllib/Kea-1.1.4/Kea@ 9156

#!/usr/bin/perl -w

#    Kea
#    Version 1.1.4

#    Kea -- Automatic Keyphrase Extraction
#    Copyright 1998-1999 by Gordon Paynter and Eibe Frank
#    Contact [email protected] or [email protected]
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# Version history
#
# 1.0   Witten et.al.
# 1.0.1 Bug: stopword file loaded as model file
# 1.0.2 Java paths explicit for nikau; JIT compiler
# 1.0.3 Include tf.idf in output if -t is set
# 1.0.4 Allow optional keyphrase frequency file
# 1.0.5 Use $perl_command and $java_command for system-indepence
# 1.0.6 -C argument selects model, stopword file, df file 
# 1.0.7 Changes to Kea.pl.
#       This is Phillip's version
# 1.0.8 Accepts .htm as well as .html
# 1.0.9 Accepts .text as well as .txt
# 1.1   First Distribution.  GPL added.  Documentation added.
# 1.1.1 -E argument sets output extension (default is .kea)
# 1.1.2 Documented java variables
#       Maximum phrase length can be set at command-line.
#       Note: default=3; NOT the length for the model. Sorry.
# 1.1.3 Moved Lynx command into separate script that checks
#       for circumstances that are likely to crash it.
# 1.1.4 Updated documentation and added a few extra files.

print STDERR "Kea (version 1.1.4): automatic keyphrase extraction\n";

$gsdlhome = $ENV{'GSDLHOME'};

if (! -x "$gsdlhome/perllib/Kea-1.1.4/stemmer") {
    print STDERR "** need to compile stemmer **\n";
    system("gcc", ("-o", "$gsdlhome/perllib/Kea-1.1.4/stemmer",
           "$gsdlhome/perllib/Kea-1.1.4/Iterated-Lovins-stemmer/stem-Lovins-iterated.c",
           "$gsdlhome/perllib/Kea-1.1.4/Iterated-Lovins-stemmer/stem.c"
           ));
}

# Kea runs other perl scripts in shells.
$perl_command = "perl -w";

# Java is a bit more difficult.
# set this variable to your java home directory
#$java_home = "/usr/local/jdk";
$java_home = "/usr/local/share/java";

# this variable will hold the CLASSPATH for java which we set at 
# the command line to incorporate jaws.jar; you shouldn't need to change it
$java_classpath = ".:$gsdlhome/perllib/Kea-1.1.4/jaws.jar:$gsdlhome/perllib/Kea-1.1.4:$java_home/lib/classes.zip";

# The name of your java just-in-time compiler.  I use TYA.
# An empty string means you don;t have a JIT compiler.
$java_JIT_compiler = "";
#$java_JIT_compiler = "tya";

# If you want to give java lots of memory or use other arguments, 
# use this variable, otherwise make it an empty string.
$java_extra_args = "";
$java_extra_args = "-ss100000000 -oss100000000 -mx200000000";

# see if java executable is on path
my $java_exec="";
if (system("which java >/dev/null 2>/dev/null")==0) {
    $java_exec=`which java`;
    chomp $java_exec;
} else {
    $java_exec="$java_home/bin/java";
}

# The actual java command is based on these other variables:
$java_command = "$java_exec -classpath \"$java_classpath\"";
$java_command .= " -Djava.compiler=$java_JIT_compiler" if ($java_JIT_compiler);
$java_command .= " $java_extra_args" if ($java_extra_args);


# Parse command line options
use Getopt::Std;
local $opt_d=0; # debug option
getopts("dtN:E:C:F:K:L:M:S:");

# What files shall we use?
if (!$ARGV[0]) { 
    die "Usage: Kea [options] text-or-html-or-cstr-files
Options:
 -d           Debug mode
 -t           Ouput TF.IDF
 -N n         Output n keyphrases
 -L n         Maximum phrases length is n (default = 3)
 -E <suffix>  Output extension is <suffix>
 -C <corpus>  Use model/df/kf/stopwords based on <corpus>  
 -F <document-frequency file>
 -K <keyphrase-frequency file>
 -M <Naive-Bayes model file>
 -S <stopword file>
See README for more detail.
";
}


# Number of phrases to extract
if (defined($opt_N) && ($opt_N =~ /^\d+$/) && ($opt_N > 0)) {
    $number_of_phrases = "-N $opt_N";
    print STDERR "Number of phrases to extract: $opt_N\n";
} elsif (defined($opt_N)) {
    die "Kea cannot understand -N argument (must be a number): $opt_N\n";
} else {
    if ($opt_d) {
    print STDERR "Number of phrases dictated by model (default)\n";
    }
    $number_of_phrases = "";
}

# -L maximum phrase length
$maximum_phrase_length = 0;
if (defined($opt_L) && ($opt_L =~ /^\d+$/) && ($opt_L > 0)) {
    $maximum_phrase_length = "$opt_L";
    print STDERR "Maximum phrase length: $opt_L\n";
} elsif (defined($opt_L)) {
    die "Kea cannot understand -L argument (must be a number): $opt_L\n";
}

# -E What extension shall we use?
$extension = "kea";
if (defined($opt_E)) {
    $extension = $opt_E;
    $extension =~ s/^\.//g;
    print STDERR "Using output extension: $extension\n";
}

# -t Should we output tfidf?  (This option is used by Kniles.)
if (defined($opt_t) && $opt_t) {
    $output_tfidf = 1;
    print STDERR "Do print tf.idf\n";
} else {
    $output_tfidf = 0;
}


# -C Corpus file stem 
$default_stem = "$gsdlhome/perllib/Kea-1.1.4/aliweb";
if (defined($opt_C)) {
    print STDERR "Corpus: $opt_C (setting default model/stopwords/df)\n";
    $default_stem = "$gsdlhome/perllib/Kea-1.1.4/$opt_C";
}

# set the default model etc
$model_file = "$default_stem.model";
$stopword_file = "$default_stem.stopwords";
$frequency_file = "$default_stem.df";
$keyword_frequency_file = "";
$keyword_frequency_file = "$default_stem.kf" if (-e "$default_stem.kf");


# -F Document Frequency file
if (defined($opt_F)) {$frequency_file = "$gsdlhome/perllib/Kea-1.1.4/$opt_F";}
if ($opt_d) {print STDERR "Document frequency file: $frequency_file\n";}
die "Document frequency file does not exist!\n" if (!(-e $frequency_file));

# -M Model file
if (defined($opt_M)) {$model_file =  "$gsdlhome/perllib/Kea-1.1.4/$opt_M";}
if ($opt_d) {print STDERR "Model file: $model_file\n";}
die "Model file does not exist!\n" if (!(-e $model_file));

# -S Stopword file
if (defined($opt_S)) {$stopword_file =  "$gsdlhome/perllib/Kea-1.1.4/$opt_S";}
if ($opt_d) {print STDERR "Stopword file: $stopword_file\n";}
die "Stopword file does not exist!\n" if (!(-e $stopword_file));

# -K Keyword frequency file
if (defined($opt_K)) {
    $keyword_frequency_file =  "$gsdlhome/perllib/Kea-1.1.4/$opt_K";
}
if ($keyword_frequency_file) {
    print STDERR "Keyword frequency file: $keyword_frequency_file\n";
    die "Keyword frequency file does not exist!\n" if (! -e $keyword_frequency_file);
} else {
    if ($opt_d) {print STDERR "No keyword frequency file (default)\n";}
}

# Count the number of files
$number_of_files = $#ARGV + 1;
if ($opt_d) {print STDERR "Number of files: $number_of_files\n\n";}



# Set up working files

$stem = "$gsdlhome/tmp/kea.$$";

$data = "$stem.data";
$arff = "$stem.arff";
$out = "$stem.out";
$err = "$stem.err";

`mkdir -m 777 $data`;


# Process each input file into a working file
if ($opt_d) {print STDERR "Preparing input files in: $data\n";}

for ($f = 0; $f <= $#ARGV; $f++) {
    print STDERR "  document ", ($f+1), ": $ARGV[$f]\n";

    $file = $ARGV[$f];
    $temp = "$data/$f";
    $temp =~ s@//@/@g;
    $original_filename{$temp} = $file;

    # copy the file to the data directory & coerce into a clause file
    if ($file =~ /.*\.clauses/i) {
    system("cp","$file","$temp.clauses");
    } else {
    if ($file =~ /.*\.te?xt/i) {
        system("cp","$file","$temp.txt");
    } elsif ($file =~ /.*\.html?/i) {
        system("cp", "$file", "$temp.html");
        `$perl_command $gsdlhome/perllib/Kea-1.1.4/convert-html-to-text.pl $temp.html > $temp.txt`;
    } elsif ($file =~ /.*\.cstr/) {
        `cp $file $temp.cstr`;
        `$perl_command $gsdlhome/perllib/Kea-1.1.4/cstr-to-text.pl $temp.cstr $temp.txt`;
    } else {
        die "Unknown file type: $file\n";
    }
    # prepare the file
    `$perl_command $gsdlhome/perllib/Kea-1.1.4/prepare-clauses.pl $temp.txt $temp.clauses`;
    }
}
#print STDERR "\n\n";

# Build the arff file
$command = "$perl_command $gsdlhome/perllib/Kea-1.1.4/k4.pl -S $stopword_file -f $frequency_file";
$command .= " -K $keyword_frequency_file" if ($keyword_frequency_file);
$command .= " -L $maximum_phrase_length" if ($maximum_phrase_length);
$command .= " $data $arff";

if ($opt_d) {
    print STDERR "** $command **\n";
}
system ("$command");


# use KEP.java to extract phrases
$command = "$java_command KEP -m $model_file -T $arff";
$command .= " $number_of_phrases" if ($number_of_phrases);
# -R is correctly set by default in the java file when the model is loaded
# $command .= " -R tfidf,first_occurrence,keyword_freq,class" if ($keyword_frequency_file);
$command .= " > $out 2> $err";

if ($opt_d) {
    print STDERR "** $command  **\n";
}
system ("$command");



# Read output file and create .kea files

open(KEP, "<$out");

$document = "";

while (<KEP>) {

    chomp;
    $line = $_;
    
    # new document
    if ($line =~ "^Current document") {
    # close the old document
    if ($document) {
        close(DOC);
    }
    # open start the new document
    ($doc) = $line =~ /Current document: (.*)\.clauses$/;
    $doc =~ s@//@/@g;
    $document = $original_filename{$doc};
    $document =~ s/\.[^\.]+$//;
    $document .= ".$extension";
    #print STDERR "Opening DOC: $original_filename{$doc} => $document\n";
    open(DOC, ">$document");

    } elsif ($line =~ "^Miss:") {
    die "Trying to write with no current document!" if (!$document);
    ($phrase, $tfidf, $evidence) = $line =~ /Miss: ([^,]+).*,([^,]+),no (.+)$/;
    ($stemmed, $unstemmed) = $phrase =~ /\'(.+) \((.+)\)\'/;
    if ($output_tfidf) {
        print DOC "$unstemmed\t$stemmed\t$evidence\t$tfidf\n";       
    } else {
        print DOC "$unstemmed\t$stemmed\t$evidence\n";       
    }
    }
}


#get rid of temporary files
if (!$opt_d) {
    system ("rm", ("-r", "$data", "$arff", "$out", "$err"));
}