Index: /trunk/gsdl/perllib/DateExtract.pm
===================================================================
--- /trunk/gsdl/perllib/DateExtract.pm	(revision 1953)
+++ /trunk/gsdl/perllib/DateExtract.pm	(revision 1954)
@@ -7,12 +7,12 @@
 #75% of the instances of the word century use full name ordinals
 my %ordinals = ("first" => 1, "second" => 2, "third" => 3, "fourth" => 4,
-	     "fifth" => 5, "sixth" => 6, "seventh" => 7, "eighth" => 8,
-	     "ninth" => 9, "tenth" => 10, "eleventh" => 11, "twelfth" => 12,
-	     "thirteenth" => 13, "fourteenth" => 14, "fifteenth" => 15,
-	     "sixteenth" => 16, "seventeenth" => 17, "eighteenth" => 18,
-	     "nineteenth" => 19, "twentieth" => 20);
-
-
-	     
+             "fifth" => 5, "sixth" => 6, "seventh" => 7, "eighth" => 8,
+             "ninth" => 9, "tenth" => 10, "eleventh" => 11, "twelfth" => 12,
+             "thirteenth" => 13, "fourteenth" => 14, "fifteenth" => 15,
+             "sixteenth" => 16, "seventeenth" => 17, "eighteenth" => 18,
+             "nineteenth" => 19, "twentieth" => 20);
+
+
+             
 
 #definitions for a date grammar.
@@ -20,5 +20,5 @@
 
 my @months = ("january","february","march","april","may","june","july",
-	      "august","september","october","november","december");
+              "august","september","october","november","december");
 
 my $shortmth = "";
@@ -77,8 +77,8 @@
     if($max_century =~ /B/)
     {
-	$max_century = $`;
-	$max_century =~ /\d+/;
-	$max_century = $&;
-	$max_century *=-1
+        $max_century = $`;
+        $max_century =~ /\d+/;
+        $max_century = $&;
+        $max_century *=-1
     }
    
@@ -87,5 +87,5 @@
     $extr = &remove_tags($extr);
     if(!$keep_bib){
-	$extr = &remove_biblio($extr);
+        $extr = &remove_biblio($extr);
     }
   
@@ -94,69 +94,69 @@
     while($extr =~ m!($range)|($millenium)|($qualified)|($centurydate)|($tri_digit)!i)
     { 
-	$extr = $';
-	my $fulldate = $&;
-	if ($fulldate =~ /$centurydate/i)
-	{
-	    if($max_century!=-1)
-	    {
-
-		local $date = $fulldate; if($date =~ /\d+/) {$date = $&;} 
-		else 
-		{
-		    $date=$fulldate; $date =~ m! ($Century)!i; $date = $`;
-		    $date =~ tr/A-Z/a-z/;
-		    $date = $ordinals{$date};
-		}
-		if($max_century >= $date){
-		    $date = ($date-1)*100 +1;
-		    #if it BC, make it negative
-		    $date = &convert_bc($fulldate,$date);
-		    $end = $date + 99; 
-		    @century = ($date..$end);
-		    @datelist = (@datelist,@century);
-		}
-	    }
-	}
-	
-	elsif($fulldate =~ /$range/)
-	{
-	    $fulldate =~ /$sep/;
-	    my @addlist = ();
-	    #print "Range: $fulldate\n";
-	    $fullfirst = $`; 
-	    $fullsecond = $'; 
-	    $fullfirst =~ /\d+/; $first = $&; 
-	    $fullsecond =~ /\d+/; $second = $&;
-	    $len1 = length($first);
-	    $len2 = length($second);
-	    $second = (substr($first,0,($len1-$len2))).$second;
-	    $first = &convert_bc($fullfirst,$first);
-	    $second = &convert_bc($fullsecond,$second);
-	    @addlist = ($first..$second);
-	    @datelist = (@datelist,@addlist);
-	    
-	}
-	else {
-	    
-	    my $date = $fulldate; $date =~ /\d+/; $date = $&;  
-	    $date = &convert_bc($fulldate,$date);
-	    #add the date metadata
-	    push(@datelist,$date);
-	    #print "datelist @datelist\n"
-	}
-	
+        $extr = $';
+        my $fulldate = $&;
+        if ($fulldate =~ /$centurydate/i)
+        {
+            if($max_century!=-1)
+            {
+
+                local $date = $fulldate; if($date =~ /\d+/) {$date = $&;} 
+                else 
+                {
+                    $date=$fulldate; $date =~ m! ($Century)!i; $date = $`;
+                    $date =~ tr/A-Z/a-z/;
+                    $date = $ordinals{$date};
+                }
+                if($max_century >= $date){
+                    $date = ($date-1)*100 +1;
+                    #if it BC, make it negative
+                    $date = &convert_bc($fulldate,$date);
+                    $end = $date + 99; 
+                    @century = ($date..$end);
+                    @datelist = (@datelist,@century);
+                }
+            }
+        }
+        
+        elsif($fulldate =~ /$range/)
+        {
+            $fulldate =~ /$sep/;
+            my @addlist = ();
+            #print "Range: $fulldate\n";
+            $fullfirst = $`; 
+            $fullsecond = $'; 
+            $fullfirst =~ /\d+/; $first = $&; 
+            $fullsecond =~ /\d+/; $second = $&;
+            $len1 = length($first);
+            $len2 = length($second);
+            $second = (substr($first,0,($len1-$len2))).$second;
+            $first = &convert_bc($fullfirst,$first);
+            $second = &convert_bc($fullsecond,$second);
+            @addlist = ($first..$second);
+            @datelist = (@datelist,@addlist);
+            
+        }
+        else {
+            
+            my $date = $fulldate; $date =~ /\d+/; $date = $&;  
+            $date = &convert_bc($fulldate,$date);
+            #add the date metadata
+            push(@datelist,$date);
+            #print "datelist @datelist\n"
+        }
+        
     }
     
     if(@datelist){
-	@datelist = sort { $a <=> $b } @datelist;
-	@datelist = &post_process($max_year, @datelist);
-	foreach $date (@datelist)
-	{
-	    if($date>0){
-		$doc->add_metadata($cursection,"Coverage",$date);}
-	    else{
-		$doc->add_metadata($cursection,"Coverage","bc".(-1*$date));}
-		
-	}
+        @datelist = sort { $a <=> $b } @datelist;
+        @datelist = &post_process($max_year, @datelist);
+        foreach $date (@datelist)
+        {
+            if($date>0){
+                $doc->add_metadata($cursection,"Coverage",$date);}
+            else{
+                $doc->add_metadata($cursection,"Coverage","bc".(-1*$date));}
+                
+        }
     }
 }
@@ -172,8 +172,8 @@
     $prev = 0;
     foreach $e (@list) {
-	if ($e!=$prev && $e <= $max_year) {
-	    push(@cleanlist, $e);
-	}
-	$prev = $e;
+        if ($e!=$prev && $e <= $max_year) {
+            push(@cleanlist, $e);
+        }
+        $prev = $e;
     }
     @cleanlist;
@@ -191,6 +191,6 @@
     while($tmp=~ m!<([^>])*(>|$)! && $tmp ne "")
     {
-	$parsed .= $`;#keep all that is not in a tag
-	$tmp = $';    #restart the search after then end of the tag
+        $parsed .= $`;#keep all that is not in a tag
+        $tmp = $';    #restart the search after then end of the tag
     }
     $parsed .= $tmp; #add anything after the last match
@@ -206,26 +206,26 @@
     if(($tmp =~ m!($spurious)|($lookalikes)!i) == 0 )
     { 
-	$parsed = $tmp;
+        $parsed = $tmp;
     }
     else { 
-	while ($tmp =~ m!($spurious)|($lookalikes)!i 
-	       && $tmp ne "")
-	{
-	    $parsed .= $`;
-	    $storage = $&;
-	    $tmp = $';
-	    #match the pattern which indicates most recent alteration 
-	    if ($storage =~ m!$lastaltered!i)
-	    {
-		#match a four digit year or up until the first / 
-		#(as in last edited 3/97). 
-		$tmp =~ m!($millenium)|(\/)!;
-		$tmp = $';
-	    }
-		
-	}
-	
-	$parsed .= $tmp;
-	   
+        while ($tmp =~ m!($spurious)|($lookalikes)!i 
+               && $tmp ne "")
+        {
+            $parsed .= $`;
+            $storage = $&;
+            $tmp = $';
+            #match the pattern which indicates most recent alteration 
+            if ($storage =~ m!$lastaltered!i)
+            {
+                #match a four digit year or up until the first / 
+                #(as in last edited 3/97). 
+                $tmp =~ m!($millenium)|(\/)!;
+                $tmp = $';
+            }
+                
+        }
+        
+        $parsed .= $tmp;
+           
     }
     #print "Parsed:\n $parsed\n\n";
@@ -240,5 +240,5 @@
     if($tmp =~ m!$bibheader!i)
     {
-	$tmp=$`;
+        $tmp=$`;
     }
     
@@ -246,26 +246,26 @@
     if(($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|($seasonref) ($millenium)!i) == 0)
     {
-	$parsed = $tmp;
+        $parsed = $tmp;
     }
     else{
 
-	#print "removing bib\n";
-	while ($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|(($seasonref) ($millenium))|($bibheader)!i && $tmp ne "")
-	{
-	    
-	    $parsed .= $`;
-	    $tmp = $';
-	    if($&=~m!($comma)|($fullstop)!)
-	    {
-		
-		local  $date = $&;
-		if($parsed =~ m!((\d)($Ord)$)|(($shortmth)$)|(($longmth)$)!i)
-		{
-		    $parsed .= $date;
-		}
-	    }  
-	    
-	}
-		   $parsed .= $tmp;
+        #print "removing bib\n";
+        while ($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|(($seasonref) ($millenium))|($bibheader)!i && $tmp ne "")
+        {
+            
+            $parsed .= $`;
+            $tmp = $';
+            if($&=~m!($comma)|($fullstop)!)
+            {
+                
+                local  $date = $&;
+                if($parsed =~ m!((\d)($Ord)$)|(($shortmth)$)|(($longmth)$)!i)
+                {
+                    $parsed .= $date;
+                }
+            }  
+            
+        }
+                   $parsed .= $tmp;
     }
     $parsed;
@@ -292,2 +292,10 @@
 
 
+
+
+
+
+
+
+
+
Index: /trunk/gsdl/perllib/Kea.pm
===================================================================
--- /trunk/gsdl/perllib/Kea.pm	(revision 1954)
+++ /trunk/gsdl/perllib/Kea.pm	(revision 1954)
@@ -0,0 +1,85 @@
+package Kea;
+
+
+use BasPlug;
+
+#This function is called by BasPlug.pm when a flag in a collection configuration
+#document specifies that keyphrase metadata must be gathered for that collection.
+#It is passed as arguments, the documents text and possibly some options for how
+#the keyphrase data is to be collected if the keyphrase option flag was set in
+#the collection configuration file.  This module then writes the documents text
+#to a file because the stand-alone program Kea which will be called to do the
+#actual extraction of the keyphrases expects a file argument. Once Kea has been
+#called upon, the file containing the keyphrase data gathered by Kea should be
+#stored in gsdl/tmp and this file is read, the data we are interested in is extracted 
+#and passed back to BasPlug.pm in an appropriate format.
+
+sub extract_KeyPhrases {
+
+    my $gsdlhome = $ENV{'GSDLHOME'}; 
+    my $doc = shift(@_); #documents text  
+    my $args = shift(@_); #any options
+    my @optionlist = split(/ +/, $args) if (defined($args)); #list of options
+    my $suffix = 'kea'; #default file will be called .kea
+    my $command = ""; 
+    my @keylist; 
+    my @stemlist;
+
+    print STDERR "optionlist: @optionlist\n";
+       
+    foreach $element (@optionlist){ #for each option
+	my ($option, $file) = split(/,/, $element); #split option letter and file (if file exist)
+
+	$option  = "-".$option; #place dash in front of option
+	$file = "" if(!defined($file)); #no file options specified
+	$suffix = $file if($option eq '-E'); #if option is extension (suffix) option
+	$command .= " $option $file "; #add to list of commands
+    }
+
+    print STDERR "Using output suffix: $suffix\n"; 
+
+    # remove all HTML tags
+    $doc =~ s/<P[^>]*>/\n/sgi;
+    $doc =~ s/<H[^>]*>/\n/sgi;
+    $doc =~ s/<[^>]*>//sgi;
+    $doc =~ tr/\n/\n/s;
+
+    #write text to a file eg doc.txt 
+    open(OUT, ">$gsdlhome/tmp/doc.txt") or die "In Kea.pm doc.txt could not be created\n";  
+    print OUT $doc;
+    close(OUT);
+
+    #call Kea with specifed options
+    `$gsdlhome/perllib/Kea-1.1.4/Kea $command $gsdlhome/tmp/doc.txt`; 
+
+    #read doc.kea with keywords
+    open(IN, "<$gsdlhome/tmp/doc.$suffix") or return @emptykeylist;
+                                          #this means doc.kea does not exist
+                                          #either because an option was wrongly specified
+	                                  #or no keyphrases were found
+    while(<IN>){
+	chomp;
+	@key = split(/\t/); #split into array separated by a tab
+	push(@keylist, $key[0]); #add to list of keywords
+	push(@stemlist, $key[1]); #add to list of stems  
+    }
+    close(IN);
+
+    #put data into appropriate format 
+    $keylist = join(", ", @keylist);
+    $stemlist = join(", ", @stemlist);	
+ 
+    #delete doc.extension so that in future it will not be opened and read
+    `rm $gsdlhome/tmp/doc.$suffix`;
+
+    #return keywords + stems to basplug
+    my @keystemlist = ($keylist, $stemlist);
+    return @keystemlist;
+
+}
+
+
+
+1;
+
+
Index: /trunk/gsdl/perllib/parsargv.pm
===================================================================
--- /trunk/gsdl/perllib/parsargv.pm	(revision 1953)
+++ /trunk/gsdl/perllib/parsargv.pm	(revision 1954)
@@ -73,4 +73,5 @@
 
     my @rest = @_;
+
 
     # if the last argument is the string "allow_extra_options" then options
@@ -92,4 +93,5 @@
     while (($spec, $var) = splice(@rest, 0, 2))
         {
+	  
 	die "Variable for $spec is not a valid type."
 	    unless ref($var) eq 'SCALAR' || ref($var) eq 'ARRAY';
@@ -107,6 +109,7 @@
 	my ($name, $regex, $default) = split(/$delimiter/, $spec, 3);
 	
+	
 	if ($name)
-	{
+	{   
 	    if ($default && $default !~ /$regex/)
 	    {
@@ -225,2 +228,10 @@
 
 1;
+
+
+
+
+
+
+
+
Index: /trunk/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1953)
+++ /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1954)
@@ -25,5 +25,5 @@
 
 package BasPlug;
-
+use Kea;
 use parsargv;
 use multiread;
@@ -130,15 +130,17 @@
     $self->{'outhandle'} = STDERR;
     my $year = (localtime)[5]+1900;
-
+    
+  
     # general options available to all plugins
     if (!parsargv::parse(\@_,
 			 q^process_exp/.*/^, \$self->{'process_exp'},
 			 q^block_exp/.*/^, \$self->{'block_exp'},
+			 q^extract_acronyms^, \$self->{'extract_acronyms'}, 
+			 q^extract_keyphrases^, \$self->{'kea'}, #with extra options 
+			 q^extract_keyphrase_options/.*/^, \$self->{'kea_options'}, #no extra options
 			 qq^input_encoding/$enc/auto^, \$self->{'input_encoding'},
 			 qq^default_encoding/$denc/iso_8859_1^, \$self->{'default_encoding'},
-			 q^extract_acronyms^, \$self->{'extract_acronyms'},
 			 q^extract_email^, \$self->{'extract_email'},
 			 q^markup_acronyms^, \$self->{'markup_acronyms'},
-			 q^extract_language^, \$self->{'extract_language'},
 			 q^default_language/.{2}/en^, \$self->{'default_language'},
 			 q^first/.*/^, \$self->{'first'},
@@ -233,5 +235,6 @@
 
 sub read {
-    my $self = shift (@_);
+    my $self = shift (@_);  
+  
     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
 
@@ -283,5 +286,5 @@
 	return 0;
     }
-
+   
     # include any metadata passed in from previous plugins 
     # note that this metadata is associated with the top level section
@@ -290,8 +293,8 @@
     # do plugin specific processing of doc_obj
     return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
-
+   
     # do any automatic metadata extraction
     $self->auto_extract_metadata ($doc_obj);
-
+   
     # add an OID
     $doc_obj->set_OID();
@@ -501,5 +504,7 @@
 
 # extract metadata
-sub auto_extract_metadata {
+sub auto_extract_metadata { 
+ 
+
     my $self = shift (@_);
     my ($doc_obj) = @_;
@@ -512,5 +517,37 @@
 	    $thissection = $doc_obj->get_next_section ($thissection);
 	}
-    }   
+    } 
+
+
+#adding kea keyphrases
+    if ($self->{'kea'}) {  
+	
+	my $thissection = $doc_obj->get_top_section();
+	my $text = "";
+	my @list;
+
+	while (defined $thissection) { #loop through sections to gather whole doc
+	    my $sectiontext = $doc_obj->get_text($thissection);   
+	    $text = $text.$sectiontext;
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	} 
+       
+	if($self->{'kea_options'}) { #if kea options flag is set, call Kea with specified options 
+	    @list = &Kea::extract_KeyPhrases ($text, $self->{'kea_options'}); 
+	} else { #otherwise call Kea with no options
+	    @list = &Kea::extract_KeyPhrases ($text); 
+	}
+	 
+	if(@list){ #if a list of kea keyphrases was returned (ie not empty)
+	    my $keyphrases = $list[0]; #first arg is keyphrase list
+	    my $stems = $list[1]; #second  arg is stemmed keyphrase list
+	    print STDERR "keyphrases: $keyphrases\n"; 
+	    print STDERR "stems: $stems\n";
+	    $thissection = $doc_obj->get_top_section(); #add metadata to top section
+	    $doc_obj->add_metadata($thissection, "kea", $keyphrases);
+	    $doc_obj->add_metadata($thissection, "stems", $stems); 
+	}
+    } #end of kea 
+
     if ($self->{'first'}) {
 	my $thissection = $doc_obj->get_top_section();
Index: /trunk/gsdl/perllib/plugins/ConvertToPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/ConvertToPlug.pm	(revision 1953)
+++ /trunk/gsdl/perllib/plugins/ConvertToPlug.pm	(revision 1954)
@@ -73,5 +73,9 @@
 
     my $generate_format;
-    if (!parsargv::parse($args, 
+    my $kea_arg;
+
+    if (!parsargv::parse($args,  
+			 q^extract_keyphrases^, \$kea_arg->{'kea'}, #with extra options 
+			 q^extract_keyphrase_options/.*/^, \$kea_arg->{'kea_options'}, #no extra options
 			 q^convert_to/(html|text)/html^, \$generate_format,
 			 "allow_extra_options")) {
@@ -82,11 +86,11 @@
 	die "\n";
     }
-
-    return ($plugin_name,$generate_format);
+   
+    return ($plugin_name,$generate_format, $kea_arg);
 }
 
 sub new {
     my $class = shift (@_);
-    my ($plugin_name,$generate_format) = $class->parse_args(\@_);
+    my ($plugin_name,$generate_format, $kea_arg) = $class->parse_args(\@_);
     my $self;
 
@@ -107,4 +111,8 @@
     }
 
+    #if kea data to be extracted...
+    $self->{'kea'} = 1 if($kea_arg->{'kea'});
+    $self->{'kea_options'} = 1 if($kea_arg->{'kea_options'});
+ 
     return bless $self, $class;
 }
@@ -212,5 +220,5 @@
     
     my $ret_val = BasPlug::read($self,@_);
-
+ 
     $self->cleanup_tmp_area();
     
Index: /trunk/gsdl/perllib/plugins/GMLPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/GMLPlug.pm	(revision 1953)
+++ /trunk/gsdl/perllib/plugins/GMLPlug.pm	(revision 1954)
@@ -41,6 +41,5 @@
     my $self = new BasPlug ("GMLPlug", @_);
 
-    return bless $self, $class;
-}
+    return bless $self, $class;}
 
 sub get_default_process_exp {
Index: /trunk/gsdl/perllib/plugins/PDFPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/PDFPlug.pm	(revision 1953)
+++ /trunk/gsdl/perllib/plugins/PDFPlug.pm	(revision 1954)
@@ -37,5 +37,5 @@
 sub new {
     my $class = shift (@_);
-
+     
     my $self = new ConvertToPlug ($class, @_, "--", "-title_sub", 'Page\s+\d+');
     
@@ -70,2 +70,5 @@
 1;
 
+
+
+
