Index: /trunk/gsdl/perllib/acronym.pm
===================================================================
--- /trunk/gsdl/perllib/acronym.pm	(revision 1392)
+++ /trunk/gsdl/perllib/acronym.pm	(revision 1393)
@@ -47,10 +47,9 @@
 my $min_acro_length = 3;
 #minimum acronym length saving
-my $min_length_saving = 3;
+my $min_length_saving = 4;
 #allow recusive acronyms
 my $allow_recursive = "";
 
-my @stop_words = split / /, "A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND";
-#my @stop_words = split / /, "OF AT THE IN TO AND";
+my @stop_words = split / /, "OF AT THE IN TO AND";
 
 #the text split into an array, one word per element
@@ -58,4 +57,186 @@
 my @acronym_list = ();
 
+#the file to collate acronyms into
+my $acronym_accumulate_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_definitions.pm";
+my $acronym_options_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_options.pm";
+
+my %acronyms_found_in_collection = ();
+my %acronyms_banned_from_collection = ();
+
+my $writing_acronyms = 1;
+my $accumulate_acronyms = 1;
+my $markup_accumulate_acronyms = 1;
+my $markup_local_acronyms = 1;
+
+
+
+###########################################################################
+#   file saving / loading stuff
+###########################################################################
+
+sub init_acronyms {
+    
+    my $file_text = "";
+    if (open ACRONYM_HANDLE, "<$acronym_options_file")
+    {
+	$file_text = do { local $/; <ACRONYM_HANDLE> };  
+    }
+    if ($file_text eq "")
+    {
+	print STDERR "failed to open $acronym_options_file\n";
+	open ACRONYM_HANDLE, ">$acronym_options_file\n";
+	print ACRONYM_HANDLE "#Config file for acronym extraction. EDIT THIS FILE, it should\n";
+	print ACRONYM_HANDLE "#not be overridden by the software. It's read by GSDL using perl's\n";
+	print ACRONYM_HANDLE "#'eval' function, so pretty much anything that's valid in perl is \n";
+	print ACRONYM_HANDLE "#valid here.\n\n";
+	print ACRONYM_HANDLE "#Quite a few things here are defined in terms of recall and precision\n";
+	print ACRONYM_HANDLE "#which are the key measures from Information Retreval (IR). If you\n";
+	print ACRONYM_HANDLE "#don't understand recall and precision, any good IR textbook should\n";
+	print ACRONYM_HANDLE "#explain them fully \n\n";
+	print ACRONYM_HANDLE "#the maximum range to look for acronyms (raise to raise precision)\n"; 
+	print ACRONYM_HANDLE "\$max_offset = 30;\n\n"; 
+	print ACRONYM_HANDLE "#acronyms must be upper case (0 = false, 1 = true)\n";
+	print ACRONYM_HANDLE "\$upper_case = 1;\n\n";
+	print ACRONYM_HANDLE "#acronym case must match (0 = false, 1 = true)\n";
+	print ACRONYM_HANDLE "\$case_match = 1;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$min_def_length = 3;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$min_acro_length = 3;\n\n";
+	print ACRONYM_HANDLE "#minimum acronym length saving (raise to raise precision)\n";
+	print ACRONYM_HANDLE "\$min_length_saving = 4;\n\n";
+	print ACRONYM_HANDLE "#allow recusive acronyms (0 = false, 1 = true)\n";
+	print ACRONYM_HANDLE "\$allow_recursive = 0;\n\n";
+	print ACRONYM_HANDLE "#stop words-words allowed in acronyms (the multi-lingual version\n";
+	print ACRONYM_HANDLE "#slows down acronym extraction slightly so is not the default)\n";
+	print ACRONYM_HANDLE "#\@stop_words = split / /, \"A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND DE DU A LA LE LES L DANS ET S\";\n";
+	print ACRONYM_HANDLE "\@stop_words = split / /, \"OF AT THE IN TO AND\";\n"; 
+	print ACRONYM_HANDLE "\n"; 
+	print ACRONYM_HANDLE "#the file to collate acronyms into\n";
+	print ACRONYM_HANDLE "\$acronym_accumulate_file = \"$ENV{'GSDLCOLLECTDIR'}\" . \"/etc/acronym_definitions.pm\";\n";
+	print ACRONYM_HANDLE "\n";
+	print ACRONYM_HANDLE "\$accumulate_acronyms = 1;\n\n";
+	print ACRONYM_HANDLE "# any acronym definitions which should always be marked up can be copied here\n";
+	print ACRONYM_HANDLE "# from the acronym_accumulate_file file ...\n";
+	print ACRONYM_HANDLE "# \n";
+	print ACRONYM_HANDLE "# \n";
+	print ACRONYM_HANDLE "# \n";
+    }
+    eval $file_text;
+    #promotes warnings/errors from evaluated file to current context 
+    warn $@ if $@;
+
+
+
+    &read_all_acronyms_from_file();
+#    rename $acronym_file, $acronym_file . "." . int(rand (2<<7)). 
+#	int(rand (2<<7)). int(rand (2<<7)). int(rand (2<<7));
+    if ($writing_acronyms && open ACRONYM_HANDLE, ">$acronym_accumulate_file")
+    {
+	print ACRONYM_HANDLE "#This is an automatically generated file.\n";
+	print ACRONYM_HANDLE "#\n";
+	print ACRONYM_HANDLE "#If you edit this file and it will be overwritten the next\n";
+	print ACRONYM_HANDLE "#time the acronym code runs unless you set the file to \n";
+	print ACRONYM_HANDLE "#read-only. \n";
+	print ACRONYM_HANDLE "#\n";
+	print ACRONYM_HANDLE "#start of acronyms...\n";
+	$writing_acronyms = 1;
+    } else {
+	warn "failed to open $acronym_accumulate_file for writing\n";
+	$writing_acronyms = 0;
+    }
+}
+
+#close the list of accumulated acronyms
+END {
+    if ($writing_acronyms)
+    {
+	print ACRONYM_HANDLE "#end of acronyms.\n";  
+	close ACRONYM_HANDLE;
+    }
+}
+
+#eval a file of accumulated acronyms
+sub read_all_acronyms_from_file {
+    
+    my $file_text = "";
+    if (open ACRONYM_HANDLE, "<$acronym_accumulate_file")
+    {
+	$file_text = do { local $/; <ACRONYM_HANDLE> };  
+    } else {
+	print STDERR "failed to open $acronym_accumulate_file for reading (this is the first pass?).\n";
+    }
+    eval $file_text;
+    #promotes warnings/errors from evaluated file to current context 
+    warn $@ if $@;
+}
+
+#called from within the file of accumulated acronyms to indicate a good acronym
+sub add {
+    my $self = shift (@_);
+    if (defined ($acronyms_found_in_collection{$self->[0]}))
+    {
+	my $def = $self->to_def_string();
+	if ($acronyms_found_in_collection{$self->[0]} =~ m/(^|\|)$def(\||$)/)
+	{
+	    return;
+	}
+	$acronyms_found_in_collection{$self->[0]} = 
+	    $acronyms_found_in_collection{$self->[0]} . "|" . $self->to_def_string();
+    } else {
+	$acronyms_found_in_collection{$self->[0]} = $self->to_def_string();
+    }
+}
+
+#called from within the file of accumulated acronyms to indicate a bad acronym
+sub ban {
+    my $self = shift (@_);
+    
+    if (!defined $acronyms_banned_from_collection{$self->[0]})
+    {
+	$acronyms_banned_from_collection{$self->[0]} = $self->to_def_string();
+    } else {
+	$acronyms_banned_from_collection{$self->[0]} = $acronyms_banned_from_collection{$self->[0]} . "|" . $self->to_def_string();
+    }
+}
+
+
+#write a good acronym to the accumulated acronyms file
+sub write_to_file {
+    my $self = shift (@_);
+    if ($writing_acronyms)
+    {
+	print ACRONYM_HANDLE "new acronym(\"$self->[0]\",\"" . 
+	    $self->to_def_string() . 
+		"\")->add();\n";
+    }
+}
+
+
+###########################################################################
+# mark functionality    
+###########################################################################
+
+#small routine to sort by length 
+sub sort_by_length {
+    length($b) <=> length($a) or $a cmp $b
+}
+
+sub markup_acronyms {
+    my  $text = shift (@_);
+    my  $verbosity_obj = shift (@_);
+    if (defined $text)
+    {
+	for my $acro (sort sort_by_length keys %acronyms_found_in_collection)
+	{
+	    $text  =~ s/^((?:[^\<\n]|(?:\<[^\>\n]*\>))*)$acro([^\<A-Z])/$1$acro\<img src=\"\" width=8 height=8 alt=\"$acronyms_found_in_collection{$acro}\"\>$2/gm;
+	    printf STDERR " " .  $acro . ","
+		if ($verbosity_obj->{'verbosity'} >= 2);
+	}
+    }
+    return $text;
+}
+
+
 
 ###########################################################################
@@ -65,12 +246,18 @@
 
 sub new {
+    my $trash = shift (@_); 
+    my $acro = shift (@_); 
+    my $def  = shift (@_); 
+    
     my $self = [
-	"", # 0 acronym
-	[], # 1 definition
-    ];
+		"", # 0 acronym
+		[], # 1 definition
+               ];
+    
+    $self->[0] = $acro                    if defined $acro;
+    push @{$self->[1]},  split / /, $def  if defined $def;
+    
     bless $self;
 }
-
-
 
 sub clone  {
@@ -226,6 +413,6 @@
 #	print "acronym " . $self->to_string() . " rejected (too short III)\n";
 #	print "" . $min_length_saving .
-	    "|" . $self->letters_in_acronym() .
-	    "|" . $self->letters_in_acronym_definition() . "\n";
+#	    "|" . $self->letters_in_acronym() .
+#	    "|" . $self->letters_in_acronym_definition() . "\n";
 	return 0;
     }
@@ -446,4 +633,6 @@
 #&test();
 
+&init_acronyms();
+
 1;
 
Index: /trunk/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1392)
+++ /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1393)
@@ -67,4 +67,5 @@
     print STDERR "                     file extensions.\n";
     print STDERR "   -extract_acronyms Extract acronyms from within text and set as metadata\n\n";
+    print STDERR "   -markup_acronyms  Added acronym metadata into document text\n\n";
     print STDERR "   -extract_langauge Identify the language of the text and set as metadata\n\n";
 }
@@ -90,4 +91,5 @@
 			 q^block_exp/.*/^, \$self->{'block_exp'},
 			 q^extract_acronyms^, \$self->{'extract_acronyms'},
+			 q^markup_acronyms^, \$self->{'markup_acronyms'},
 			 q^extract_language^, \$self->{'extract_language'},
 			 "allow_extra_options")) {
@@ -297,4 +299,15 @@
     }
 
+    if ($self->{'markup_acronyms'}) {
+	my $thissection = $doc_obj->get_top_section();
+	while (defined $thissection) {
+	    my $text = $doc_obj->get_text($thissection);
+	    $text = $self->markup_acronyms ($text, $doc_obj, $thissection);
+	    $doc_obj->delete_text($thissection);
+	    $doc_obj->add_text($thissection, $text);
+	    $thissection = $doc_obj->get_next_section ($thissection);
+	}
+    }
+
     if ($self->{'extract_language'}) {
 	my $thissection = $doc_obj->get_top_section();
@@ -341,5 +354,5 @@
     my ($textref, $doc_obj, $thissection) = @_;
 
-    print STDERR " checking for acronyms ...\n" 
+    print STDERR " extracting acronyms ...\n" 
 	if ($self->{'verbosity'} >= 2);
 
@@ -362,4 +375,7 @@
 	if ($seen_before eq "false")
 	{
+	    #write it to the file ...
+	    $acro->write_to_file();
+
 	    #do the normal acronym
 	    $doc_obj->add_utf8_metadata($thissection, "Acronym",  $acro->to_string());
@@ -367,16 +383,32 @@
 		    if ($self->{'verbosity'} >= 1);
 	    
-	    # do the KWIC (Key Word In Context) acronym
-	    my @kwic = $acro->to_string_kwic();
-	    foreach my $kwic (@kwic) {
-		$doc_obj->add_utf8_metadata($thissection, "AcronymKWIC",  $kwic);
-		print STDERR "   adding ".  $kwic . "\n" 
-		    if ($self->{'verbosity'} >= 2);
-	    }
-	}
-    }
-    print STDERR " done with acronyms. \n" 
+#	    # do the KWIC (Key Word In Context) acronym
+#	    my @kwic = $acro->to_string_kwic();
+#	    foreach my $kwic (@kwic) {
+#		$doc_obj->add_utf8_metadata($thissection, "AcronymKWIC",  $kwic);
+#		print STDERR "   adding ".  $kwic . "\n" 
+#		    if ($self->{'verbosity'} >= 2);
+#	    }
+	}
+    }
+    print STDERR " done extracting acronyms. \n" 
 	if ($self->{'verbosity'} >= 2);
 }
 
+sub markup_acronyms {
+    my $self = shift (@_);
+    my ($text, $doc_obj, $thissection) = @_;
+
+    print STDERR " marking up acronyms ...\n" 
+	if ($self->{'verbosity'} >= 2);
+
+    #self is passed in to check for verbosity ...
+    $text = &acronym::markup_acronyms($text, $self);
+
+    print STDERR " done marking up acronyms. \n" 
+	if ($self->{'verbosity'} >= 2);
+
+    return $text;
+}
+
 1;
