Changeset 1393

Timestamp:

2000-08-11T11:12:15+12:00 (24 years ago)

Author:

say1

Message:

acronym markup functionality

Location:

trunk/gsdl/perllib

Files:

• acronym.pm (modified) (5 diffs)
• plugins/BasPlug.pm (modified) (6 diffs)

trunk/gsdl/perllib/acronym.pm

@@ -47 +47 @@
 my $min_acro_length = 3;
 #minimum acronym length saving
-my $min_length_saving = 3;
+my $min_length_saving = 4;
 #allow recusive acronyms
 my $allow_recursive = "";
 
-my @stop_words = split / /, "A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND";
-#my @stop_words = split / /, "OF AT THE IN TO AND";
+my @stop_words = split / /, "OF AT THE IN TO AND";
 
 #the text split into an array, one word per element
@@ -58 +57 @@
 my @acronym_list = ();
 
+#the file to collate acronyms into
+my $acronym_accumulate_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_definitions.pm";
+my $acronym_options_file = $ENV{'GSDLCOLLECTDIR'} . "/etc/acronym_options.pm";
+
+my %acronyms_found_in_collection = ();
+my %acronyms_banned_from_collection = ();
+
+my $writing_acronyms = 1;
+my $accumulate_acronyms = 1;
+my $markup_accumulate_acronyms = 1;
+my $markup_local_acronyms = 1;
+
+
+
+###########################################################################
+#   file saving / loading stuff
+###########################################################################
+
+sub init_acronyms {
+    
+    my $file_text = "";
+    if (open ACRONYM_HANDLE, "<$acronym_options_file")
+    {
+    $file_text = do { local $/; <ACRONYM_HANDLE> };  
+    }
+    if ($file_text eq "")
+    {
+    print STDERR "failed to open $acronym_options_file\n";
+    open ACRONYM_HANDLE, ">$acronym_options_file\n";
+    print ACRONYM_HANDLE "#Config file for acronym extraction. EDIT THIS FILE, it should\n";
+    print ACRONYM_HANDLE "#not be overridden by the software. It's read by GSDL using perl's\n";
+    print ACRONYM_HANDLE "#'eval' function, so pretty much anything that's valid in perl is \n";
+    print ACRONYM_HANDLE "#valid here.\n\n";
+    print ACRONYM_HANDLE "#Quite a few things here are defined in terms of recall and precision\n";
+    print ACRONYM_HANDLE "#which are the key measures from Information Retreval (IR). If you\n";
+    print ACRONYM_HANDLE "#don't understand recall and precision, any good IR textbook should\n";
+    print ACRONYM_HANDLE "#explain them fully \n\n";
+    print ACRONYM_HANDLE "#the maximum range to look for acronyms (raise to raise precision)\n"; 
+    print ACRONYM_HANDLE "\$max_offset = 30;\n\n"; 
+    print ACRONYM_HANDLE "#acronyms must be upper case (0 = false, 1 = true)\n";
+    print ACRONYM_HANDLE "\$upper_case = 1;\n\n";
+    print ACRONYM_HANDLE "#acronym case must match (0 = false, 1 = true)\n";
+    print ACRONYM_HANDLE "\$case_match = 1;\n\n";
+    print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+    print ACRONYM_HANDLE "\$min_def_length = 3;\n\n";
+    print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
+    print ACRONYM_HANDLE "\$min_acro_length = 3;\n\n";
+    print ACRONYM_HANDLE "#minimum acronym length saving (raise to raise precision)\n";
+    print ACRONYM_HANDLE "\$min_length_saving = 4;\n\n";
+    print ACRONYM_HANDLE "#allow recusive acronyms (0 = false, 1 = true)\n";
+    print ACRONYM_HANDLE "\$allow_recursive = 0;\n\n";
+    print ACRONYM_HANDLE "#stop words-words allowed in acronyms (the multi-lingual version\n";
+    print ACRONYM_HANDLE "#slows down acronym extraction slightly so is not the default)\n";
+    print ACRONYM_HANDLE "#\@stop_words = split / /, \"A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND DE DU A LA LE LES L DANS ET S\";\n";
+    print ACRONYM_HANDLE "\@stop_words = split / /, \"OF AT THE IN TO AND\";\n"; 
+    print ACRONYM_HANDLE "\n"; 
+    print ACRONYM_HANDLE "#the file to collate acronyms into\n";
+    print ACRONYM_HANDLE "\$acronym_accumulate_file = \"$ENV{'GSDLCOLLECTDIR'}\" . \"/etc/acronym_definitions.pm\";\n";
+    print ACRONYM_HANDLE "\n";
+    print ACRONYM_HANDLE "\$accumulate_acronyms = 1;\n\n";
+    print ACRONYM_HANDLE "# any acronym definitions which should always be marked up can be copied here\n";
+    print ACRONYM_HANDLE "# from the acronym_accumulate_file file ...\n";
+    print ACRONYM_HANDLE "# \n";
+    print ACRONYM_HANDLE "# \n";
+    print ACRONYM_HANDLE "# \n";
+    }
+    eval $file_text;
+    #promotes warnings/errors from evaluated file to current context 
+    warn $@ if $@;
+
+
+
+    &read_all_acronyms_from_file();
+#    rename $acronym_file, $acronym_file . "." . int(rand (2<<7)). 
+#   int(rand (2<<7)). int(rand (2<<7)). int(rand (2<<7));
+    if ($writing_acronyms && open ACRONYM_HANDLE, ">$acronym_accumulate_file")
+    {
+    print ACRONYM_HANDLE "#This is an automatically generated file.\n";
+    print ACRONYM_HANDLE "#\n";
+    print ACRONYM_HANDLE "#If you edit this file and it will be overwritten the next\n";
+    print ACRONYM_HANDLE "#time the acronym code runs unless you set the file to \n";
+    print ACRONYM_HANDLE "#read-only. \n";
+    print ACRONYM_HANDLE "#\n";
+    print ACRONYM_HANDLE "#start of acronyms...\n";
+    $writing_acronyms = 1;
+    } else {
+    warn "failed to open $acronym_accumulate_file for writing\n";
+    $writing_acronyms = 0;
+    }
+}
+
+#close the list of accumulated acronyms
+END {
+    if ($writing_acronyms)
+    {
+    print ACRONYM_HANDLE "#end of acronyms.\n";  
+    close ACRONYM_HANDLE;
+    }
+}
+
+#eval a file of accumulated acronyms
+sub read_all_acronyms_from_file {
+    
+    my $file_text = "";
+    if (open ACRONYM_HANDLE, "<$acronym_accumulate_file")
+    {
+    $file_text = do { local $/; <ACRONYM_HANDLE> };  
+    } else {
+    print STDERR "failed to open $acronym_accumulate_file for reading (this is the first pass?).\n";
+    }
+    eval $file_text;
+    #promotes warnings/errors from evaluated file to current context 
+    warn $@ if $@;
+}
+
+#called from within the file of accumulated acronyms to indicate a good acronym
+sub add {
+    my $self = shift (@_);
+    if (defined ($acronyms_found_in_collection{$self->[0]}))
+    {
+    my $def = $self->to_def_string();
+    if ($acronyms_found_in_collection{$self->[0]} =~ m/(^|\|)$def(\||$)/)
+    {
+        return;
+    }
+    $acronyms_found_in_collection{$self->[0]} = 
+        $acronyms_found_in_collection{$self->[0]} . "|" . $self->to_def_string();
+    } else {
+    $acronyms_found_in_collection{$self->[0]} = $self->to_def_string();
+    }
+}
+
+#called from within the file of accumulated acronyms to indicate a bad acronym
+sub ban {
+    my $self = shift (@_);
+    
+    if (!defined $acronyms_banned_from_collection{$self->[0]})
+    {
+    $acronyms_banned_from_collection{$self->[0]} = $self->to_def_string();
+    } else {
+    $acronyms_banned_from_collection{$self->[0]} = $acronyms_banned_from_collection{$self->[0]} . "|" . $self->to_def_string();
+    }
+}
+
+
+#write a good acronym to the accumulated acronyms file
+sub write_to_file {
+    my $self = shift (@_);
+    if ($writing_acronyms)
+    {
+    print ACRONYM_HANDLE "new acronym(\"$self->[0]\",\"" . 
+        $self->to_def_string() . 
+        "\")->add();\n";
+    }
+}
+
+
+###########################################################################
+# mark functionality    
+###########################################################################
+
+#small routine to sort by length 
+sub sort_by_length {
+    length($b) <=> length($a) or $a cmp $b
+}
+
+sub markup_acronyms {
+    my  $text = shift (@_);
+    my  $verbosity_obj = shift (@_);
+    if (defined $text)
+    {
+    for my $acro (sort sort_by_length keys %acronyms_found_in_collection)
+    {
+        $text  =~ s/^((?:[^\<\n]|(?:\<[^\>\n]*\>))*)$acro([^\<A-Z])/$1$acro\<img src=\"\" width=8 height=8 alt=\"$acronyms_found_in_collection{$acro}\"\>$2/gm;
+        printf STDERR " " .  $acro . ","
+        if ($verbosity_obj->{'verbosity'} >= 2);
+    }
+    }
+    return $text;
+}
+
+
 
 ###########################################################################
@@ -65 +246 @@
 
 sub new {
+    my $trash = shift (@_); 
+    my $acro = shift (@_); 
+    my $def  = shift (@_); 
+    
     my $self = [
-    "", # 0 acronym
-    [], # 1 definition
-    ];
+        "", # 0 acronym
+        [], # 1 definition
+               ];
+    
+    $self->[0] = $acro                    if defined $acro;
+    push @{$self->[1]},  split / /, $def  if defined $def;
+    
     bless $self;
 }
-
-
 
 sub clone  {
@@ -226 +413 @@
 #   print "acronym " . $self->to_string() . " rejected (too short III)\n";
 #   print "" . $min_length_saving .
-        "|" . $self->letters_in_acronym() .
-        "|" . $self->letters_in_acronym_definition() . "\n";
+#       "|" . $self->letters_in_acronym() .
+#       "|" . $self->letters_in_acronym_definition() . "\n";
     return 0;
     }
@@ -446 +633 @@
 #&test();
 
+&init_acronyms();
+
 1;
 

trunk/gsdl/perllib/plugins/BasPlug.pm

@@ -67 +67 @@
     print STDERR "                     file extensions.\n";
     print STDERR "   -extract_acronyms Extract acronyms from within text and set as metadata\n\n";
+    print STDERR "   -markup_acronyms  Added acronym metadata into document text\n\n";
     print STDERR "   -extract_langauge Identify the language of the text and set as metadata\n\n";
 }
@@ -90 +91 @@
              q^block_exp/.*/^, \$self->{'block_exp'},
              q^extract_acronyms^, \$self->{'extract_acronyms'},
+             q^markup_acronyms^, \$self->{'markup_acronyms'},
              q^extract_language^, \$self->{'extract_language'},
              "allow_extra_options")) {
@@ -297 +299 @@
     }
 
+    if ($self->{'markup_acronyms'}) {
+    my $thissection = $doc_obj->get_top_section();
+    while (defined $thissection) {
+        my $text = $doc_obj->get_text($thissection);
+        $text = $self->markup_acronyms ($text, $doc_obj, $thissection);
+        $doc_obj->delete_text($thissection);
+        $doc_obj->add_text($thissection, $text);
+        $thissection = $doc_obj->get_next_section ($thissection);
+    }
+    }
+
     if ($self->{'extract_language'}) {
     my $thissection = $doc_obj->get_top_section();
@@ -341 +354 @@
     my ($textref, $doc_obj, $thissection) = @_;
 
-    print STDERR " checking for acronyms ...\n" 
+    print STDERR " extracting acronyms ...\n" 
     if ($self->{'verbosity'} >= 2);
 
@@ -362 +375 @@
     if ($seen_before eq "false")
     {
+        #write it to the file ...
+        $acro->write_to_file();
+
         #do the normal acronym
         $doc_obj->add_utf8_metadata($thissection, "Acronym",  $acro->to_string());
@@ -367 +383 @@
             if ($self->{'verbosity'} >= 1);
         
-        # do the KWIC (Key Word In Context) acronym
-        my @kwic = $acro->to_string_kwic();
-        foreach my $kwic (@kwic) {
-        $doc_obj->add_utf8_metadata($thissection, "AcronymKWIC",  $kwic);
-        print STDERR "   adding ".  $kwic . "\n" 
-            if ($self->{'verbosity'} >= 2);
-        }
-    }
-    }
-    print STDERR " done with acronyms. \n" 
+#       # do the KWIC (Key Word In Context) acronym
+#       my @kwic = $acro->to_string_kwic();
+#       foreach my $kwic (@kwic) {
+#       $doc_obj->add_utf8_metadata($thissection, "AcronymKWIC",  $kwic);
+#       print STDERR "   adding ".  $kwic . "\n" 
+#           if ($self->{'verbosity'} >= 2);
+#       }
+    }
+    }
+    print STDERR " done extracting acronyms. \n" 
     if ($self->{'verbosity'} >= 2);
 }
 
+sub markup_acronyms {
+    my $self = shift (@_);
+    my ($text, $doc_obj, $thissection) = @_;
+
+    print STDERR " marking up acronyms ...\n" 
+    if ($self->{'verbosity'} >= 2);
+
+    #self is passed in to check for verbosity ...
+    $text = &acronym::markup_acronyms($text, $self);
+
+    print STDERR " done marking up acronyms. \n" 
+    if ($self->{'verbosity'} >= 2);
+
+    return $text;
+}
+
 1;