Changeset 3856 for trunk/gsdl/perllib

Timestamp:

2003-03-12T11:53:28+13:00 (21 years ago)

Author:

davidb

Message:

General improvement to the translator facility.

File:

• trunk/gsdl/perllib/plugins/MACROPlug.pm (modified) (10 diffs)

trunk/gsdl/perllib/plugins/MACROPlug.pm

@@ -55 +55 @@
 }
 
+
+sub load_language_table
+{
+    my $lang_table = {};
+
+    my $lang_fname = util::filename_cat($ENV{'GSDLHOME'},"tmp","lang",
+                    "package_forms","languages.log");
+    open (LANGFILE, "<$lang_fname") 
+    || die ("Unable to open $lang_fname: $!\n");
+
+    my $full_name;
+    my $abbr_name;
+
+    while (defined ($full_name=<LANGFILE>)) {
+    chomp($full_name);
+    
+    $abbr_name = <LANGFILE>;
+    chomp($abbr_name);
+
+    $lang_table->{$full_name} = $abbr_name;
+
+    my $fourchar_name = substr($full_name,0,4);
+    if (!defined $lang_table->{$fourchar_name}) {
+        $lang_table->{$fourchar_name} = $abbr_name;
+    }
+    else {
+        print STDERR "Warning: Clash on four character abbreviation for language $fourchar_name\n";
+    }
+    }
+
+    close LANGFILE;
+
+    return $lang_table;
+}
+
+
+
 sub new {
     my ($class) = @_;
@@ -63 +100 @@
     push( @{$option_list}, $options );
 
+    $self->{'lang_abbr'} = load_language_table();
+
     return bless $self, $class;
 }
@@ -71 +110 @@
     return q^(?i)\.dm$^;
 }
+
+
+sub read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $lang_table = $self->{'lang_abbr'};
+    my $fn = $file;
+    $fn =~ s/.*\/(.*)\..*/$1/;
+    $fn =~ s/\d+$//; # remove any digits from end of filename
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ((!-d $filename) && ($file !~ m/doc.xml$/) && (!defined $lang_table->{$fn})) {
+    print $outhandle "MACROPlug: blocking $file\n" 
+        if $self->{'verbosity'} > 2;
+    $self->{'num_blocked'} ++;
+    return 0;
+    }
+
+    return $self->SUPER::read(@_);
+}
+
 
 # do plugin specific processing of doc_obj
@@ -94 +159 @@
 }
 
-1;
 
 sub extract_macronames {
@@ -103 +167 @@
     my $outhandle = $self->{'outhandle'};
 
-    print $outhandle " extracting macronames ...\n";
+    print $outhandle " extracting macronames ...\n"
+    if ($self->{'verbosity'}>3);
     
-    my @textarray = split ("\n", $$$textref);
+    my @textarray = split ("\n", $$$textref)
+    if ($self->{'verbosity'}>3);
 
     my $macro_text = "";
@@ -276 +342 @@
     }
    
-    print $outhandle "done extracting macros\n";
-}
-
-
-sub get_language_encoding {
+    print $outhandle "done extracting macros\n"
+    if ($self->{'verbosity'}>3);
+
+}
+
+
+sub get_language_encoding_old {
     my $self = shift (@_);
     my ($filename) = @_;
@@ -300 +368 @@
     my $results = [];
 
-    if ($filename =~ m/spanish\.dm/) {
-    $results->[0] = "es-utf8";
-    }
-    else {
-    
-    # get the language/encoding
-    $results = $self->{'textcat'}->classify(\$text);
-    }
+    # get the language/encoding
+    $results = $self->{'textcat'}->classify(\$text);
 
     foreach $r (@$results) {
-    print $outhandle "MY1 $r\n";
+    print $outhandle "Results: $r\n";
     
     }
@@ -317 +379 @@
     # first one in the list - otherwise use the defaults
     if (scalar @$results > 3) {
-                                        
-    open (LANGFILE, "</research/kde2/gsdl/tmp/lang/package_forms/languages.log") or die ("MURGH LANGFILE\n");
+                                
+        my $lang_fname = util::filename_cat($ENV{'GSDLHOME'},"tmp","lang","package_forms",
+                        "languages.log");
+    open (LANGFILE, "<$lang_fname") or die ("Unable to open $lang_fname: $!\n");
 
     while (<LANGFILE>) {
@@ -397 +461 @@
     }
 
+    print STDERR "**** forcing encoding to be utf8\n";
+    $encoding = "utf8";
+
+    print STDERR "**** forcing language to be first two letters\n";
+    my $lfname = $filename;
+    $lfname =~ s/^.*\///;
+    $language = substr($lfname,0,2);
+
     print $outhandle "RETURNING VALUES $language $encoding\n";
 
@@ -403 +475 @@
 
 
-
-
-
-
+sub find_language {
+    my ($self,$fn) = @_;
+
+    my $lang_table = $self->{'lang_abbr'};
+
+    if (!defined $lang_table->{$fn}) {
+
+    # try and find it with shorter string name
+    
+    my $try_len = length($fn);
+
+    while ($try_len>=4) {
+        $try_fn = substr($fn,0,$try_len);
+
+        if (defined $lang_table->{$try_fn}) {
+        $fn = $try_fn;
+        last;
+        }
+        $try_len--;
+    }
+    }
+
+    return $fn;
+}
+
+
+sub get_language_encoding {
+    my $self = shift (@_);
+    my ($filename) = @_;
+    my $outhandle = $self->{'outhandle'};
+
+    my $fn = $filename;
+    $fn =~ s/.*\/(.*)\..*/$1/;
+    $fn =~ s/\d+$//; # remove any digits from end of filename
+
+    my $languge;
+    my $encoding = "utf8";
+
+    ## my $lang_lookup = $self->find_language($fn);
+
+    my $lang_table = $self->{'lang_abbr'};
+
+    if (!defined $lang_table->{$fn}) {
+
+    print $outhandle "Warning: Macro file name $filename not in list of languages.\n";
+    print $outhandle "         Using default language.\n";
+    $language = $self->{'default_language'};
+    }
+    else {
+    $language = $lang_table->{$fn};
+    }
+
+    ## print $outhandle "Storing $filename as $language $encoding\n";
+
+    return ($language, $encoding);
+}
+
+
+1;