Changeset 1894

Timestamp:

2001-02-01T17:43:27+13:00 (23 years ago)

Author:

jrm21

Message:

updated by copying BasPlug's new language/encoding stuff over for the read()
stuff.

File:

• trunk/gsdl/perllib/plugins/SplitPlug.pm (modified) (3 diffs)

trunk/gsdl/perllib/plugins/SplitPlug.pm

@@ -109 +109 @@
     $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
     
+    my ($language, $encoding);
+    if ($self->{'input_encoding'} eq "auto") {
+    # use textcat to automatically work out the input encoding and language
+    ($language, $encoding) = $self->get_language_encoding ($filename);
+    
+    } elsif ($self->{'extract_language'}) {
+    # use textcat to get language metadata
+    ($language, $extracted_encoding) = $self->get_language_encoding ($filename);
+    $encoding = $self->{'input_encoding'};
+    
+    if ($extracted_encoding ne $encoding && $self->{'verbosity'}) {
+        print $outhandle "$plugin_name: WARNING: $file was read using $encoding encoding but ";
+        print $outhandle "appears to be encoded as $extracted_encoding.\n";
+    }
+    
+    } else {
+    $language = $self->{'default_language'};
+    $encoding = $self->{'input_encoding'};
+    }
+
     # Read in file ($text will be in utf8)
     my $text = "";
-    $self->read_file ($filename, \$text);
+    $self->read_file ($filename, $encoding, \$text);
 
     if ($text !~ /\w/) {
@@ -119 +139 @@
     return 0;
     }
-
+    
+    
     # Split the text into several smaller segments
     my $split_exp = $self->{'split_exp'};
@@ -135 +156 @@
     # create a new document
     my $doc_obj = new doc ($filename, "indexed_doc");
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
 
     # Calculate a "base" document ID.