Changeset 2027

Timestamp:

2001-02-20T15:57:37+13:00 (23 years ago)

Author:

jrm21

Message:

read() is now completely independent of BasPlug::read(), as the latter
does textcat over the raw file to determine lang/enc. Our read does this
after conversion by gsConvert.

File:

• trunk/gsdl/perllib/plugins/ConvertToPlug.pm (modified) (1 diff)

trunk/gsdl/perllib/plugins/ConvertToPlug.pm

@@ -215 +215 @@
 
 # Override BasPlug read
-
+# We don't want to get language encoding stuff until after we've converted
+# our file to either TEXT or HTML.
 sub read {
     my $self = shift (@_);
-    
-    my $ret_val = BasPlug::read($self,@_);
- 
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+#    if ($self->is_recursive()) {
+#        die "BasPlug::read function must be implemented in sub-class for recursive plugins\n";
+#    }
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+        return undef;
+    }
+    my $plugin_name = ref ($self);
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+#    $self->read_file ($filename, $encoding, \$text);
+### was read_file
+#    my $self = shift (@_);
+#    my ($src_filename, $encoding, $textref) = @_;
+
+    my $output_ext = $self->{'convert_to_ext'};
+    my $conv_filename = $self->tmp_area_convert_file($output_ext,$filename);
+# change following to return undef?
+    if ("$conv_filename" eq "") {return "";} # allows continue on errors
+    $self->{'conv_filename'} = $conv_filename;
+
+### was read_file
+
+
+
+
+# Do encoding stuff
+    my ($language, $encoding);
+    if ($self->{'input_encoding'} eq "auto") {
+        # use textcat to automatically work out the input encoding and language
+        ($language, $encoding) = $self->get_language_encoding ($conv_filename);
+    } elsif ($self->{'extract_language'}) {
+        # use textcat to get language metadata
+
+        my ($language, $extracted_encoding) = $self->get_language_encoding ($conv_filename);
+        $encoding = $self->{'input_encoding'};
+        if ($extracted_encoding ne $encoding && $self->{'verbosity'}) {
+            print $outhandle "$plugin_name: WARNING: $file was read using $encoding encoding but ";
+            print $outhandle "appears to be encoded as $extracted_encoding.\n";
+        }
+    } else {
+        $language = $self->{'default_language'};
+        $encoding = $self->{'input_encoding'};
+    }
+
+    BasPlug::read_file($self,$conv_filename, $encoding, \$text);
+    if (!length ($text)) {
+        print $outhandle "$plugin_name: ERROR: $file contains no text\n" if $self->{'verbosity'};
+        return 0;
+    }
+
+    # create a new document
+    my $doc_obj = new doc ($conv_filename, "indexed_doc");
+
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", 
+                $language);
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", 
+                $encoding);
+
+
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    # do plugin specific processing of doc_obj
+    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+    # add an OID
+    $doc_obj->set_OID();
+    # process the document
+    $processor->process($doc_obj);
     $self->cleanup_tmp_area();
-    
-    return $ret_val;
+
+
+    return 1;
 }