Changeset 9357

Timestamp:

2005-03-10T12:14:13+13:00 (19 years ago)

Author:

davidb

Message:

SplitPlug enhanced to support the two pass: metadata_read, read algorithm.
This is largely done by the support of MetadataPass.pm, a new "plugin" module
that contains much of the necessary code.

File:

• trunk/gsdl/perllib/plugins/SplitPlug.pm (modified) (5 diffs)

trunk/gsdl/perllib/plugins/SplitPlug.pm

@@ -82 +82 @@
     }
 
+    $self->{'textcat_store'} = {};
+    $self->{'metapass_srcdoc'} = {}; # which segments have valid metadata_srcdoc
     return bless $self, $class;
 }
@@ -116 +118 @@
     return q^\n\s*\n^;
 }
+
+sub metadata_read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    my $matched = $self->SUPER::metadata_read($pluginfo, $base_dir, $file, 
+                          $metadata, $extrametakeys, 
+                          $extrametadata, $processor, 
+                          $maxdocs, $gli);
+    $split_matched = undef;
+
+    if ($matched) {
+
+    my $outhandle = $self->{'outhandle'};
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    my $plugin_name = ref ($self);
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    $self->{'metapass_srcdoc'}->{$file} = {};
+
+    # Do encoding stuff
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+    my $le_rec = { 'language' => $language, 'encoding' => $encoding };
+    $self->{'textcat_store'}->{$file} = $le_rec;
+
+    # Read in file ($text will be in utf8)
+    my $text = "";
+    $self->read_file ($filename, $encoding, $language, \$text);
+
+    if ($text !~ /\w/) {
+        gsprintf($outhandle, "$plugin_name: {BasPlug.file_has_no_text}\n",
+             $file)
+        if $self->{'verbosity'};
+        
+        my $failhandle = $self->{'failhandle'};
+        print $failhandle "$file: " . ref($self) . ": file contains no text\n";
+        $self->{'num_not_processed'} ++;
+
+        $self->{'textcat_store'}->{$file} = undef;
+
+        return 0; 
+    }
+    
+    
+    # Split the text into several smaller segments
+    my $split_exp = $self->{'split_exp'};
+    my @segments = split(/$split_exp/, $text);
+    print $outhandle "SplitPlug found " . (scalar @segments) . " documents in $filename\n" 
+        if $self->{'verbosity'};
+    
+    $self->{'split_segments'} = \@segments;
+    $split_matched = scalar(@segments);
+    }
+    
+    return $split_matched;
+}
+
 
 
@@ -136 +196 @@
     return undef;
     }
-    my $plugin_name = ref ($self);
     $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
 
-    # Do encoding stuff
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
-
-    # Read in file ($text will be in utf8)
-    my $text = "";
-    $self->read_file ($filename, $encoding, $language, \$text);
-
-    if ($text !~ /\w/) {
-    my $outhandle = $self->{'outhandle'};
-    gsprintf($outhandle, "$plugin_name: {BasPlug.file_has_no_text}\n",
-            $file)
-        if $self->{'verbosity'};
-
-    my $failhandle = $self->{'failhandle'};
-    print $failhandle "$file: " . ref($self) . ": file contains no text\n";
-    $self->{'num_not_processed'} ++;
-
+    my $le_rec = $self->{'textcat_store'}->{$file};
+    if (!defined $le_rec) {
+    # means no text was found;
     return 0; # not processed but no point in passing it on
     }
-    
-    
-    # Split the text into several smaller segments
-    my $split_exp = $self->{'split_exp'};
-    my @segments = split(/$split_exp/, $text);
-    print $outhandle "SplitPlug found " . (scalar @segments) . " documents in $filename\n" 
-    if $self->{'verbosity'};
+
+    my $language = $le_rec->{'language'};
+    my $encoding = $le_rec->{'encoding'};
+    $self->{'textcat_store'}->{$file} = undef;
+
+    my $segments = $self->{'split_segments'};
+
+    $self->{'split_segments'} = undef;
 
     # Process each segment in turn
@@ -170 +216 @@
     $segment = 0;
     $count = 0;
-    foreach $segtext (@segments) {
+    foreach $segtext (@$segments) {
     $segment++;
+
+    if (defined $self->{'metapass_srcdoc'}->{$file}->{$segment}) {
+        # metadata is attached to a srcdoc
+        next;
+    }
 
     # create a new document
@@ -223 +274 @@
     }
 
+    delete $self->{'metapass_srcdoc'}->{$file};
+
     # Return number of document objects produced
     return $count;