Index: /trunk/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 10279)
+++ /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 10280)
@@ -213,6 +213,4 @@
 
 
-
-
 sub get_arguments
 {
@@ -386,4 +384,5 @@
 	die "\n";
     }
+
 
     # else parsing was successful.
@@ -431,5 +430,5 @@
 	    $self->{'places'} = $places_ref;
 	}
-    }	
+    }
     return bless $self, $class;
     
@@ -673,31 +672,9 @@
 
 
-# The BasPlug read() function. This function does all the right things
-# to make general options work for a given plugin. It calls the process()
-# function which does all the work specific to a plugin (like the old
-# read functions used to do). Most plugins should define their own 
-# process() function and let this read() function keep control.
-#
-# recursive plugins (e.g. RecPlug) and specialized plugins like those
-# capable of processing many documents within a single file (e.g.
-# GMLPlug) should normally implement their own version of read()
-#
-# Return number of files processed, undef if can't recognise, -1 if can't
-# process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-
-sub read {
+sub read_block {
     my $self = shift (@_);  
   
     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
 
-    if ($self->is_recursive()) {
-	gsprintf(STDERR, "{BasPlug.read_must_be_implemented}") && die "\n";
-    }
-
-    my $outhandle   = $self->{'outhandle'};
-    my $smart_block = $self->{'smart_block'};
-    my $smart_block_BN = $self->{'smart_block_BN'};
 
     my $filename = $file;
@@ -707,6 +684,9 @@
 	# a form of smart block
 	$self->{'num_blocked'} ++;
-	return 0; # blocked
-    }
+	return (0,undef); # blocked
+    }
+
+    my $smart_block = $self->{'smart_block'};
+    my $smart_block_BN = $self->{'smart_block_BN'};
 
     if ($smart_block || $smart_block_BN) {
@@ -714,15 +694,59 @@
 	if (defined $self->{'file_blocks'}->{$filename} && $self->{'file_blocks'}->{$filename} == 1){
 	    $self->{'num_blocked'} ++;
-	    return 0; # blocked
+	    return (0,undef); # blocked
 	}
     } elsif ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
 	$self->{'num_blocked'} ++;
-	return 0; # blocked
+	return (0,undef); # blocked
     }
     
     if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
-	return undef; # can't recognise
-    }
+	return (undef,undef); # can't recognise
+    }
+    
+    return (1,$filename);
+}
+
+sub read_tidy_file {
+
+    my $self = shift (@_);  
+  
+    my ($file) = @_;
+
     $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    return $file;
+}
+
+
+
+# The BasPlug read_into_doc_obj() function. This function does all the
+# right things to make general options work for a given plugin.  It reads in
+# a file and sets up a slew of metadata all saved in doc_obj, which
+# it then returns as part of a tuple (process_status,doc_obj)
+#
+# Much of this functionality used to reside in read, but it was broken
+# down into a supporting routine to make the code more flexible.  
+#
+# recursive plugins (e.g. RecPlug) and specialized plugins like those
+# capable of processing many documents within a single file (e.g.
+# GMLPlug) will normally want to implement their own version of
+# read_into_doc_obj()
+#
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read_into_doc_obj {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    if ($self->is_recursive()) {
+	gsprintf(STDERR, "{BasPlug.read_must_be_implemented}") && die "\n";
+    }
+
+    my $outhandle   = $self->{'outhandle'};
+
+    my ($block_status,$filename) = $self->read_block(@_);    
+    return $block_status if ((!defined $block_status) || ($block_status==0));
+    $file = $self->read_tidy_file($file);
 
     # Do encoding stuff
@@ -760,5 +784,5 @@
 	$self->{'num_not_processed'} ++;
 
-	return 0; # what should we return here?? error but don't want to pass it on
+	return (0,undef); # what should we return here?? error but don't want to pass it on
     }
    
@@ -773,5 +797,5 @@
 	undef $text;
 	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1;
+	return (-1,undef);
     }
     $text='';
@@ -790,16 +814,46 @@
 	$doc_obj->set_OID();
     }
-
-    # process the document
-    $processor->process($doc_obj);
-
-    if(defined($self->{'places_filename'})){
-	&util::rm($self->{'places_filename'});
-	$self->{'places_filename'} = undef;
-    }
-
-    $self->{'num_processed'} ++;
-    undef $doc_obj;
-    return 1; # processed the file
+    
+    return (1,$doc_obj);
+}
+
+
+# The BasPlug read() function. This function calls read_into_doc_obj()
+# to ensure all the right things to make general options work for a
+# given plugin are done. It then calls the process() function which
+# does all the work specific to a plugin (like the old read functions
+# used to do). Most plugins should define their own process() function
+# and let this read() function keep control.  
+#
+# recursive plugins (e.g. RecPlug) and specialized plugins like those
+# capable of processing many documents within a single file (e.g.
+# GMLPlug) might want to implement their own version of read(), but
+# more likely need to implement their own version of read_into_doc_obj()
+#
+# Return number of files processed, undef if can't recognise, -1 if can't
+# process
+
+sub read {
+    my $self = shift (@_);  
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
+    
+    if ((defined $process_status) && ($process_status == 1)) {
+	# process the document
+	$processor->process($doc_obj);
+	
+	if(defined($self->{'places_filename'})){
+	    &util::rm($self->{'places_filename'});
+	    $self->{'places_filename'} = undef;
+	}
+	
+	$self->{'num_processed'} ++;
+	undef $doc_obj; 
+    }
+
+    # if process_status == 1, then the file has been processed.
+    return $process_status;
+
 }
 
@@ -830,12 +884,10 @@
 	return;
     }
-
     $$textref = "";
-
     if (!open (FILE, $filename)) {
 	gsprintf(STDERR, "BasPlug::read_file {BasPlug.could_not_open_for_reading} ($!)\n", $filename);
-	 die "\n";
-     }
-
+	die "\n";
+    }
+     
     if ($encoding eq "ascii") {
 	undef $/;
@@ -847,6 +899,5 @@
 	$reader->set_encoding ($encoding);
 	$reader->read_file ($textref);
-
-	#Now segments chinese if the separate_cjk option is set
+        #Now segments chinese if the separate_cjk option is set
 	if ($self->{'separate_cjk'}) {
 	    # segment the Chinese words
@@ -854,7 +905,22 @@
 	}
     }
-
     close FILE;
 }
+
+# write_file -- used by ConvertToPlug, for example in post processing
+#
+sub utf8_write_file {
+    my $self = shift (@_);
+    my ($textref, $filename) = @_;
+    
+    if (!open (FILE, ">$filename")) {
+	gsprintf(STDERR, "ConvertToPlug::write_file {ConvertToPlug.could_not_open_for_writing} ($!)\n", $filename);
+	 die "\n";
+     }
+    print FILE $$textref;
+    
+    close FILE;
+}
+
 
 sub filename_based_title
@@ -887,4 +953,5 @@
     my ($filename) = @_;
 
+    
     my ($language, $encoding, $extracted_encoding);
     if ($self->{'input_encoding'} eq "auto") {
@@ -910,4 +977,5 @@
         $encoding = $self->{'input_encoding'};
     }
+
     return ($language, $encoding);
 }
@@ -942,5 +1010,5 @@
 	}
     }
-
+   
 
     # remove <title>stuff</title> -- as titles tend often to be in English
