Index: /trunk/gsdl/perllib/plugins/ConvertToRogPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/ConvertToRogPlug.pm	(revision 3737)
+++ /trunk/gsdl/perllib/plugins/ConvertToRogPlug.pm	(revision 3737)
@@ -0,0 +1,479 @@
+###########################################################################
+#
+# ConvertToRogPlug.pm -- plugin that inherits from RogPlug
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package ConvertToRogPlug;
+
+use BasPlug;
+use RogPlug;
+
+
+sub BEGIN {
+    @ISA = ('RogPlug');
+}
+
+sub print_usage {
+    my ($plugin_name) = @_;
+    
+    # for when this function is called directly by pluginfo.pl
+    if (ref ($plugin_name)) {
+	$plugin_name = ref ($plugin_name);
+    }
+
+    print STDERR "\n  usage: plugin $plugin_name [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   [none at present]\n";
+}
+
+sub parse_args
+{
+    my $class = shift (@_);
+    my ($args) = @_;
+
+    my $plugin_name = $class;
+    $plugin_name =~ s/\.pm$//;
+
+    if (!parsargv::parse($args,  
+			 "allow_extra_options")) {
+
+	print STDERR "\nIncorrect options passed to $plugin_name, ";
+	print STDERR "check your collect.cfg configuration file\n";
+	&print_usage($plugin_name);
+	die "\n";
+    }
+   
+    return ($plugin_name);
+}
+
+sub new {
+    my $class = shift (@_);
+    if ($class eq "ConvertToRogPlug") {$class = shift (@_);}
+    my $self;
+    # parsargv::parse might modify the list, so we do this by creating a copy
+    # of the argument list.
+    my @arglist = @_;
+    my ($plugin_name) = $class->parse_args(\@_);
+
+    $self = new RogPlug ($class, @arglist);
+    $self->{'convert_to'} = "Rog";
+    $self->{'convert_to_ext'} = "rog";
+
+    return bless $self, $class;
+}
+
+
+sub begin {
+    my $self = shift (@_);
+    
+    $self->SUPER::begin(@_);
+
+    $self->{'docnum'} = 0;
+}
+
+sub end {
+    my ($self) = @_;
+
+    # nothing to do, but keep symmetric with begin function
+    $self->SUPER::end(@_);
+}
+
+
+# Run conversion utility on the input file.  
+#
+# The conversion takes place in a collection specific 'tmp' directory so 
+# that we don't accidentally damage the input.
+#
+# The desired output type is indicated by $output_ext.  This is usually
+# something like "html" or "word", but can be "best" (or the empty string)
+# to indicate that the conversion utility should do the best it can.
+
+sub tmp_area_convert_file {
+    my $self = shift (@_);
+    my ($output_ext, $input_filename, $textref) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $convert_to = $self->{'convert_to'};
+    my $failhandle = $self->{'failhandle'};
+
+    # softlink to collection tmp dir
+    my $tmp_dirname 
+	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
+
+    # derive tmp filename from input filename
+    my ($tailname, $dirname, $suffix)
+	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
+
+    # Remove any white space from filename -- no risk of name collision, and
+    # makes later conversion by utils simpler. Leave spaces in path...
+    $tailname =~ s/\s+//g; 
+
+    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
+
+    &util::soft_link($input_filename, $tmp_filename);
+
+    my $verbosity = $self->{'verbosity'};
+    if ($verbosity > 0) {
+	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
+    }
+
+    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
+    
+    # Execute the conversion command and get the type of the result,
+    # making sure the converter gives us the appropriate output type
+    my $output_type = lc($convert_to);
+    my $cmd = "perl -S gsMusicConvert.pl -verbose $verbosity -errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
+    $output_type = `$cmd`;
+
+    # remove symbolic link to original file
+    &util::rm($tmp_filename);
+
+    # Check STDERR here
+    chomp $output_type;
+    if ($output_type eq "fail") {
+	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
+	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
+	$self->{'num_not_processed'} ++;
+	if (-s "$errlog") {
+	    open(ERRLOG, "$errlog");
+	    while (<ERRLOG>) {
+		print $outhandle "$_";
+	    }
+	    print $outhandle "\n";
+	    close ERRLOG;
+	}
+	&util::rm("$errlog") if (-e "$errlog");
+	return "";
+    }
+
+    # store the *actual* output type and return the output filename
+    # it's possible we requested conversion to html, but only to text succeeded
+
+    $self->{'convert_to_ext'} = $output_type;
+    $self->{'converted_to'} = "Rog";
+
+    my $output_filename = $tmp_filename;
+
+    $output_filename =~ s/$suffix$//;
+
+    return $output_filename;
+}
+
+
+# Remove collection specific tmp directory and all its contents.
+
+sub cleanup_tmp_area {
+    my $self = shift (@_);
+
+    my $tmp_dirname 
+	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    &util::rm_r($tmp_dirname);
+    &util::mk_dir($tmp_dirname);
+}
+
+
+# Exact copy of read_rog_record from RogPlug 
+# Needed for FILE in right scope
+
+sub read_rog_record
+{
+    my ($self,$file_buffer, $docnum, $seclevel) = @_;
+
+    my $next_line = $file_buffer->{'next_line'};
+
+    return 0 if (!defined $next_line);
+
+    if ($next_line eq "")
+    {
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	    $next_line = $line;
+	    last;
+	}
+    }
+	    
+    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
+    {
+	print STDERR "Error: Malformed Rog file: $next_line";
+	return 0;
+    }
+    else
+    {
+	# init default values
+	$file_buffer->{'song'}->{'tempo'}    = 120;
+	$file_buffer->{'song'}->{'ks_type'}  = 0;
+	$file_buffer->{'song'}->{'ks_num'}   = 0;
+	$file_buffer->{'song'}->{'metadata'} = [];
+	$file_buffer->{'song'}->{'content'}  = "";
+	
+	$file_buffer->{'song'}->{'subcol'} = $2;
+	$file_buffer->{'song'}->{'title'}  = $4;
+	$file_buffer->{'song'}->{'tval'}   = $6;
+
+	chomp($next_line);
+	my $content = $next_line;
+	if (defined $docnum) 
+	{
+	    $content.= " $docnum $seclevel";
+	}
+	$content .= "\n";
+
+	$file_buffer->{'song'}->{'content'} = $content;
+
+
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	
+	    if ($line =~ m/^song/) 
+	    {	
+		$file_buffer->{'next_line'} = $line;
+		return 1;
+	    }
+	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'tempo'} = $2;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    } 
+	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ks_type'} = $2;
+		$file_buffer->{'song'}->{'ks_num'}  = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;	   
+	    } 
+	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ts_numer'} = $2;
+		$file_buffer->{'song'}->{'ts_denom'} = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
+	    {
+		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    else
+	    {
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	}
+	
+	$file_buffer->{'next_line'} = undef;
+    }
+
+    return 1;
+}
+
+# Override RogPlug function so rog files are stored as sections (not docs)
+
+sub process_rog_record
+{
+    my ($self,$doc_obj,$cursection,$song) = @_;
+
+    $cursection = 
+	$doc_obj->insert_section($cursection);
+    $self->{'docnum'}++;
+
+    my $title = $song->{'title'};
+    my $title_safe = $title;
+    $title_safe =~ s/\'/\\\\&apos;/g;
+
+    # add metadata 
+    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
+    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
+    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
+    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
+    $doc_obj->add_metadata($cursection, "Title",         $title); 
+    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
+    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
+
+    foreach $md ( @{$song->{'metadata'}} )
+    {
+	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
+    }
+
+    # add contents as text
+    $doc_obj->add_text($cursection,$song->{'content'});
+
+    return $cursection;
+}
+
+
+
+# Override BasPlug read
+# We don't want to get language encoding stuff until after we've converted
+# our file to Rog format
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+#    if ($self->is_recursive()) {
+#        die "BasPlug::read function must be implemented in sub-class for recursive plugins\n";
+#    }
+
+    my $outhandle = $self->{'outhandle'};
+
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
+	$self->{'num_blocked'} ++;
+	return 0;
+    }
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+        return undef;
+    }
+    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+
+    my $output_ext = $self->{'convert_to_ext'};
+    my $conv_filename = $self->tmp_area_convert_file($output_ext, $filename);
+
+    if ("$conv_filename" eq "") {return 0;} # allows continue on errors
+    $self->{'conv_filename'} = $conv_filename;
+
+
+    # create a new document
+    my $doc_obj = new doc ($conv_filename, "indexed_doc");
+    my $topsection = $doc_obj->get_top_section();
+    my $cursection = $topsection;
+
+    $self->{'docnum'}++;
+    my $docnum = $self->{'docnum'};
+
+    $doc_obj->set_OIDtype ($processor->{'OIDtype'});
+    my ($filemeta) = $file =~ /([^\\\/]+)$/;
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+    if ($self->{'cover_image'}) {
+	$self->associate_cover_image($doc_obj, $filename);
+    }
+
+
+    my $track_no = "1";
+    my $rog_filename = "$conv_filename$track_no.$output_ext";
+    while (1)
+    {
+	last unless open (FILE, $rog_filename) ;
+
+	my $file_buffer = { line_no => 0, next_line => "", song => {} };
+	
+	while ($self->read_rog_record($file_buffer, $docnum, $track_no))
+	{
+	    my $song = $file_buffer->{'song'};
+	    my $content = $song->{'content'};
+	    $content =~ s/^song\w+(.*)$/song $1 X.$track_no/;
+	    
+	    $cursection 
+		= $self->process_rog_record($doc_obj,$cursection,
+					    $file_buffer->{'song'});
+	}
+
+	close FILE;
+
+	$track_no++;
+	$rog_filename = "$conv_filename$track_no.$output_ext";
+    }
+
+    print STDERR "\n";
+
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    # do plugin specific processing of doc_obj
+    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+    # add an OID
+    $doc_obj->set_OID();
+
+    my $oid = $doc_obj->get_OID();
+    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
+
+    $doc_obj->add_utf8_metadata ($topsection, "audiolink",  $appletlink); 
+    $doc_obj->add_utf8_metadata ($topsection, "audioicon",  "_iconaudio_"); 
+    $doc_obj->add_utf8_metadata ($topsection, "/audiolink", "</a>"); 
+
+    # if no title metadata defined, set it to filename minus extension
+    my $existing_title = $doc_obj->get_metadata_element($topsection,"Title");
+    if (!defined $existing_title) 
+    {
+	my $title = $doc_obj->get_metadata_element($topsection,"Source");
+	$title =~ s/\..*?$//g;
+	$doc_obj->add_utf8_metadata ($topsection, "Title", $title); 
+
+	my $title_safe = $title;
+	$title_safe =~ s/\'/\\\\&apos;/g;
+	$doc_obj->add_utf8_metadata ($topsection, "TitleSafe", $title_safe); 
+    }
+
+    # process the document
+    $processor->process($doc_obj);
+    $self->cleanup_tmp_area();
+
+    $self->{'num_processed'} ++;
+
+    return 1;
+}
+
+
+# do plugin specific processing of doc_obj for HTML type
+sub process_type {
+    my $self = shift (@_);
+    my ($doc_ext, $textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
+    
+    my $conv_filename = $self->{'conv_filename'};
+    my $tmp_dirname = File::Basename::dirname($conv_filename);
+    my $tmp_tailname = File::Basename::basename($conv_filename);
+    
+    my $converted_to = $self->{'converted_to'};
+    my $ret_val = 1;    
+
+#   $ret_val = &RogPlug::process($self, $textref, $pluginfo,
+#				 $tmp_dirname, $tmp_tailname,
+#				 $metadata, $doc_obj);
+
+    # associate original file with doc object
+    my $cursection = $doc_obj->get_top_section();
+    my $filename = &util::filename_cat($base_dir, $file);
+    $doc_obj->associate_file($filename, "doc.$doc_ext", undef, $cursection);
+
+    my $doclink = "<a href=\"_httpcollection_/index/assoc/[archivedir]/doc.$doc_ext\">";
+    $doc_obj->add_utf8_metadata ($cursection, "srclink",  $doclink); 
+    $doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icon".$doc_ext."_"); 
+    $doc_obj->add_utf8_metadata ($cursection, "/srclink", "</a>"); 
+    return $ret_val;
+}
+
+1;
Index: /trunk/gsdl/perllib/plugins/RogPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/RogPlug.pm	(revision 3737)
+++ /trunk/gsdl/perllib/plugins/RogPlug.pm	(revision 3737)
@@ -0,0 +1,264 @@
+###########################################################################
+#
+# RogPlug.pm -- simple text plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .rog or .mdb files 
+
+package RogPlug;
+
+use BasPlug;
+use sorttools;
+use doc; 
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# This plugin processes files with the suffix ".mdb" or ".rog"
+sub get_default_process_exp {
+    return q^(?i)\.(mdb|rog)$^;
+}
+
+sub read_rog_record
+{
+    my ($self,$file_buffer, $seclevel) = @_;
+
+    my $next_line = $file_buffer->{'next_line'};
+
+    return 0 if (!defined $next_line);
+
+    if ($next_line eq "")
+    {
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	    $next_line = $line;
+	    last;
+	}
+    }
+	    
+    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
+    {
+	print STDERR "Error: Malformed Rog file: $next_line";
+	return 0;
+    }
+    else
+    {
+	# init default values
+	$file_buffer->{'song'}->{'tempo'}    = 120;
+	$file_buffer->{'song'}->{'ks_type'}  = 0;
+	$file_buffer->{'song'}->{'ks_num'}   = 0;
+	$file_buffer->{'song'}->{'metadata'} = [];
+	$file_buffer->{'song'}->{'content'}  = "";
+	
+	$file_buffer->{'song'}->{'subcol'} = $2;
+	$file_buffer->{'song'}->{'title'}  = $4;
+	$file_buffer->{'song'}->{'tval'}   = $6;
+
+	chomp($next_line);
+	my $content = $next_line;
+	if (defined $seclevel) 
+	{
+	    $content.= " $seclevel";
+	}
+	$content .= "\n";
+
+	$file_buffer->{'song'}->{'content'} = $content;
+
+
+	my $line;
+	while(defined($line=<FILE>))  
+	{
+	    $line =~ s/\r$//;
+
+	    $file_buffer->{'line_no'}++;
+	    next if ($line =~ m/^\#/);
+	
+	    if ($line =~ m/^song/) 
+	    {	
+		$file_buffer->{'next_line'} = $line;
+		return 1;
+	    }
+	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'tempo'} = $2;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    } 
+	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ks_type'} = $2;
+		$file_buffer->{'song'}->{'ks_num'}  = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;	   
+	    } 
+	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
+	    {
+		$file_buffer->{'song'}->{'ts_numer'} = $2;
+		$file_buffer->{'song'}->{'ts_denom'} = $4;
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
+	    {
+		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	    else
+	    {
+		$file_buffer->{'song'}->{'content'} .= $line;
+	    }
+	}
+	
+	$file_buffer->{'next_line'} = undef;
+    }
+
+    return 1;
+}
+
+sub process_rog_record
+{
+    my ($self,$file,$metadata,$song,$processor) = @_;
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    my $cursection = $doc_obj->get_top_section();
+
+    my $title = $song->{'title'};
+    my $title_safe = $title;
+    $title_safe =~ s/\'/\\\\&apos;/g;
+
+    # add metadata 
+    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
+    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
+    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
+    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
+    $doc_obj->add_metadata($cursection, "Title",         $title); 
+    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
+    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
+
+    foreach $md ( @{$song->{'metadata'}} )
+    {
+	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
+    }
+
+    # add contents as text
+    $doc_obj->add_text($cursection,$song->{'content'});
+
+    $self->extra_metadata($doc_obj,$cursection, $metadata);
+    
+    # add OID
+    $doc_obj->set_OID ();
+
+    my $oid = $doc_obj->get_OID();
+    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
+
+    $doc_obj->add_utf8_metadata ($cursection, "audiolink",  $appletlink); 
+    $doc_obj->add_utf8_metadata ($cursection, "audioicon",  "_iconaudio_"); 
+    $doc_obj->add_utf8_metadata ($cursection, "/audiolink", "</a>"); 
+
+    # process the document
+    $processor->process($doc_obj);
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    return undef unless ($filename =~ /\.((rog|mdb)(\.gz)?)$/i && (-e $filename));
+
+    my $gz = (defined $3) ? 1: 0;
+
+    print STDERR "RogPlug: processing $filename\n" if $processor->{'verbosity'};
+    
+    if ($gz) {
+	open (FILE, "zcat $filename |") 
+	    || die "RogPlug::read - zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) 
+	    || die "RogPlug::read - can't open $filename\n";
+    }
+
+    my $doc_count = 0;
+    my $dot_count = 0;
+    my $file_buffer = { line_no => 0, next_line => "", song => {} };
+
+    while ($self->read_rog_record($file_buffer))
+    {
+	$self->process_rog_record($file,$metadata,$file_buffer->{'song'},$processor);
+	$doc_count++;
+
+        if (defined $maxdocs && $maxdocs =~ /\d/)
+	{
+	    last if ($maxdocs >=0 && $doc_count >= $maxdocs);
+	}
+
+	if (($doc_count % 10) == 0)
+	{
+	    print STDERR ".";
+	    $dot_count++;
+	    print STDERR "\n" if (($dot_count % 80) == 0);
+	}
+    }
+
+    close FILE;
+
+    print STDERR "\n";
+
+    $self->{'num_processed'} = $doc_count;
+
+    return 1; # processed the file
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
