Changeset 9853
- Timestamp:
- 2005-05-10T16:55:00+12:00 (19 years ago)
- Location:
- trunk/gsdl/perllib
- Files:
-
- 29 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/gsdl/perllib/lucenebuilder.pm
r9669 r9853 131 131 $self->{'buildproc'}, $self->{'maxdocs'}); 132 132 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 133 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );133 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 134 134 &plugin::end($self->{'pluginfo'}); 135 135 close ($handle) unless $self->{'debug'}; … … 313 313 $self->{'buildproc'}->reset(); 314 314 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 315 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );315 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 316 316 close ($handle) unless $self->{'debug'}; 317 317 -
trunk/gsdl/perllib/mgbuilder.pm
r9669 r9853 278 278 $self->{'buildproc'}, $self->{'maxdocs'}); 279 279 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 280 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );280 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 281 281 &plugin::end($self->{'pluginfo'}); 282 282 … … 316 316 317 317 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 318 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );318 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 319 319 320 320 close ($handle) unless $self->{'debug'}; … … 605 605 $self->{'buildproc'}->reset(); 606 606 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 607 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );607 "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'}); 608 608 close ($handle) unless $self->{'debug'}; 609 609 … … 639 639 $self->{'buildproc'}->reset(); 640 640 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 641 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );641 "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'}); 642 642 643 643 … … 810 810 811 811 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 812 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );812 "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'}); 813 813 814 814 # output classification information -
trunk/gsdl/perllib/mgppbuilder.pm
r9669 r9853 329 329 } 330 330 } 331 331 $mgpp_passes_sections .= "-K SENT "; 332 332 print $outhandle "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1); 333 333 print STDERR "<Stage name='CompressText'>\n" if $self->{'gli'}; … … 365 365 $self->{'buildproc'}, $self->{'maxdocs'}); 366 366 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 367 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );367 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 368 368 &plugin::end($self->{'pluginfo'}); 369 369 close (PIPEOUT); … … 407 407 408 408 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 409 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );409 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 410 410 close ($handle) unless $self->{'debug'}; 411 411 … … 705 705 $self->{'buildproc'}->reset(); 706 706 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 707 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );707 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 708 708 close ($handle) unless $self->{'debug'}; 709 709 … … 740 740 $self->{'buildproc'}->reset(); 741 741 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 742 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );742 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 743 743 744 744 $self->print_stats (); … … 939 939 940 940 &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 941 "", {}, $self->{'buildproc'}, $self->{'maxdocs'} );941 "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 942 942 943 943 # output classification information -
trunk/gsdl/perllib/plugin.pm
r9584 r9853 155 155 156 156 sub read { 157 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli, $aux) = @_;157 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli, $aux) = @_; 158 158 159 159 $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/; 160 $total_count = 0 unless defined $total_count && $total_count =~ /\d/; 160 161 $gli = 0 unless defined $gli; 161 162 … … 187 188 188 189 $rv = $plugobj->read($pluginfo, $base_dir, $file, 189 $metadata, $processor, $maxdocs, $ gli, $aux);190 $metadata, $processor, $maxdocs, $total_count, $gli, $aux); 190 191 191 192 if (defined $rv) { -
trunk/gsdl/perllib/plugins/ArcPlug.pm
r8716 r9853 75 75 sub read { 76 76 my $self = shift (@_); 77 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;77 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs,$total_count, $gli) = @_; 78 78 my $outhandle = $self->{'outhandle'}; 79 79 … … 96 96 # process each file 97 97 foreach my $subfile (@$file_list) { 98 last if ($maxdocs != -1 && $count>= $maxdocs);98 last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs); 99 99 100 100 my $tmp = &util::filename_cat ($file, $subfile->[0]); 101 101 next if $tmp eq $file; 102 102 # note: metadata is not carried on to the next level 103 $count += &plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor, $maxdocs );103 $count += &plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor, $maxdocs, ($total_count+$count), $gli); 104 104 } 105 105 -
trunk/gsdl/perllib/plugins/BNContentePlug.pm
r9462 r9853 148 148 sub metadata_read { 149 149 my $self = shift (@_); 150 my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs ) = @_;150 my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_; 151 151 152 152 my $outhandle = $self->{'outhandle'}; … … 203 203 sub readxxx { 204 204 my $self = shift (@_); 205 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;205 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 206 206 my $outhandle = $self->{'outhandle'}; 207 207 -
trunk/gsdl/perllib/plugins/BasPlug.pm
r9703 r9853 662 662 my $self = shift (@_); 663 663 664 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;664 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 665 665 666 666 if ($self->is_recursive()) { -
trunk/gsdl/perllib/plugins/ConvertToPlug.pm
r9584 r9853 267 267 sub read { 268 268 my $self = shift (@_); 269 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;269 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 270 270 # if ($self->is_recursive()) { 271 271 # die "BasPlug::read function must be implemented in sub-class for recursive plugins\n"; -
trunk/gsdl/perllib/plugins/ConvertToRogPlug.pm
r9584 r9853 334 334 sub read { 335 335 my $self = shift (@_); 336 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;336 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 337 337 # if ($self->is_recursive()) { 338 338 # die "BasPlug::read function must be implemented in sub-class for recursive plugins\n"; -
trunk/gsdl/perllib/plugins/DBPlug.pm
r8121 r9853 90 90 sub read { 91 91 my $self = shift (@_); 92 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;92 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 93 93 94 94 # see if we can handle the passed file... -
trunk/gsdl/perllib/plugins/DSpacePlug.pm
r9694 r9853 245 245 sub metadata_read { 246 246 my $self = shift (@_); 247 my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs ) = @_;247 my ($pluginfo, $base_dir, $file, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_; 248 248 249 249 my $only_first_doc = $self->{'only_first_doc'}; … … 336 336 sub read { 337 337 my $self = shift (@_); 338 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;338 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 339 339 my $outhandle = $self->{'outhandle'}; 340 340 -
trunk/gsdl/perllib/plugins/FOXPlug.pm
r9703 r9853 73 73 sub read { 74 74 my $self = shift (@_); 75 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;75 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 76 76 my $fullname = &util::filename_cat ($base_dir, $file); 77 77 -
trunk/gsdl/perllib/plugins/GMLPlug.pm
r9703 r9853 73 73 sub read { 74 74 my $self = shift (@_); 75 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;75 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 76 76 my $outhandle = $self->{'outhandle'}; 77 77 … … 227 227 228 228 $no_docs++; 229 last if ($maxdocs > -1 && $no_docs>= $maxdocs);229 last if ($maxdocs > -1 && ($total_count+$no_docs) >= $maxdocs); 230 230 last unless defined $gml && $gml =~ /\w/; 231 231 } # while(1) document level -
trunk/gsdl/perllib/plugins/HBPlug.pm
r8166 r9853 237 237 sub read { 238 238 my $self = shift (@_); 239 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;239 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 240 240 my $outhandle = $self->{'outhandle'}; 241 241 -
trunk/gsdl/perllib/plugins/ImagePlug.pm
r9703 r9853 362 362 sub read { 363 363 my $self = shift (@_); 364 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;364 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 365 365 366 366 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/IndexPlug.pm
r6408 r9853 91 91 sub read { 92 92 my $self = shift (@_); 93 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;93 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 94 94 my $outhandle = $self->{'outhandle'}; 95 95 … … 117 117 my $count = 0; 118 118 foreach my $docfile (keys (%$list)) { 119 last if ($maxdocs != -1 && $count>= $maxdocs);119 last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs); 120 120 $metadata = {}; # at present we can do this as metadata 121 121 # will always be empty when it arrives … … 141 141 } 142 142 } 143 $count += &plugin::read ($pluginfo, $index_base_dir, $docfile, $metadata, $processor, $maxdocs );143 $count += &plugin::read ($pluginfo, $index_base_dir, $docfile, $metadata, $processor, $maxdocs, ($total_count +$count), $gli); 144 144 } 145 145 } -
trunk/gsdl/perllib/plugins/MACROPlug.pm
r8166 r9853 106 106 sub read { 107 107 my $self = shift (@_); 108 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;108 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 109 109 110 110 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/MP3Plug.pm
r9167 r9853 308 308 sub read { 309 309 my $self = shift (@_); 310 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;310 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 311 311 312 312 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/NULPlug.pm
r9584 r9853 91 91 sub read { 92 92 my $self = shift (@_); 93 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;93 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 94 94 95 95 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/OAIPlug.pm
r9738 r9853 79 79 my $self = shift (@_); 80 80 81 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;81 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 82 82 83 83 my $outhandle = $self->{'outhandle'}; … … 133 133 134 134 return &plugin::read ($pluginfo, $url_base_dir, $url_array->[0], 135 $metadata, $processor, $maxdocs, $ gli);135 $metadata, $processor, $maxdocs, $total_count, $gli); 136 136 } 137 137 else -
trunk/gsdl/perllib/plugins/OggVorbisPlug.pm
r8365 r9853 87 87 { 88 88 my $self = shift (@_); 89 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;89 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 90 90 91 91 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/PagedImgPlug.pm
r9584 r9853 501 501 sub read { 502 502 my $self = shift (@_); 503 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;503 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 504 504 505 505 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/RecPlug.pm
r9703 r9853 190 190 sub read { 191 191 my $self = shift (@_); 192 my ($pluginfo, $base_dir, $file, $in_metadata, $processor, $maxdocs, $ gli) = @_;192 my ($pluginfo, $base_dir, $file, $in_metadata, $processor, $maxdocs, $total_count, $gli) = @_; 193 193 194 194 my $outhandle = $self->{'outhandle'}; … … 360 360 my $subfile = $dir[$i]; 361 361 my $this_file_base_dir = $base_dir; 362 last if ($maxdocs != -1 && $count>= $maxdocs);362 last if ($maxdocs != -1 && ($count + $total_count) >= $maxdocs); 363 363 next if ($subfile =~ /^\.\.?$/); 364 364 next if ($read_metadata_files && $subfile =~ /metadata\.xml$/); … … 419 419 $count += &plugin::read ($pluginfo, $this_file_base_dir, 420 420 &util::filename_cat($file, $subfile), 421 $out_metadata, $processor, $maxdocs, $gli);421 $out_metadata, $processor, $maxdocs, ($total_count + $count), $gli); 422 422 } 423 423 -
trunk/gsdl/perllib/plugins/RogPlug.pm
r8166 r9853 223 223 sub read { 224 224 my $self = shift (@_); 225 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;225 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 226 226 227 227 my $filename = &util::filename_cat($base_dir, $file); … … 251 251 $doc_count++; 252 252 253 if (defined $maxdocs && $maxdocs =~ /\d/) 254 { 255 last if ($maxdocs >=0 && $doc_count >= $maxdocs); 256 } 257 253 last if ($maxdocs !=-1 && ($total_count+$doc_count) >= $maxdocs); 254 258 255 if (($doc_count % 10) == 0) 259 256 { -
trunk/gsdl/perllib/plugins/SplitPlug.pm
r9493 r9853 185 185 sub read { 186 186 my $self = shift (@_); 187 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;187 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 188 188 my $outhandle = $self->{'outhandle'}; 189 189 my $verbosity = $self->{'verbosity'}; -
trunk/gsdl/perllib/plugins/UnknownPlug.pm
r9706 r9853 175 175 sub read { 176 176 my $self = shift (@_); 177 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;177 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 178 178 179 179 my $outhandle = $self->{'outhandle'}; -
trunk/gsdl/perllib/plugins/W3ImgPlug.pm
r8121 r9853 416 416 # include directories 417 417 sub read { 418 my ($self, $pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = (@_);418 my ($self, $pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = (@_); 419 419 my ($doc_obj, $section, $filepath, $imgtag, $pos, $context, $numdocs, $tndir, $imgs); 420 420 # forward normal read (runs HTMLPlug if index_pages T) 421 my $ok = $self->SUPER::read($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs );421 my $ok = $self->SUPER::read($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli); 422 422 if ( ! $ok ) { return $ok } # what is this returning?? 423 423 -
trunk/gsdl/perllib/plugins/XMLPlug.pm
r9584 r9853 81 81 $self = shift (@_); 82 82 83 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $ gli) = @_;83 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 84 84 85 85 my $filename = $file; -
trunk/gsdl/perllib/plugins/ZIPPlug.pm
r9229 r9853 113 113 sub read { 114 114 my $self = shift (@_); 115 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs ) = @_;115 my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $total_count, $gli) = @_; 116 116 my $outhandle = $self->{'outhandle'}; 117 117 … … 150 150 chdir ($cwd) || die "Unable to change back to $cwd"; 151 151 152 my $numdocs = &plugin::read ($pluginfo, "", $tmpdir, $metadata, $processor, $maxdocs );152 my $numdocs = &plugin::read ($pluginfo, "", $tmpdir, $metadata, $processor, $maxdocs, $total_count, $gli); 153 153 &util::rm_r ($tmpdir); 154 154
Note:
See TracChangeset
for help on using the changeset viewer.