source: trunk/gsdl/perllib/mgbuildproc.pm@ 3148

Last change on this file since 3148 was 3148, checked in by jrm21, 20 years ago

If a document has associated files that are also given a subdirectory, that subdirectory is now a subdir of the HASH... dir and not just a subdir of the assoc dir. If the given subdir starts with a '/' then it will be a subdir of the assoc dir.

Eg
"1.jpg" will be put into building/assoc/HASHnnn.dir/1.jpg
"0/1.jpg" will be put into building/assoc/HASHnnn.dir/0/1.jpg
"/foo/1.jpg" -> building/assoc/foo/1.jpg

This is because HTMLPlug was putting first 1000 assoc files into "assoc/0", next thousand into "assoc/1", etc. But this meant that if you updated a collection, files in "0" would get overridden.

The only change to HTMLPlug is to use the _httpdocimg_ macro instead of the _httpcolling_ so we link to the right place.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 12.9 KB
Line 
1###########################################################################
2#
3# mgbuildproc.pm --
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# This document processor outputs a document
27# for mg to process
28
29
30package mgbuildproc;
31
32use classify;
33use doc;
34use docproc;
35use util;
36
37BEGIN {
38 @ISA = ('docproc');
39}
40
41
42sub new {
43 my ($class, $collection, $source_dir, $build_dir,
44 $verbosity, $outhandle) = @_;
45 my $self = new docproc ();
46
47 # outhandle is where all the debugging info goes
48 # output_handle is where the output of the plugins is piped
49 # to (i.e. mg, gdbm etc.)
50 $outhandle = STDERR unless defined $outhandle;
51
52 $self->{'collection'} = $collection;
53 $self->{'source_dir'} = $source_dir;
54 $self->{'build_dir'} = $build_dir;
55 $self->{'verbosity'} = $verbosity;
56 $self->{'classifiers'} = [];
57 $self->{'mode'} = "text";
58 $self->{'assocdir'} = $build_dir;
59 $self->{'dontgdbm'} = {};
60 $self->{'index'} = "section:text";
61 $self->{'indexexparr'} = [];
62 $self->{'output_handle'} = "STDOUT";
63 $self->{'num_docs'} = 0;
64 $self->{'num_sections'} = 0;
65 $self->{'num_bytes'} = 0;
66 $self->{'num_processed_bytes'} = 0;
67 $self->{'store_text'} = 1;
68 $self->{'outhandle'} = $outhandle;
69
70 #used by browse interface
71 $self->{'doclist'} = [];
72
73 $self->{'indexing_text'} = 0;
74
75 return bless $self, $class;
76}
77
78sub reset {
79 my $self = shift (@_);
80
81 $self->{'num_docs'} = 0;
82 $self->{'num_sections'} = 0;
83 $self->{'num_processed_bytes'} = 0;
84 $self->{'num_bytes'} = 0;
85}
86
87sub get_num_docs {
88 my $self = shift (@_);
89
90 return $self->{'num_docs'};
91}
92
93sub get_num_sections {
94 my $self = shift (@_);
95
96 return $self->{'num_sections'};
97}
98
99# num_bytes is the actual number of bytes in the collection
100# this is normally the same as what's processed during text compression
101sub get_num_bytes {
102 my $self = shift (@_);
103
104 return $self->{'num_bytes'};
105}
106
107# num_processed_bytes is the number of bytes actually passed
108# to mg for the current index
109sub get_num_processed_bytes {
110 my $self = shift (@_);
111
112 return $self->{'num_processed_bytes'};
113}
114
115sub set_output_handle {
116 my $self = shift (@_);
117 my ($handle) = @_;
118
119 $self->{'output_handle'} = $handle;
120}
121
122sub set_mode {
123 my $self = shift (@_);
124 my ($mode) = @_;
125
126 $self->{'mode'} = $mode;
127}
128
129sub set_assocdir {
130 my $self = shift (@_);
131 my ($assocdir) = @_;
132
133 $self->{'assocdir'} = $assocdir;
134}
135
136sub set_dontgdbm {
137 my $self = shift (@_);
138 my ($dontgdbm) = @_;
139
140 $self->{'dontgdbm'} = $dontgdbm;
141}
142
143sub set_index {
144 my $self = shift (@_);
145 my ($index, $indexexparr) = @_;
146
147 $self->{'index'} = $index;
148 $self->{'indexexparr'} = $indexexparr if defined $indexexparr;
149}
150
151sub get_index {
152 my $self = shift (@_);
153
154 return $self->{'index'};
155}
156
157sub set_classifiers {
158 my $self = shift (@_);
159 my ($classifiers) = @_;
160
161 $self->{'classifiers'} = $classifiers;
162}
163
164sub set_indexing_text {
165 my $self = shift (@_);
166 my ($indexing_text) = @_;
167
168 $self->{'indexing_text'} = $indexing_text;
169}
170
171sub get_indexing_text {
172 my $self = shift (@_);
173
174 return $self->{'indexing_text'};
175}
176
177sub set_store_text {
178 my $self = shift (@_);
179 my ($store_text) = @_;
180
181 $self->{'store_text'} = $store_text;
182}
183
184sub get_doc_list {
185 my $self = shift(@_);
186
187 return @{$self->{'doclist'}};
188}
189
190
191sub process {
192 my $self = shift (@_);
193 my $method = $self->{'mode'};
194
195 $self->$method(@_);
196}
197
198# use 'Paged' if document has no more than 2 levels
199# and each section at second level has a number for
200# Title metadata
201sub get_document_type {
202 my $self = shift (@_);
203 my ($doc_obj) = @_;
204
205 my $thistype = "VList";
206 my $childtype = "VList";
207 my $title;
208 my @tmp = ();
209
210 my $section = $doc_obj->get_top_section ();
211 my $first = 1;
212 while (defined $section) {
213 @tmp = split /\./, $section;
214 if (scalar(@tmp) > 1) {
215 return ($thistype, $childtype);
216 }
217 if (!$first) {
218 $title = $doc_obj->get_metadata_element ($section, "Title");
219 if (!defined $title || $title !~ /^\d+$/) {
220 return ($thistype, $childtype);
221 }
222 }
223 $first = 0;
224 $section = $doc_obj->get_next_section($section);
225 }
226 if ($doc_obj->get_text_length ($doc_obj->get_top_section())) {
227 $thistype = "Paged";
228 } else {
229 $thistype = "Invisible";
230 }
231 $childtype = "Paged";
232 return ($thistype, $childtype);
233}
234
235sub assoc_files {
236 my $self = shift (@_);
237 my ($doc_obj, $archivedir) = @_;
238 my ($afile);
239
240 foreach my $assoc_file (@{$doc_obj->get_assoc_files()}) {
241 # if assoc file starts with a slash, we put it relative to the assoc
242 # dir, otherwise it is relative to the HASH... directory
243 if ($assoc_file->[1] =~ m@^[/\\]@) {
244 $afile = &util::filename_cat($self->{'assocdir'},$assoc_file->[1]);
245 } else {
246 $afile = &util::filename_cat($self->{'assocdir'}, $archivedir, $assoc_file->[1]);
247 }
248 &util::hard_link ($assoc_file->[0], $afile);
249 }
250}
251
252sub infodb {
253 my $self = shift (@_);
254 my ($doc_obj, $filename) = @_;
255 my $handle = $self->{'output_handle'};
256# $handle = "main::STDOUT";
257
258 my $doctype = $doc_obj->get_doc_type();
259
260 # only output this document if it is one to be indexed
261 return if ($doctype ne "indexed_doc");
262
263 my ($archivedir) = $filename =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
264 $archivedir = "" unless defined $archivedir;
265 $archivedir =~ s/\\/\//g;
266 $archivedir =~ s/^\/+//;
267 $archivedir =~ s/\/+$//;
268
269 $self->assoc_files ($doc_obj, $archivedir);
270
271 # this is another document
272 $self->{'num_docs'} += 1 unless ($doctype eq "classification");
273
274 # is this a paged or a hierarchical document
275 my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
276
277 my $section = $doc_obj->get_top_section ();
278 my $doc_OID = $doc_obj->get_OID();
279 my $first = 1;
280 my $url = "";
281 while (defined $section) {
282 # update a few statistics
283 $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
284 $self->{'num_sections'} += 1 unless ($doctype eq "classification");
285
286 # output the section name
287 if ($section eq "") { print $handle "[$doc_OID]\n"; }
288 else { print $handle "[$doc_OID.$section]\n"; }
289
290 # output the fact that this document is a document (unless doctype
291 # has been set to something else from within a plugin
292 my $dtype = $doc_obj->get_metadata_element ($section, "doctype");
293 if (!defined $dtype || $dtype !~ /\w/) {
294 print $handle "<doctype>doc\n";
295 }
296
297 # output whether this node contains text
298 if ($doc_obj->get_text_length($section) > 0) {
299 print $handle "<hastxt>1\n";
300 } else {
301 print $handle "<hastxt>0\n";
302 }
303
304 # output all the section metadata
305 my $metadata = $doc_obj->get_all_metadata ($section);
306 foreach my $pair (@$metadata) {
307 my ($field, $value) = (@$pair);
308
309 if ($field ne "Identifier" && $field !~ /^gsdl/ &&
310 defined $value && $value ne "") {
311
312 # escape problematic stuff
313 $value =~ s/\\/\\\\/g;
314 $value =~ s/\n/\\n/g;
315 $value =~ s/\r/\\r/g;
316
317 # special case for URL metadata
318 if ($field =~ /^URL$/i) {
319 $url .= "[$value]\n";
320 if ($section eq "") {$url .= "<section>$doc_OID\n";}
321 else {$url .= "<section>$doc_OID.$section\n";}
322 $url .= '-' x 70 . "\n";
323 }
324
325 if (!defined $self->{'dontgdbm'}->{$field}) {
326 print $handle "<$field>$value\n";
327 }
328 }
329 }
330
331 # output archivedir if at top level
332 if ($section eq $doc_obj->get_top_section()) {
333 print $handle "<archivedir>$archivedir\n";
334 }
335
336 # output document display type
337 if ($first) {
338 print $handle "<thistype>$thistype\n";
339 }
340
341 # output a list of children
342 my $children = $doc_obj->get_children ($section);
343 if (scalar(@$children) > 0) {
344 print $handle "<childtype>$childtype\n";
345 print $handle "<contains>";
346 my $firstchild = 1;
347 foreach my $child (@$children) {
348 print $handle ";" unless $firstchild;
349 $firstchild = 0;
350 if ($child =~ /^.*?\.(\d+)$/) {
351 print $handle "\".$1";
352 } else {
353 print $handle "\".$child";
354 }
355# if ($child eq "") { print $handle "$doc_OID"; }
356# elsif ($section eq "") { print $handle "$doc_OID.$child"; }
357# else { print $handle "$doc_OID.$section.$child"; }
358 }
359 print $handle "\n";
360 }
361
362 # output the matching document number
363 print $handle "<docnum>$self->{'num_sections'}\n";
364
365 print $handle '-' x 70, "\n";
366
367
368 # output a database entry for the document number
369 print $handle "[$self->{'num_sections'}]\n";
370 if ($section eq "") { print $handle "<section>$doc_OID\n"; }
371 else { print $handle "<section>$doc_OID.$section\n"; }
372 print $handle '-' x 70, "\n";
373
374 # output entry for url
375 if ($url ne "") {
376 print $handle $url;
377 }
378
379 $first = 0;
380 $section = $doc_obj->get_next_section($section);
381 }
382
383 #add this document to the browse structure
384 push(@{$self->{'doclist'}},$doc_obj->get_OID())
385 unless ($doctype eq "classification");
386
387 # classify this document
388 &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
389
390}
391
392sub find_paragraphs {
393 $_[1] =~ s/(<p\b)/\cC$1/gi;
394}
395
396sub filter_text {
397 # $self->filter_text ($field, $new_text);
398 # don't want to do anything for this version, however,
399 # in a particular collection you might want to override
400 # this method to post-process certain fields depending on
401 # the field, or whether we are outputting it for indexing
402}
403
404sub text {
405 my $self = shift (@_);
406 my ($doc_obj) = @_;
407 my $handle = $self->{'output_handle'};
408 my $indexed_doc = 1;
409
410 # only output this document if it is one to be indexed
411 return if ($doc_obj->get_doc_type() ne "indexed_doc");
412
413 # see if this document belongs to this subcollection
414 foreach my $indexexp (@{$self->{'indexexparr'}}) {
415 $indexed_doc = 0;
416 my ($field, $exp, $options) = split /\//, $indexexp;
417 if (defined ($field) && defined ($exp)) {
418 my ($bool) = $field =~ /^(.)/;
419 $field =~ s/^.// if $bool eq '!';
420 if ($field =~ /^filename$/i) {
421 $field = $doc_obj->get_source_filename();
422 } else {
423 $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
424 }
425 next unless defined $field;
426 if ($bool eq '!') {
427 if ($options =~ /^i$/i) {
428 if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
429 } else {
430 if ($field !~ /$exp/) {$indexed_doc = 1; last;}
431 }
432 } else {
433 if ($options =~ /^i$/i) {
434 if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
435 } else {
436 if ($field =~ /$exp/) {$indexed_doc = 1; last;}
437 }
438 }
439 }
440 }
441
442 # this is another document
443 $self->{'num_docs'} += 1;
444
445 # get the parameters for the output
446 my ($level, $fields) = split (/:/, $self->{'index'});
447 $fields =~ s/\ball\b/Title,Creator,text/;
448 $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;
449
450 my $doc_section = 0; # just for this document
451 my $text = "";
452 my $text_extra = "";
453
454 # get the text for this document
455 my $section = $doc_obj->get_top_section();
456 while (defined $section) {
457 # update a few statistics
458 $doc_section++;
459 $self->{'num_sections'} += 1;
460
461 if ($indexed_doc) {
462 $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
463 foreach my $field (split (/,/, $fields)) {
464 # only deal with this field if it doesn't start with top or
465 # this is the first section
466 my $real_field = $field;
467 if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
468 my $new_text = "";
469 if ($real_field eq "text") {
470 $new_text = $doc_obj->get_text ($section) if $self->{'store_text'};
471 $self->{'num_processed_bytes'} += length ($new_text);
472 $new_text =~ s/[\cB\cC]//g;
473 $self->find_paragraphs($new_text);
474
475 } else {
476 my $first = 1;
477 foreach $meta (@{$doc_obj->get_metadata ($section, $real_field)}) {
478 $meta =~ s/[\cB\cC]//g;
479 $self->{'num_processed_bytes'} += length ($meta);
480 $new_text .= "\cC" unless $first;
481 $new_text .= $meta if $self->{'store_text'};
482 $first = 0;
483 }
484 }
485
486 # filter the text
487 $self->filter_text ($field, $new_text);
488
489 $text .= "$new_text\cC";
490 }
491 }
492 }
493
494 if ($level eq "document") { $text_extra .= "\cB"; }
495 else { $text .= "\cB"; }
496
497 $section = $doc_obj->get_next_section($section);
498 }
499
500 print $handle "$text$text_extra";
501}
502
5031;
504
Note: See TracBrowser for help on using the repository browser.