source: trunk/gsdl/perllib/mgppbuildproc.pm@ 3148

Last change on this file since 3148 was 3148, checked in by jrm21, 20 years ago

If a document has associated files that are also given a subdirectory, that subdirectory is now a subdir of the HASH... dir and not just a subdir of the assoc dir. If the given subdir starts with a '/' then it will be a subdir of the assoc dir.

Eg
"1.jpg" will be put into building/assoc/HASHnnn.dir/1.jpg
"0/1.jpg" will be put into building/assoc/HASHnnn.dir/0/1.jpg
"/foo/1.jpg" -> building/assoc/foo/1.jpg

This is because HTMLPlug was putting first 1000 assoc files into "assoc/0", next thousand into "assoc/1", etc. But this meant that if you updated a collection, files in "0" would get overridden.

The only change to HTMLPlug is to use the _httpdocimg_ macro instead of the _httpcolling_ so we link to the right place.

  • Property svn:keywords set to Author Date Id Revision
File size: 17.8 KB
Line 
1###########################################################################
2#
3# mgppbuildproc.pm --
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# This document processor outputs a document
27# for mgpp to process
28
29
30package mgppbuildproc;
31
32use classify;
33use doc;
34use docproc;
35use util;
36
37
38BEGIN {
39 @ISA = ('docproc');
40}
41
42
43sub new {
44 my ($class, $collection, $source_dir, $build_dir,
45 $verbosity, $outhandle) = @_;
46 my $self = new docproc ();
47
48 # outhandle is where all the debugging info goes
49 # output_handle is where the output of the plugins is piped
50 # to (i.e. mg, gdbm etc.)
51 $outhandle = STDERR unless defined $outhandle;
52
53 $self->{'collection'} = $collection;
54 $self->{'source_dir'} = $source_dir;
55 $self->{'build_dir'} = $build_dir;
56 $self->{'verbosity'} = $verbosity;
57 $self->{'classifiers'} = [];
58 $self->{'mode'} = "text";
59 $self->{'assocdir'} = $build_dir;
60 $self->{'dontgdbm'} = {};
61 $self->{'index'} = "text";
62 $self->{'indexexparr'} = [];
63 $self->{'output_handle'} = "STDOUT";
64 $self->{'num_docs'} = 0;
65 $self->{'num_sections'} = 0;
66 $self->{'num_bytes'} = 0;
67 $self->{'num_processed_bytes'} = 0;
68 $self->{'store_text'} = 1;
69 $self->{'outhandle'} = $outhandle;
70
71 #used by browse interface
72 $self->{'doclist'} = [];
73
74 $self->{'indexing_text'} = 0;
75
76 #new ones for mgpp
77 $self->{'dontindex'} = {};
78 $self->{'indexfieldmap'} = {};
79 $self->{'indexfields'} = {};
80 $self->{'strip_html'}=1;
81
82
83 return bless $self, $class;
84}
85
86sub reset {
87 my $self = shift (@_);
88
89 $self->{'num_docs'} = 0;
90 $self->{'num_sections'} = 0;
91 $self->{'num_processed_bytes'} = 0;
92 $self->{'num_bytes'} = 0;
93}
94
95sub get_num_docs {
96 my $self = shift (@_);
97
98 return $self->{'num_docs'};
99}
100
101sub get_num_sections {
102 my $self = shift (@_);
103
104 return $self->{'num_sections'};
105}
106
107# num_bytes is the actual number of bytes in the collection
108# this is normally the same as what's processed during text compression
109sub get_num_bytes {
110 my $self = shift (@_);
111
112 return $self->{'num_bytes'};
113}
114
115# num_processed_bytes is the number of bytes actually passed
116# to mgpp for the current index
117sub get_num_processed_bytes {
118 my $self = shift (@_);
119
120 return $self->{'num_processed_bytes'};
121}
122
123sub set_output_handle {
124 my $self = shift (@_);
125 my ($handle) = @_;
126
127 $self->{'output_handle'} = $handle;
128}
129
130sub set_mode {
131 my $self = shift (@_);
132 my ($mode) = @_;
133
134 $self->{'mode'} = $mode;
135}
136
137sub set_assocdir {
138 my $self = shift (@_);
139 my ($assocdir) = @_;
140
141 $self->{'assocdir'} = $assocdir;
142}
143
144sub set_dontgdbm {
145 my $self = shift (@_);
146 my ($dontgdbm) = @_;
147
148 $self->{'dontgdbm'} = $dontgdbm;
149}
150
151sub set_index {
152 my $self = shift (@_);
153 my ($index, $indexexparr) = @_;
154
155 $self->{'index'} = $index;
156 $self->{'indexexparr'} = $indexexparr if defined $indexexparr;
157}
158
159sub get_index {
160 my $self = shift (@_);
161
162 return $self->{'index'};
163}
164
165sub set_classifiers {
166 my $self = shift (@_);
167 my ($classifiers) = @_;
168
169 $self->{'classifiers'} = $classifiers;
170}
171
172sub set_indexing_text {
173 my $self = shift (@_);
174 my ($indexing_text) = @_;
175
176 $self->{'indexing_text'} = $indexing_text;
177}
178
179sub get_indexing_text {
180 my $self = shift (@_);
181
182 return $self->{'indexing_text'};
183}
184
185sub set_store_text {
186 my $self = shift (@_);
187 my ($store_text) = @_;
188
189 $self->{'store_text'} = $store_text;
190}
191
192sub get_doc_list {
193 my $self = shift(@_);
194
195 return @{$self->{'doclist'}};
196}
197
198sub set_indexfieldmap {
199 my $self = shift (@_);
200 my ($indexmap) = @_;
201
202 $self->{'indexfieldmap'} = $indexmap;
203}
204
205sub get_indexfieldmap {
206 my $self = shift (@_);
207
208 return $self->{'indexfieldmap'};
209}
210
211sub set_levels {
212 my $self = shift (@_);
213 my ($levels) = @_;
214
215 $self->{'levels'} = $levels;
216}
217
218sub set_strip_html {
219 my $self = shift (@_);
220 my ($strip) = @_;
221 $self->{'strip_html'}=$strip;
222}
223
224sub process {
225 my $self = shift (@_);
226 my $method = $self->{'mode'};
227
228 $self->$method(@_);
229}
230
231# use 'Paged' if document has no more than 2 levels
232# and each section at second level has a number for
233# Title metadata
234sub get_document_type {
235 my $self = shift (@_);
236 my ($doc_obj) = @_;
237
238 my $thistype = "VList";
239 my $childtype = "VList";
240 my $title;
241 my @tmp = ();
242
243 my $section = $doc_obj->get_top_section ();
244 my $first = 1;
245 while (defined $section) {
246 @tmp = split /\./, $section;
247 if (scalar(@tmp) > 1) {
248 return ($thistype, $childtype);
249 }
250 if (!$first) {
251 $title = $doc_obj->get_metadata_element ($section, "Title");
252 if (!defined $title || $title !~ /^\d+$/) {
253 return ($thistype, $childtype);
254 }
255 }
256 $first = 0;
257 $section = $doc_obj->get_next_section($section);
258 }
259 if ($doc_obj->get_text_length ($doc_obj->get_top_section())) {
260 $thistype = "Paged";
261 } else {
262 $thistype = "Invisible";
263 }
264 $childtype = "Paged";
265 return ($thistype, $childtype);
266}
267
268sub assoc_files {
269 my $self = shift (@_);
270 my ($doc_obj, $archivedir) = @_;
271 my ($afile);
272
273 foreach my $assoc_file (@{$doc_obj->get_assoc_files()}) {
274 # if assoc file starts with a slash, we put it relative to the assoc
275 # dir, otherwise it is relative to the HASH... directory
276 if ($assoc_file->[1] =~ m@^[/\\]@) {
277 $afile = &util::filename_cat($self->{'assocdir'},$assoc_file->[1]);
278 } else {
279 $afile = &util::filename_cat($self->{'assocdir'}, $archivedir, $assoc_file->[1]);
280 }
281 &util::hard_link ($assoc_file->[0], $afile);
282 }
283}
284
285sub infodb {
286 my $self = shift (@_);
287 my ($doc_obj, $filename) = @_;
288 my $handle = $self->{'output_handle'};
289
290 my $doctype = $doc_obj->get_doc_type();
291
292 # only output this document if it is one to be indexed
293 return if ($doctype ne "indexed_doc");
294
295 #if a Section level index is not built, the gdbm file should be at doc
296 #level not Section
297 my $docs_only = 1;
298 if ($self->{'levels'}->{'Section'}) {
299 $docs_only = 0;
300 }
301
302 my ($archivedir) = $filename =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
303 $archivedir = "" unless defined $archivedir;
304 $archivedir =~ s/\\/\//g;
305 $archivedir =~ s/^\/+//;
306 $archivedir =~ s/\/+$//;
307
308 $self->assoc_files ($doc_obj, $archivedir);
309
310 # this is another document
311 $self->{'num_docs'} += 1 unless ($doctype eq "classification");
312
313 # is this a paged or a hierarchical document
314 my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
315
316 my $section = $doc_obj->get_top_section ();
317 my $doc_OID = $doc_obj->get_OID();
318 my $first = 1;
319 my $url = "";
320 while (defined $section) {
321 # update a few statistics
322 $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
323 $self->{'num_sections'} += 1 unless ($doctype eq "classification");
324
325 # output the section name
326 if ($section eq "") { print $handle "[$doc_OID]\n"; }
327 else { print $handle "[$doc_OID.$section]\n"; }
328
329 # output the fact that this document is a document (unless doctype
330 # has been set to something else from within a plugin
331 my $dtype = $doc_obj->get_metadata_element ($section, "doctype");
332 if (!defined $dtype || $dtype !~ /\w/) {
333 print $handle "<doctype>doc\n";
334 }
335
336 # output whether this node contains text
337 if ($doc_obj->get_text_length($section) > 0) {
338 print $handle "<hastxt>1\n";
339 } else {
340 print $handle "<hastxt>0\n";
341 }
342
343 # output all the section metadata
344 my $metadata = $doc_obj->get_all_metadata ($section);
345 foreach my $pair (@$metadata) {
346 my ($field, $value) = (@$pair);
347
348 if ($field ne "Identifier" && $field !~ /^gsdl/ &&
349 defined $value && $value ne "") {
350
351 # escape problematic stuff
352 $value =~ s/\\/\\\\/g;
353 $value =~ s/\n/\\n/g;
354 $value =~ s/\r/\\r/g;
355
356 # special case for URL metadata
357 if ($field =~ /^URL$/i) {
358 $url .= "[$value]\n";
359 if ($section eq "") {$url .= "<section>$doc_OID\n";}
360 else {$url .= "<section>$doc_OID.$section\n";}
361 $url .= '-' x 70 . "\n";
362 }
363
364 if (!defined $self->{'dontgdbm'}->{$field}) {
365 print $handle "<$field>$value\n";
366 }
367 }
368 }
369
370 # output archivedir if at top level
371 if ($section eq $doc_obj->get_top_section()) {
372 print $handle "<archivedir>$archivedir\n";
373 }
374
375 # output document display type
376 if ($first) {
377 print $handle "<thistype>$thistype\n";
378 }
379
380 if (!$docs_only) {
381 # output a list of children
382 my $children = $doc_obj->get_children ($section);
383 if (scalar(@$children) > 0) {
384 print $handle "<childtype>$childtype\n";
385 print $handle "<contains>";
386 my $firstchild = 1;
387 foreach $child (@$children) {
388 print $handle ";" unless $firstchild;
389 $firstchild = 0;
390 if ($child =~ /^.*?\.(\d+)$/) {
391 print $handle "\".$1";
392 } else {
393 print $handle "\".$child";
394 }
395# if ($child eq "") { print $handle "$doc_OID"; }
396# elsif ($section eq "") { print $handle "$doc_OID.$child"; }
397# else { print $handle "$doc_OID.$section.$child"; }
398 }
399 print $handle "\n";
400 }
401 #output the matching doc number
402 print $handle "<docnum>$self->{'num_sections'}\n";
403
404 } # if (!$docs_only)
405 else { #docs only, doc num is num_docs not num_sections
406 # output the matching document number
407 print $handle "<docnum>$self->{'num_docs'}\n";
408 }
409
410 print $handle '-' x 70, "\n";
411
412
413 # output a database entry for the document number
414 if ($docs_only) {
415 print $handle "[$self->{'num_docs'}]\n";
416 print $handle "<section>$doc_OID\n";
417 }
418 else {
419 print $handle "[$self->{'num_sections'}]\n";
420 if ($section eq "") { print $handle "<section>$doc_OID\n"; }
421 else { print $handle "<section>$doc_OID.$section\n"; }
422 }
423 print $handle '-' x 70, "\n";
424
425 # output entry for url
426 if ($url ne "") {
427 print $handle $url;
428 }
429
430 $first = 0;
431 $section = $doc_obj->get_next_section($section);
432 last if ($docs_only); # if no sections wanted, only gdbm the docs
433 }
434
435 #add this document to the browse structure
436 push(@{$self->{'doclist'}},$doc_obj->get_OID())
437 unless ($doctype eq "classification");
438
439 # classify this document
440 &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
441
442}
443
444sub find_paragraphs {
445 $_[1] =~ s/(<p\b)/<Paragraph>$1/gi;
446}
447
448#this function strips the html tags from the doc if ($strip_html) and
449# if ($para) replaces <p> with <Paragraph> tags.
450# if both are false, the original text is returned
451#assumes that <pre> and </pre> have no spaces, and removes all < and > inside
452#these tags
453sub preprocess_text {
454 my $self = shift (@_);
455 my ($text, $strip_html, $para) = @_;
456
457 my ($outtext) = "";
458 if ($strip_html) {
459 while ($text =~ /<([^>]*)>/ && $text ne "") {
460
461 $tag = $1;
462 $outtext .= $`." "; #add everything before the matched tag
463 $text = $'; #everything after the matched tag
464 if ($para && $tag =~ /^\s*p\s/) {
465 $outtext .= "<Paragraph> ";
466 }
467 elsif ($tag =~ /^pre$/) { # a pre tag
468 $text =~ /<\/pre>/; # find the closing pre tag
469 my $tmp_text = $`; #everything before the closing pre tag
470 $text = $'; #everything after the </pre>
471 $tmp_text =~ s/[<>]//g; # remove all < and >
472 $outtext.= $tmp_text . " ";
473 }
474 }
475
476 $outtext .= $text; # add any remaining text
477 return $outtext;
478 } #if strip_html
479
480 if ($para) {
481 $text =~ s/(<p\b)/<Paragraph>$1/gi;
482 return $text;
483 }
484 return $text;
485}
486
487
488
489sub filter_text {
490 # $self->filter_text ($field, $new_text);
491 # don't want to do anything for this version, however,
492 # in a particular collection you might want to override
493 # this method to post-process certain fields depending on
494 # the field, or whether we are outputting it for indexing
495}
496
497sub text {
498 my $self = shift (@_);
499 my ($doc_obj) = @_;
500 my $handle = $self->{'output_handle'};
501 my $indexed_doc = 1;
502
503 # only output this document if it is one to be indexed
504 return if ($doc_obj->get_doc_type() ne "indexed_doc");
505
506 # see if this document belongs to this subcollection
507 foreach my $indexexp (@{$self->{'indexexparr'}}) {
508 $indexed_doc = 0;
509 my ($field, $exp, $options) = split /\//, $indexexp;
510 if (defined ($field) && defined ($exp)) {
511 my ($bool) = $field =~ /^(.)/;
512 $field =~ s/^.// if $bool eq '!';
513 if ($field =~ /^filename$/i) {
514 $field = $doc_obj->get_source_filename();
515 } else {
516 $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
517 }
518 next unless defined $field;
519 if ($bool eq '!') {
520 if ($options =~ /^i$/i) {
521 if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
522 } else {
523 if ($field !~ /$exp/) {$indexed_doc = 1; last;}
524 }
525 } else {
526 if ($options =~ /^i$/i) {
527 if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
528 } else {
529 if ($field =~ /$exp/) {$indexed_doc = 1; last;}
530 }
531 }
532 }
533 }
534
535 # this is another document
536 $self->{'num_docs'} += 1;
537
538 # get the parameters for the output
539 my ($fields) = $self->{'index'};
540
541 my ($sectiontag) = "";
542 if ($self->{'levels'}->{'Section'}) {
543 $sectiontag = "\n<Section>\n";
544 }
545 my ($paratag) = "";
546 if ($self->{'levels'}->{'Paragraph'}) {
547 $paratag = "<Paragraph>";
548 }
549 my $doc_section = 0; # just for this document
550 my $text = "<Document>\n";
551
552 # get the text for this document
553 my $section = $doc_obj->get_top_section();
554 while (defined $section) {
555 # update a few statistics
556 $doc_section++;
557 $self->{'num_sections'} += 1;
558 $text .= $sectiontag;
559
560 if ($indexed_doc) {
561 $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
562 foreach my $field (split (/,/, $fields)) {
563 # only deal with this field if it doesn't start with top or
564 # this is the first section
565 my $real_field = $field;
566 if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
567 my $new_text = "";
568 my $tmp_text = "";
569 if ($real_field eq "text") {
570 if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed
571 $new_text .= "<TX>\n";
572 $tmp_text .= $doc_obj->get_text ($section) if $self->{'store_text'};
573 $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, $self->{'levels'}->{'Paragraph'});
574
575 $new_text .= "$tmp_text</TX>\n";
576 if (!defined $self->{'indexfields'}->{'TextOnly'}) {
577 $self->{'indexfields'}->{'TextOnly'} = 1;
578 }
579 }
580 else { # leave html stuff in, and dont add Paragraph tags - never retrieve paras at the moment
581 $new_text .= $doc_obj->get_text ($section) if $self->{'store_text'};
582 #if ($self->{'levels'}->{'Paragraph'}) {
583 #$self->find_paragraphs($new_text);
584 #}
585 }
586 } else { # metadata field
587 if ($real_field eq "metadata") { # insert all metadata
588 #except gsdl stuff
589 my $shortname = "";
590 my $metadata = $doc_obj->get_all_metadata ($section);
591 foreach $pair (@$metadata) {
592 my ($mfield, $mvalue) = (@$pair);
593 # check fields here, maybe others dont want - change to use dontindex!!
594 if ($mfield ne "Identifier" && $mfield ne "classifytype" && $mfield ne "assocfilepath" &&
595 $mfield !~ /^gsdl/ && defined $mvalue && $mvalue ne "") {
596
597 if (defined $self->{'indexfieldmap'}->{$mfield}) {
598 $shortname = $self->{'indexfieldmap'}->{$mfield};
599 }
600 else {
601 $shortname = $self->create_shortname($mfield);
602 $self->{'indexfieldmap'}->{$mfield} = $shortname;
603 $self->{'indexfieldmap'}->{$shortname} = 1;
604 }
605 $new_text .= "$paratag<$shortname>$mvalue</$shortname>\n";
606 if (!defined $self->{'indexfields'}->{$mfield}) {
607 $self->{'indexfields'}->{$mfield} = 1;
608 }
609 }
610 }
611
612 }
613 else { #individual metadata specified
614 my $shortname="";
615 if (!defined $self->{'indexfields'}->{$real_field}) {
616 $self->{'indexfields'}->{$real_field} = 1;
617 }
618 if (defined $self->{'indexfieldmap'}->{$real_field}) {
619 $shortname = $self->{'indexfieldmap'}->{$real_field};
620 }
621 else {
622 $shortname = $self->create_shortname($real_field);
623 $self->{'indexfieldmap'}->{$real_field} = $shortname;
624 $self->{'indexfieldmap'}->{$shortname} = 1;
625 }
626 foreach $item (@{$doc_obj->get_metadata ($section, $real_field)}) {
627 $new_text .= "$paratag<$shortname>$item</$shortname>\n";
628 }
629 }
630
631 }
632
633 # filter the text
634 $self->filter_text ($field, $new_text);
635
636 $self->{'num_processed_bytes'} += length ($new_text);
637 $text .= "$new_text";
638 }
639 }
640 } # if (indexed_doc)
641
642 $section = $doc_obj->get_next_section($section);
643 } #while defined section
644 print $handle "$text\n</Document>\n";
645}
646
647sub create_shortname {
648 $self = shift(@_);
649
650 my ($realname) = @_;
651 #take the first two chars
652 my ($shortname) = $realname =~ /^(\w\w)/;
653 $shortname =~ tr/a-z/A-Z/;
654
655 #if already used, take the first and third letters and so on
656 $count = 1;
657 while (defined $self->{'indexfieldmap'}->{$shortname}) {
658 if ($realname =~ /^(\w).{$count}(\w)/) {
659 $shortname = "$1$2";
660 $count++;
661 $shortname =~ tr/a-z/A-Z/;
662
663 }
664 else {
665 $realname =~ s/^.//;
666 $count = 0;
667 }
668 }
669
670 return $shortname;
671}
672
6731;
674
Note: See TracBrowser for help on using the repository browser.