source: trunk/niupepa/perllib/plugins/NPPlug.pm@ 1554

Last change on this file since 1554 was 1554, checked in by sjboddie, 24 years ago

* empty log message *
  • Property svn:keywords set to Author Date Id Revision
File size: 13.2 KB
Line 
1###########################################################################
2#
3# NPPlug.pm -- Plugin for the niupepa collection
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# The niupepa collection has a file structure as follows:
27# Each niupepa series has its own directory containing some/all
28# of the following:
29
30# meta.txt - file contains metadata to be associated with all documents
31# in series.
32# *.issue - each issue should have a .issue file which may or may not
33# contain metadata to associate with the issue. also contains the list
34# of filenames that make up the issue (i.e. one for each page). meta.txt
35# is read before *.issue so metadata in .issue files will override that in
36# meta.txt
37# *.commentary - the commentary of the niupepa series (1 per series)
38# text/*.txt/htm - text/html files of issue pages (1 per page) -
39# text files are expected to be either .htm or .txt (lower case).
40# images/*.gif - image files of issue pages (1 per page)
41# abstracts/*.abstract - html files of issue abstracts (1 per issue)
42
43
44package NPPlug;
45
46use BasPlug;
47use util;
48
49sub BEGIN {
50 @ISA = ('BasPlug');
51}
52
53use strict;
54
55sub print_usage {
56 print STDERR "\nIncorrect options passed to NPPlug, check your collect.cfg configuration file\n";
57
58 print STDERR "\n usage: plugin NPPlug [options]\n\n";
59 print STDERR " options:\n";
60 print STDERR " -create_log Creates a log file containing info about which portions\n";
61 print STDERR " of papers are missing\n";
62 print STDERR " -logfile Path of logfile (defaults to ./log.txt)\n\n";
63}
64
65sub new {
66 my ($class) = @_;
67 my $self = new BasPlug ();
68
69 if (!parsargv::parse(\@_,
70 q^create_log^, \$self->{'create_log'},
71 q^logfile/.*/./log.txt^, \$self->{'logfile'})) {
72 &print_usage();
73 die "\n";
74 }
75
76 $self->{'commentaries'} = {};
77 $self->{'num_issues'} = 0;
78 $self->{'num_text_pages'} = 0;
79 $self->{'num_image_pages'} = 0;
80 $self->{'num_abstracts'} = 0;
81 return bless $self, $class;
82}
83
84sub is_recursive {
85 my $self = shift (@_);
86
87 return 0; # this is not a recursive plugin
88}
89
90sub begin {
91 my $self = shift (@_);
92 my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
93
94 # open up logfile
95 # note that we append to logfile as building will otherwise
96 # overwrite a file generated at import time
97 if ($self->{'create_log'}) {
98 open (LOGFILE, ">>$self->{'logfile'}") || die
99 "NPPlug.pm: Couldn't open log file $self->{'logfile'}\n";
100
101 my @time = localtime (time);
102
103 print LOGFILE "------------------------------------------------------------\n";
104 print LOGFILE "Log start $time[3]/$time[4]/" . (1900 + $time[5]) . "\n";
105 print LOGFILE "------------------------------------------------------------\n";
106 }
107}
108
109sub end {
110 my $self = shift (@_);
111
112 if ($self->{'create_log'}) {
113 my $numseries = 0;
114 my $numcommentaries = 0;
115
116 # record missing commentaries in logfile
117 foreach my $key (keys %{$self->{'commentaries'}}) {
118 $numseries ++;
119 if (!$self->{'commentaries'}->{$key}) {
120 print LOGFILE "Commentary missing for series $key\n";
121 } else {
122 $numcommentaries ++;
123 }
124 }
125
126 print LOGFILE "\n\nStatistics:\n";
127 print LOGFILE "series: $numseries\n";
128 print LOGFILE "commentaries: $numcommentaries\n";
129 print LOGFILE "issues: $self->{'num_issues'}\n";
130 print LOGFILE "abstracts: $self->{'num_abstracts'}\n";
131 print LOGFILE "text pages: $self->{'num_text_pages'}\n";
132 print LOGFILE "image pages: $self->{'num_image_pages'}\n";
133
134 # close logfile
135 close LOGFILE;
136 }
137}
138
139# return number of files processed, undef if can't process
140# Note that $base_dir might be "" and that $file might
141# include directories
142sub read {
143 my $self = shift (@_);
144 my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
145
146 my $filename = &util::filename_cat($base_dir, $file);
147
148 # we don't want RecPlug to go recursing into the text, images or
149 # abstracts directories
150 return 0 if (-d $filename && $filename =~ /(abstracts|images|text)/);
151
152 return 0 if $filename =~ /meta\.txt$/i;
153
154 return undef unless ($filename =~ /\.(issue|commentary)$/i && (-e $filename));
155
156 $self->{'verbosity'} = $processor->{'verbosity'};
157 print STDERR "NPPlug: processing $filename\n" if $self->{'verbosity'};
158
159 my ($dir);
160 ($dir, $file) = $filename =~ /^(.*?)([^\/\\]*)$/;
161 my ($issuekey) = $file =~ /^([^\.\_]*)/;
162
163 if ($filename =~ /\.commentary$/i) {
164 # commentary
165 return $self->process_commentary ($filename, $issuekey, $dir, $file, $processor);
166 }
167
168 my $numprocessed = 0;
169 $self->{'commentaries'}->{$issuekey} = 0 unless defined $self->{'commentaries'}->{$issuekey};
170
171 my ($abstractfile) = $file =~ /^([^\.]*)\.issue/i;
172 my $abstractOID = $abstractfile . "abstract";
173 $abstractfile .= ".abstract";
174 my $afile = &util::filename_cat($dir, "abstracts", $abstractfile);
175 my $hasabstract = 0;
176 if (-e $afile) {$hasabstract = 1;}
177 else {$abstractOID = undef;}
178
179 # process the .issue file
180 my %meta = ();
181 $numprocessed += $self->process_issue ($filename, $issuekey, $dir, $file,
182 $abstractOID, $processor, \%meta);
183
184 # process abstract of this issue
185 if ($hasabstract) {
186 $numprocessed += $self->process_abstract ($afile, $issuekey, $dir,
187 $abstractfile, $processor, \%meta);
188 }
189
190 return $numprocessed;
191}
192
193sub process_issue {
194 my $self = shift (@_);
195 my ($filename, $issuekey, $dir, $file, $abstract, $processor, $meta) = @_;
196
197 $self->{'num_issues'} ++;
198 my $doc_obj = new doc ($file, "indexed_doc");
199 my $topsection = $doc_obj->get_top_section();
200 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
201 $doc_obj->set_metadata_element ($topsection, 'Title', $self->get_title_string($file));
202 $doc_obj->set_metadata_element ($topsection, 'abstract', $abstract) if defined $abstract;
203 $self->set_main_metadata ($doc_obj, $dir);
204
205 open (ISSUEFILE, $filename) || die "couldn't open $filename\n";
206 my $line = "";
207 while (defined ($line = <ISSUEFILE>)) {
208 next unless $line =~ /\w/;
209 chomp $line;
210 if ($line =~ /^<([^>]*)>(.*?)\s*$/) {
211 $doc_obj->set_metadata_element ($topsection, $1, $2);
212 $meta->{$1} = $2;
213 } else {
214 # should be a section name
215 $line =~ s/^\s+//;
216 $line =~ s/\s+$//;
217 my ($pagenum) = $line =~ /(\d+)$/;
218 $doc_obj->create_named_section($pagenum);
219 $doc_obj->set_metadata_element($pagenum, 'Title', $pagenum);
220 $self->process_text ($dir, $line, $doc_obj, $pagenum);
221 $self->process_images ($dir, $line, $doc_obj, $pagenum);
222 }
223 }
224 $file =~ s/\.issue//i;
225 $doc_obj->set_OID ($file);
226 $processor->process ($doc_obj);
227 return 1;
228}
229
230sub process_images {
231 my $self = shift (@_);
232 my ($dir, $page, $doc_obj, $cursection) = @_;
233
234 my $filename = &util::filename_cat ($dir, "images", $page);
235
236 if (-e "$filename.gif") {
237 $self->{'num_image_pages'} ++;
238 $doc_obj->set_metadata_element ($cursection, "hasimg", "1");
239 $doc_obj->set_metadata_element ($cursection, "Source", $page);
240 $doc_obj->associate_file("$filename.gif", "$page.gif", "image/gif");
241 } elsif ($self->{'create_log'}) {
242 $doc_obj->set_metadata_element ($cursection, "hasimg", "0");
243 print LOGFILE "no fullsize image file for $page\n";
244 }
245
246 if (-e "${filename}_p.gif") {
247 $doc_obj->set_metadata_element ($cursection, "hasprevimg", "1");
248 $doc_obj->set_metadata_element ($cursection, "Source", $page);
249 $doc_obj->associate_file("${filename}_p.gif", "${page}_p.gif", "image/gif");
250 } elsif ($self->{'create_log'}) {
251 $doc_obj->set_metadata_element ($cursection, "hasprevimg", "0");
252 print LOGFILE "no preview image file for $page\n";
253 }
254}
255
256sub process_text {
257 my $self = shift (@_);
258 my ($dir, $page, $doc_obj, $cursection) = @_;
259 my ($text);
260
261 my $filename = &util::filename_cat ($dir, "text", $page);
262 if (-e "$filename.htm") {
263 $text = $self->get_text ("$filename.htm");
264 } elsif (-e "$filename.txt") {
265 $text = $self->get_text ("$filename.txt");
266 }
267
268 if (defined $text) {
269 $self->{'num_text_pages'} ++;
270 $doc_obj->add_text ($cursection, $text);
271 } elsif ($self->{'create_log'}) {
272 print LOGFILE "no txt or htm file for $page\n";
273 }
274}
275
276sub process_abstract {
277 my $self = shift (@_);
278 my ($filename, $issuekey, $dir, $file, $processor, $meta) = @_;
279
280 my $text = $self->get_text ($filename);
281 if (defined $text) {
282 # remove html header and stuff (assumes all future abstracts
283 # are same format as those for niupepa 01)
284 $text =~ s/^.*?<b>.*?<\/b>//is;
285
286 $self->{'num_abstracts'} ++;
287 my $doc_obj = new doc ($file, "indexed_doc");
288 my $cursection = $doc_obj->get_top_section();
289 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
290 $doc_obj->set_metadata_element ($cursection, 'Title', $self->get_title_string($file));
291 $self->set_main_metadata ($doc_obj, $dir);
292 map { $doc_obj->set_metadata_element ($cursection, $_, $meta->{$_}); } keys %$meta;
293 $doc_obj->set_metadata_element ($cursection, "doctype", "Description");
294 $doc_obj->add_text ($cursection, $text);
295 $file =~ s/\.abstract//i;
296 $doc_obj->set_OID ($file . "abstract");
297 $processor->process ($doc_obj);
298 return 1;
299 }
300
301 if ($self->{'create_log'}) {
302 print LOGFILE "abstract file $filename doesn't exist\n";
303 }
304 return 0;
305}
306
307sub process_commentary {
308 my $self = shift (@_);
309 my ($filename, $issuekey, $dir, $file, $processor) = @_;
310
311 my $text = $self->get_text ($filename);
312
313 return 0 unless defined $text;
314
315 $self->{'commentaries'}->{$issuekey} = 1;
316 my $doc_obj = new doc ($file, "indexed_doc");
317 my $cursection = $doc_obj->get_top_section();
318 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
319 $doc_obj->set_metadata_element ($cursection, 'Title', "_commentary_");
320 $self->set_main_metadata ($doc_obj, $dir);
321 $doc_obj->set_metadata_element ($cursection, "doctype", "Commentary");
322 $doc_obj->add_text ($cursection, $text);
323 $doc_obj->set_OID ($issuekey . "commentary");
324 $processor->process ($doc_obj);
325 return 1;
326}
327
328sub associate_cover_images {
329 my $self = shift (@_);
330 my ($doc_obj, $dir, $issuekey) = @_;
331
332 my $cover = &util::filename_cat ($dir, $issuekey);
333 $doc_obj->associate_file("${cover}on.gif", "${issuekey}/coveron.gif", "image/gif");
334 $doc_obj->associate_file("${cover}of.gif", "${issuekey}/coverof.gif", "image/gif");
335}
336
337# reads in the meta.txt file and sets metadata
338sub set_main_metadata {
339 my $self = shift (@_);
340 my ($doc_obj, $dir) = @_;
341
342 my $metafile = &util::filename_cat ($dir, "meta.txt");
343 return unless (-e $metafile);
344
345 if (!open (METAFILE, $metafile)) {
346 print STDERR "NPPlug: Couldn't read $metafile\n" if $self->{'verbosity'};
347 return;
348 }
349
350 my $cursection = $doc_obj->get_top_section();
351 my $line = "";
352 while (defined ($line = <METAFILE>)) {
353 next unless $line =~ /\w/;
354 chomp $line;
355 if ($line =~ /<([^>]*)>(.*)$/) {
356 # note we're using set_metadata_element (not add_metadata_element)
357 # this will override any previously set metadata of the same name
358 $doc_obj->set_metadata_element ($cursection, $1, $2);
359 } elsif ($self->{'verbosity'}) {
360 print STDERR "NPPlug: Badly formatted line in $metafile\n";
361 print STDERR "meta.txt lines should be formatted '<metaname>metavalue'\n";
362 }
363 }
364}
365
366sub get_text {
367 my $self = shift (@_);
368 my ($filename) = @_;
369
370 if (open (FILE, $filename)) {
371 my $text = "";
372 my $line = "";
373 if ($filename =~ /\.(htm|commentary|abstract)$/i) {
374 my $savedtext = "";
375 my $foundbody = 0;
376 while (defined ($line = <FILE>)) {
377 if ($line =~ s/.*?<body[^>]*>//i) {
378 $foundbody = 1;
379 }
380 $line =~ s/(<\/?html[^>]*>|<\/?head[^>]*>|<\/p>|<\/?font[^>]*>|<\/?body[^>]*>)//ig;
381 if ($foundbody) {
382 $text .= $line;
383 } else {
384 $savedtext .= $line;
385 }
386 }
387 close FILE;
388 if ($foundbody) {return $text;}
389 else {return $savedtext;}
390
391 } else {
392 while (defined ($line = <FILE>)) {
393 $line = "<p>\n" unless $line =~ /\w/;
394 $text .= $line;
395 }
396 close FILE;
397 return $text;
398 }
399
400 } else {
401 print STDERR "NPPlug: Warning: get_text() couldn't open $filename\n"
402 if $self->{'verbosity'};
403 return undef;
404 }
405}
406
407sub get_title_string {
408 my $self = shift (@_);
409 my ($filename) = @_;
410
411 $filename =~ s/\.(issue|abstract)$//i;
412 my ($series, $vol, $num) = split /\_/, $filename;
413 my $title = "";
414 $title .= "_vol_ $vol" if defined $vol && $vol =~ /\w/;
415 if (defined $num && $num =~ /\w/) {
416 $title .= ", " if defined $vol && $vol =~ /\w/;
417 $title .= "_num_ $num";
418 }
419 return $title;
420}
421
4221;
Note: See TracBrowser for help on using the repository browser.