source: trunk/niupepa/perllib/plugins/NPPlug.pm@ 1023

Last change on this file since 1023 was 1023, checked in by sjboddie, 24 years ago

Initial revision
  • Property svn:keywords set to Author Date Id Revision
File size: 13.3 KB
Line 
1###########################################################################
2#
3# NPPlug.pm -- Plugin for the niupepa collection
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# The niupepa collection has a file structure as follows:
27# Each niupepa series has its own directory containing some/all
28# of the following:
29
30# meta.txt - file contains metadata to be associated with all documents
31# in series.
32# *.issue - each issue should have a .issue file which may or may not
33# contain metadata to associate with the issue. also contains the list
34# of filenames that make up the issue (i.e. one for each page). meta.txt
35# is read before *.issue so metadata in .issue files will override that in
36# meta.txt
37# *.commentary - the commentary of the niupepa series (1 per series)
38# text/*.txt/htm - text/html files of issue pages (1 per page) -
39# text files are expected to be either .htm or .txt (lower case).
40# images/*.gif - image files of issue pages (1 per page)
41# abstracts/*.abstract - html files of issue abstracts (1 per issue)
42
43
44package NPPlug;
45
46use BasPlug;
47use util;
48
49sub BEGIN {
50 @ISA = ('BasPlug');
51}
52
53use strict;
54
55sub print_usage {
56 print STDERR "\nIncorrect options passed to NPPlug, check your collect.cfg configuration file\n";
57
58 print STDERR "\n usage: plugin NPPlug [options]\n\n";
59 print STDERR " options:\n";
60 print STDERR " -create_log Creates a log file containing info about which portions\n";
61 print STDERR " of papers are missing\n";
62 print STDERR " -logfile Path of logfile (defaults to ./log.txt)\n\n";
63}
64
65sub new {
66 my ($class) = @_;
67 my $self = new BasPlug ();
68
69 if (!parsargv::parse(\@_,
70 q^create_log^, \$self->{'create_log'},
71 q^logfile/.*/./log.txt^, \$self->{'logfile'})) {
72 &print_usage();
73 die "\n";
74 }
75
76 $self->{'commentaries'} = {};
77 $self->{'num_issues'} = 0;
78 $self->{'num_text_pages'} = 0;
79 $self->{'num_images_pages'} = 0;
80 $self->{'num_abstracts'} = 0;
81 return bless $self, $class;
82}
83
84sub is_recursive {
85 my $self = shift (@_);
86
87 return 0; # this is not a recursive plugin
88}
89
90sub begin {
91 my $self = shift (@_);
92 my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
93
94 # open up logfile
95 # note that we append to logfile as building will otherwise
96 # overwrite a file generated at import time
97 if ($self->{'create_log'}) {
98 open (LOGFILE, ">>$self->{'logfile'}") || die
99 "NPPlug.pm: Couldn't open log file $self->{'logfile'}\n";
100
101 my @time = localtime (time);
102
103 print LOGFILE "------------------------------------------------------------\n";
104 print LOGFILE "Log start $time[3]/$time[4]/" . (1900 + $time[5]) . "\n";
105 print LOGFILE "------------------------------------------------------------\n";
106 }
107}
108
109sub end {
110 my $self = shift (@_);
111
112 if ($self->{'create_log'}) {
113 my $numseries = 0;
114 my $numcommentaries = 0;
115
116 # record missing commentaries in logfile
117 foreach my $key (keys %{$self->{'commentaries'}}) {
118 $numseries ++;
119 if (!$self->{'commentaries'}->{$key}) {
120 print LOGFILE "Commentary missing for series $key\n";
121 } else {
122 $numcommentaries ++;
123 }
124 }
125
126 print LOGFILE "\n\nStatistics:\n";
127 print LOGFILE "series: $numseries\n";
128 print LOGFILE "commentaries: $numcommentaries\n";
129 print LOGFILE "issues: $self->{'num_issues'}\n";
130 print LOGFILE "abstracts: $self->{'num_abstracts'}\n";
131 print LOGFILE "text pages: $self->{'num_text_pages'}\n";
132 print LOGFILE "image pages: $self->{'num_image_pages'}\n";
133
134 # close logfile
135 close LOGFILE;
136 }
137}
138
139# return number of files processed, undef if can't process
140# Note that $base_dir might be "" and that $file might
141# include directories
142sub read {
143 my $self = shift (@_);
144 my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
145
146 my $filename = &util::filename_cat($base_dir, $file);
147
148 # we don't want RecPlug to go recursing into the text, images or
149 # abstracts directories
150 return 0 if (-d $filename && $filename =~ /(abstracts|images|text)/);
151
152 return 0 if $filename =~ /meta\.txt$/i;
153
154 return undef unless ($filename =~ /\.(issue|commentary)$/i && (-e $filename));
155
156 $self->{'verbosity'} = $processor->{'verbosity'};
157 print STDERR "NPPlug: processing $filename\n" if $self->{'verbosity'};
158
159 my ($dir);
160 ($dir, $file) = $filename =~ /^(.*?)([^\/\\]*)$/;
161 my ($issuekey) = $file =~ /^([^\.\_]*)/;
162
163 if ($filename =~ /\.commentary$/i) {
164 # commentary
165 return $self->process_commentary ($filename, $issuekey, $dir, $file, $processor);
166 }
167
168 my $numprocessed = 0;
169 $self->{'commentaries'}->{$issuekey} = 0 unless defined $self->{'commentaries'}->{$issuekey};
170
171 my ($abstractfile) = $file =~ /^([^\.]*)\.issue/i;
172 my $abstractOID = $abstractfile . "abstract";
173 $abstractfile .= ".abstract";
174 my $afile = &util::filename_cat($dir, "abstracts", $abstractfile);
175 my $hasabstract = 0;
176 if (-e $afile) {$hasabstract = 1;}
177 else {$abstractOID = undef;}
178
179 # process the .issue file
180 my %meta = ();
181 $numprocessed += $self->process_issue ($filename, $issuekey, $dir, $file,
182 $abstractOID, $processor, \%meta);
183
184 # process abstract of this issue
185 if ($hasabstract) {
186 $numprocessed += $self->process_abstract ($afile, $issuekey, $dir,
187 $abstractfile, $processor, \%meta);
188 }
189
190 return $numprocessed;
191}
192
193sub process_issue {
194 my $self = shift (@_);
195 my ($filename, $issuekey, $dir, $file, $abstract, $processor, $meta) = @_;
196
197 $self->{'num_issues'} ++;
198 my $doc_obj = new doc ($file, "indexed_doc");
199 my $topsection = $doc_obj->get_top_section();
200 my $cursection = $topsection;
201 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
202 $doc_obj->set_metadata_element ($topsection, 'Title', $self->get_title_string($file));
203 $doc_obj->set_metadata_element ($topsection, 'abstract', $abstract) if defined $abstract;
204 $self->set_main_metadata ($doc_obj, $dir);
205
206 open (ISSUEFILE, $filename) || die "couldn't open $filename\n";
207 my $line = "";
208 while (defined ($line = <ISSUEFILE>)) {
209 next unless $line =~ /\w/;
210 chomp $line;
211 if ($line =~ /^<([^>]*)>(.*?)\s*$/) {
212 $doc_obj->set_metadata_element ($topsection, $1, $2);
213 $meta->{$1} = $2;
214 } else {
215 # should be a section name
216 $line =~ s/^\s+//;
217 $line =~ s/\s+$//;
218 $cursection = $doc_obj->insert_section($doc_obj->get_end_child($topsection));
219 my ($pagenum) = $line =~ /(\d+)$/;
220 $doc_obj->set_metadata_element($cursection, 'Title', $pagenum);
221 $self->process_text ($dir, $line, $doc_obj, $cursection);
222 $self->process_images ($dir, $line, $doc_obj, $cursection);
223 }
224 }
225 $file =~ s/\.issue//i;
226 $doc_obj->set_OID ($file);
227 $processor->process ($doc_obj);
228 return 1;
229}
230
231sub process_images {
232 my $self = shift (@_);
233 my ($dir, $page, $doc_obj, $cursection) = @_;
234
235 my $filename = &util::filename_cat ($dir, "images", $page);
236
237 if (-e "$filename.gif") {
238 $self->{'num_image_pages'} ++;
239 $doc_obj->set_metadata_element ($cursection, "hasimg", "1");
240 $doc_obj->set_metadata_element ($cursection, "Source", $page);
241 $doc_obj->associate_file("$filename.gif", "$page.gif", "image/gif");
242 } elsif ($self->{'create_log'}) {
243 $doc_obj->set_metadata_element ($cursection, "hasimg", "0");
244 print LOGFILE "no fullsize image file for $page\n";
245 }
246
247 if (-e "${filename}_p.gif") {
248 $doc_obj->set_metadata_element ($cursection, "hasprevimg", "1");
249 $doc_obj->set_metadata_element ($cursection, "Source", $page);
250 $doc_obj->associate_file("${filename}_p.gif", "${page}_p.gif", "image/gif");
251 } elsif ($self->{'create_log'}) {
252 $doc_obj->set_metadata_element ($cursection, "hasprevimg", "0");
253 print LOGFILE "no preview image file for $page\n";
254 }
255}
256
257sub process_text {
258 my $self = shift (@_);
259 my ($dir, $page, $doc_obj, $cursection) = @_;
260 my ($text);
261
262 my $filename = &util::filename_cat ($dir, "text", $page);
263 if (-e "$filename.htm") {
264 $text = $self->get_text ("$filename.htm");
265 } elsif (-e "$filename.txt") {
266 $text = $self->get_text ("$filename.txt");
267 }
268
269 if (defined $text) {
270 $self->{'num_text_pages'} ++;
271 $doc_obj->add_text ($cursection, $text);
272 } elsif ($self->{'create_log'}) {
273 print LOGFILE "no txt or htm file for $page\n";
274 }
275}
276
277sub process_abstract {
278 my $self = shift (@_);
279 my ($filename, $issuekey, $dir, $file, $processor, $meta) = @_;
280
281 my $text = $self->get_text ($filename);
282 if (defined $text) {
283 # remove html header and stuff (assumes all future abstracts
284 # are same format as those for niupepa 01)
285 $text =~ s/^.*?<b>.*?<\/b>//is;
286
287 $self->{'num_abstracts'} ++;
288 my $doc_obj = new doc ($file, "indexed_doc");
289 my $cursection = $doc_obj->get_top_section();
290 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
291 $doc_obj->set_metadata_element ($cursection, 'Title', "_abstract_ " .
292 $self->get_title_string($file));
293 $self->set_main_metadata ($doc_obj, $dir);
294 map { $doc_obj->set_metadata_element ($cursection, $_, $meta->{$_}); } keys %$meta;
295 $doc_obj->set_metadata_element ($cursection, "doctype", "Description");
296 $doc_obj->add_text ($cursection, $text);
297 $file =~ s/\.abstract//i;
298 $doc_obj->set_OID ($file . "abstract");
299 $processor->process ($doc_obj);
300 return 1;
301 }
302
303 if ($self->{'create_log'}) {
304 print LOGFILE "abstract file $filename doesn't exist\n";
305 }
306 return 0;
307}
308
309sub process_commentary {
310 my $self = shift (@_);
311 my ($filename, $issuekey, $dir, $file, $processor) = @_;
312
313 my $text = $self->get_text ($filename);
314
315 return 0 unless defined $text;
316
317 $self->{'commentaries'}->{$issuekey} = 1;
318 my $doc_obj = new doc ($file, "indexed_doc");
319 my $cursection = $doc_obj->get_top_section();
320 $self->associate_cover_images ($doc_obj, $dir, $issuekey);
321 $doc_obj->set_metadata_element ($cursection, 'Title', "_commentary_");
322 $self->set_main_metadata ($doc_obj, $dir);
323 $doc_obj->set_metadata_element ($cursection, "doctype", "Commentary");
324 $doc_obj->add_text ($cursection, $text);
325 $doc_obj->set_OID ($issuekey . "commentary");
326 $processor->process ($doc_obj);
327 return 1;
328}
329
330sub associate_cover_images {
331 my $self = shift (@_);
332 my ($doc_obj, $dir, $issuekey) = @_;
333
334 my $cover = &util::filename_cat ($dir, $issuekey);
335 $doc_obj->associate_file("${cover}on.gif", "${issuekey}/coveron.gif", "image/gif");
336 $doc_obj->associate_file("${cover}of.gif", "${issuekey}/coverof.gif", "image/gif");
337}
338
339# reads in the meta.txt file and sets metadata
340sub set_main_metadata {
341 my $self = shift (@_);
342 my ($doc_obj, $dir) = @_;
343
344 my $metafile = &util::filename_cat ($dir, "meta.txt");
345 return unless (-e $metafile);
346
347 if (!open (METAFILE, $metafile)) {
348 print STDERR "NPPlug: Couldn't read $metafile\n" if $self->{'verbosity'};
349 return;
350 }
351
352 my $cursection = $doc_obj->get_top_section();
353 my $line = "";
354 while (defined ($line = <METAFILE>)) {
355 next unless $line =~ /\w/;
356 chomp $line;
357 if ($line =~ /<([^>]*)>(.*)$/) {
358 # note we're using set_metadata_element (not add_metadata_element)
359 # this will override any previously set metadata of the same name
360 $doc_obj->set_metadata_element ($cursection, $1, $2);
361 } elsif ($self->{'verbosity'}) {
362 print STDERR "NPPlug: Badly formatted line in $metafile\n";
363 print STDERR "meta.txt lines should be formatted '<metaname>metavalue'\n";
364 }
365 }
366}
367
368sub get_text {
369 my $self = shift (@_);
370 my ($filename) = @_;
371
372 if (open (FILE, $filename)) {
373 my $text = "";
374 my $line = "";
375 if ($filename =~ /\.(htm|commentary|abstract)$/i) {
376 my $savedtext = "";
377 my $foundbody = 0;
378 while (defined ($line = <FILE>)) {
379 if ($line =~ s/.*?<body[^>]*>//i) {
380 $foundbody = 1;
381 }
382 $line =~ s/(<\/?html[^>]*>|<\/?head[^>]*>|<\/p>|<\/?font[^>]*>|<\/?body[^>]*>)//ig;
383 if ($foundbody) {
384 $text .= $line;
385 } else {
386 $savedtext .= $line;
387 }
388 }
389 close FILE;
390 if ($foundbody) {return $text;}
391 else {return $savedtext;}
392
393 } else {
394 while (defined ($line = <FILE>)) {
395 $line = "<p>\n" unless $line =~ /\w/;
396 $text .= $line;
397 }
398 close FILE;
399 return $text;
400 }
401
402 } else {
403 print STDERR "NPPlug: Warning: get_text() couldn't open $filename\n"
404 if $self->{'verbosity'};
405 return undef;
406 }
407}
408
409sub get_title_string {
410 my $self = shift (@_);
411 my ($filename) = @_;
412
413 $filename =~ s/\.(issue|abstract)$//i;
414 my ($series, $vol, $num) = split /\_/, $filename;
415 my $title = "";
416 $title .= "_vol_ $vol" if defined $vol && $vol =~ /\w/;
417 if (defined $num && $num =~ /\w/) {
418 $title .= ": " if defined $vol && $vol =~ /\w/;
419 $title .= "_num_ $num";
420 }
421 return $title;
422}
423
4241;
Note: See TracBrowser for help on using the repository browser.