source: gsdl/trunk/perllib/plugins/MARCXMLPlugin.pm@ 16697

Last change on this file since 16697 was 16697, checked in by kjdon, 16 years ago

if marc mapping file cannot be located, print a warning about can't extract metadata but continue to process the file - get the record in the text so at least get something in the collection

  • Property svn:keywords set to Author Date Id Revision
File size: 13.0 KB
Line 
1###########################################################################
2#
3# MARCXMLPlugin.pm
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 2001 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# Processes MARCXML documents. Note that this plugin does no
27# syntax checking (though the XML::Parser module tests for
28# well-formedness).
29
30package MARCXMLPlugin;
31
32use ReadXMLFile;
33use ReadTextFile;
34use marcmapping;
35
36use strict;
37no strict 'refs'; # allow filehandles to be variables and viceversa
38
39sub BEGIN {
40 @MARCXMLPlugin::ISA = ('ReadXMLFile', 'ReadTextFile');
41}
42
43my $arguments = [{'name' => "metadata_mapping_file",
44 'desc' => "{MARCXMLPlugin.metadata_mapping_file}",
45 'type' => "string",
46 'deft' => "marctodc.txt",
47 'reqd' => "no" }];
48
49my $options = { 'name' => "MARCXMLPlugin",
50 'desc' => "{MARCXMLPlugin.desc}",
51 'abstract' => "no",
52 'inherits' => "yes",
53 'args' => $arguments
54 };
55
56sub new {
57 my ($class) = shift (@_);
58 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
59 push(@$pluginlist, $class);
60
61 push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
62 push(@{$hashArgOptLists->{"OptList"}},$options);
63
64 # we want to be able to use the textcat methods from ReadTextFile
65 # to get the language and encoding
66 new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists, 1);
67
68 my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
69
70 $self->{'content'} = "";
71 $self->{'xmlcontent'} = "";
72 $self->{'record_count'} = 1;
73 $self->{'language'} = "";
74 $self->{'encoding'} = "";
75 $self->{'marc_mapping'} = {};
76 $self->{'current_code'} = "";
77 $self->{'current_tag'} = "";
78 $self->{'current_element'} = "";
79 $self->{'metadata_mapping'} = undef;
80 $self->{'num_processed'} = 0;
81 $self->{'indent'} = 0;
82
83 return bless $self, $class;
84}
85
86
87
88sub get_doctype {
89 my $self = shift(@_);
90
91 return "collection";
92}
93
94
95sub init {
96 my $self = shift (@_);
97 my ($verbosity, $outhandle, $failhandle) = @_;
98
99 ## the mapping file has already been loaded
100 if (defined $self->{'metadata_mapping'} ){
101 $self->SUPER::init(@_);
102 return;
103 }
104
105 # read in the metadata mapping files
106 my $mm_files = &util::locate_config_files($self->{'metadata_mapping_file'});
107
108
109 if (scalar(@$mm_files)==0)
110 {
111 my $msg = "MARCXMLPlugin ERROR: Can't locate mapping file \"" .
112 $self->{'metadata_mapping_file'} . "\".\n " .
113 " No metadata will be extracted from MARCXML files.\n";
114
115 print $outhandle $msg;
116 print $failhandle $msg;
117 $self->{'metadata_mapping'} = undef;
118 # We pick up the error in process() if there is no $mm_file
119 # If we exit here, then pluginfo.pl will exit too!
120 }
121 else {
122 $self->{'metadata_mapping'} = &marcmapping::parse_marc_metadata_mapping($mm_files, $outhandle);
123 }
124
125
126 ##map { print STDERR $_."=>".$self->{'metadata_mapping'}->{$_}."\n"; } keys %{$self->{'metadata_mapping'}};
127
128 $self->SUPER::init(@_);
129}
130
131# Called for DOCTYPE declarations - use die to bail out if this doctype
132# is not meant for this plugin
133sub xml_doctype {
134 my $self = shift(@_);
135
136 my ($expat, $name, $sysid, $pubid, $internal) = @_;
137 return;
138
139}
140
141
142sub xml_start_document {
143 my $self = shift(@_);
144
145 my ($expat, $name, $sysid, $pubid, $internal) = @_;
146
147
148 my $file = $self->{'file'};
149 my $filename = $self->{'filename'};
150
151 my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
152
153 $self->{'language'} = $language;
154 $self->{'encoding'} = $encoding;
155 $self->{'element_count'} = 1;
156 $self->{'indent'} = 0;
157 my $outhandle = $self->{'outhandle'};
158 print $outhandle "MARCXMLPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
159 print STDERR "<Processing n='$self->{'file'}' p='MARCXMLPlugin'>\n" if $self->{'gli'};
160
161}
162
163sub xml_end_document {
164
165}
166
167sub xml_start_tag {
168 my $self = shift;
169 my $expat = shift;
170 my $element = shift;
171
172 my $text = $_;
173 my $escaped_text = $self->escape_text($_);
174
175 $self->{'current_element'} = $element;
176
177 ##get all atributes of this element and store it in a map name=>value
178 my %attr_map = ();
179 my $attrstring = $_;
180 while ($attrstring =~ /(\w+)=\"(\w+)\"/){
181 $attr_map{$1}=$2;
182 $attrstring = $'; #'
183 }
184
185
186 my $processor = $self->{'processor'};
187
188 ##create a new document for each record
189 if ($element eq "record") {
190 my $filename = $self->{'filename'};
191 my $language = $self->{'language'};
192 my $encoding = $self->{'encoding'};
193 my $file = $self->{'file'};
194 my $doc_obj = new doc($filename);
195 $doc_obj->set_OIDtype ($processor->{'OIDtype'}, $processor->{'OIDmetadata'});
196 $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
197 $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
198 my ($filemeta) = $file =~ /([^\\\/]+)$/;
199 $self->set_Source_metadata($doc_obj, $filemeta, $encoding);
200 $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$self->{'record_count'}");
201 if ($self->{'cover_image'}) {
202 $self->associate_cover_image($doc_obj, $filename);
203 }
204 $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
205 $doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "MARCXML");
206
207 my $outhandle = $self->{'outhandle'};
208 print $outhandle "Record $self->{'record_count'}\n" if $self->{'verbosity'} > 1;
209
210 $self->{'record_count'}++;
211 $self->{'doc_obj'} = $doc_obj;
212 $self->{'num_processed'}++;
213
214 }
215
216 ## get the marc code, for example 520
217 if ($element eq "datafield") {
218 if (defined $attr_map{'tag'} and $attr_map{'tag'} ne ""){
219 $self->{'current_tag'} = $attr_map{tag};
220 }
221 }
222
223
224 ## append the subcode to the marc code for example 520a or 520b
225 if ($element eq "subfield"){
226 if (defined $attr_map{'code'} and $attr_map{'code'} ne "" and $self->{'current_tag'} ne ""){
227 $self->{'current_code'} = $attr_map{'code'};
228 }
229 }
230
231 if ($element eq "record"){
232 $self->{'indent'} = 0;
233 $self->{'content'} = "";
234 $self->{'xmlcontent'} = "";
235 }
236 else {
237 if ($element ne "subfield"){
238 $self->{'indent'} = 1;
239 }
240 else{
241 $self->{'indent'} = 2;
242 }
243 }
244
245
246 if ($element eq "collection") {
247 # remember the full start tag for <collection ...>
248 # This is needed to wrap around each <record> when generating its associate MARCXML file
249
250 $self->{'xmlcollectiontag'} = $text;
251 }
252 else {
253 $self->{'content'} .= "<br/>" if ($element ne "record");
254 $self->{'content'} .= $self->calculate_indent($self->{'indent'}).$escaped_text;
255 $self->{'xmlcontent'} .= $text;
256 }
257
258}
259
260
261
262sub xml_end_tag {
263 my $self = shift(@_);
264 my ($expat, $element) = @_;
265
266 my $text = $_;
267 my $escaped_text = $self->escape_text($_);
268
269 if ($element eq "record" and defined $self->{'doc_obj'}) {
270 # process the document
271 my $processor = $self->{'processor'};
272 my $doc_obj = $self->{'doc_obj'};
273 $self->{'content'} .= "<br/>".$escaped_text;
274 $self->{'xmlcontent'} .= $text;
275
276
277 my $top_section = $doc_obj->get_top_section();
278
279 my $tmp_marcxml_filename = &util::get_tmp_filename("xml");
280 if (open (XMLOUT,">$tmp_marcxml_filename")) {
281
282 print XMLOUT "<?xml-stylesheet type=\"text/xsl\" href=\"MARC21slim2English.xsl\"?>\n";
283 my $xml_content = $self->{'xmlcontent'};
284
285 $xml_content = $self->{'xmlcollectiontag'}.$xml_content."</collection>";
286
287 print XMLOUT $xml_content;
288
289 close(XMLOUT);
290
291 $doc_obj->associate_file($tmp_marcxml_filename,"marcxml.xml","text/xml", $top_section);
292
293 # assicate xsl style file for presentation as HTML
294 my $xsl_filename = &util::filename_cat($ENV{'GSDLHOME'},"etc","MARC21slim2English.xsl");
295 $doc_obj->associate_file($xsl_filename,"MARC21slim2English.xsl","text/xml", $top_section);
296
297 }
298 else {
299 my $outhandle = $self->{'outhandle'};
300 print $outhandle "Warning: Unable for write out associated MARCXML file $tmp_marcxml_filename\n";
301 }
302
303 # include any metadata passed in from previous plugins
304 # note that this metadata is associated with the top level section
305
306 $self->extra_metadata ($doc_obj,
307 $doc_obj->get_top_section(),
308 $self->{'metadata'});
309
310
311 $self->add_OID($doc_obj, $self->{'record_count'});
312
313 $doc_obj->add_utf8_text($doc_obj->get_top_section(),$self->{'content'});
314 $processor->process($doc_obj);
315
316 ##clean up
317 $self->{'content'} = "";
318 $self->{'xmlcontent'} = "";
319 $self->{'doc_obj'} = undef;
320 return;
321 }
322
323 ## map the xmlmarc to gsdl metadata
324 if ($element eq "datafield" and defined $self->{'doc_obj'} and defined $self->{'marc_mapping'} and defined $self->{'metadata_mapping'}){
325 my $metadata_mapping = $self->{'metadata_mapping'};
326 my $marc_mapping = $self->{'marc_mapping'};
327 my $doc_obj = $self->{'doc_obj'};
328
329## print STDERR "**** Marc Record\n";
330## map { print STDERR $_."=>".$marc_mapping->{$_}."\n"; } keys %$marc_mapping;
331## print STDERR "**** Metadata Mapping\n";
332## map { print STDERR $_."=>".$metadata_mapping->{$_}."\n"; } keys %$metadata_mapping;
333
334
335 foreach my $marc_field (keys %$metadata_mapping){
336
337 ## test whether this field has subfield
338 my $subfield = undef;
339 if ($marc_field =~ /(\d\d\d)(?:\$|\^)?(\w)/){
340 $marc_field = $1;
341 $subfield = $2;
342 }
343
344 my $matched_field = $marc_mapping->{$marc_field};
345
346 if (defined $matched_field) {
347
348 my $meta_name = undef;
349 my $meta_value = undef;
350
351 if (defined $subfield){
352 $meta_name = $metadata_mapping->{$marc_field."\$".$subfield};
353
354 $meta_value = $matched_field->{$subfield};
355
356 if (!defined $meta_value) {
357 # record read in does not have the specified subfield
358 next;
359 }
360 }
361 else {
362 $meta_name = $metadata_mapping->{$marc_field};
363
364 # no subfield => get all the values
365 foreach my $value (sort keys %{$matched_field}) {
366 $meta_value .= $matched_field->{$value} ." ";
367 }
368
369 }
370
371 ## escape [ and ]
372 $meta_value =~ s/\[/\\\[/g;
373 $meta_value =~ s/\]/\\\]/g;
374 ##print STDERR "$meta_name=$meta_value\n";
375 $doc_obj->add_utf8_metadata($doc_obj->get_top_section(),$meta_name, $meta_value);
376
377 }
378
379 }
380
381 ##clean up
382 $self->{'marc_mapping'} = undef;
383 $self->{'current_tag'} = "";
384 }
385
386 if ($element eq "datafield"){
387 $self->{'indent'} = 1;
388 $self->{'content'} .= "<br/>".$self->calculate_indent($self->{'indent'}).$escaped_text;
389 $self->{'xmlcontent'} .= $text;
390 }
391 else{
392 $self->{'content'} .= $escaped_text;
393 $self->{'xmlcontent'} .= $text;
394 }
395
396}
397
398
399sub set_OID {
400 my $self = shift (@_);
401 my ($doc_obj, $record_number) = @_;
402
403 # first set it to generate hash value
404 $doc_obj->set_OID();
405
406 # then top it up with an "r" + record-number suffix
407 my $id = $doc_obj->get_OID();
408 $doc_obj->set_OID($id . "r" . $record_number);
409}
410
411sub xml_text {
412 my $self = shift(@_);
413 my ($expat) = @_;
414
415 my $text = $_;
416 my $escaped_text = $self->escape_text($_);
417
418 # protect against & in raw text file
419 $text =~ s/&/&amp;/g; # can't have & in raw form, even in 'raw' xml text
420
421 ## store the text of a marc code, for exapmle 520a=>A poem about....
422 if ($self->{'current_element'} eq "subfield" and $self->{'current_code'} ne "" and $_ ne "" ){
423 ##stored it in the marc_mapping
424
425 my $current_tag = $self->{'current_tag'};
426 my $current_code = $self->{'current_code'};
427
428 $self->{'marc_mapping'}->{$current_tag}->{$current_code} .= $_;
429
430 $self->{'current_code'} = "";
431 }
432
433 $self->{'content'} .= $escaped_text;
434 $self->{'xmlcontent'} .= $text;
435
436}
437
438sub calculate_indent{
439 my ($self,$num) = @_;
440
441 my $indent ="";
442
443 for (my $i=0; $i<$num;$i++){
444 $indent .= "&nbsp;&nbsp;&nbsp;&nbsp;";
445 }
446
447 return $indent;
448
449}
450
451sub escape_text {
452 my ($self,$text) = @_;
453 # special characters in the xml encoding
454 $text =~ s/&/&amp;/g; # this has to be first...
455 $text =~ s/</&lt;/g;
456 $text =~ s/>/&gt;/g;
457 $text =~ s/\"/&quot;/g;
458
459 return $text;
460}
461
462
463sub unescape_text {
464 my ($self,$text) = @_;
465 # special characters in the xml encoding
466 $text =~ s/&lt;/</g;
467 $text =~ s/&gt;/>/g;
468 $text =~ s/&quot;/\"/g;
469
470 $text =~ s/&/&amp;/g; # can't have & in raw form, even in unescaped xml!
471
472 return $text;
473}
474
475
4761;
477
478
Note: See TracBrowser for help on using the repository browser.