source: gsdl/trunk/perllib/downloaders/OAIDownload.pm@ 14941

Last change on this file since 14941 was 14941, checked in by dmn, 16 years ago

updates for qualified DC

  • Property svn:keywords set to Author Date Id Revision
File size: 11.3 KB
Line 
1###########################################################################
2#
3# WebDownload.pm -- base class for all the import plugins
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package OAIDownload;
27
28eval {require bytes};
29
30# suppress the annoying "subroutine redefined" warning that various
31# plugins cause under perl 5.6
32$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
33
34use strict;
35
36use WgetDownload;
37use XMLParser;
38
39use POSIX qw(tmpnam);
40use util;
41
42sub BEGIN {
43 @OAIDownload::ISA = ('WgetDownload');
44}
45
46my $arguments =
47 [ { 'name' => "url",
48 'disp' => "{OAIDownload.url_disp}",
49 'desc' => "{OAIDownload.url}",
50 'type' => "string",
51 'reqd' => "yes"},
52 { 'name' => "metadata_prefix",
53 'disp' => "{OAIDownload.metadata_prefix_disp}",
54 'desc' => "{OAIDownload.metadata_prefix}",
55 'type' => "string",
56 'deft' => "oai_dc",
57 'reqd' => "no"},
58 { 'name' => "set",
59 'disp' => "{OAIDownload.set_disp}",
60 'desc' => "{OAIDownload.set}",
61 'type' => "string",
62 'reqd' => "no"},
63 { 'name' => "get_doc",
64 'disp' => "{OAIDownload.get_doc_disp}",
65 'desc' => "{OAIDownload.get_doc}",
66 'type' => "flag",
67 'reqd' => "no"},
68 { 'name' => "max_records",
69 'disp' => "{OAIDownload.max_records_disp}",
70 'desc' => "{OAIDownload.max_records}",
71 'type' => "int",
72 'deft' => "500",
73 'range' => "1,",
74 'reqd' => "no"} ];
75
76my $options = { 'name' => "OAIDownload",
77 'desc' => "{OAIDownload.desc}",
78 'abstract' => "no",
79 'inherits' => "yes",
80 'args' => $arguments };
81
82my $self;
83
84my $strWgetOptions="";
85
86sub new
87{
88 my ($class) = shift (@_);
89 my ($getlist,$inputargs,$hashArgOptLists) = @_;
90 push(@$getlist, $class);
91
92 if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
93 if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
94
95 $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
96
97 if ($self->{'info_only'}) {
98 # don't worry about any options etc
99 return bless $self, $class;
100 }
101
102 my $parser = new XML::Parser('Style' => 'Stream',
103 'Handlers' => {'Char' => \&Char,
104 'Start' => \&OAI_StartTag,
105 'End' => \&OAI_EndTag
106 });
107 $self->{'parser'} = $parser;
108
109 # make sure the tmp directory that we will use later exists
110 my $tmp_dir = "$ENV{GSDLHOME}/tmp";
111 if (! -e $tmp_dir) {
112 &util::mk_dir($tmp_dir);
113 }
114
115 return bless $self, $class;
116}
117
118sub download
119{
120 my ($self) = shift (@_);
121 my ($hashGeneralOptions) = @_;
122
123 print STDERR "here2";
124
125 $strWgetOptions = $self->getWgetOptions();
126 my $cmdWget = $strWgetOptions;
127
128 my $strOutputDir ="";
129 $strOutputDir = $hashGeneralOptions->{"cache_dir"};
130 my $strBasURL = $self->{'url'};
131 my $intMaxRecords = $self->{'max_records'};
132 my $blnDownloadDoc = $self->{'get_doc'};
133
134 print STDERR "<<Defined Maximum>>\n";
135
136 my $strIDs = $self->getOAIIDs($strBasURL);
137
138 if($strIDs eq "")
139 {
140 print STDERR "Error: No ID being found\n";
141 return 0;
142 }
143 my $aryIDs = $self->parseOAIIDs($strIDs);
144 my $intIDs = 0;
145 if($self->{'max_records'} < scalar(@$aryIDs))
146 {
147 $intIDs = $self->{'max_records'};
148 }
149 else
150 {
151 $intIDs = scalar(@$aryIDs);
152 }
153 print STDERR "<<Total number of record(s):$intIDs>>\n";
154
155 $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
156
157 my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
158 &util::rm($tmp_file);
159
160 return 1;
161}
162
163sub getOAIIDs
164{
165 my ($self,$strBasURL) = @_;
166 my ($cmdWget);
167
168 my $wgetOptions = $self->getWgetOptions();
169
170 $cmdWget = $wgetOptions;
171
172 print STDERR "Gathering OAI identifiers.....\n";
173
174 my $metadata_prefix = $self->{'metadata_prefix'};
175 $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=$metadata_prefix";
176
177
178 # if $set specified, add it in to URL
179 my $set = $self->{'set'};
180 $cmdWget .= "&set=$set" if ($set ne "");
181
182 $cmdWget .= "\" ";
183
184
185 my $strIDs = $self->useWget($cmdWget);
186
187 if (!defined $strIDs or $strIDs eq "" ){
188 print STDERR "Server information is unavailable.\n";
189 print STDERR "<<Finished>>\n";
190 return;
191 }
192
193 print STDERR "<<Download Information>>\n";
194
195 $self->parse_xml($strIDs);
196
197 return $strIDs;
198}
199
200sub parseOAIIDs
201{
202 my ($self,$strIDs) = @_;
203
204 print STDERR "Parsing OAI identifiers.....\n";
205 $strIDs =~ s/^.*?<identifier>/<identifier>/s;
206 $strIDs =~ s/^(.*<\/identifier>).*$/$1/s;
207
208 my @aryIDs = ();
209
210 while ($strIDs =~ m/<identifier>(.*?)<\/identifier>(.*)$/s)
211 {
212 $strIDs = $2;
213 push(@aryIDs,$1);
214 }
215
216 return \@aryIDs;
217}
218
219sub dirFileSplit
220{
221 my ($self,$strFile) = @_;
222
223 my @aryDirs = split("[/\]",$strFile);
224
225 my $strLocalFile = pop(@aryDirs);
226 my $strSubDirs = join("/",@aryDirs);
227
228 return ($strSubDirs,$strLocalFile);
229}
230
231sub getOAIDoc
232{
233 my ($self,$strRecord, $strSubDirPath) = @_;
234
235 print STDERR "Gathering source documents.....\n";
236 # look out for identifier tag in metadata section
237
238 if ($strRecord =~ m/<metadata>(.*)<\/metadata>/s)
239 {
240 my $strMetaTag = $1;
241
242 if ($strMetaTag =~ m/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>/s)
243 {
244 my $strDocURL = $2;
245
246 my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
247
248 my $strSoureDirPath ="";
249
250 $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
251
252 &util::mk_dir($strSoureDirPath) if (!-e "$strSoureDirPath");
253
254 my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
255
256 my $wget_cmd = $strWgetOptions." -q -O \"$strFullDocFilePath\" \"$strDocURL\"";
257
258 my $strResponse = $self->useWget($wget_cmd,1);
259
260 if($strResponse ne "")
261 {
262 print STDERR "Error occured while retriving OAI souce documents: $strResponse\n";
263 exit(-1);
264 }
265
266 $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$strDocURL<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigURL>$strDocURL<\/OrigURL>\n <identifier>srcdocs\/$strDocFile<\/identifier>$4<\/metadata>/s;
267 }
268 else
269 {
270 print STDERR "\tNo souce document URL is specified in the OAI record (No (dc:)?identifier is provided)\n";
271 }
272 }
273 else
274 {
275 print STDERR "\tNo souce document URL is specified in the OAI record (No metadata field is provided)\n";
276 }
277
278}
279
280sub getOAIRecords
281{
282 my ($self,$aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
283
284 my $intDocCounter = 0;
285
286 my $metadata_prefix = $self->{'metadata_prefix'};
287
288 foreach my $strID ( @$aryIDs)
289 {
290 print STDERR "Gathering OAI record with ID:$strID.....\n";
291
292 my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
293
294 my $strRecord = $self->useWget($cmdWget);
295
296
297 my @fileDirs = split(":",$strID);
298
299 # setup directories
300
301 $strOutputDir =~ s/"//g; #"
302
303 my $host =$self->{'url'};
304
305 $host =~ s/http:\/\///g;
306
307 $host =~ s/:.*//g;
308
309 my $midDir = join ("/",@fileDirs);
310 my $strFileURL = "$strOutputDir/$host/".$midDir.".oai";
311
312 # prepare subdirectory for record (if needed)
313 my ($strSubDirPath,$unused) = ("", "");
314
315 ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
316
317 &util::mk_all_dir($strSubDirPath);
318
319 my $ds = &util::get_dirsep();
320
321 if($blnDownloadDoc)
322 {
323 $self->getOAIDoc($strRecord,$strSubDirPath);
324 }
325
326 # save record
327 open (OAIOUT,">$strFileURL")
328 || die "Unable to save oai metadata record: $!\n";
329 print OAIOUT $strRecord;
330 close(OAIOUT);
331
332 print STDERR "Saving records to $strFileURL\n";
333 print STDERR "<<Done>>\n";
334 $intDocCounter ++;
335 last if ($intDocCounter >= $intMaxRecords);
336 }
337
338 ($intDocCounter >= $intMaxRecords) ?
339 print STDERR "Reached maximum download records, use -max_records to set the maximum.\n":
340 print STDERR "Complete download meta record from $strBasURL\n";
341
342 print STDERR "<<Finished>>\n";
343}
344
345sub url_information
346{
347 my ($self) = shift (@_);
348 if(!defined $self){ die "System Error: No \$self defined for url_information in OAIDownload\n";}
349
350 my $wgetOptions = $self->getWgetOptions();
351 my $strBaseCMD = $wgetOptions." -q -O - \"$self->{'url'}?_OPTS_\"";
352
353 my $strIdentify = "verb=Identify";
354 my $strListSets = "verb=ListSets";
355
356 my $strIdentifyCMD = $strBaseCMD;
357 $strIdentifyCMD =~ s/_OPTS_/$strIdentify/;
358
359 my $strIdentifyText = $self->useWget($strIdentifyCMD);
360
361 if (!defined $strIdentifyText or $strIdentifyText eq "" ){
362 print STDERR "Server information is unavailable.\n";
363 print STDERR "<<Finished>>\n";
364 return;
365 }
366
367 print STDERR "General information:\n";
368 $self->parse_xml($strIdentifyText);
369
370 my $strListSetCMD = $strBaseCMD;
371 $strListSetCMD =~ s/_OPTS_/$strListSets/;
372 my $strListSetsText = $self->useWget($strListSetCMD);
373
374
375 print STDERR "List Information:\n";
376 $self->parse_xml($strListSetsText);
377}
378
379sub parse_xml
380{
381 my ($self) = shift (@_);
382 my ($strOutputText) = @_;
383
384 #Open a temporary file to store OAI information, and store the information to the temp file
385 my $name = "$ENV{GSDLHOME}/tmp/oai.tmp";
386
387 open(*OAIOUT,"> $name");
388
389 print OAIOUT $strOutputText;
390 close(OAIOUT);
391
392 $self->{'temp_file_name'} = $name;
393
394 eval {
395 $self->{'parser'}->parsefile("$name");
396 };
397
398 if ($@) {
399 die "OAI: $name is not a well formed XML file ($@)\n";
400 }
401}
402
403END{
404 if($self->{'info'})
405 {
406 unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
407 }
408}
409
410# This Char function overrides the one in XML::Parser::Stream to overcome a
411# problem where $expat->{Text} is treated as the return value, slowing
412# things down significantly in some cases.
413sub Char {
414 use bytes; # Necessary to prevent encoding issues with XML::Parser 2.31+
415 $_[0]->{'Text'} .= $_[1];
416 if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
417 $self->{'text'} .= $_[1];
418 $self->{'text'} =~ s/[\n]|([ ]{2,})//g;
419 if($self->{'text'} ne "")
420 {
421 print STDERR " $self->{'subfield'}:($self->{'text'})\n";
422 }
423 }
424 return undef;
425}
426
427sub OAI_StartTag
428{
429 my ($expat, $element, %attr) = @_;
430
431 $self->{'subfield'} = $element;
432
433}
434
435sub OAI_EndTag
436{
437 my ($expat, $element) = @_;
438 $self->{'text'} = "";
439 $self->{'subfield'} = "";
440}
441
442sub error
443{
444 my ($self,$strFunctionName,$strError) = @_;
445 {
446 print "Error occoured in OAIDownload.pm\n".
447 "In Function:".$strFunctionName."\n".
448 "Error Message:".$strError."\n";
449 exit(-1);
450 }
451}
452
453
454
4551;
Note: See TracBrowser for help on using the repository browser.