source: gsdl/trunk/perllib/plugins/AutoExtractMetadata.pm@ 17026

Last change on this file since 17026 was 17026, checked in by kjdon, 12 years ago

OID generation modifications: OIDtype and OIDmetadata options now available for plugins as well as import. OIDtype for plugins defaults to auto - if set to auto, then use the values from import. All plugins now call self->add_OID instead of doc_obj->set_OID. This sets the doc_obj OIDtype so that doesn't need to be donein other places any more. all plugins have the get_oid_hash_type method - normally returns hash_on_file, but can be overridden to return hash_on_ga_xml for those plugins that don't want hashing on file (MP3,OggVorbis...)

  • Property svn:executable set to *
File size: 4.8 KB
Line 
1###########################################################################
2#
3# AutoExtractMetadata.pm -- base plugin for all plugins that want to do metadata extraction from text and/or metadata
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 2008 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# This plugin uses the supporting Extractors to add metadata extraction
27# functionality to BasePlugin.
28
29
30package AutoExtractMetadata;
31
32use strict;
33no strict 'subs';
34no strict 'refs'; # allow filehandles to be variables and viceversa
35
36use BasePlugin;
37use AcronymExtractor;
38use KeyphraseExtractor;
39use EmailAddressExtractor;
40use DateExtractor;
41use GISExtractor;
42use CJKTextSegmenter;
43
44
45sub BEGIN {
46 @AutoExtractMetadata::ISA = ( 'BasePlugin', 'AcronymExtractor', 'KeyphraseExtractor', 'EmailAddressExtractor', 'DateExtractor', 'CJKTextSegmenter','GISExtractor' );
47}
48
49my $arguments = [
50 {'name' => "first",
51 'desc' => "{AutoExtractMetadata.first}",
52 'type' => "string",
53 'reqd' => "no" }
54 ];
55
56
57my $options = { 'name' => "AutoExtractMetadata",
58 'desc' => "{AutoExtractMetadata.desc}",
59 'abstract' => "yes",
60 'inherits' => "no",
61 'args' => $arguments };
62
63
64sub new {
65
66 # Start the AutoExtractMetadata Constructor
67 my $class = shift (@_);
68 my ($pluginlist,$inputargs,$hashArgOptLists,$auxiliary) = @_;
69 push(@$pluginlist, $class);
70
71 push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
72 push(@{$hashArgOptLists->{"OptList"}},$options);
73
74 # load up the options and args for the supporting plugins
75 new AcronymExtractor($pluginlist, $inputargs, $hashArgOptLists);
76 new KeyphraseExtractor($pluginlist, $inputargs, $hashArgOptLists);
77 new EmailAddressExtractor($pluginlist, $inputargs, $hashArgOptLists);
78 new DateExtractor($pluginlist, $inputargs, $hashArgOptLists);
79 new GISExtractor($pluginlist, $inputargs, $hashArgOptLists);
80 new CJKTextSegmenter($pluginlist, $inputargs, $hashArgOptLists);
81 my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
82
83 return bless $self, $class;
84
85}
86
87sub begin {
88 my $self = shift (@_);
89 my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
90
91 $self->SUPER::begin(@_);
92
93 #initialise those extractors that need initialisation
94 $self->initialise_acronym_extractor();
95 $self->initialise_gis_extractor();
96
97}
98
99sub end {
100 # potentially called at the end of each plugin pass
101 # import.pl only has one plugin pass, but buildcol.pl has multiple ones
102
103 my ($self) = @_;
104 # finalise those extractors that need finalisation
105 $self->finalise_acronym_extractor();
106}
107
108# here is where we call methods from the supporting extractor plugins
109sub auto_extract_metadata {
110 my $self = shift(@_);
111 my ($doc_obj) = @_;
112
113 if ($self->{'first'}) {
114 my $thissection = $doc_obj->get_top_section();
115 while (defined $thissection) {
116 my $text = $doc_obj->get_text($thissection);
117 $self->extract_first_NNNN_characters (\$text, $doc_obj, $thissection) if $text =~ /./;
118 $thissection = $doc_obj->get_next_section ($thissection);
119 }
120 }
121 $self->extract_acronym_metadata($doc_obj);
122 $self->extract_keyphrase_metadata($doc_obj);
123 $self->extract_email_metadata($doc_obj);
124 $self->extract_date_metadata($doc_obj);
125 $self->extract_gis_metadata($doc_obj);
126 $self->separate_cjk_text($doc_obj);
127
128}
129
130
131# FIRSTNNN: extract the first NNN characters as metadata
132sub extract_first_NNNN_characters {
133 my $self = shift (@_);
134 my ($textref, $doc_obj, $thissection) = @_;
135
136 foreach my $size (split /,/, $self->{'first'}) {
137 my $tmptext = $$textref;
138 $tmptext =~ s/^\s+//;
139 $tmptext =~ s/\s+$//;
140 $tmptext =~ s/\s+/ /gs;
141 $tmptext = substr ($tmptext, 0, $size);
142 $tmptext =~ s/\s\S*$/…/;
143 $doc_obj->add_utf8_metadata ($thissection, "First$size", $tmptext);
144 }
145}
146
147sub clean_up_after_doc_obj_processing {
148 my $self = shift(@_);
149
150 $self->SUPER::clean_up_after_doc_obj_processing();
151 $self->GISExtractor::clean_up_temp_files();
152}
153
1541;
Note: See TracBrowser for help on using the repository browser.