root/gsdl/trunk/perllib/plugins/AutoExtractMetadata.pm @ 17026

Revision 17026, 4.8 KB (checked in by kjdon, 11 years ago)

OID generation modifications: OIDtype and OIDmetadata options now available for plugins as well as import. OIDtype for plugins defaults to auto - if set to auto, then use the values from import. All plugins now call self->add_OID instead of doc_obj->set_OID. This sets the doc_obj OIDtype so that doesn't need to be donein other places any more. all plugins have the get_oid_hash_type method - normally returns hash_on_file, but can be overridden to return hash_on_ga_xml for those plugins that don't want hashing on file (MP3,OggVorbis?...)

  • Property svn:executable set to *
Line 
1###########################################################################
2#
3# AutoExtractMetadata.pm -- base plugin for all plugins that want to do metadata extraction from text and/or metadata
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 2008 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26# This plugin uses the supporting Extractors to add metadata extraction
27# functionality to BasePlugin.
28
29
30package AutoExtractMetadata;
31
32use strict;
33no strict 'subs';
34no strict 'refs'; # allow filehandles to be variables and viceversa
35
36use BasePlugin;
37use AcronymExtractor;
38use KeyphraseExtractor;
39use EmailAddressExtractor;
40use DateExtractor;
41use GISExtractor;
42use CJKTextSegmenter;
43
44
45sub BEGIN {
46    @AutoExtractMetadata::ISA = ( 'BasePlugin', 'AcronymExtractor', 'KeyphraseExtractor', 'EmailAddressExtractor', 'DateExtractor', 'CJKTextSegmenter','GISExtractor' );
47}
48
49my $arguments = [
50         {'name' => "first",
51          'desc' => "{AutoExtractMetadata.first}",
52          'type' => "string",
53          'reqd' => "no" }
54         ];
55
56
57my $options = { 'name'     => "AutoExtractMetadata",
58        'desc'     => "{AutoExtractMetadata.desc}",
59        'abstract' => "yes",
60        'inherits' => "no",
61        'args'     => $arguments };
62
63
64sub new {
65
66    # Start the AutoExtractMetadata Constructor
67    my $class = shift (@_);
68    my ($pluginlist,$inputargs,$hashArgOptLists,$auxiliary) = @_;
69    push(@$pluginlist, $class);
70   
71    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
72    push(@{$hashArgOptLists->{"OptList"}},$options);
73
74    # load up the options and args for the supporting plugins
75    new AcronymExtractor($pluginlist, $inputargs, $hashArgOptLists);
76    new KeyphraseExtractor($pluginlist, $inputargs, $hashArgOptLists);
77    new EmailAddressExtractor($pluginlist, $inputargs, $hashArgOptLists);
78    new DateExtractor($pluginlist, $inputargs, $hashArgOptLists);
79    new GISExtractor($pluginlist, $inputargs, $hashArgOptLists);
80    new CJKTextSegmenter($pluginlist, $inputargs, $hashArgOptLists);
81    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
82
83    return bless $self, $class;
84   
85}
86
87sub begin {
88    my $self = shift (@_);
89    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
90
91    $self->SUPER::begin(@_);
92
93    #initialise those extractors that need initialisation
94    $self->initialise_acronym_extractor();
95    $self->initialise_gis_extractor();
96
97}
98
99sub end {
100    # potentially called at the end of each plugin pass
101    # import.pl only has one plugin pass, but buildcol.pl has multiple ones
102
103    my ($self) = @_;
104    # finalise those extractors that need finalisation
105    $self->finalise_acronym_extractor();
106}
107
108# here is where we call methods from the supporting extractor plugins
109sub auto_extract_metadata {
110    my $self = shift(@_);
111    my ($doc_obj) = @_;
112
113    if ($self->{'first'}) {
114    my $thissection = $doc_obj->get_top_section();
115    while (defined $thissection) {
116        my $text = $doc_obj->get_text($thissection);
117        $self->extract_first_NNNN_characters (\$text, $doc_obj, $thissection) if $text =~ /./;
118        $thissection = $doc_obj->get_next_section ($thissection);
119    }
120    }
121    $self->extract_acronym_metadata($doc_obj);
122    $self->extract_keyphrase_metadata($doc_obj);
123    $self->extract_email_metadata($doc_obj);
124    $self->extract_date_metadata($doc_obj);
125    $self->extract_gis_metadata($doc_obj);
126    $self->separate_cjk_text($doc_obj);
127
128}
129
130
131# FIRSTNNN: extract the first NNN characters as metadata
132sub extract_first_NNNN_characters {
133    my $self = shift (@_);
134    my ($textref, $doc_obj, $thissection) = @_;
135   
136    foreach my $size (split /,/, $self->{'first'}) {
137    my $tmptext =  $$textref;
138    $tmptext =~ s/^\s+//;
139    $tmptext =~ s/\s+$//;
140    $tmptext =~ s/\s+/ /gs;
141    $tmptext = substr ($tmptext, 0, $size);
142    $tmptext =~ s/\s\S*$/…/;
143    $doc_obj->add_utf8_metadata ($thissection, "First$size", $tmptext);
144    }
145}
146
147sub clean_up_after_doc_obj_processing {
148    my $self = shift(@_);
149
150    $self->SUPER::clean_up_after_doc_obj_processing();
151    $self->GISExtractor::clean_up_temp_files();
152}
153
1541;
Note: See TracBrowser for help on using the browser.