source: main/trunk/model-sites-dev/atea/collect/digital-nz/perllib/plugins/DNZJSONPlugin.pm@ 33166

###########################################################################
#
# DNZJSONPlugin.pm -- A plugin for JSON files resulting from downloadin
#                     records using the Digital NZ API
#
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright 2016 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package DNZJSONPlugin;


use BaseImporter;
use ReadTextFile;

use strict;
no strict 'refs';
use multiread;

use Encode;
use JSON;

# methods with identical signatures take precedence in the order given in the ISA list.
sub BEGIN {
#    @DNZJSONPlugin::ISA = ('MetadataRead', 'ReadTextFile');
    @DNZJSONPlugin::ISA = ('ReadTextFile');
}


my $arguments = [
      { 'name' => "process_exp",
    'desc' => "{BaseImporter.process_exp}",
    'type' => "regexp",
    'reqd' => "no",
    'deft' => &get_default_process_exp() }

];


my $options = { 'name'     => "DNZJSONPlugin",
        'desc'     => "{DNZJSONPlugin.desc}",
        'abstract' => "no",
        'inherits' => "yes",
        'args'     => $arguments };


sub new
{
    my ($class) = shift (@_);
    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
    push(@$pluginlist, $class);

    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
    push(@{$hashArgOptLists->{"OptList"}},$options);

    my $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);

    return bless $self, $class;
}


sub get_default_process_exp
{
    return q^(?i)\.json$^;
}

sub file_block_read {
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;

    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);

    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
    return undef; # can't recognise
    }

    # set this so we know this is a metadata file - needed for incremental 
    # build
    # if this file changes, then we need to reimport everything
    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;

    return 1;
}



sub read_file
{
    my $self = shift (@_);
    my ($filename, $encoding, $language, $textref) = @_;

    # Read in file the usual ReadTextFile way
    # This ensure that $textref is a unicode aware string
    $self->SUPER::read_file(@_);

    #
    # Now top-up the processing of the text with what this plugin
    # needs
    #

    my $json_metadata = JSON->new->utf8->decode($$textref);

    $self->{'json_metadata'} = $json_metadata;
}

sub process
{
    my $self = shift (@_);
    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    my $outhandle = $self->{'outhandle'};

    my $section = $doc_obj->get_top_section();

    my $json_metadata = $self->{'json_metadata'};

    # Add the raw line as the document text
    $doc_obj->add_utf8_text($section, $$textref);

    # Process each metadata entry
    foreach my $k (keys %$json_metadata) {
    my $md_vals = $json_metadata->{$k};
    if (ref($md_vals) eq "ARRAY") {
        foreach my $md_val (@{$md_vals}) {
        if (defined $md_val && ($md_val ne "")) {
            $doc_obj->add_utf8_metadata($section, "dnz.$k", $md_val);
        }
        }
    }
    else {
        my $md_val = $md_vals;
        if (defined $md_val && ($md_val ne "")) {
            $doc_obj->add_utf8_metadata($section, "dnz.$k", $md_val);
        }
    }

    }


    # Record was processed successfully
    return 1;
}


1;