source: main/trunk/greenstone2/perllib/cgiactions/metadataaction.pm@ 23766

###########################################################################
#
# metadataaction.pm -- 
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 2009 New Zealand Digital Library Project
#
# This program is free software; you can redistr   te it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package metadataaction;

use strict;

use cgiactions::baseaction;

use dbutil;
use ghtml;


BEGIN {
#    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/perl-5.8");
    require XML::Rules;
}


@metadataaction::ISA = ('baseaction');


my $action_table =
{ 
    "get-live-metadata"     => { 'compulsory-args' => [ "d", "metaname" ],
                     'optional-args'   => [] }, 

    "get-metadata"          => { 'compulsory-args' => [ "d", "metaname" ],
                     'optional-args'   => [ "metapos" ] }, 

    "set-live-metadata"     => { 'compulsory-args' => [ "d", "metaname", "metavalue" ],
                     'optional-args'   => [ ] },

    "set-metadata"          => { 'compulsory-args' => [ "d", "metaname", "metavalue" ],
                     'optional-args'   => [ "metapos" ] },

    "set-archives-metadata" => { 'compulsory-args' => [ "d", "metaname", "metavalue" ],
                     'optional-args'   => [ "metapos", "metamode" ]
                      # metamode can be "accumulate", "override",
                },

    "set-import-metadata"   => { 'compulsory-args' => [ "metaname", "metavalue" ],
                     'optional-args'   => [ "d", "f", "metamode" ]
                    # metamode can be "accumulate", "override", or "unique-id"
                 },


    "remove-live-metadata"  => { 'compulsory-args' => [ "d", "metaname" ],
                     'optional-args'   => [ ] },

    "remove-metadata"       => { 'compulsory-args' => [ "d", "metaname" ],
                     'optional-args'   => [ "metapos" ] },

    "insert-metadata"       => { 'compulsory-args' => [ "d", "metaname", "metavalue" ],
                     'optional-args'   => [ ]
                   }
};


sub new 
{
    my $class = shift (@_);
    my ($gsdl_cgi,$iis6_mode) = @_;

    # Treat metavalue specially.  To transmit this through a GET request
    # the Javascript side has url-encoded it, so here we need to decode
    # it before proceeding

    my $url_encoded_metavalue = $gsdl_cgi->param("metavalue");
    my $url_decoded_metavalue = &unicode::url_decode($url_encoded_metavalue,1);

    my $unicode_array = &unicode::utf82unicode($url_decoded_metavalue);

    $url_decoded_metavalue = join("",map(chr($_),@$unicode_array));

    $gsdl_cgi->param("metavalue",$url_decoded_metavalue);

    my $self = new baseaction($action_table,$gsdl_cgi,$iis6_mode);

    return bless $self, $class;
}


sub get_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    # live metadata gets/saves value scoped (prefixed) by the current usename 
    # so (for now) let's not bother to enforce authentication

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid  = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
       $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }

    # Generate the dbkey
    my $metaname  = $self->{'metaname'};
    my $dbkey = "$docid.$metaname";

    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);
    
    # Obtain the content of the key
    my $cmd = "gdbmget $infodb_file_path $dbkey";
    if (open(GIN,"$cmd |") == 0) {
        # Catch error if gdbmget failed
    my $mess = "Failed to get metadata key: $metaname\n";
    $mess .= "$!\n";

    $gsdl_cgi->generate_error($mess);
    }
    else {
    binmode(GIN,":utf8");
        # Read everything in and concatenate them into $metavalue
    my $metavalue = "";
    my $line;
    while (defined ($line=<GIN>)) {
        $metavalue .= $line;
    }
    close(GIN);
    chomp($metavalue); # Get rid off the tailing newlines
    $gsdl_cgi->generate_ok_message("$metavalue");
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub get_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};

    # Authenticate user if it is enabled
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    &authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    ##my $collect_dir = &util::filename_cat($gsdlhome, "collect");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid     = $self->{'d'};
    my $metaname  = $self->{'metaname'};
    my $metapos   = $self->{'metapos'};
    my $infodbtype = $self->{'infodbtype'};

    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);
  
    # Basically loop through and unescape_html the values
    foreach my $k (keys %$doc_rec) {
    my @escaped_v = ();
    foreach my $v (@{$doc_rec->{$k}}) {
        my $ev = &ghtml::unescape_html($v);
        push(@escaped_v, $ev);
    }
    $doc_rec->{$k} = \@escaped_v;
    }

    # Obtain the specified metadata value
    $metapos = 0 if (!defined $metapos);
    my $metavalue = $doc_rec->{$metaname}->[$metapos];
    $gsdl_cgi->generate_ok_message("$metavalue");
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub set_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
  
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    &authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    ##my $collect_dir = &util::filename_cat($gsdlhome, "collect");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    my $metavalue = $self->{'metavalue'};
  

    # Generate the dbkey    
    my $metaname  = $self->{'metaname'};
    my $dbkey = "$docid.$metaname";

    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    # Set the new value
    my $cmd = "gdbmset \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
    my $status = system($cmd);
    if ($status != 0) {
        # Catch error if gdbmget failed
    my $mess = "Failed to set metadata key: $dbkey\n";

    $mess .= "PATH: $ENV{'PATH'}\n";
    $mess .= "cmd = $cmd\n";
    $mess .= "Exit status: $status\n";
    $mess .= "System Error Message: $!\n";

    $gsdl_cgi->generate_error($mess);
    }
    else {
    $gsdl_cgi->generate_ok_message("set-live-metadata successful: Key[$metaname]=$metavalue");
    }
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub set_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};

    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    &authenticate_user($gsdl_cgi, $username, $collect);
    }

    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    $gsdl_cgi->checked_chdir($collect_dir);

    # Obtain the collect dir
    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid     = $self->{'d'};
    my $metaname  = $self->{'metaname'};
    my $metapos   = $self->{'metapos'};
    my $metavalue = $self->{'metavalue'};
    my $infodbtype = $self->{'infodbtype'};
    
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);
    
    print STDERR "**** infodb file path = $infodb_file_path\n";
    print STDERR "***** infodb type = $infodbtype\n";
    
    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);
    
    # Set the metadata value
    if (defined $metapos) {
    $doc_rec->{$metaname}->[$metapos] = $metavalue;
    }
    else {
    $doc_rec->{$metaname} = [ $metavalue ];
    }
 
    my $status = &dbutil::set_infodb_entry($infodbtype, $infodb_file_path,$docid,$doc_rec);
    if ($status != 0) {
        # Catch error if set infodb entry failed
    my $mess = "Failed to set metadata key: $docid\n";
    
    $mess .= "PATH: $ENV{'PATH'}\n";
    $mess .= "Exit status: $status\n";
    $mess .= "System Error Message: $!\n";
    
    $gsdl_cgi->generate_error($mess);
    }
    else {
    my $mess = "set-document-metadata successful: Key[$docid]\n";
    $mess .= "  $metaname";
    $mess .= "->[$metapos]" if (defined $metapos);
    $mess .= " = $metavalue";
    
    $gsdl_cgi->generate_ok_message($mess);
    }
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub dxml_metadata
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metamode = $parser->{'parameters'}->{'metamode'};
    
    my $opt_doc_secnum = $parser->{'parameters'}->{'secnum'};
    
    # Find the right metadata tag and checks if we are going to
    # override it
    #
    # Note: This over writes the first metadata block it
    # encountered. If there are multiple Sections in the doc.xml, it
    # might not behave as you would expect

    my $curr_secnum = $parser->{'parameters'}->{'curr_section_num'};
##    print STDERR "**** checking $opt_doc_secnum <=> $curr_secnum\n";
##    print STDERR "**** metamode = $metamode\n";
    
    if ((!defined $opt_doc_secnum) || ($opt_doc_secnum eq $curr_secnum)) {
    my $name_attr = $attrHash->{'name'};
    if (($name_attr eq $metaname) && ($metamode eq "override")) {
##      print STDERR "**** got match!!\n";
        # Get the value and override the current value
        my $metavalue = $parser->{'parameters'}->{'metavalue'};
        $attrHash->{'_content'} = $metavalue;
        
        # Don't want it to wipe out any other pieces of metadata
        $parser->{'parameters'}->{'metamode'} = "done";
    }
    }

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub dxml_description
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metamode = $parser->{'parameters'}->{'metamode'};

    # Accumulate the metadata
    # NOTE: This appends new metadata element to all description fields.
    # If there are multiple Sections/SubSections, the new metadata block will get appended to all of them
    if (($metamode eq "accumulate") || ($metamode eq "override")) {
    # If get to here and metamode is override, the this means there 
    # was no existing value to overide => treat as an append operation

    # Tack a new metadata tag on to the end of the <Metadata>+ block
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metavalue = $parser->{'parameters'}->{'metavalue'};
    
    my $metadata_attr = { '_content' => $metavalue, 
                  'name'     => $metaname, 
                  'mode'     => "accumulate" };

    my $append_metadata = [ "Metadata" => $metadata_attr ];
    my $description_content = $attrHash->{'_content'};

##  print STDERR "**** appending to doc.xml\n";

    push(@$description_content,"    ", $append_metadata ,"\n        ");
    $parser->{'parameters'}->{'metamode'} = "done";
    }


    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}



sub dxml_start_section
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

    my $new_depth = scalar(@$contextArray);

    if ($new_depth == 1) {
    $parser->{'parameters'}->{'curr_section_depth'} = 1;
    $parser->{'parameters'}->{'curr_section_num'}   = "";
    }

    my $old_depth  = $parser->{'parameters'}->{'curr_section_depth'};
    my $old_secnum = $parser->{'parameters'}->{'curr_section_num'};

    my $new_secnum;

    if ($new_depth > $old_depth) {
    # child subsection
    $new_secnum = "$old_secnum.1";
    }
    elsif ($new_depth == $old_depth) {
    # sibling section => increase it's value by 1
    my ($tail_num) = ($old_secnum =~ m/\.(\d+)$/);
    $tail_num++;
    $new_secnum = $old_secnum;
    $new_secnum =~ s/\.(\d+)$/\.$tail_num/;
    }
    else {
    # back up to parent section => lopp off tail
    $new_secnum = $old_secnum;
    $new_secnum =~ s/\.\d+$//;
    }

    $parser->{'parameters'}->{'curr_section_depth'} = $new_depth;
    $parser->{'parameters'}->{'curr_section_num'}   = $new_secnum;

    print STDERR "*** In Section: $new_secnum\n";
}

sub edit_xml_file
{
    my $self = shift @_;
    my ($gsdl_cgi, $filename, $start_rules, $rules, $options) = @_;

    # use XML::Rules to add it in (read in and out again)
    my $parser = XML::Rules->new(start_rules     => $start_rules,
                 rules           => $rules, 
                 style           => 'filter',
                                 output_encoding => 'utf8' );

    my $xml_in = "";
    if (!open(MIN,"<$filename")) {
    $gsdl_cgi->generate_error("Unable to read in $filename: $!");
    }
    else {
        # Read all the text in
    my $line;
    while (defined ($line=<MIN>)) {
        $xml_in .= $line;
    }
    close(MIN);
    
    my $MOUT;    
    if (!open($MOUT,">$filename")) {
        $gsdl_cgi->generate_error("Unable to write out to $filename: $!");
    }
    else {
        # Matched lines will get handled by the call backs
##      my $xml_out = "";

        binmode($MOUT,":utf8");
        $parser->filter($xml_in,$MOUT, $options);

#       binmode(MOUT,":utf8");
#       print MOUT $xml_out;
        close($MOUT);       
    }
    }
}


sub edit_doc_xml
{
    my $self = shift @_;
    my ($gsdl_cgi, $doc_xml_filename, $metaname, $metavalue, $metapos, $metamode, $opt_secnum) = @_;

    # To monitor which section/subsection number we are in
    my @start_rules = 
    ( 'Section'    => \&dxml_start_section );

    # use XML::Rules to add it in (read in and out again)
    # Set the call back functions
    my @rules = 
    ( _default => 'raw',
      'Metadata'    => \&dxml_metadata,
      'Description' => \&dxml_description);
      
    # Sets the parameters
    my $options = { 'metaname'  => $metaname,
            'metapos'   => $metapos,
            'metavalue' => $metavalue,
            'metamode'  => $metamode };
            
    if (defined $opt_secnum) {
    $options->{'secnum'} = $opt_secnum;
    }

    $self->edit_xml_file($gsdl_cgi,$doc_xml_filename,\@start_rules,\@rules,$options);
}


sub set_archives_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    $self->authenticate_user($username, $collect);
    }

    # Obtain the collect and archive dir   
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site); 
    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");
    
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid  = $self->{'d'};
    my $metaname   = $self->{'metaname'};
    my $metavalue  = $self->{'metavalue'};
    
    my $metapos    = $self->{'metapos'};
    $metapos = 0 if (!defined $metapos);

    my $metamode   = $self->{'metamode'};
    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
    # make "accumulate" the default (less destructive, as won't actually 
    # delete any existing values)
    $metamode = "accumulate";
    }
    
    # Obtain the doc.xml path for the specified docID
    my ($docid_root,$docid_secnum) = ($docid =~ m/^(.*?)(\..*)?$/);

    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid_root);
    my $doc_xml_file = $doc_rec->{'doc-file'}->[0];
    
    # The $doc_xml_file is relative to the archives, and now let's get the full path
    my $archives_dir = &util::filename_cat($collect_dir,$collect,"archives");    
    my $doc_xml_filename = &util::filename_cat($archives_dir,$doc_xml_file);
    
    # Edit the doc.xml file with the specified metadata name, value and position.
    # TODO: there is a potential problem here as this edit_doc_xml function 
    # is assuming the simple doc.xml situation where there is only one Section and no SubSections.
    # Running import.pl -groupsize will cause this to have multiple sections in one doc.xml
    $self->edit_doc_xml($gsdl_cgi,$doc_xml_filename,
            $metaname,$metavalue,$metapos,$metamode,$docid_secnum);
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);

    my $mess = "set-archives-metadata successful: Key[$docid]\n";
    $mess .= "  $metaname";
    $mess .= "->[$metapos]" if (defined $metapos);
    $mess .= " = $metavalue";
    $mess .= " ($metamode)\n";
    
    $gsdl_cgi->generate_ok_message($mess);  
}


sub mxml_metadata
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metamode = $parser->{'parameters'}->{'metamode'};

    # Report error if we don't see FileName tag before this
    die "Fatel Error: Unexpected metadata.xml structure. Undefind current_file, possiblely encountered Description before FileName" if (!defined($parser->{'parameters'}->{'current_file'}));
    
    # Don't do anything if we are not in the right FileSet
    my $file_regexp = $parser->{'parameters'}->{'current_file'};
    if ($file_regexp =~ /\.\*/) {
    # Only interested in a file_regexp if it specifies precisely one
    # file.  
    # So, skip anything with a .* in it as it is too general
    return [$tagname => $attrHash];
    }
    my $src_file = $parser->{'parameters'}->{'src_file'};
    if (!($src_file =~ /$file_regexp/)) {
    return [$tagname => $attrHash];
    }
##    print STDERR "*** mxl metamode = $metamode\n";

    # Find the right metadata tag and checks if we are going to override it
    my $name_attr = $attrHash->{'name'};
    if (($name_attr eq $metaname) && ($metamode eq "override")) {
        # Get the value and override the current value
    my $metavalue = $parser->{'parameters'}->{'metavalue'};
    $attrHash->{'_content'} = $metavalue;

##  print STDERR "**** overrideing metadata.xml\n";

    # Don't want it to wipe out any other pieces of metadata
    $parser->{'parameters'}->{'metamode'} = "done";
    }

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub mxml_description
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metamode = $parser->{'parameters'}->{'metamode'};    

    # Failed... Report error if we don't see FileName tag before this
    die "Fatel Error: Unexpected metadata.xml structure. Undefind current_file, possiblely encountered Description before FileName" if (!defined($parser->{'parameters'}->{'current_file'}));

    # Don't do anything if we are not in the right FileSet
    my $file_regexp = $parser->{'parameters'}->{'current_file'};
    if ($file_regexp =~ /\.\*/) {
    # Only interested in a file_regexp if it specifies precisely one
    # file.  
    # So, skip anything with a .* in it as it is too general
    return [$tagname => $attrHash];
    }
    my $src_file = $parser->{'parameters'}->{'src_file'};
    if (!($src_file =~ /$file_regexp/)) {
    return [$tagname => $attrHash];
    }

    # Accumulate the metadata block to the end of the description block
    # Note: This adds metadata block to all description blocks, so if there are 
    # multiple FileSets, it will add to all of them
    if (($metamode eq "accumulate") || ($metamode eq "override")) {
    # if metamode was "override" but get to here then it failed to
    # find an item to override, in which case it should append its 
    # value to the end, just like the "accumulate" mode

    # tack a new metadata tag on to the end of the <Metadata>+ block
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metavalue = $parser->{'parameters'}->{'metavalue'};
    
    my $metadata_attr = { '_content' => $metavalue, 
                  'name'     => $metaname, 
                  'mode'     => "accumulate" };

    my $append_metadata = [ "Metadata" => $metadata_attr ];
    my $description_content = $attrHash->{'_content'};

##  print STDERR "*** appending to metadata.xml\n";

    # append the new metadata element to the end of the current
    # content contained inside this tag
    push(@$description_content,"    ", $append_metadata ,"\n        ");

    $parser->{'parameters'}->{'metamode'} = "done";
    }

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub mxml_filename
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

    # Store the filename of the Current Fileset
    # Note: According to http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd
    # FileName tag must come before Description tag
    $parser->{'parameters'}->{'current_file'} = $attrHash->{'_content'};

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub mxml_fileset
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

    # Initilise the current_file
    # Note: According to http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd
    # FileName tag must come before Description tag
    $parser->{'parameters'}->{'current_file'} = "";

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub edit_metadata_xml
{
    my $self = shift @_;
    my ($gsdl_cgi, $metadata_xml_filename, $metaname, $metavalue, $metamode, $src_file) = @_;

    # Set the call-back functions for the metadata tags
    my @rules = 
    ( _default => 'raw',
          'FileName' => \&mxml_filename,
      'Metadata' => \&mxml_metadata,
      'Description' => \&mxml_description,
          'FileSet' => \&mxml_fileset);

    # use XML::Rules to add it in (read in and out again)
    my $parser = XML::Rules->new(rules => \@rules, 
                 style => 'filter',
                                 output_encoding => 'utf8');

    my $xml_in = "";
    if (!open(MIN,"<$metadata_xml_filename")) {
    $gsdl_cgi->generate_error("Unable to read in $metadata_xml_filename: $!");
    }
    else {
        # Read them in
    my $line;
    while (defined ($line=<MIN>)) {
        $xml_in .= $line;
    }
    close(MIN); 

        # Filter with the call-back functions
    my $xml_out = "";

    my $MOUT;
    if (!open($MOUT,">$metadata_xml_filename")) {
        $gsdl_cgi->generate_error("Unable to write out to $metadata_xml_filename: $!");
    }
    else {
        binmode($MOUT,":utf8");

            # Some wise person please find out how to keep the DTD and encode lines in after it gets filtered by this XML::Rules
            # At the moment, I will just hack it!
            #my $header_with_utf8_dtd = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
        #$header_with_utf8_dtd .= "<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">";
            #$xml_out =~ s/\<\?xml\sversion\=\"1.0\"\?\>/$header_with_utf8_dtd/;
        #print MOUT $xml_out;

        $parser->filter($xml_in, $MOUT, { metaname => $metaname,
                          metavalue => $metavalue,
                          metamode => $metamode,
                          src_file => $src_file,
                          current_file => undef} );
        close($MOUT);       
    }
    }
}


sub set_import_metadata
{
    my $self = shift @_;
    
    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    $self->authenticate_user($username, $collect);
    }


    # Obtain the collect and archive dir    
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);
    
    # look up additional args
    # want either d= or f=
    my $docid  = $self->{'d'};
    my $import_file  = $self->{'f'};
    if ((!defined $docid) && (!defined $import_file)) {
    $gsdl_cgi->generate_error("No docid (d=...) or import file (f=) specified.");
    } 

    # Get the parameters and set default mode to "accumulate"
    my $metaname   = $self->{'metaname'};
    my $metavalue  = $self->{'metavalue'};
##    $metavalue =~ s/&amp;lt;(.*?)&amp;gt;/<$1>/g;
    $metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
    print STDERR "*** set import meta: val = $metavalue\n";
    
    my $metamode   = $self->{'metamode'};
    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
    # make "accumulate" the default (less destructive, as won't actually 
    # delete any existing values)
    $metamode = "accumulate";
    }

    # Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
    # If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
    my $metadata_xml_file;
    my $import_filename = undef;
    if (defined $docid) {
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

    # This now stores the full pathname
    $import_filename = $doc_rec->{'src-file'}->[0]; 
    }
    else {
        $import_filename = &util::filename_cat($collect_dir,$collect,$import_file);
    }

    # figure out correct metadata.xml file [?]
    # Assuming the metadata.xml file is next to the source file
    # Note: This will not work if it is using the inherited metadata from the parent folder
    my ($import_tailname, $import_dirname) 
    = File::Basename::fileparse($import_filename);
    my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");

    # Edit the metadata.xml
    # Modified by Jeffrey from DL Consulting
    # Handle the case where there is one metadata.xml file for multiple FileSets
    # The XML filter needs to know whether it is in the right FileSet
    # TODO: This doesn't fix the problem where the metadata.xml is not next to the src file.
    # TODO: This doesn't handle the common metadata (where FileName doesn't point to a single file)
    $self->edit_metadata_xml($gsdl_cgi, $metadata_xml_filename,
                             $metaname, $metavalue, $metamode, $import_tailname);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);

    my $mess = "set-import-metadata successful: Key[$docid] -> $metadata_xml_filename\n";
    $mess .= "  $metaname";
    $mess .= " = $metavalue";
    $mess .= " ($metamode)\n";
    
    $gsdl_cgi->generate_ok_message($mess);
    
}


sub remove_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    &authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    
    # Generate the dbkey
    my $metaname  = $self->{'metaname'};
    my $dbkey = "$docid.$metaname";

    # To people who know $collect_tail please add some comments
    # Obtain the live gdbm_db path 
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    # Remove the key
    my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\"";
    my $status = system($cmd);
    if ($status != 0) {
        # Catch error if gdbmdel failed
    my $mess = "Failed to set metadata key: $dbkey\n";
    
    $mess .= "PATH: $ENV{'PATH'}\n";
    $mess .= "cmd = $cmd\n";
    $mess .= "Exit status: $status\n";
    $mess .= "System Error Message: $!\n";

    $gsdl_cgi->generate_error($mess);
    }
    else {
    $gsdl_cgi->generate_ok_message("DB remove successful: Key[$metaname]");
    }

}


sub remove_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    &authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    ## my $collect_dir = &util::filename_cat($gsdlhome, "collect");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    my $metaname  = $self->{'metaname'};
    my $metapos   = $self->{'metapos'};

    # To people who know $collect_tail please add some comments
    # Obtain the path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);

    # Basically loop through and unescape_html the values
    foreach my $k (keys %$doc_rec) {
    my @escaped_v = ();
    foreach my $v (@{$doc_rec->{$k}}) {
        if ($k eq "contains") {
        # protect quotes in ".2;".3 etc
        $v =~ s/\"/\\\"/g;
        push(@escaped_v, $v);
        }
        else {
        my $ev = &ghtml::unescape_html($v);
        $ev =~ s/\"/\\\"/g;
        push(@escaped_v, $ev);
        }
    }
    $doc_rec->{$k} = \@escaped_v;
    }

    # Check to make sure the key does exist
    if (!defined ($doc_rec->{$metaname})) {
        $gsdl_cgi->generate_error("No metadata field \"" . $metaname . "\" in the specified document: [" . $docid . "]");
    }

    # Obtain the specified metadata pos
    $metapos = 0 if (!defined $metapos);

    # consider check key is defined before deleting?
    # Loop through the metadata array and ignore the specified position
    my $filtered_metadata = [];
    my $num_metadata_vals = scalar(@{$doc_rec->{$metaname}});    
    for (my $i=0; $i<$num_metadata_vals; $i++) {
    my $metavalue = shift(@{$doc_rec->{$metaname}});

    if ($i != $metapos) {
        push(@$filtered_metadata,$metavalue)
    }
    }
    $doc_rec->{$metaname} = $filtered_metadata;

    # Turn the record back to string
    my $serialized_doc_rec = &dbutil::convert_infodb_hash_to_string($doc_rec);

    # Store it back to the database
    my $cmd = "gdbmset \"$infodb_file_path\" \"$docid\" \"$serialized_doc_rec\"";
    my $status = system($cmd);
    if ($status != 0) {
    my $mess = "Failed to set metadata key: $docid\n";
    
    $mess .= "PATH: $ENV{'PATH'}\n";
    $mess .= "cmd = $cmd\n";
    $mess .= "Exit status: $status\n";
    $mess .= "System Error Message: $!\n";

    $gsdl_cgi->generate_error($mess);
    }
    else {
    my $mess = "DB set (with item deleted) successful: Key[$docid]\n";
    $mess .= "  $metaname";
    $mess .= "->[$metapos]" if (defined $metapos);

    $gsdl_cgi->generate_ok_message($mess);
    }
}


# Was trying to reused the codes, but the functions need to be broken
# down more before they can be reused, otherwise there will be too
# much overhead and duplicate process...
sub insert_metadata
{
    my $self = shift @_;
    
    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $gsdlhome  = $self->{'gsdlhome'};
    my $infodbtype = $self->{'infodbtype'};
    
    # If the import metadata and gdbm database have been updated, we
    # need to insert some notification to warn user that the the text
    # they see at the moment is not indexed and require a rebuild.
    my $rebuild_pending_macro = "_rebuildpendingmessage_";

    if ($baseaction::authentication_enabled) {
    # Ensure the user is allowed to edit this collection
    $self->authenticate_user($username, $collect);
    }

    # Obtain the collect and archive dir   
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    ##my $collect_dir = &util::filename_cat($gsdlhome, "collect");
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);
    
    # Check additional args
    my $docid = $self->{'d'};
    if (!defined($docid)) {
    $gsdl_cgi->generate_error("No document id is specified: d=...");
    } 
    my $metaname = $self->{'metaname'};
    if (!defined($metaname)) {
    $gsdl_cgi->generate_error("No metaname is specified: metadataname=...");
    } 
    my $metavalue = $self->{'metavalue'};
    if (!defined($metavalue) || $metavalue eq "") {
    $gsdl_cgi->generate_error("No metavalue or empty metavalue is specified: metadataname=...");
    } 
    # make "accumulate" the default (less destructive, as won't actually 
    # delete any existing values)
    my $metamode = "accumulate";

    #=======================================================================#
    # set_import_metadata [START]
    #=======================================================================#
    # Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
    # If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
    my $metadata_xml_file;
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $archive_doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);
    
    # This now stores the full pathname
    my $import_filename = $archive_doc_rec->{'src-file'}->[0];
    
    # figure out correct metadata.xml file [?]
    # Assuming the metadata.xml file is next to the source file
    # Note: This will not work if it is using the inherited metadata from the parent folder
    my ($import_tailname, $import_dirname) 
    = File::Basename::fileparse($import_filename);
    my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");

    # Shane's escape characters
    $metavalue = pack "U0C*", unpack "C*", $metavalue;
    $metavalue =~ s/\,/&#44;/g;
    $metavalue =~ s/\:/&#58;/g;
    $metavalue =~ s/\|/&#124;/g;
    $metavalue =~ s/\(/&#40;/g;
    $metavalue =~ s/\)/&#41;/g;
    $metavalue =~ s/\[/&#91;/g;
    $metavalue =~ s/\\/&#92;/g;
    $metavalue =~ s/\]/&#93;/g;
    $metavalue =~ s/\{/&#123;/g;
    $metavalue =~ s/\}/&#125;/g;
    $metavalue =~ s/\"/&#34;/g;
    $metavalue =~ s/\`/&#96;/g;
    $metavalue =~ s/\n/_newline_/g;

    # Edit the metadata.xml
    # Modified by Jeffrey from DL Consulting
    # Handle the case where there is one metadata.xml file for multiple FileSets
    # The XML filter needs to know whether it is in the right FileSet
    # TODO: This doesn't fix the problem where the metadata.xml is not next to the src file.
    # TODO: This doesn't handle the common metadata (where FileName doesn't point to a single file)
    $self->edit_metadata_xml($gsdl_cgi, $metadata_xml_filename,
                             $metaname, $metavalue, $metamode, $import_tailname);
    #=======================================================================#
    # set_import_metadata [END]
    #=======================================================================#


    #=======================================================================#
    # set_metadata (accumulate version) [START]
    #=======================================================================#
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);
   
    foreach my $k (keys %$doc_rec) {
    my @escaped_v = ();
    foreach my $v (@{$doc_rec->{$k}}) {
        if ($k eq "contains") {
        # protect quotes in ".2;".3 etc
        $v =~ s/\"/\\\"/g;
        push(@escaped_v, $v);
        }
        else {
        my $ev = &ghtml::unescape_html($v);
        $ev =~ s/\"/\\\"/g;
        push(@escaped_v, $ev);
        }
    }
    $doc_rec->{$k} = \@escaped_v;
    }

    # Protect the quotes
    $metavalue =~ s/\"/\\\"/g;

    # Adds the pending macro
    my $macro_metavalue = $rebuild_pending_macro . $metavalue;

    # If the metadata doesn't exist, create a new one
    if (!defined($doc_rec->{$metaname})){    
    $doc_rec->{$metaname} = [ $macro_metavalue ];
    }
    # Else, let's acculumate the values
    else {
        push(@{$doc_rec->{$metaname}},$macro_metavalue);
    }

    # Generate the record string
    my $serialized_doc_rec = &dbutil::convert_infodb_hash_to_string($doc_rec);

    # Store it into GDBM
    my $cmd = "gdbmset \"$infodb_file_path\" \"$docid\" \"$serialized_doc_rec\"";
    my $status = system($cmd);
    if ($status != 0) {
        # Catch error if gdbmget failed
    my $mess = "Failed to set metadata key: $docid\n";
    
    $mess .= "PATH: $ENV{'PATH'}\n";
    $mess .= "cmd = $cmd\n";
    $mess .= "Exit status: $status\n";
    $mess .= "System Error Message: $!\n";

    $gsdl_cgi->generate_error($mess);
    }
    else {
    my $mess = "insert-metadata successful: Key[$docid]\n";
    $mess .= "  [In metadata.xml] $metaname";
    $mess .= " = $metavalue\n";
    $mess .= "  [In database] $metaname";
    $mess .= " = $macro_metavalue\n";
    $mess .= "  The new text has not been indexed, rebuilding collection is required\n";
        $gsdl_cgi->generate_ok_message($mess);
    }    
    #=======================================================================#
    # set_metadata (accumulate version) [END]
    #=======================================================================#

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

1;