Changeset 22413 for main/trunk/greenstone2/perllib/inexport.pm

Timestamp:

2010-07-16T14:13:01+12:00 (14 years ago)

Author:

davidb

Message:

Initial pass at getting the main code to import.pl (and the very similar export.pl) structured as a shared module

File:

• main/trunk/greenstone2/perllib/inexport.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/inexport.pm

@@ -1 +1 @@
 ###########################################################################
 #
-# inexport.pm -- useful utilities to support import.pl and export.pl
+# inexport.pm -- useful class to support import.pl and export.pl
 # A component of the Greenstone digital library software
 # from the New Zealand Digital Library Project at the 
@@ -28 +28 @@
 use strict;
 
-use File::Basename;
-
+no strict 'refs'; # allow filehandles to be variables and vice versa
+no strict 'subs'; # allow barewords (eg STDERR) as function arguments
+
+use arcinfo;
+use colcfg;
+use dbutil;
+use plugin;
+use plugout;
+use manifest;
+use inexport;
 use dbutil;
 use util;
+use scriptutil;
+use FileHandle;
+use gsprintf 'gsprintf';
+use printusage;
+use parse2;
+
+use File::Basename;
+
+sub new 
+{
+    my $class = shift (@_);
+    my ($argv,$options) = @_;
+
+    my $self = { 'xml' => 0 };
+
+    # general options available to all plugins
+    my $arguments = $options->{'args'};
+    my $intArgLeftinAfterParsing = parse2::parse($argv,$arguments,$self,"allow_extra_options");
+    # Parse returns -1 if something has gone wrong
+    if ($intArgLeftinAfterParsing == -1)
+    {
+    &PrintUsage::print_txt_usage($options, "{import.params}");
+    die "\n";
+    }
+    
+    my $language = $self->{'language'};
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language && $language =~ /\S/) {
+    &gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($self->{'xml'}) {
+        &PrintUsage::print_xml_usage($options);
+    print "\n";
+    return;
+    }
+
+    if ($self->{'gli'}) { # the gli wants strings to be in UTF-8
+    &gsprintf::output_strings_in_UTF8; 
+    }
+    
+    # now check that we had exactly one leftover arg, which should be 
+    # the collection name. We don't want to do this earlier, cos 
+    # -xml arg doesn't need a collection name
+    # Or if the user specified -h, then we output the usage also
+    if ($intArgLeftinAfterParsing != 1 || (@$argv && $argv->[0] =~ /^\-+h/))
+    {
+    &PrintUsage::print_txt_usage($options, "{import.params}");
+    die "\n";
+    }
+
+    $self->{'close_out'} = 0;
+    my $out = $self->{'out'};
+    if ($out !~ /^(STDERR|STDOUT)$/i) {
+    open (OUT, ">$out") ||
+        (&gsprintf(STDERR, "{common.cannot_open_output_file}: $!\n", $out) && die);
+    $out = 'import::OUT';
+    $self->{'close_out'} = 1;
+    }
+    $out->autoflush(1);
+    $self->{'out'} = $out;
+
+    # @ARGV should be only one item, the name of the collection
+    $self->{'collection'} = shift @$argv;
+
+    return bless $self, $class;
+}
+
+sub get_collection
+{
+    my $self = shift @_;
+    
+    return $self->{'collection'};
+}
+
+
+sub read_collection_cfg
+{
+    my $self = shift @_;
+    my ($collection,$options) = @_;
+
+    my $collectdir = $self->{'collectdir'};
+    my $site       = $self->{'site'};
+    my $out        = $self->{'out'};
+
+    if (($collection = &colcfg::use_collection($site, $collection, $collectdir)) eq "") {
+    &PrintUsage::print_txt_usage($options, "{import.params}");
+    die "\n";
+    }
+
+    # add collection's perllib dir  into include path in 
+    # case we have collection specific modules
+    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
+
+    # check that we can open the faillog
+    my $faillog = $self->{'faillog'};
+    if ($faillog eq "") {
+    $faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
+    }
+    open (FAILLOG, ">$faillog") ||
+    (&gsprintf(STDERR, "{import.cannot_open_fail_log}\n", $faillog) && die);
+
+    
+    my $faillogname = $faillog;
+    $faillog = 'inexport::FAILLOG';
+    $faillog->autoflush(1);
+    $self->{'faillog'} = $faillog;
+    $self->{'faillogname'} = $faillogname;
+
+    # Read in the collection configuration file.
+    my ($configfilename, $gs_mode) = &colcfg::get_collect_cfg_name($out);
+    my $collectcfg = &colcfg::read_collection_cfg ($configfilename, $gs_mode);
+
+    return $collectcfg;
+}
+
+sub set_collection_options
+{
+    my $self = shift @_;
+    my ($inexport_mode,$collectcfg) = @_;
+
+    my $verbosity  = $self->{'verbosity'};
+    print STDERR "**** verbosity = $verbosity\n\n\n";
+
+    my $debug      = $self->{'debug'};
+    my $importdir  = $self->{'importdir'};
+    my $archivedir = $self->{'archivedir'};
+    my $out        = $self->{'out'};
+
+    # If the infodbtype value wasn't defined in the collect.cfg file, use the default
+    if (!defined($collectcfg->{'infodbtype'}))
+    {
+      $collectcfg->{'infodbtype'} = &dbutil::get_default_infodb_type();
+    }
+
+    if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+    $importdir = $collectcfg->{'importdir'};
+    }
+    if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+    $archivedir = $collectcfg->{'archivedir'};
+    }
+    # fill in the default import and archives directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+    if (!-e $importdir) {
+    &gsprintf($out, "{import.no_import_dir}\n\n", $importdir);
+    die "\n";
+    }
+    $self->{'importdir'} = $importdir;
+
+    if ($archivedir eq "") {
+    if ($inexport_mode eq "import") {
+        $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
+    }
+    elsif ($inexport_mode eq "export") {
+        $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "export");
+    }
+    else {
+        print STDERR "Warning: Unrecognized import/export mode '$inexport_mode'\n";
+        print STDERR "         Defaulting to 'archives' for file output\n";
+        $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
+    }
+    }
+
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+    $self->{'archivedir'} = $archivedir;
+
+    if ($verbosity !~ /\d+/) {
+    if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
+        $verbosity = $collectcfg->{'verbosity'};
+    } else {
+        $verbosity = 2; # the default
+    }
+    }
+    if (defined $collectcfg->{'manifest'} && $self->{'manifest'} eq "") {
+    $self->{'manifest'} = $collectcfg->{'manifest'};
+    }
+
+    if (defined $collectcfg->{'gzip'} && !$self->{'gzip'}) {
+    if ($collectcfg->{'gzip'} =~ /^true$/i) {
+        $self->{'gzip'} = 1;
+    }
+    }
+
+    if ($self->{'maxdocs'} !~ /\-?\d+/) {
+    if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
+        $self->{'maxdocs'} = $collectcfg->{'maxdocs'};
+    } else {
+        $self->{'maxdocs'} = -1; # the default
+    }
+    }
+    if ($self->{'groupsize'} == 1) {
+    if (defined $collectcfg->{'groupsize'} && $collectcfg->{'groupsize'} =~ /\d+/) {
+        $self->{'groupsize'} = $collectcfg->{'groupsize'};
+    }
+    }
+
+    if (!defined $self->{'OIDtype'} 
+    || ($self->{'OIDtype'} !~ /^(hash|incremental|assigned|dirname)$/ )) {
+    if (defined $collectcfg->{'OIDtype'} 
+        && $collectcfg->{'OIDtype'} =~ /^(hash|incremental|assigned|dirname)$/) {
+        $self->{'OIDtype'} = $collectcfg->{'OIDtype'};
+    } else {
+        $self->{'OIDtype'} = "hash"; # the default
+    }
+    }
+
+    if ((!defined $self->{'OIDmetadata'}) || ($self->{'OIDmetadata'} eq "")) {
+    if (defined $collectcfg->{'OIDmetadata'}) {
+        $self->{'OIDmetadata'} = $collectcfg->{'OIDmetadata'};
+    } else {
+        $self->{'OIDmetadata'} = "dc.Identifier"; # the default
+    }
+    }
+
+    my $sortmeta = $self->{'sortmeta'};
+    if (defined $collectcfg->{'sortmeta'} && (!defined $sortmeta || $sortmeta eq "")) {
+    $sortmeta = $collectcfg->{'sortmeta'};
+    }
+    # sortmeta cannot be used with group size
+    $sortmeta = undef unless defined $sortmeta && $sortmeta =~ /\S/;
+    if (defined $sortmeta && $self->{'groupsize'} > 1) {
+    &gsprintf($out, "{import.cannot_sort}\n\n");
+    $sortmeta = undef;
+    }
+    $self->{'sortmeta'} = $sortmeta;
+
+    if (defined $collectcfg->{'removeprefix'} && $self->{'removeprefix'} eq "") {
+    $self->{'removeprefix'} = $collectcfg->{'removeprefix'};
+    }
+    
+    if (defined $collectcfg->{'removesuffix'} && $self->{'removesuffix'} eq "") {
+    $self->{'removesuffix'} = $collectcfg->{'removesuffix'};
+    }
+    if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
+    $self->{'debug'} = 1;
+    }
+    if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
+    $self->{'gli'} = 1;
+    }
+    $self->{'gli'} = 0 unless defined $self->{'gli'};
+       
+    # check keepold and removeold
+    my ($removeold, $keepold, $incremental, $incremental_mode) 
+    = &scriptutil::check_removeold_and_keepold($self->{'removeold'}, $self->{'keepold'}, 
+                           $self->{'incremental'}, "archives", 
+                           $collectcfg);
+
+    $self->{'removeold'}        = $removeold;
+    $self->{'keepold'}          = $keepold;
+    $self->{'incremental'}      = $incremental;
+    $self->{'incremental_mode'} = $incremental_mode;
+}
+
+sub process_files
+{
+    my $self = shift @_;
+    my ($inexport_mode,$collectcfg) = @_;
+
+    my $verbosity   = $self->{'verbosity'};
+    my $debug       = $self->{'debug'};
+
+    my $importdir   = $self->{'importdir'};
+    my $archivedir  = $self->{'archivedir'};
+
+    my $incremental = $self->{'incremental'};
+    my $incremental_mode = $self->{'incremental_mode'};
+
+    my $removeold   = $self->{'removeold'};
+    my $keepold     = $self->{'keepold'};
+
+    my $saveas      = $self->{'saveas'};
+    my $OIDtype     = $self->{'OIDtype'};
+    my $OIDmetadata = $self->{'OIDmetadata'};
+
+    my $out         = $self->{'out'};
+    my $faillog     = $self->{'faillog'};
+
+    my $maxdocs     = $self->{'maxdocs'};
+    my $gzip        = $self->{'gzip'};
+    my $groupsize   = $self->{'groupsize'};
+    my $sortmeta    = $self->{'sortmeta'};
+
+    my $removeprefix = $self->{'removeprefix'};
+    my $removesuffix = $self->{'removesuffix'};
+
+    my $gli         = $self->{'gli'};
+
+    print STDERR "<Import>\n" if $gli;
+    
+    my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
+    if ($self->{'manifest'} ne "") {    
+    my $manifest_filename = $self->{'manifest'};
+
+    if (!&util::filename_is_absolute($manifest_filename)) {
+        $manifest_filename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, $manifest_filename);
+    }
+
+    $self->{'manifest'} =~ s/[\\\/]+/\//g;
+    $self->{'manifest'} =~ s/\/$//;
+
+    $manifest_lookup->parse($manifest_filename);
+    }
+
+    my $manifest = $self->{'manifest'};
+
+    # load all the plugins
+    my $plugins = [];
+    if (defined $collectcfg->{'plugin'}) {
+    $plugins = $collectcfg->{'plugin'};
+    }
+
+    #some global options for the plugins
+    my @global_opts = ();
+
+
+    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $incremental_mode);
+    if (scalar(@$pluginfo) == 0) {
+    &gsprintf($out, "{import.no_plugins_loaded}\n");
+    die "\n";
+    }
+
+    # remove the old contents of the archives directory (and tmp directory) if needed
+    if ($removeold) {
+    if (-e $archivedir) {
+        &gsprintf($out, "{import.removing_archives}\n");
+        &util::rm_r ($archivedir);
+    }
+    my $tmpdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "tmp");
+    $tmpdir =~ s/[\\\/]+/\//g;
+    $tmpdir =~ s/\/$//;
+    if (-e $tmpdir) {
+        &gsprintf($out, "{import.removing_tmpdir}\n");
+        &util::rm_r ($tmpdir);
+    }
+    }
+
+    # create the archives dir if needed
+    &util::mk_all_dir($archivedir);
+
+    # read the archive information file
+##  my $arcinfo_doc_filename = &util::filename_cat ($archivedir, "archives.inf");
+
+    # BACKWARDS COMPATIBILITY: Just in case there are old .ldb/.bdb files (won't do anything for other infodbtypes)
+    &util::rename_ldb_or_bdb_file(&util::filename_cat($archivedir, "archiveinf-doc"));
+    &util::rename_ldb_or_bdb_file(&util::filename_cat($archivedir, "archiveinf-src"));
+
+    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-doc", $archivedir);
+    my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir);
+                            
+    my $archive_info = new arcinfo ($collectcfg->{'infodbtype'});
+    $archive_info->load_info ($arcinfo_doc_filename);
+
+    if ($manifest eq "") {
+    # Load in list of files in import folder from last import (if present)
+    $archive_info->load_prev_import_filelist ($arcinfo_src_filename);
+    }
+
+    ####Use Plugout####
+    my ($plugout); 
+    if (defined $collectcfg->{'plugout'}) {
+    # If a plugout was specified in the collect.cfg file, assume it is sensible
+    # We can't check the name because it could be anything, if it is a custom plugout
+    $plugout = $collectcfg->{'plugout'};
+    }
+    else{
+    if ($saveas !~ /^(GreenstoneXML|GreenstoneMETS)$/) {
+        push @$plugout,"GreenstoneXMLPlugout";
+    }
+    else{
+        push @$plugout,$saveas."Plugout";
+    }
+    }
+
+    push @$plugout,("-output_info",$archive_info) if (defined $archive_info); 
+    push @$plugout,("-verbosity",$verbosity)      if (defined $verbosity);
+    push @$plugout,("-gzip_output")               if ($gzip);
+    push @$plugout,("-group_size",$groupsize)     if (defined $groupsize);
+    push @$plugout,("-output_handle",$out)        if (defined);
+    push @$plugout,("-debug")                     if ($debug);
+    
+    my $processor = &plugout::load_plugout($plugout);                        
+    $processor->setoutputdir ($archivedir);
+    $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
+    $processor->set_OIDtype ($OIDtype, $OIDmetadata);
+    
+    &plugin::begin($pluginfo, $importdir, $processor, $maxdocs, $gli);
+    
+    if ($removeold) {
+        # occasionally, plugins may want to do something on remove old, eg pharos image indexing
+    &plugin::remove_all($pluginfo, $importdir, $processor, $maxdocs, $gli);
+    }
+    if ($manifest eq "") {
+    # process the import directory
+    my $block_hash = {};
+    my $metadata = {};
+    # gobal blocking pass may set up some metadata
+    &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+
+
+    if ($incremental || $incremental_mode eq "onlyadd") {
+
+        prime_doc_oid_count($archivedir);
+
+
+        # Can now work out which files were new, already existed, and have
+        # been deleted
+        
+        new_vs_old_import_diff($archive_info,$block_hash,$importdir,
+                   $archivedir,$verbosity,$incremental_mode);
+        
+        my @new_files = sort keys %{$block_hash->{'new_files'}};
+        if (scalar(@new_files>0)) {
+        print STDERR "New files and modified metadata files since last import:\n  ";
+        print STDERR join("\n  ",@new_files), "\n";
+        }
+
+        if ($incremental) {
+               # only look for deletions if we are truely incremental
+        my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
+        # Filter out any in gsdl/tmp area
+        my @filtered_deleted_files = ();
+        my $gsdl_tmp_area = &util::filename_cat($ENV{'GSDLHOME'}, "tmp");
+        my $collect_tmp_area = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+        $gsdl_tmp_area = &util::filename_to_regex($gsdl_tmp_area);
+        $collect_tmp_area = &util::filename_to_regex($collect_tmp_area);
+                  
+        foreach my $df (@deleted_files) {
+            next if ($df =~ m/^$gsdl_tmp_area/);
+            next if ($df =~ m/^$collect_tmp_area/);
+            
+            push(@filtered_deleted_files,$df);
+        }       
+        
+
+        @deleted_files = @filtered_deleted_files;
+        
+        if (scalar(@deleted_files)>0) {
+            print STDERR "Files deleted since last import:\n  ";
+            print STDERR join("\n  ",@deleted_files), "\n";
+        
+        
+            &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@deleted_files);
+            
+            mark_docs_for_deletion($archive_info,$block_hash,\@deleted_files, $archivedir,$verbosity, "delete");
+        }
+        
+        my @reindex_files = sort keys %{$block_hash->{'reindex_files'}};
+        
+        if (scalar(@reindex_files)>0) {
+            print STDERR "Files to reindex since last import:\n  ";
+            print STDERR join("\n  ",@reindex_files), "\n";
+            &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@reindex_files);
+            mark_docs_for_deletion($archive_info,$block_hash,\@reindex_files, $archivedir,$verbosity, "reindex");
+        }
+                
+        }
+        
+        # Play it safe, and run through the entire folder, only processing new or edited files
+        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+
+    }
+    else {
+        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    }
+
+    }
+    else
+    {
+    # 
+    # 1. Process delete files first
+    # 
+
+    my @deleted_files = keys %{$manifest_lookup->{'delete'}};
+    my @full_deleted_files = ();
+
+    # ensure all filenames are absolute
+    foreach my $df (@deleted_files) {       
+        my $full_df =
+        (&util::filename_is_absolute($df)) 
+        ? $df
+        : &util::filename_cat($importdir,$df);
+
+        push(@full_deleted_files,$full_df);
+    }
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
+    mark_docs_for_deletion($archive_info,{},
+                      \@full_deleted_files,
+                      $archivedir, $verbosity, "delete");
+
+
+    # 
+    # 2. Now files for reindexing
+    # 
+
+    my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
+    my @full_reindex_files = ();
+
+    # ensure all filenames are absolute
+    foreach my $rf (@reindex_files) {       
+        my $full_rf =
+        (&util::filename_is_absolute($rf)) 
+        ? $rf
+        : &util::filename_cat($importdir,$rf);
+
+        push(@full_reindex_files,$full_rf);
+    }
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
+    mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
+
+    # And now ensure the new version of the file processed by appropriate
+    # plugin
+    foreach my $full_rf (@full_reindex_files) {
+        &plugin::read ($pluginfo, "", $full_rf, {}, {}, $processor, $maxdocs, 0, $gli);
+    }
+
+
+    # 
+    # 3. Now finally any new files
+    # 
+
+    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
+        &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
+    }
+
+
+    }
+
+    &plugin::end($pluginfo, $processor);
+
+    &plugin::deinit($pluginfo, $processor);
+
+    # Store the value of OIDCount (used in doc.pm) so it can be
+    # restored correctly to this value on an incremental build
+    store_doc_oid_count($archivedir);
+
+    # write out the archive information file
+    $processor->close_file_output() if $groupsize > 1;
+    $processor->close_group_output() if $processor->is_group();
+
+# The following 'if' statement is in the export.pl version of the script,
+# The reason for the 'if' statement is now given in export.pl
+# Unclear at this point if the same should be done here
+##    if (($saveas =~ m/^.*METS$/) || ($saveas eq "MARC")) {
+    # Not all export types need this (e.g. DSpace)
+
+    # should we still do this in debug mode??
+
+    # for backwards compatability with archvies.inf file
+    if ($arcinfo_doc_filename =~ m/(contents)|(\.inf)$/) {
+    $archive_info->save_info($arcinfo_doc_filename);
+    }
+    else {
+    $archive_info->save_revinfo_db($arcinfo_src_filename);
+    }
+
+
+##    }
+
+    return $pluginfo;
+}
+
+
+sub generate_statistics
+{
+    my $self = shift @_;
+    my ($inexport_mode,$pluginfo) = @_;
+
+    my $statsfile = $self->{'statsfile'};
+    my $out       = $self->{'out'};
+    my $faillogname = $self->{'faillogname'};
+    my $gli       = $self->{'gli'};
+
+    # write out import stats
+    my $close_stats = 0;
+    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
+    if (open (STATS, ">$statsfile")) {
+        $statsfile = 'import::STATS';
+        $close_stats = 1;
+    } else {
+        &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
+        &gsprintf($out, "{import.stats_backup}\n");
+        $statsfile = 'STDERR';
+    }
+    }
+
+    &gsprintf($out, "\n");
+    &gsprintf($out, "*********************************************\n");
+    &gsprintf($out, "{import.complete}\n");
+    &gsprintf($out, "*********************************************\n");
+
+    &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
+    if ($close_stats) {
+    close STATS;
+    }
+
+    close OUT if $self->{'close_out'};
+    close FAILLOG;
+}
+
+
+
+
+