Index: /main/trunk/greenstone2/bin/script/buildcol.pl
===================================================================
--- /main/trunk/greenstone2/bin/script/buildcol.pl	(revision 27304)
+++ /main/trunk/greenstone2/bin/script/buildcol.pl	(revision 27305)
@@ -4,6 +4,7 @@
 #
 # buildcol.pl --
+#
 # A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
+# from the New Zealand Digital Library Project at the
 # University of Waikato, New Zealand.
 #
@@ -26,62 +27,67 @@
 ###########################################################################
 
-
 # This program will build a particular collection.
-
 package buildcol;
 
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/XML/XPath");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
-
-    if (defined $ENV{'GSDL-RUN-SETUP'}) {
-	require util;
-	&util::setup_greenstone_env($ENV{'GSDLHOME'}, $ENV{'GSDLOS'});
+# Environment
+BEGIN
+{
+  die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+  die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan/XML/XPath');
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugins');
+  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/classify');
+
+  if (defined $ENV{'GSDL-RUN-SETUP'})
+  {
+    require util;
+    &util::setup_greenstone_env($ENV{'GSDLHOME'}, $ENV{'GSDLOS'});
+  }
+
+  if (defined $ENV{'GSDLEXTS'})
+  {
+    my @extensions = split(/:/, $ENV{'GSDLEXTS'});
+    foreach my $e (@extensions)
+    {
+      my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
+
+      unshift(@INC, $ext_prefix . '/perllib');
+      unshift(@INC, $ext_prefix . '/perllib/cpan');
+      unshift(@INC, $ext_prefix . '/perllib/plugins');
+      unshift(@INC, $ext_prefix . '/perllib/classify');
     }
-	
-    if (defined $ENV{'GSDLEXTS'}) {
-	my @extensions = split(/:/,$ENV{'GSDLEXTS'});
-	foreach my $e (@extensions) {
-	    my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e";
-
-	    unshift (@INC, "$ext_prefix/perllib");
-	    unshift (@INC, "$ext_prefix/perllib/cpan");
-	    unshift (@INC, "$ext_prefix/perllib/plugins");
-	    unshift (@INC, "$ext_prefix/perllib/classify");
-	}
+  }
+  if (defined $ENV{'GSDL3EXTS'})
+  {
+    my @extensions = split(/:/, $ENV{'GSDL3EXTS'});
+    foreach my $e (@extensions)
+    {
+      my $ext_prefix = $ENV{'GSDL3SRCHOME'} . '/ext/' . $e;
+
+      unshift(@INC, $ext_prefix . '/perllib');
+      unshift(@INC, $ext_prefix . '/perllib/cpan');
+      unshift(@INC, $ext_prefix . '/perllib/plugins');
+      unshift(@INC, $ext_prefix . '/perllib/classify');
     }
-    if (defined $ENV{'GSDL3EXTS'}) {
-	my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
-	foreach my $e (@extensions) {
-	    my $ext_prefix = "$ENV{'GSDL3SRCHOME'}/ext/$e";
-
-	    unshift (@INC, "$ext_prefix/perllib");
-	    unshift (@INC, "$ext_prefix/perllib/cpan");
-	    unshift (@INC, "$ext_prefix/perllib/plugins");
-	    unshift (@INC, "$ext_prefix/perllib/classify");
-	}
-    }
-
+  }
 }
 
-use colcfg;
-use dbutil;
-use util;
-use scriptutil;
-use FileHandle;
-use gsprintf;
-use printusage;
-use parse2;
-
+# Pragma
 use strict;
 no strict 'refs'; # allow filehandles to be variables and vice versa
 no strict 'subs'; # allow barewords (eg STDERR) as function arguments
 
-
+# Modules
+use Symbol qw<qualify>; # Needed for runtime loading of modules [jmt12]
+
+# Greenstone Modules
+use buildcolutils;
+use FileUtils;
+use util;
+
+# Globals
+# - build up arguments list/control
 my $mode_list =
     [ { 'name' => "all",
@@ -94,5 +100,5 @@
         'desc' => "{buildcol.mode.infodb}" } ];
 
-my $sec_index_list = 
+my $sec_index_list =
     [ {'name' => "never",
        'desc' => "{buildcol.sections_index_document_metadata.never}" },
@@ -127,5 +133,5 @@
 	'type' => "string",
 	# parsearg left "" as default
-	#'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	#'deft' => &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "collect"),
 	'reqd' => "no",
         'hiddengli' => "yes" },
@@ -145,5 +151,5 @@
 	'type' => "string",
 	# parsearg left "" as default
-	#'deft' => &util::filename_cat("<collectdir>", "colname", "etc", "fail.log"),
+	#'deft' => &FileUtils::filenameConcatenate("<collectdir>", "colname", "etc", "fail.log"),
 	'reqd' => "no",
 	'modegli' => "3" },
@@ -243,4 +249,14 @@
 	'reqd' => "no",
 	'hiddengli' => "yes" },
+      { 'name' => "indexname",
+	'desc' => "{buildcol.index}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "3" },
+      { 'name' => "indexlevel",
+	'desc' => "{buildcol.indexlevel}",
+	'type' => "string",
+	'reqd' => "no",
+	'modegli' => "3" },
       ];
 
@@ -249,489 +265,134 @@
 		'args' => $arguments };
 
-
-# globals
-my $collection;
-my $configfilename;
-my $out;
-
-# used to signify "gs2"(default) or "gs3"
-my $gs_mode = "gs2";
-
-## @method gsprintf()
-#  Print a string to the screen after looking it up from a locale dependant
-#  strings file. This function is losely based on the idea of resource
-#  bundles as used in Java.
-#
-#  @param  $error The STDERR stream.
-#  @param  $text The string containing GS keys that should be replaced with
-#                their locale dependant equivilents.
-#  @param  $out The output stream.
-#  @return The locale-based string to output.
-#
-sub gsprintf()
-{
-    return &gsprintf::gsprintf(@_);
-}
-## gsprintf() ##
-
+# The hash maps between argument and the buildcolutils subclass supporting that
+# argument - allowing for extensions to override the normal buildcolutils as
+# necessary
+my $function_to_subclass_mappings = {};
+
+# Lets get the party rolling... or ball started... hmmm
 &main();
 
-## @method main()
-#
-#  [Parses up and validates the arguments to the build process before creating
-#  the appropriate build process to do the actual work - John]
-#
-#  @note Added true incremental support - John Thompson, DL Consulting Ltd.
-#  @note There were several bugs regarding using directories other than 
-#        "import" or "archives" during import and build quashed. - John 
-#        Thompson, DL Consulting Ltd.
-#
-#  @param  $incremental If true indicates this build should not regenerate all
-#                       the index and metadata files, and should instead just
-#                       append the information found in the archives directory
-#                       to the existing files. If this requires some complex
-#                       work so as to correctly insert into a classifier so be
-#                       it. Of course none of this is done here - instead the
-#                       incremental argument is passed to the document
-#                       processor.
-#
+exit;
+
 sub main
 {
-    # command line args
-    my ($verbosity, $archivedir, $cachedir, $builddir, $site, $maxdocs, 
-	$debug, $mode, $indexname, $removeold, $keepold, 
-	$incremental, $incremental_mode,
-	$remove_empty_classifications,
-	$collectdir, $build, $type, $textindex,
-	$no_strip_html, $store_metadata_coverage,
-	$no_text, $faillog, $gli, $index, $language,
-	$sections_index_document_metadata, $maxnumeric, $activate);
-	
-    my $xml = 0;
-    my $hashParsingResult = {};
-    # general options available to all plugins
-    my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
-
-    # If parse returns -1 then something has gone wrong
-    if ($intArgLeftinAfterParsing == -1)
+  # Dynamically include arguments from any subclasses of buildcolutils we find
+  # in the extensions directory
+  if (defined $ENV{'GSDLEXTS'})
+  {
+    &_scanForSubclasses($ENV{'GSDLHOME'}, $ENV{'GSDLEXTS'});
+  }
+  if (defined $ENV{'GSDL3EXTS'})
+  {
+    &_scanForSubclasses($ENV{'GSDL3SRCHOME'}, $ENV{'GSDL3EXTS'});
+  }
+
+  # Loop through arguments, checking to see if any depend on a specific
+  # subclass of buildcolutils. Note that we load the first subclass we
+  # encounter so only support a single 'override' ATM.
+  my $subclass;
+  foreach my $argument (@ARGV)
+  {
+    # proper arguments start with a hyphen
+    if ($argument =~ /^-/ && defined $function_to_subclass_mappings->{$argument})
     {
-	&PrintUsage::print_txt_usage($options, "{buildcol.params}");
-	die "\n";
+      my $required_subclass = $function_to_subclass_mappings->{$argument};
+      if (!defined $subclass)
+      {
+        $subclass = $required_subclass;
+      }
+      # Oh noes! The user has included specific arguments from two different
+      # subclasses... this isn't supported
+      elsif ($subclass ne $required_subclass)
+      {
+        print STDERR "Error! You cannot specify arguments from two different extention specific buildcolutils modules: " . $subclass . " != " . $required_subclass . "\n";
+        exit;
+      }
     }
-    
-    foreach my $strVariable (keys %$hashParsingResult)
+  }
+
+  my $buildcolutils;
+  if (defined $subclass)
+  {
+    print "* Loading overriding buildcolutils module: " . $subclass . "\n";
+    require $subclass . '.pm';
+    $buildcolutils = new $subclass(\@ARGV, $options);
+  }
+  # We don't have a overridden buildcolutils, or the above command failed
+  # somehow so load the base class
+  if (!defined $buildcolutils)
+  {
+    $buildcolutils = new buildcolutils(\@ARGV, $options);
+  }
+
+  my $collection = $buildcolutils->get_collection();
+  if (defined $collection)
+  {
+    my ($config_filename,$collect_cfg) = $buildcolutils->read_collection_cfg($collection, $options);
+    $buildcolutils->set_collection_options($collect_cfg);
+
+    my $builders_ref = $buildcolutils->prepare_builders($config_filename, $collect_cfg);
+    $buildcolutils->build_collection($builders_ref);
+    $buildcolutils->build_auxiliary_files($builders_ref);
+    $buildcolutils->complete_builders($builders_ref);
+
+    # The user may have requested the collection be activated
+    $buildcolutils->activate_collection();
+  }
+
+  # Cleanup
+  $buildcolutils->deinit();
+}
+# main()
+
+# @function _scanForSubclasses()
+# @param $dir The extension directory to look within
+# @param $exts A list of the available extensions (as a colon separated string)
+# @return The number of subclasses of buildcolutils found as an Integer
+sub _scanForSubclasses
+{
+  my ($dir, $exts) = @_;
+  my $class_count = 0;
+  my $ext_prefix = &FileUtils::filenameConcatenate($dir, "ext");
+  my @extensions = split(/:/, $exts);
+  foreach my $e (@extensions)
+  {
+    # - any subclass must be prefixed with the name of the ext
+    my $package_name = $e . 'buildcolutils';
+    $package_name =~ s/[^a-z]//gi; # package names have limited characters
+    my $file_name = $package_name . '.pm';
+    my $file_path = &FileUtils::filenameConcatenate($ext_prefix, $e, 'perllib', $file_name);
+    # see if we have a subclass lurking in that extension folder
+    if (&FileUtils::fileExists($file_path))
     {
-	eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
+      # - note we load the filename (with pm) unlike normal modules
+      require $file_name;
+      # - make call to the newly created package
+      my $symbol = qualify('getSupportedArguments', $package_name);
+      # - strict prevents strings being used as function calls, so temporarily
+      #   disable that pragma
+      no strict;
+      # - lets check that the function we are about to call actually exists
+      if ( defined &{$symbol} )
+      {
+        my $extra_arguments = &{$symbol}();
+        foreach my $argument (@{$extra_arguments})
+        {
+          # - record a mapping from each extra arguments to the subclass
+          #   that supports it. We put the hyphen on here to make comparing
+          #   with command line arguments even easier
+          $function_to_subclass_mappings->{'-' . $argument->{'name'}} = $package_name;
+          # - and them add them as acceptable arguments to import.pl
+          push(@{$options->{'args'}}, $argument);
+        }
+        $class_count++;
+      }
+      else
+      {
+        print "Warning! A subclass of buildcolutils module (named '" . $file_name . "') does not implement the required getSupportedArguments() function - ignoring. Found in: " . $file_path . "\n";
+      }
     }
-
-    # If $language has been specified, load the appropriate resource bundle
-    # (Otherwise, the default resource bundle will be loaded automatically)
-    if ($language && $language =~ /\S/) {
-	&gsprintf::load_language_specific_resource_bundle($language);
-    }
-
-    if ($xml) {
-        &PrintUsage::print_xml_usage($options);
-	print "\n";
-	return;
-    }
-
-    if ($gli) { # the gli wants strings to be in UTF-8
-	&gsprintf::output_strings_in_UTF8; 
-    }
-
-    # now check that we had exactly one leftover arg, which should be 
-    # the collection name. We don't want to do this earlier, cos 
-    # -xml arg doesn't need a collection name
-    # Or if the user specified -h, then we output the usage also
-    if ($intArgLeftinAfterParsing != 1 || (@ARGV && $ARGV[0] =~ /^\-+h/))
-    {
-	&PrintUsage::print_txt_usage($options, "{buildcol.params}");
-	die "\n";
-    }
-    
-    $textindex = "";
-    my $close_out = 0;
-    if ($out !~ /^(STDERR|STDOUT)$/i) {
-	open (OUT, ">$out") ||
-	    (&gsprintf(STDERR, "{common.cannot_open_output_file}\n", $out) && die);
-	$out = "buildcol::OUT";
-	$close_out = 1;
-    }
-    $out->autoflush(1);
-
-    # get and check the collection
-    if (($collection = &colcfg::use_collection($site, @ARGV, $collectdir)) eq "") {
-	&PrintUsage::print_txt_usage($options, "{buildcol.params}");
-	die "\n";
-    }
-
-    if ($faillog eq "") {
-	$faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
-    }
-    # note that we're appending to the faillog here (import.pl clears it each time)
-    # this could potentially create a situation where the faillog keeps being added 
-    # to over multiple builds (if the import process is being skipped)
-    open (FAILLOG, ">>$faillog") ||
-	(&gsprintf(STDERR, "{common.cannot_open_fail_log}\n", $faillog) && die);
-    $faillog = 'buildcol::FAILLOG';
-    $faillog->autoflush(1);
-
-    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
-    # Don't know why this didn't already happen, but now collection specific
-    # classify and plugins directory also added to include path
-    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib/classify");
-    unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib/plugins");
-
-    # Read in the collection configuration file.
-    my ($collectcfg, $buildtype, $orthogonalbuildtypes); 
-    if ((defined $site) && ($site ne "")) { # GS3
-	$gs_mode = "gs3";
-    }
-    $configfilename = &colcfg::get_collect_cfg_name($out, $gs_mode);
-    $collectcfg = &colcfg::read_collection_cfg ($configfilename, $gs_mode);
-    
-    # If the infodbtype value wasn't defined in the collect.cfg file, use the default
-    if (!defined($collectcfg->{'infodbtype'}))
-    {
-      $collectcfg->{'infodbtype'} = &dbutil::get_default_infodb_type();
-    }
-
-    if ($verbosity !~ /\d+/) {
-	if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
-	    $verbosity = $collectcfg->{'verbosity'};
-	} else {
-	    $verbosity = 2; # the default
-	}
-    }
-    # we use searchtype for determining buildtype, but for old versions, use buildtype
-    if (defined $collectcfg->{'buildtype'}) {
-	$buildtype = $collectcfg->{'buildtype'};
-    } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
-	$buildtype = "mgpp";
-    } else {
-	$buildtype = "mg"; #mg is the default
-    }
-
-    if (defined $collectcfg->{'orthogonalbuildtypes'}) {
-	$orthogonalbuildtypes = $collectcfg->{'orthogonalbuildtypes'};
-    }
-
-    if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
-	$archivedir = $collectcfg->{'archivedir'};
-    }
-    if (defined $collectcfg->{'cachedir'} && $cachedir eq "") {
-	$cachedir = $collectcfg->{'cachedir'};
-    }
-    if (defined $collectcfg->{'builddir'} && $builddir eq "") {
-	$builddir = $collectcfg->{'builddir'};
-    }
-    if ($maxdocs !~ /\-?\d+/) {
-	if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
-	    $maxdocs = $collectcfg->{'maxdocs'};
-	} else {
-	    $maxdocs = -1; # the default
-	}
-    }
-    if (defined $collectcfg->{'maxnumeric'} && $collectcfg->{'maxnumeric'} =~ /\d+/) {
-	$maxnumeric = $collectcfg->{'maxnumeric'};
-    } 
-    
-    if ($maxnumeric < 4 || $maxnumeric > 512) {
-	$maxnumeric = 4;
-    }
-    
-    if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
-	$debug = 1;
-    }
-    if ($mode !~ /^(all|compress_text|build_index|infodb)$/) {
-	if (defined $collectcfg->{'mode'} && $collectcfg->{'mode'} =~ /^(all|compress_text|build_index|infodb)$/) {
-	    $mode = $collectcfg->{'mode'};
-	} else {
-	    $mode = "all"; # the default
-	}
-    }
-    if (defined $collectcfg->{'index'} && $indexname eq "") {
-	$indexname = $collectcfg->{'index'};
-    }
-    if (defined $collectcfg->{'no_text'} && $no_text == 0) {
-	if ($collectcfg->{'no_text'} =~ /^true$/i) {
-	    $no_text = 1;
-	}
-    }
-    if (defined $collectcfg->{'no_strip_html'} && $no_strip_html == 0) {
-	if ($collectcfg->{'no_strip_html'} =~ /^true$/i) {
-	    $no_strip_html = 1;
-	}
-    }
-    if (defined $collectcfg->{'store_metadata_coverage'} && $store_metadata_coverage == 0) {
-	if ($collectcfg->{'store_metadata_coverage'} =~ /^true$/i) {
-	    $store_metadata_coverage = 1;
-	}
-    }
-    if (defined $collectcfg->{'remove_empty_classifications'} && $remove_empty_classifications == 0) {
-	if ($collectcfg->{'remove_empty_classifications'} =~ /^true$/i) {
-	    $remove_empty_classifications = 1;
-	}
-    }
-    
-    if ($buildtype eq "mgpp" && defined $collectcfg->{'textcompress'}) {
-	$textindex = $collectcfg->{'textcompress'};
-    }
-    if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
-	$gli = 1;
-    }
-
-    if ($sections_index_document_metadata !~ /\S/ && defined $collectcfg->{'sections_index_document_metadata'}) {
-	$sections_index_document_metadata = $collectcfg->{'sections_index_document_metadata'};
-    }
-    
-    if ($sections_index_document_metadata !~ /^(never|always|unless_section_metadata_exists)$/) {
-	$sections_index_document_metadata = "never";
-    }
-    
-    ($removeold, $keepold, $incremental, $incremental_mode) 
-	= &scriptutil::check_removeold_and_keepold($removeold, $keepold, 
-						   $incremental, "building", 
-						   $collectcfg);
- 
-    $gli = 0 unless defined $gli;
-    
-    # New argument to track whether build is incremental
-    $incremental = 0 unless defined $incremental;
-
-    print STDERR "<Build>\n" if $gli;
-
-    #set the text index
-    if (($buildtype eq "mgpp") || ($buildtype eq "lucene") || ($buildtype eq "solr")) {
-	if ($textindex eq "") {
-	    $textindex = "text";
-	}
-    }
-    else {
-	$textindex = "section:text";
-    }
-
-    # fill in the default archives and building directories if none
-    # were supplied, turn all \ into / and remove trailing /
-
-    my ($realarchivedir, $realbuilddir);
-    # Modified so that the archivedir, if provided as an argument, is made
-    # absolute if it isn't already
-    if ($archivedir eq "")
-      {
-        $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
-      }
-    else
-      {
-        $archivedir = &util::make_absolute($ENV{'GSDLCOLLECTDIR'}, $archivedir);
-      }
-    # End Mod
-    $archivedir =~ s/[\\\/]+/\//g;
-    $archivedir =~ s/\/$//;
-
-    if ($builddir eq "") {
-	$builddir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "building");
-	if ($incremental) {
-	    &gsprintf($out, "{buildcol.incremental_default_builddir}\n");
-	}
-    }
-    $builddir =~ s/[\\\/]+/\//g;
-    $builddir =~ s/\/$//;
-
-    # update the archive cache if needed
-    if ($cachedir) {
-	&gsprintf($out, "{buildcol.updating_archive_cache}\n")
-	    if ($verbosity >= 1);
-
-	$cachedir =~ s/[\\\/]+$//;
-	$cachedir .= "/collect/$collection" unless 
-	    $cachedir =~ /collect\/$collection/;
-
-	$realarchivedir = "$cachedir/archives";
-	$realbuilddir = "$cachedir/building";
-	&util::mk_all_dir ($realarchivedir);
-	&util::mk_all_dir ($realbuilddir);
-	&util::cachedir ($archivedir, $realarchivedir, $verbosity);
-
-    } else {
-	$realarchivedir = $archivedir;
-	$realbuilddir = $builddir;
-    }
-
-    # build it in realbuilddir
-    &util::mk_all_dir ($realbuilddir);
-
-    my ($buildertype, $builderdir,  $builder);
-    # if a builder class has been created for this collection, use it
-    # otherwise, use the mg or mgpp builder
-    if (-e "$ENV{'GSDLCOLLECTDIR'}/custom/${collection}/perllib/custombuilder.pm") {
-	$builderdir = "$ENV{'GSDLCOLLECTDIR'}/custom/${collection}/perllib";
-	$buildertype = "custombuilder";
-    } elsif (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/custombuilder.pm") {
-	$builderdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
-	$buildertype = "custombuilder";
-    } elsif (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}builder.pm") {
-	$builderdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
-	$buildertype = "${collection}builder";
-    } else {	
-
-	$builderdir = undef;
-	if ($buildtype ne "") {
-	    # caters for extension-based build types, such as 'solr'
-	    $buildertype = $buildtype."builder";
-	}
-	else {
-	    # Default to mgpp
-	    $buildertype = "mgppbuilder";
-	}
-    }
-    # check for extension specific builders 
-    # (that will then be run after main builder.pm
-
-    my @builderdir_list = ($builderdir);
-    my @buildertype_list = ($buildertype);
-    
-    if (defined $orthogonalbuildtypes) {
-	foreach my $obt (@$orthogonalbuildtypes) {
-
-	    push(@builderdir_list,undef); # rely on @INC to find it
-	    push(@buildertype_list,$obt."Builder");
-	}
-    }
-
-    # Set up array of the main builder.pm, followed by any ones
-    # from the extension folders
-    
-    my $num_builders = scalar(@buildertype_list);
-    my @builders = ();
-
-    for (my $i=0; $i<$num_builders; $i++) {
-	my $this_builder;
-	my $this_buildertype = $buildertype_list[$i];
-	my $this_builderdir  = $builderdir_list[$i];
-
-	if ((defined $this_builderdir) && ($this_builderdir ne "")) {
-	    require "$this_builderdir/$this_buildertype.pm";
-	}
-	else {
-	    require "$this_buildertype.pm";
-	}
-
-	eval("\$this_builder = new $this_buildertype(\$site, \$collection, " .
-	     "\$realarchivedir, \$realbuilddir, \$verbosity, " .
-	     "\$maxdocs, \$debug, \$keepold, \$incremental, \$incremental_mode, " .
-	     "\$remove_empty_classifications, " .
-	     "\$out, \$no_text, \$faillog, \$gli)");
-	die "$@" if $@;
-
-	push(@builders,$this_builder);
-    }
-    
-    # Init phase for builders
-    for (my $i=0; $i<$num_builders; $i++) {
-	my $this_buildertype = $buildertype_list[$i];
-	my $this_builderdir  = $builderdir_list[$i];
-	my $this_builder     = $builders[$i];
-
-	$this_builder->init();
-	$this_builder->set_maxnumeric($maxnumeric);
-    
-	if (($this_buildertype eq "mgppbuilder") && $no_strip_html) {
-	    $this_builder->set_strip_html(0);
-	}
-
-	if ($sections_index_document_metadata ne "never") {
-	    $this_builder->set_sections_index_document_metadata($sections_index_document_metadata);
-	}
-	
-	if ($store_metadata_coverage) {
-	    $this_builder->set_store_metadata_coverage(1);
-	}
-    }
-        
-    # Run the requested passes
-
-    if ($mode =~ /^all$/i) {
-
-	# 'map' modifies the elements of the original array, so calling
-	# methods -- as done below -- will cause (by default) @builders
-	# to be changed to whatever these functions return (which is *not*
-	# what we want -- we want to leave the values unchanged) 
-	# => Use 'local' (dynamic scoping) to give each 'map' call its
-	#    own local copy This could also be done with: 
-	#      (my $new =$_)->method(); $new 
-	#    but is a bit more cumbersome to write
-	
-	map { local $_=$_; $_->compress_text($textindex); } @builders;
-	map { local $_=$_; $_->build_indexes($indexname); } @builders;
-	map { local $_=$_; $_->make_infodatabase(); }  @builders;
-	map { local $_=$_; $_->collect_specific(); } @builders;
-    } elsif ($mode =~ /^compress_text$/i) {
-	map { local $_=$_; $_->compress_text($textindex); } @builders;
-    } elsif ($mode =~ /^build_index$/i) {
-	map { local $_=$_; $_->build_indexes($indexname); } @builders;
-    } elsif ($mode =~ /^infodb$/i) {
-	map { local $_=$_; $_->make_infodatabase(); } @builders;
-    } else {
-	(&gsprintf(STDERR, "{buildcol.unknown_mode}\n", $mode) && die);
-    }
-
-    if (!$debug) {
-	map {local $_=$_; $_->make_auxiliary_files(); } @builders;
-    }
-    map {local $_=$_; $_->deinit(); } @builders;
-    
-    if (($realbuilddir ne $builddir) && !$debug) {
-	&gsprintf($out, "{buildcol.copying_back_cached_build}\n")
-	    if ($verbosity >= 1);
-	&util::rm_r ($builddir);
-	&util::cp_r ($realbuilddir, $builddir);
-    }
-
-
-    # for RSS support: Need rss-items.rdf file in index folder
-    #  check if a file called rss-items.rdf exists in archives, then copy it into the building folder
-    #  so that when building is moved to index, this file will then also be in index as desired
-    my $collection_dir = &util::resolve_collection_dir($collectdir, $collection, $site);
-    my $rss_items_rdf_file = &util::filename_cat($archivedir, "rss-items.rdf");
-    if(defined $builddir && -d $builddir && -f $rss_items_rdf_file) {
-	&gsprintf($out, "{buildcol.copying_rss_items_rdf}\n") if ($verbosity >= 1);
-	&util::cp ($rss_items_rdf_file, $builddir);
-    }
-
-	# if buildcol.pl was run with -activate, need to run activate.pl 
-	# now that building's complete	
-	if($activate) {
-	
-		#my $quoted_argv = join(" ", map { "\"$_\"" } @ARGV);
-		
-		my @activate_argv = ();
-		push(@activate_argv,"-collectdir",$collectdir) if($collectdir);
-		push(@activate_argv,"-builddir",$builddir) if($builddir);
-		push(@activate_argv,"-site",$site) if($site);
-		push(@activate_argv,"-verbosity",$verbosity) if($verbosity);
-		push(@activate_argv,"-removeold") if($removeold);
-		push(@activate_argv,"-keepold") if($keepold);
-		push(@activate_argv,"-incremental") if($incremental);
-		my $quoted_argv = join(" ", map { "\"$_\"" } @activate_argv);
-		
-		my $activatecol_cmd = "\"".&util::get_perl_exec()."\" -S activate.pl $quoted_argv \"$collection\"";
-		
-		my $activatecol_status = system($activatecol_cmd)/256;
-
-		if ($activatecol_status != 0) {
-			print STDERR "Error: Failed to run: $activatecol_cmd\n";
-			print STDERR "       $!\n" if ($! ne "");
-			exit(-1);
-		}
-	}
-    
-    close OUT if $close_out;
-    close FAILLOG;
-
-    print STDERR "</Build>\n" if $gli;
+  }
+  return $class_count;
 }
-
-
-
+# _scanForSubclasses()
Index: /main/trunk/greenstone2/bin/script/import.pl
===================================================================
--- /main/trunk/greenstone2/bin/script/import.pl	(revision 27304)
+++ /main/trunk/greenstone2/bin/script/import.pl	(revision 27305)
@@ -67,8 +67,17 @@
 }
 
+# Pragma
 use strict;
+use warnings;
+
+# Modules
+use Symbol qw<qualify>; # Needed for runtime loading of modules [jmt12]
+
+# Greenstone Modules
+use FileUtils;
 use inexport;
-
-my $oidtype_list = 
+use util;
+
+my $oidtype_list =
     [ { 'name' => "hash",
         'desc' => "{import.OIDtype.hash}" },
@@ -129,5 +138,5 @@
 	'type' => "string",
 	# parsearg left "" as default
-	#'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+	#'deft' => &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "collect"),
 	'deft' => "",
 	'reqd' => "no",
@@ -154,5 +163,5 @@
 	'type' => "string",
 	# parsearg left "" as default
-	#'deft' => &util::filename_cat("&lt;collectdir&gt;", "colname", "etc", "fail.log"),
+	#'deft' => &FileUtils::filenameConcatenate("&lt;collectdir&gt;", "colname", "etc", "fail.log"),
 	'deft' => "",
 	'reqd' => "no",
@@ -267,24 +276,125 @@
 		'args' => $arguments };
 
-
-
-sub main 
+my $function_to_inexport_subclass_mappings = {};
+
+sub main
 {
-    my $inexport = new inexport("import",\@ARGV,$options);
-    
-    my $collection = $inexport->get_collection();
-
-    if (defined $collection) {
-	my ($config_filename,$collect_cfg) 
-	    = $inexport->read_collection_cfg($collection,$options);    
-
-	$inexport->set_collection_options($collect_cfg);
-	
-	my $pluginfo = $inexport->process_files($config_filename,$collect_cfg);
-	
-	$inexport->generate_statistics($pluginfo);
+  # Dynamically include arguments from any subclasses of inexport we find
+  # in the extensions directory
+  if (defined $ENV{'GSDLEXTS'})
+  {
+    &_scanForSubclasses($ENV{'GSDLHOME'}, $ENV{'GSDLEXTS'});
+  }
+  if (defined $ENV{'GSDL3EXTS'})
+  {
+    &_scanForSubclasses($ENV{'GSDL3SRCHOME'}, $ENV{'GSDL3EXTS'});
+  }
+
+  # Loop through arguments, checking to see if any depend on a specific
+  # subclass of InExport. Note that we load the first subclass we encounter
+  # so only support a single 'override' ATM.
+  my $inexport_subclass;
+  foreach my $argument (@ARGV)
+  {
+    # proper arguments start with a hyphen
+    if ($argument =~ /^-/ && defined $function_to_inexport_subclass_mappings->{$argument})
+    {
+      my $required_inexport_subclass = $function_to_inexport_subclass_mappings->{$argument};
+      if (!defined $inexport_subclass)
+      {
+        $inexport_subclass = $required_inexport_subclass;
+      }
+      # Oh noes! The user has included specific arguments from two different
+      # inexport subclasses... this isn't supported
+      elsif ($inexport_subclass ne $required_inexport_subclass)
+      {
+        print STDERR "Error! You cannot specify arguments from two different extention specific inexport modules: " . $inexport_subclass . " != " . $required_inexport_subclass . "\n";
+        exit;
+      }
     }
+  }
+
+  my $inexport;
+  if (defined $inexport_subclass)
+  {
+    print "* Loading Overriding InExport Module: " . $inexport_subclass . "\n";
+    require $inexport_subclass . '.pm';
+    $inexport = new $inexport_subclass("import",\@ARGV,$options);
+  }
+  # We don't have a overridden inexport, or the above command failed somehow
+  # so load the base inexport class
+  if (!defined $inexport)
+  {
+    $inexport = new inexport("import",\@ARGV,$options);
+  }
+
+  my $collection = $inexport->get_collection();
+
+  if (defined $collection)
+  {
+    my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options);
+
+    $inexport->set_collection_options($collect_cfg);
+
+    my $pluginfo = $inexport->process_files($config_filename,$collect_cfg);
+
+    $inexport->generate_statistics($pluginfo);
+  }
+
+  $inexport->deinit();
 }
-
+# main()
+
+# @function _scanForSubclasses()
+# @param $dir The extension directory to look within
+# @param $exts A list of the available extensions (as a colon separated string)
+# @return The number of subclasses of InExport found as an Integer
+sub _scanForSubclasses
+{
+  my ($dir, $exts) = @_;
+  my $inexport_class_count = 0;
+  my $ext_prefix = &FileUtils::filenameConcatenate($dir, "ext");
+  my @extensions = split(/:/, $exts);
+  foreach my $e (@extensions)
+  {
+    # - any subclass of InExport must be prefixed with the name of the ext
+    my $package_name = $e . 'inexport';
+    $package_name =~ s/[^a-z]//gi; # package names have limited characters
+    my $inexport_filename = $package_name . '.pm';
+    my $inexport_path = &FileUtils::filenameConcatenate($ext_prefix, $e, 'perllib', $inexport_filename);
+    # see if we have a subclass of InExport lurking in that extension folder
+    if (-f $inexport_path)
+    {
+      # - note we load the filename (with pm) unlike normal modules
+      require $inexport_filename;
+      # - make call to the newly created package
+      my $symbol = qualify('getSupportedArguments', $package_name);
+      # - strict prevents strings being used as function calls, so temporarily
+      #   disable that pragma
+      no strict;
+      # - lets check that the function we are about to call actually exists
+      if ( defined &{$symbol} )
+      {
+        my $extra_arguments = &{$symbol}();
+        foreach my $argument (@{$extra_arguments})
+        {
+          # - record a mapping from each extra arguments to the inexport class
+          #   that supports it. We put the hyphen on here to make comparing
+          #   with command line arguments even easier
+          $function_to_inexport_subclass_mappings->{'-' . $argument->{'name'}} = $package_name;
+          # - and them add them as acceptable arguments to import.pl
+          push(@{$options->{'args'}}, $argument);
+        }
+        $inexport_class_count++;
+      }
+      else
+      {
+        print "Warning! A subclass of InExport module (named '" . $inexport_filename . "') does not implement the required getSupportedArguments() function - ignoring. Found in: " . $inexport_path . "\n";
+      }
+    }
+  }
+  return $inexport_class_count;
+}
+# _scanForInExportModules()
 
 &main();
