Index: /trunk/gsdl/perllib/lucenebuilder.pm
===================================================================
--- /trunk/gsdl/perllib/lucenebuilder.pm	(revision 13589)
+++ /trunk/gsdl/perllib/lucenebuilder.pm	(revision 13590)
@@ -156,12 +156,10 @@
 	$handle = lucenebuilder::PIPEOUT;
     }
-    my $levels = $self->{'levels'};
-    my $gdbm_level = "document";
-    if ($levels->{'section'})
-    {
-	$gdbm_level = "section";
-    }
-
-    undef $levels->{'paragraph'}; # get rid of para if we had it.
+
+    # stored text is always Doc and Sec levels    
+    my $levels = { 'document' => 1, 'section' => 1 };
+    # always do gdbm at section level
+    my $gdbm_level = "section"; 
+
     # set up the document processr
     $self->{'buildproc'}->set_output_handle ($handle);
@@ -232,4 +230,178 @@
 }
 
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index,$llevel) = @_;
+    my $outhandle = $self->{'outhandle'};
+    my $build_dir = $self->{'build_dir'};
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($build_dir, $indexdir));
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $scriptdir = "$ENV{'GSDLHOME'}/bin/script";
+
+    # Find the perl script to call to run lucene
+    my $full_lucene_passes = $self->{'full_lucene_passes'};
+    my $full_lucene_passes_exe = $self->{'full_lucene_passes_exe'};
+
+    # define the section names for lucenepasses
+    # define the section names and possibly the doc name for lucenepasses
+    my $lucene_passes_sections = $llevel;
+
+    my $opt_create_index = ($self->{'keepold'}) ? "" : "-create";
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$build_dir =~ s@/@\\@g;
+    } else {
+	if ($outhandle ne "STDERR") {
+	    # so lucene_passes doesn't print to stderr if we redirect output
+	    $osextra .= " 2>/dev/null";
+	}
+    }
+
+    # get the index expression if this index belongs
+    # to a subcollection
+    my $indexexparr = [];
+    my $langarr = [];
+
+    # there may be subcollection info, and language info.
+    my ($fields, $subcollection, $language) = split (":", $index);
+    my @subcollections = ();
+    @subcollections = split /,/, $subcollection if (defined $subcollection);
+
+    foreach $subcollection (@subcollections) {
+	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
+	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
+	}
+    }
+
+    # add expressions for languages if this index belongs to
+    # a language subcollection - only put languages expressions for the
+    # ones we want in the index
+    my @languages = ();
+    my $language_metadata = "Language";
+    if (defined ($self->{'collect_cfg'}->{'language_metadata'})) {
+	$language_metadata = $self->{'collect_cfg'}->{'language_metadata'};
+    }
+    @languages = split /,/, $language if (defined $language);
+    foreach my $language (@languages) {
+	my $not=0;
+	if ($language =~ s/^\!//) {
+	    $not = 1;
+	}
+	if($not) {
+	    push (@$langarr, "!$language");
+	} else {
+	    push (@$langarr, "$language");
+	}
+    }
+
+    # Build index dictionary. Uses verbatim stem method
+    print $outhandle "\n    creating index dictionary (lucene_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
+    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
+    my ($handle);
+
+    if ($self->{'debug'}) {
+	$handle = STDOUT;
+    } else {
+	print STDERR "Cmd: $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra\n";
+	if (!-e "$full_lucene_passes" ||
+	    !open (PIPEOUT, "| $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
+	    print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
+	    die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";
+	}
+	$handle = lucenebuilder::PIPEOUT;
+    }
+
+    my $store_levels = $self->{'levels'};
+    my $gdbm_level = "section"; #always
+    my $dom_level = "";
+    foreach my $key (keys %$store_levels) {
+	if ($mgppbuilder::level_map{$key} eq $llevel) {
+	    $dom_level = $key;
+	}
+    }
+    if ($dom_level eq "") {
+	print STDERR "Warning: unrecognized tag level $llevel\n";
+	$dom_level = "document";
+    }
+
+    my $local_levels = { $dom_level => 1 }; # work on one level at a time
+
+    # set up the document processr
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index, $indexexparr);
+    $self->{'buildproc'}->set_index_languages ($language_metadata, $langarr) if (defined $language);
+    $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($local_levels);
+    $self->{'buildproc'}->set_gdbm_level($gdbm_level);
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'},
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
+    close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
+
+    $self->{'buildproc'}->set_levels ($store_levels);
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
+# /** A modified version of the basebuilder.pm's function that generates the
+#  *  information database (GDBM) from the GA documents. We need to change this
+#  *  so that if we've been asked to do an incremental build we only add
+#  *  metadata to autohierarchy classifiers via the IncrementalBuildUtils
+#  *  module. All other classifiers and metadata will be ignored.
+#  */
+sub make_infodatabase
+{
+    my $self = shift (@_);
+    my $outhandle = $self->{'outhandle'};
+
+    my $dbext = ".bdb";
+    $dbext = ".ldb" if &util::is_little_endian();
+    my $infodb_file = &util::filename_cat($self->{'build_dir'}, "text", $self->{'collection'} . $dbext);
+
+    # If we aren't doing an incremental addition, then we just call the super-
+    # classes version
+    # Note: Incremental addition can only occur if a text/<collection>.ldb
+    #       already exists. If it doesn't, let the super classes function be
+    #       called once to generate it.
+    if (!$self->{'incremental_dlc'} || !(-e $infodb_file))
+    {
+        # basebuilder::make_infodatabase(@_);
+        # Note: this doesn't work as the direct reference means all the $self
+        #       data is lost.
+        $self->basebuilder::make_infodatabase(@_);
+        return;
+    }
+
+    # Carry on with an incremental addition
+    print $outhandle "\n*** performing an incremental addition to the info database\n" if ($self->{'verbosity'} >= 1);
+    print STDERR "<Stage name='CreateInfoData'>\n" if $self->{'gli'};
+
+    # 1. Init all the classifiers
+    &classify::init_classifiers ($self->{'classifiers'});
+    # 2. Init the buildproc settings.
+    #    Note: we still need this to process any associated files - but we
+    #    don't expect to pipe anything to txt2db so we can do away with the
+    #    complex output handle.
+    my $assocdir = &util::filename_cat($self->{'build_dir'}, "assoc");
+    &util::mk_all_dir ($assocdir);
+    $self->{'buildproc'}->set_mode ('incinfodb'); # Very Important
+    $self->{'buildproc'}->set_assocdir ($assocdir);
+    # 3. Read in all the metadata from the files in the archives directory using
+    #    the GAPlug and using ourselves as the document processor!
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
+
+    print STDERR "</Stage>\n" if $self->{'gli'};
+}
+
 # /** Lucene specific document removal function. This works by calling lucene_passes.pl with
 #  *  -remove and the document id on the command line.
@@ -249,180 +421,4 @@
 # /** remove_document_from_database **/
 
-sub build_index {
-    my $self = shift (@_);
-    my ($index,$llevel) = @_;
-    my $outhandle = $self->{'outhandle'};
-    my $build_dir = $self->{'build_dir'};
-
-    # get the full index directory path and make sure it exists
-    my $indexdir = $self->{'index_mapping'}->{$index};
-    &util::mk_all_dir (&util::filename_cat($build_dir, $indexdir));
-
-    # get any os specific stuff
-    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
-    my $scriptdir = "$ENV{'GSDLHOME'}/bin/script";
-
-    # Find the perl script to call to run lucene
-    my $full_lucene_passes = $self->{'full_lucene_passes'};
-    my $full_lucene_passes_exe = $self->{'full_lucene_passes_exe'};
-
-    # define the section names for lucenepasses
-    # define the section names and possibly the doc name for lucenepasses
-    my $lucene_passes_sections = $llevel;
-
-    my $opt_create_index = ($self->{'keepold'}) ? "" : "-create";
-
-    my $osextra = "";
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-	$build_dir =~ s@/@\\@g;
-    } else {
-	if ($outhandle ne "STDERR") {
-	    # so lucene_passes doesn't print to stderr if we redirect output
-	    $osextra .= " 2>/dev/null";
-	}
-    }
-
-    # get the index expression if this index belongs
-    # to a subcollection
-    my $indexexparr = [];
-    my $langarr = [];
-
-    # there may be subcollection info, and language info.
-    my ($fields, $subcollection, $language) = split (":", $index);
-    my @subcollections = ();
-    @subcollections = split /,/, $subcollection if (defined $subcollection);
-
-    foreach $subcollection (@subcollections) {
-	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
-	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
-	}
-    }
-
-    # add expressions for languages if this index belongs to
-    # a language subcollection - only put languages expressions for the
-    # ones we want in the index
-    my @languages = ();
-    my $language_metadata = "Language";
-    if (defined ($self->{'collect_cfg'}->{'language_metadata'})) {
-	$language_metadata = $self->{'collect_cfg'}->{'language_metadata'};
-    }
-    @languages = split /,/, $language if (defined $language);
-    foreach my $language (@languages) {
-	my $not=0;
-	if ($language =~ s/^\!//) {
-	    $not = 1;
-	}
-	if($not) {
-	    push (@$langarr, "!$language");
-	} else {
-	    push (@$langarr, "$language");
-	}
-    }
-
-    # Build index dictionary. Uses verbatim stem method
-    print $outhandle "\n    creating index dictionary (lucene_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
-    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
-    my ($handle);
-
-    if ($self->{'debug'}) {
-	$handle = STDOUT;
-    } else {
-	print STDERR "Cmd: $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra\n";
-	if (!-e "$full_lucene_passes" ||
-	    !open (PIPEOUT, "| $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
-	    print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
-	    die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";
-	}
-	$handle = lucenebuilder::PIPEOUT;
-    }
-
-    my $store_levels = $self->{'levels'};
-    my $gdbm_level = "document";
-    if ($store_levels->{'section'}) {
-	$gdbm_level = "section";
-    }
-
-    my $dom_level = "";
-    foreach my $key (keys %$store_levels) {
-	if ($mgppbuilder::level_map{$key} eq $llevel) {
-	    $dom_level = $key;
-	}
-    }
-    if ($dom_level eq "") {
-	print STDERR "Warning: unrecognized tag level $llevel\n";
-	$dom_level = "document";
-    }
-
-    my $local_levels = { $dom_level => 1 }; # work on one level at a time
-
-    # set up the document processr
-    $self->{'buildproc'}->set_output_handle ($handle);
-    $self->{'buildproc'}->set_mode ('text');
-    $self->{'buildproc'}->set_index ($index, $indexexparr);
-    $self->{'buildproc'}->set_index_languages ($language_metadata, $langarr) if (defined $language);
-    $self->{'buildproc'}->set_indexing_text (1);
-    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
-    $self->{'buildproc'}->set_levels ($local_levels);
-    $self->{'buildproc'}->set_gdbm_level($gdbm_level);
-    $self->{'buildproc'}->reset();
-    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'},
-		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
-    close ($handle) unless $self->{'debug'};
-
-    $self->print_stats();
-
-    $self->{'buildproc'}->set_levels ($store_levels);
-    print STDERR "</Stage>\n" if $self->{'gli'};
-}
-
-# /** A modified version of the basebuilder.pm's function that generates the
-#  *  information database (GDBM) from the GA documents. We need to change this
-#  *  so that if we've been asked to do an incremental build we only add
-#  *  metadata to autohierarchy classifiers via the IncrementalBuildUtils
-#  *  module. All other classifiers and metadata will be ignored.
-#  */
-sub make_infodatabase
-{
-    my $self = shift (@_);
-    my $outhandle = $self->{'outhandle'};
-
-    my $dbext = ".bdb";
-    $dbext = ".ldb" if &util::is_little_endian();
-    my $infodb_file = &util::filename_cat($self->{'build_dir'}, "text", $self->{'collection'} . $dbext);
-
-    # If we aren't doing an incremental addition, then we just call the super-
-    # classes version
-    # Note: Incremental addition can only occur if a text/<collection>.ldb
-    #       already exists. If it doesn't, let the super classes function be
-    #       called once to generate it.
-    if (!$self->{'incremental_dlc'} || !(-e $infodb_file))
-    {
-        # basebuilder::make_infodatabase(@_);
-        # Note: this doesn't work as the direct reference means all the $self
-        #       data is lost.
-        $self->basebuilder::make_infodatabase(@_);
-        return;
-    }
-
-    # Carry on with an incremental addition
-    print $outhandle "\n*** performing an incremental addition to the info database\n" if ($self->{'verbosity'} >= 1);
-    print STDERR "<Stage name='CreateInfoData'>\n" if $self->{'gli'};
-
-    # 1. Init all the classifiers
-    &classify::init_classifiers ($self->{'classifiers'});
-    # 2. Init the buildproc settings.
-    #    Note: we still need this to process any associated files - but we
-    #    don't expect to pipe anything to txt2db so we can do away with the
-    #    complex output handle.
-    my $assocdir = &util::filename_cat($self->{'build_dir'}, "assoc");
-    &util::mk_all_dir ($assocdir);
-    $self->{'buildproc'}->set_mode ('incinfodb'); # Very Important
-    $self->{'buildproc'}->set_assocdir ($assocdir);
-    # 3. Read in all the metadata from the files in the archives directory using
-    #    the GAPlug and using ourselves as the document processor!
-    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, "", {}, $self->{'buildproc'}, $self->{'maxdocs'},0, $self->{'gli'});
-
-    print STDERR "</Stage>\n" if $self->{'gli'};
-}
 
 1;
Index: /trunk/gsdl/perllib/mgppbuilder.pm
===================================================================
--- /trunk/gsdl/perllib/mgppbuilder.pm	(revision 13589)
+++ /trunk/gsdl/perllib/mgppbuilder.pm	(revision 13590)
@@ -46,8 +46,4 @@
 	      'Sec'=>'_textsection_',
 	      'Para'=>'_textparagraph_');
-
-#$doc_level = "Doc";
-#$sec_level = "Sec";
-#$para_level = "Para";
 
 our %wanted_index_files = ('td'=>1,
@@ -127,13 +123,4 @@
     }
     
-    $self->{'doc_level'} = "document";
-    if (! $self->{'levels'}->{'document'}) {
-	if ($self->{'levels'}->{'section'}) {
-	    $self->{'doc_level'} = "section";
-	} else {
-	    die "you must have either document or section level specified!!\n";
-	}
-    }
-
     $self->{'buildtype'} = "mgpp";
 
@@ -229,12 +216,7 @@
     # the compressor doesn't need to know about paragraphs - never want to 
     # retrieve them
-    my $mgpp_passes_sections = "";
-    my ($doc_level) = $self->{'doc_level'};
-    $mgpp_passes_sections .= "-J " . $level_map{$doc_level} . " ";
-    foreach my $level (keys %{$self->{'levels'}}) {
-	if ($level ne $doc_level && $level ne "paragraph") {
-	    $mgpp_passes_sections .= "-K " . $level_map{$level} . " ";
-	}
-    }
+    
+    # always use Doc and Sec levels
+    my $mgpp_passes_sections = "-J ". $level_map{"document"} ." -K " . $level_map{"section"} ." ";
 
     print $outhandle "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
@@ -250,5 +232,4 @@
 	$handle = STDOUT;
     } else {
-	#print $outhandle "trying to run (compress 1) mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra\n";
 	if (!-e "$mgpp_passes_exe" || 
 	    !open (PIPEOUT, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra")) {
@@ -258,11 +239,7 @@
 	$handle = mgppbuilder::PIPEOUT;
     }
-
-    # gdbm_level
-    my $gdbm_level = "document";
-    if ($self->{'levels'}->{'section'}) {
-	$gdbm_level = "section";
-    }
-    
+    
+    my $gdbm_level = "section";
+
     $self->{'buildproc'}->set_output_handle ($handle);
     $self->{'buildproc'}->set_mode ('text');
@@ -300,5 +277,4 @@
 
 	if (!$self->{'debug'}) {
-	    #print $outhandle "trying to run (compress 2) mgpp_passes$exe $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra\n";
 	    if (!-e "$mgpp_passes_exe" || 
 		!open ($handle, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra")) {
@@ -351,8 +327,6 @@
     foreach my $index (@$indexes) {
 	my ($fields, $subcollection, $languages) = split (":", $index);
-	# the directory name starts with a processed version of index fields
-	#my ($pindex) = $self->process_field($fields);
-	#$pindex = lc ($pindex);
-	# now we only ever have one index, and its called 'idx'
+	
+	# we only ever have one index, and its called 'idx'
 	my $pindex = 'idx';
 	
@@ -441,13 +415,7 @@
 
     # define the section names for mgpasses
-    # define the section names and possibly the doc name for mgpasses
-    my $mgpp_passes_sections = "";
-    my ($doc_level) = $self->{'doc_level'};
-    $mgpp_passes_sections .= "-J " . $level_map{$doc_level} ." ";
-    
-    foreach my $level (keys %{$self->{'levels'}}) {
-	if ($level ne $doc_level) {
-	    $mgpp_passes_sections .= "-K " . $level_map{$level}. " ";
-	}
+    my $mgpp_passes_sections = "-J ". $level_map{"document"} ." -K " . $level_map{"section"} ." ";
+    if ($self->{'levels'}->{'paragraph'}) {
+	$mgpp_passes_sections .= "-K " . $level_map{'paragraph'}. " ";
     }
 
@@ -526,9 +494,6 @@
     }
 	    
-    # gdbm_level
-    my $gdbm_level = "document";
-    if ($self->{'levels'}->{'section'}) {
-	$gdbm_level = "section";
-    }
+    # gdbm_level is always section
+    my $gdbm_level = "section";
 
     # set up the document processr
@@ -878,9 +843,6 @@
     $build_cfg->{'levelmap'} = \@levelmap;
 
-    if ($self->{'levels'}->{'section'}) {
-	$build_cfg->{'textlevel'} = $level_map{'section'};
-    } else {	
-	$build_cfg->{'textlevel'} = $level_map{'document'};
-    }
+    # text level (and gdbm level) is always section
+    $build_cfg->{'textlevel'} = $level_map{'section'};
    
 }
Index: /trunk/gsdl/perllib/mgppbuildproc.pm
===================================================================
--- /trunk/gsdl/perllib/mgppbuildproc.pm	(revision 13589)
+++ /trunk/gsdl/perllib/mgppbuildproc.pm	(revision 13590)
@@ -85,17 +85,4 @@
     $self->{'strip_html'}=$strip;
 }
-
-
-sub get_gdbm_level {
-    my $self = shift (@_);
-    
-    #if a Section level index is not built, the gdbm file should be at doc 
-    #level not Section
-    if ($self->{'levels'}->{'section'}) {
-	return "section";
-    }
-    return "document";
-}
-
 
 #sub find_paragraphs {
@@ -208,14 +195,8 @@
     my ($fields) = split (/:/, $self->{'index'});
 
-    my ($documenttag) = "";
-    my($documentendtag) = "";
-    if ($self->{'levels'}->{'document'}) {
-	$documenttag = "\n<". $level_map{'document'} . ">\n";
-	$documentendtag = "\n</". $level_map{'document'} . ">\n";
-    }
-    my ($sectiontag) = "";
-    if ($self->{'levels'}->{'section'}) {
-	$sectiontag = "\n<". $level_map{'section'} . ">\n";
-    }
+    # we always do text and index on Doc and Sec levels
+    my ($documenttag) = "\n<". $level_map{'document'} . ">\n";
+    my ($documentendtag) = "\n</". $level_map{'document'} . ">\n";
+    my ($sectiontag) = "\n<". $level_map{'section'} . ">\n";
     my ($paratag) = "";
     
