Index: trunk/gsdl/perllib/mgppbuilder.pm
===================================================================
--- trunk/gsdl/perllib/mgppbuilder.pm	(revision 2477)
+++ trunk/gsdl/perllib/mgppbuilder.pm	(revision 2478)
@@ -59,5 +59,4 @@
 		       'i'=>1,
 		       'il'=>1,
-		       'tw'=>1,
 		       'w'=>1,
 		       'wa'=>1);
@@ -87,8 +86,10 @@
 sub new {
     my ($class, $collection, $source_dir, $build_dir, $verbosity, 
-	$maxdocs, $debug, $keepold, $allclassifications, $outhandle) = @_;
+	$maxdocs, $debug, $keepold, $allclassifications, 
+	$outhandle, $no_text) = @_;
 
     $outhandle = STDERR unless defined $outhandle;
-
+    $no_text = 0 unless defined $no_text;
+    
     # create an mgppbuilder object
     my $self = bless {'collection'=>$collection,
@@ -101,4 +102,5 @@
 		      'allclassifications'=>$allclassifications,
 		      'outhandle'=>$outhandle,
+		      'no_text'=>$no_text,
 		      'notbuilt'=>[],    # indexes not built
 		      'indexfieldmap'=>\%static_indexfield_map
@@ -130,5 +132,10 @@
 	foreach $language (@{$self->{'collect_cfg'}->{'languages'}}) {
 	    foreach $index (@$indexes) {
-		push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$language");
+		if (defined ($self->{'collect_cfg'}->{'indexsubcollections'})) {
+		    push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$language");
+		}
+		else { # add in an empty subcollection field
+		    push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index\:\:$language");
+		}		
 	    }
 	}
@@ -233,8 +240,8 @@
     my ($textindex) = @_;
 
-    my $exedir = "$ENV{'GSDLHOME'}/src/mgpp/text";
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
     my $exe = &util::get_os_exe ();
-    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
-    my $mg_compression_dict_exe = &util::filename_cat($exedir, "mg_compression_dict$exe");
+    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
+    my $mgpp_compression_dict_exe = &util::filename_cat($exedir, "mgpp_compression_dict$exe");
     my $outhandle = $self->{'outhandle'};
 
@@ -244,10 +251,9 @@
     my $basefilename = "text/$self->{'collection'}";
 
-# mgpp cant work on windows at the moment    
-#     if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-#	 $basefilename =~ s/\//\\/g;
-#	 $builddir =~ s/\//\\/g;
-#	
-#    } 
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$basefilename =~ s/\//\\/g;
+	$builddir =~ s/\//\\/g;
+	
+    } 
 
 
@@ -255,7 +261,7 @@
     # the compressor doesn't need to know about paragraphs - never want to 
     # retrieve them
-    my $mg_passes_sections = "";
+    my $mgpp_passes_sections = "";
     if ($self->{'levels'}->{'Section'}) {
-	$mg_passes_sections .= "-K Section ";
+	$mgpp_passes_sections .= "-K Section ";
     }
 	
@@ -264,5 +270,5 @@
     # collect the statistics for the text
     # -b $maxdocsize sets the maximum document size to be 12 meg
-    print $outhandle "\n    collecting text statistics (mg_passes -T1)\n"  if ($self->{'verbosity'} >= 1);
+    print $outhandle "\n    collecting text statistics (mgpp_passes -T1)\n"  if ($self->{'verbosity'} >= 1);
 
     my ($handle);
@@ -270,7 +276,7 @@
 	$handle = STDOUT;
     } else {
-	if (!-e "$mg_passes_exe" || 
-	    !open (PIPEOUT, "| $mg_passes_exe $mg_passes_sections  -d $builddir -f $basefilename -T1")) {
-	    die "mgppbuilder::compress_text - couldn't run $mg_passes_exe\n";
+	if (!-e "$mgpp_passes_exe" || 
+	    !open (PIPEOUT, "| mgpp_passes$exe $mgpp_passes_sections  -d $builddir -f $basefilename -T1")) {
+	    die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
 	}
 	$handle = mgppbuilder::PIPEOUT;
@@ -281,4 +287,9 @@
     $self->{'buildproc'}->set_index ($textindex);
     $self->{'buildproc'}->set_indexing_text (0);
+    if ($self->{'no_text'}) {
+	$self->{'buildproc'}->set_store_text(0);
+    } else {
+	$self->{'buildproc'}->set_store_text(1);
+    }
     $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
     $self->{'buildproc'}->set_levels ($self->{'levels'});                      
@@ -293,4 +304,6 @@
     close ($handle) unless $self->{'debug'};
 
+    $self->print_stats();
+
     # create the compression dictionary
     # the compression dictionary is built by assuming the stats are from a seed
@@ -301,14 +314,13 @@
     if (!$self->{'debug'}) {
 	print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
-	if (!-e "$mg_compression_dict_exe") {
-	    die "mgppbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
-	}
-	system ("$mg_compression_dict_exe -d $builddir -f $basefilename -S -H -2 -k 5120");
-
+	if (!-e "$mgpp_compression_dict_exe") {
+	    die "mgppbuilder::compress_text - couldn't run $mgpp_compression_dict_exe\n";
+	}
+	system ("mgpp_compression_dict$exe -d $builddir -f $basefilename -S -H -2 -k 5120");
 
 	if (!$self->{'debug'}) {
-	    if (!-e "$mg_passes_exe" || 
-		!open ($handle, "| $mg_passes_exe $mg_passes_compress_sections -f $basefilename -d $builddir -T2")) {
-		die "mgppbuilder::compress_text - couldn't run $mg_passes_exe\n";
+	    if (!-e "$mgpp_passes_exe" || 
+		!open ($handle, "| mgpp_passes$exe $mgpp_passes_sections -f $basefilename -d $builddir -T2")) {
+		die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
 	    }
 	}
@@ -317,5 +329,5 @@
     $self->{'buildproc'}->reset();
     # compress the text
-    print $outhandle "\n    compressing the text (mg_passes -T2)\n"  if ($self->{'verbosity'} >= 1);
+    print $outhandle "\n    compressing the text (mgpp_passes -T2)\n"  if ($self->{'verbosity'} >= 1);
     &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
 		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
@@ -408,18 +420,27 @@
 	}
 
+	$mapping{$index} = $dirname;
+
 	# store the mapping orders as well as the maps
+	# also put index, subcollection and language fields into the mapping thing - 
+	# (the full index name (eg document:text:subcol:lang) is not used on
+	# the query page) -these are used for collectionmeta later on
 	if (!defined $mapping{'indexmap'}{"$fields"}) {
 	    $mapping{'indexmap'}{"$fields"} = $pindex;
 	    push (@{$mapping{'indexmaporder'}}, "$fields");
+	    if (!defined $mapping{"$fields"}) {
+		$mapping{"$fields"} = $pindex;
+	    }	
 	}
 	if ($psub =~ /\w/ && !defined ($mapping{'subcollectionmap'}{$subcollection})) {
 	    $mapping{'subcollectionmap'}{$subcollection} = $psub;
 	    push (@{$mapping{'subcollectionmaporder'}}, $subcollection);
+	    $mapping{$subcollection} = $psub;
 	}
 	if ($plang =~ /\w/ && !defined ($mapping{'languagemap'}{$languages})) {
 	    $mapping{'languagemap'}{$languages} = $plang;
 	    push (@{$mapping{'languagemaporder'}}, $language);
-	}
-	$mapping{$index} = $dirname;
+	    $mapping{$languages} = $plang;
+	}
 	$dirnames{$dirname} = $index;
 	$pnames{'index'}{$pindex} = "$fields";
@@ -499,35 +520,37 @@
 
     # get any os specific stuff
-    my $exedir = "$ENV{'GSDLHOME'}/src/mgpp/text";
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
 
     my $exe = &util::get_os_exe ();
-    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
+    my $mgpp_passes_exe = &util::filename_cat($exedir, "mgpp_passes$exe");
 
     # define the section names for mgpasses
-    my $mg_passes_sections = "";
+    my $mgpp_passes_sections = "";
     foreach $level (keys (%{$self->{'levels'}})) {
 	if ($level eq "Section" || $level eq "Paragraph") {
-	    $mg_passes_sections .= "-K $level ";
-	}
-    }
-
-    my $mg_perf_hash_build_exe = 
-	&util::filename_cat($exedir, "mg_perf_hash_build$exe");
-    my $mg_weights_build_exe = 
-	&util::filename_cat ($exedir, "mg_weights_build$exe");
-    my $mg_invf_dict_exe = 
-	&util::filename_cat ($exedir, "mg_invf_dict$exe");
-    my $mg_stem_idx_exe =
-	&util::filename_cat ($exedir, "mg_stem_idx$exe");
-
-#    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-#	$builddir=~ s/\//\\/g;
-#	$basefilename =~ s/\//\\/g; 
-#    }
+	    $mgpp_passes_sections .= "-K $level ";
+	}
+    }
+
+    my $mgpp_perf_hash_build_exe = 
+	&util::filename_cat($exedir, "mgpp_perf_hash_build$exe");
+    my $mgpp_weights_build_exe = 
+	&util::filename_cat ($exedir, "mgpp_weights_build$exe");
+    my $mgpp_invf_dict_exe = 
+	&util::filename_cat ($exedir, "mgpp_invf_dict$exe");
+    my $mgpp_stem_idx_exe =
+	&util::filename_cat ($exedir, "mgpp_stem_idx$exe");
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) { 
+	$builddir=~ s/\//\\/g;
+	$basefilename =~ s/\//\\/g; 
+    }
 
     # get the index expression if this index belongs
     # to a subcollection
     my $indexexparr = [];
-    my ($fields, $subcollection) = split (":", $index);
+
+    # there may be subcollection info, and language info. 
+    my ($fields, $subcollection, $language) = split (":", $index);
     my @subcollections = ();
     @subcollections = split /,/, $subcollection if (defined $subcollection);
@@ -540,22 +563,35 @@
     
     # add expressions for languages if this index belongs to
-    # a language subcollection
-    foreach $language (@{$self->{'collect_cfg'}->{'languages'}}) {
+    # a language subcollection - only put languages expressions for the 
+    # ones we want in the index
+
+    my @languages = ();
+    @languages = split /,/, $language if (defined $language);
+    foreach $language (@languages) {
+	my $not=0;
 	if ($language =~ s/^\!//) {
-	    push (@$indexexparr, "!Language/$language/");
-	} else {
-	    push (@$indexexparr, "Language/$language/");
+	    $not = 1;
+	}
+	foreach $lang (@{$self->{'collect_cfg'}->{'languages'}}) {
+	    if ($lang eq $language) {
+		if ($not) {
+		    push (@$indexexparr, "!Language/$language/");
+		} else {
+		    push (@$indexexparr, "Language/$language/");
+		}
+		last;
+	    }
 	}
     }
 
     # Build index dictionary. Uses verbatim stem method
-    print $outhandle "\n    creating index dictionary (mg_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
+    print $outhandle "\n    creating index dictionary (mgpp_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
     my ($handle);
     if ($self->{'debug'}) {
 	$handle = STDOUT;
     } else {
-	if (!-e "$mg_passes_exe" || 
-	    !open (PIPEOUT, "| $mg_passes_exe $mg_passes_sections  -d $builddir -f $basefilename -I1")) {
-	    die "mgppbuilder::build_index - couldn't run $mg_passes_exe\n";
+	if (!-e "$mgpp_passes_exe" || 
+	    !open (PIPEOUT, "| mgpp_passes$exe $mgpp_passes_sections  -d $builddir -f $basefilename -I1")) {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
 	}
 	$handle = mgppbuilder::PIPEOUT;
@@ -567,4 +603,5 @@
     $self->{'buildproc'}->set_index ($index, $indexexparr);
     $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
     $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
     $self->{'buildproc'}->set_levels ($self->{'levels'});                       
@@ -578,17 +615,17 @@
     if (!$self->{'debug'}) {
 	# create the perfect hash function
-	if (!-e "$mg_perf_hash_build_exe") {
-	    die "mgppbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
-	}
-	system ("$mg_perf_hash_build_exe -d $builddir -f $basefilename");
-
-	if (!-e "$mg_passes_exe" || 
-	    !open ($handle, "| $mg_passes_exe $mg_passes_sections  -d $builddir -f $basefilename -I2")) {
-	    die "mgppbuilder::build_index - couldn't run $mg_passes_exe\n";
+	if (!-e "$mgpp_perf_hash_build_exe") {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_perf_hash_build_exe\n";
+	}
+	system ("mgpp_perf_hash_build$exe -d $builddir -f $basefilename");
+
+	if (!-e "$mgpp_passes_exe" || 
+	    !open ($handle, "| mgpp_passes$exe $mgpp_passes_sections  -d $builddir -f $basefilename -I2")) {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
 	}
     }
     
     # invert the text
-    print $outhandle "\n    inverting the text (mg_passes -I2)\n"  if ($self->{'verbosity'} >= 1);
+    print $outhandle "\n    inverting the text (mgpp_passes -I2)\n"  if ($self->{'verbosity'} >= 1);
 
     $self->{'buildproc'}->reset();
@@ -604,25 +641,25 @@
 	# create the weights file
 	print $outhandle "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
-	if (!-e "$mg_weights_build_exe") {
-	    die "mgppbuilder::build_index - couldn't run $mg_weights_build_exe\n";
-	}
-	system ("$mg_weights_build_exe -d $builddir -f $basefilename");
+	if (!-e "$mgpp_weights_build_exe") {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_weights_build_exe\n";
+	}
+	system ("mgpp_weights_build$exe -d $builddir -f $basefilename");
 
 	# create 'on-disk' stemmed dictionary
 	print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
-	if (!-e "$mg_invf_dict_exe") {
-	    die "mgppbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
-	}
-	system ("$mg_invf_dict_exe -d $builddir -f $basefilename");
+	if (!-e "$mgpp_invf_dict_exe") {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_invf_dict_exe\n";
+	}
+	system ("mgpp_invf_dict$exe -d $builddir -f $basefilename");
 
 
 	# creates stem index files for the various stemming methods
 	print $outhandle "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
-	if (!-e "$mg_stem_idx_exe") {
-	    die "mgppbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
-	}
-	system ("$mg_stem_idx_exe -b 4096 -s1 -d $builddir -f $basefilename");
-	system ("$mg_stem_idx_exe -b 4096 -s2 -d $builddir -f $basefilename");
-	system ("$mg_stem_idx_exe -b 4096 -s3 -d $builddir -f $basefilename");
+	if (!-e "$mgpp_stem_idx_exe") {
+	    die "mgppbuilder::build_index - couldn't run $mgpp_stem_idx_exe\n";
+	}
+	system ("mgpp_stem_idx$exe -b 4096 -s1 -d $builddir -f $basefilename");
+	system ("mgpp_stem_idx$exe -b 4096 -s2 -d $builddir -f $basefilename");
+	system ("mgpp_stem_idx$exe -b 4096 -s3 -d $builddir -f $basefilename");
 
     
@@ -664,4 +701,24 @@
     my $txt2db_exe = &util::filename_cat($exedir, "txt2db$exe");
 
+    # define the indexed field mapping if not already done so (ie if infodb called separately from build_index)
+    if (scalar(keys %{$self->{'buildproc'}->{'indexfieldmap'}}) == 0) {
+	#check build.cfg to see if indexfields have been filled in
+	$buildconfigfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "building/build.cfg");
+	if (-e $buildconfigfile) {
+	    $buildcfg = &colcfg::read_build_cfg( $buildconfigfile);
+	    if (defined $buildcfg->{'indexfields'}) {
+		foreach $field (@{$buildcfg->{'indexfields'}}) {
+		    $self->{'buildproc'}->{'indexfields'}->{$field} = 1;
+		}
+	    }
+	    if (defined $buildcfg->{'indexfieldmap'}) {
+		foreach $field (@{$buildcfg->{'indexfieldmap'}}) {
+		    ($f, $v) = $field =~ /^(.*)\-\>(.*)$/;
+		    $self->{'buildproc'}->{'indexfieldmap'}->{$f} = $v;
+		}
+	    }	    
+	}
+    }
+    
     print $outhandle "\n*** creating the info database and processing associated files\n" 
 	if ($self->{'verbosity'} >= 1);
@@ -687,5 +744,6 @@
     $self->{'buildproc'}->set_classifiers ($self->{'classifiers'});
     $self->{'buildproc'}->set_indexing_text (0);
-    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_store_text(1);
+    #$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
 
     $self->{'buildproc'}->reset();
@@ -714,7 +772,7 @@
 	}
 	#print out the indexfield mapping
-	foreach $field (keys(%{$self->{'indexfieldmap'}})) {
-	    $shortname = $self->{'indexfieldmap'}->{$field};
-	    print $handle "<$shortname>$field\n";
+	foreach $field (keys %{$self->{'buildproc'}->{'indexfields'}}) {
+	    $shortname = $self->{'buildproc'}->{'indexfieldmap'}->{$field};
+	    print $handle "<$shortname>$field\n" if defined $shortname;
 	}
 	print $handle "\n" . ('-' x 70) . "\n";
@@ -820,12 +878,12 @@
     print $outhandle "Total bytes in $index: $num_processed_bytes\n";
 
-    if ($num_processed_bytes < 50) {
+    if ($num_processed_bytes < 50 && ($indexing_text || !$self->{'no_text'})) {
 	print $outhandle "***************\n";
-	print $outhandle "WARNING: There is very little or no text to process for $index\n";
 	if ($indexing_text) {
-	    print $outhandle "This may cause an error while attempting to build the index\n";
-	} else {
-	    print $outhandle "This may cause an error while attempting to compress the text\n";
-	}
+	    print $outhandle "WARNING: There is very little or no text to process for $index\n";
+	} elsif (!$self->{'no_text'}) {
+	    print $outhandle "WARNING: There is very little or no text to compress\n";
+	}	   
+	print $outhandle "         Was this your intention?\n";
 	print $outhandle "***************\n";
     }
