Changeset 1279 for branches/New_Config_Format-branch/gsdl/perllib/mgbuilder.pm

Timestamp:

2000-07-13T10:21:53+12:00 (24 years ago)

Author:

sjboddie

Message:

merged changes to trunk into New_Config_Format branch

File:

• branches/New_Config_Format-branch/gsdl/perllib/mgbuilder.pm (modified) (8 diffs)

branches/New_Config_Format-branch/gsdl/perllib/mgbuilder.pm

@@ -99 +99 @@
     
     # load all the plugins
-    $self->{'pluginfo'} = &plugin::load_plugins ($plugins);
+    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity);
     if (scalar(@{$self->{'pluginfo'}}) == 0) {
     print STDERR "No plugins were loaded.\n";
@@ -207 +207 @@
     close ($handle) unless $self->{'debug'};
 
+    $self->print_stats();
+
     # create the compression dictionary
     # the compression dictionary is built by assuming the stats are from a seed
@@ -232 +234 @@
            "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
     close ($handle) unless $self->{'debug'};
+
+    $self->print_stats();
 }
 
@@ -481 +485 @@
     close ($handle) unless $self->{'debug'};
 
+    $self->print_stats();
+
     if (!$self->{'debug'}) {
     # create the perfect hash function
@@ -502 +508 @@
            "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
     
+    $self->print_stats ();
+
     if (!$self->{'debug'}) {
 
@@ -646 +654 @@
     $build_cfg->{'numbytes'} = $self->{'buildproc'}->get_num_bytes();
 
+    # get additional stats from mg
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mgstat_exe = &util::filename_cat($exedir, "mgstat$exe");
+    my $input_file = &util::filename_cat ("text", $self->{'collection'});
+    if (!-e "$mgstat_exe" || !open (PIPEIN, "$mgstat_exe -d $self->{'build_dir'} -f $input_file |")) {
+    print STDERR "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
+    } else {
+    my $line = "";
+    while (defined ($line = <PIPEIN>)) {
+        if ($line =~ /^Words in collection \[dict\]\s+:\s+(\d+)/) {
+        ($build_cfg->{'numwords'}) = $1;
+        } elsif ($line =~ /^Documents\s+:\s+(\d+)/) {
+        ($build_cfg->{'numsections'}) = $1;
+        }
+    }
+    close PIPEIN;
+    }
+
     # store the mapping between the index names and the directory names
     my @indexmap = ();
@@ -667 +694 @@
     $build_cfg->{'languagemap'} = \@languagemap if scalar (@languagemap);
 
-    $build_cfg->{'notbuilt'} = $self->{'notbuilt'};
+    $build_cfg->{'notbuilt'} = $self->{'notbuilt'} if scalar @{$self->{'notbuilt'}};
 
     # write out the build information
     &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
-                 '^(builddate|numdocs|numbytes)$', 
+                 '^(builddate|numdocs|numbytes|numwords|numsections)$', 
                              '^(indexmap|subcollectionmap|languagemap|notbuilt)$');
 
@@ -680 +707 @@
 }
 
+sub print_stats {
+    my $self = shift (@_);
+
+    my $indexing_text = $self->{'buildproc'}->get_indexing_text();
+    my $index = $self->{'buildproc'}->get_index();
+    my $num_bytes = $self->{'buildproc'}->get_num_bytes();
+    my $num_processed_bytes = $self->{'buildproc'}->get_num_processed_bytes();
+
+    if ($indexing_text) {
+    print STDERR "Stats (Creating index $index)\n";
+    } else {
+    print STDERR "Stats (Compressing text from $index)\n";
+    }
+    print STDERR "Total bytes in collection: $num_bytes\n";
+    print STDERR "Total bytes in $index: $num_processed_bytes\n";
+
+    if ($num_processed_bytes < 50) {
+    print STDERR "***************\n";
+    print STDERR "WARNING: There is very little or no text to process for $index\n";
+    if ($indexing_text) {
+        print STDERR "This may cause an error while attempting to build the index\n";
+    } else {
+        print STDERR "This may cause an error while attempting to compress the text\n";
+    }
+    print STDERR "***************\n";
+    }
+}
 
 1;