Changeset 1251 for trunk/gsdl/perllib/mgbuildproc.pm

Timestamp:

2000-06-29T10:34:25+12:00 (24 years ago)

Author:

sjboddie

Message:

Added some stat reporting and a warning message to the build code.
Now warns when very little or no text is to be processed for a given
index (as mg craps out in these situations). Will hopefully be useful
in realizing when an attempt is made to create an index of metadata that
is never set etc.

File:

• trunk/gsdl/perllib/mgbuildproc.pm (modified) (7 diffs)

trunk/gsdl/perllib/mgbuildproc.pm

@@ -58 +58 @@
     $self->{'num_sections'} = 0;
     $self->{'num_bytes'} = 0;
+    $self->{'num_processed_bytes'} = 0;
 
     $self->{'indexing_text'} = 0;
@@ -69 +70 @@
     $self->{'num_docs'} = 0;
     $self->{'num_sections'} = 0;
+    $self->{'num_processed_bytes'} = 0;
     $self->{'num_bytes'} = 0;
 }
@@ -84 +86 @@
 }
 
+# num_bytes is the actual number of bytes in the collection
+# this is normally the same as what's processed during text compression
 sub get_num_bytes {
     my $self = shift (@_);
 
     return $self->{'num_bytes'};
+}
+
+# num_processed_bytes is the number of bytes actually passed
+# to mg for the current index
+sub get_num_processed_bytes {
+    my $self = shift (@_);
+
+    return $self->{'num_processed_bytes'};
 }
 
@@ -126 +138 @@
 }
 
+sub get_index {
+    my $self = shift (@_);
+
+    return $self->{'index'};
+}
+
 sub set_classifiers {
     my $self = shift (@_);
@@ -138 +156 @@
 
     $self->{'indexing_text'} = $indexing_text;
+}
+
+sub get_indexing_text {
+    my $self = shift (@_);
+
+    return $self->{'indexing_text'};
 }
 
@@ -416 +440 @@
             if ($real_field eq "text") {
             $new_text = $doc_obj->get_text ($section);
+            $self->{'num_processed_bytes'} += length ($new_text);
             $new_text =~ s/[\cB\cC]//g;
             $self->find_paragraphs($new_text);
@@ -423 +448 @@
             foreach $meta (@{$doc_obj->get_metadata ($section, $real_field)}) {
                 $meta =~ s/[\cB\cC]//g;
+                $self->{'num_processed_bytes'} += length ($meta);
                 $new_text .= "\cC" unless $first;
                 $new_text .= $meta;