Changeset 2336

Timestamp:

2001-04-24 16:36:21 (8 years ago)

Author:

sjboddie

Message:

added a -no_text option to buildcol.pl to allow collections to be built
without storing compressed text (intended for use in collections where
original documents (PDFs or Word docs maybe) are returned instead of the
compressed text)

Files:

• trunk/gsdl/bin/script/buildcol.pl (modified) (5 diffs)
• trunk/gsdl/perllib/colcfg.pm (modified) (2 diffs)
• trunk/gsdl/perllib/mgbuilder.pm (modified) (16 diffs)
• trunk/gsdl/perllib/mgbuildproc.pm (modified) (4 diffs)

trunk/gsdl/bin/script/buildcol.pl

@@ -58 +58 @@
-    print STDERR "   -keepold              will not destroy the current contents of the\n";
-    print STDERR "                         building directory\n";
+    print STDERR "   -no_text              Don't store compressed text. This option is\n";
+    print STDERR "                         useful for minimizing the size of the built\n";
+    print STDERR "                         indexes if you intend always to display the\n";
+    print STDERR "                         original documents at run time (i.e. you won't\n";
+    print STDERR "                         be able to retrieve the compressed text version)\n";
-    print STDERR "   -allclassifications   Don't remove empty classifications\n";
-    print STDERR "   -create_images        Attempt to create default images for new\n";
@@ -79 +84 @@
-        $debug, $mode, $indexname, $keepold, $allclassifications,
-        $create_images, $collectdir, $out, $buildtype, $textindex,
-
+, $no_text
-    if (!parsargv::parse(\@ARGV, 
-                         'verbosity/\d+/2', \$verbosity,
@@ -89 +94 @@
-                         'mode/^(all|compress_text|build_index|infodb)$/all', \$mode,
-                         'index/.*/', \$indexname,
+                         'no_text', \$no_text,
-                         'keepold', \$keepold,
-                         'allclassifications', \$allclassifications,
@@ -136 +142 @@
-            $builddir = $collectcfg->{'builddir'};
-        }
+        if (defined $collectcfg->{'collectdir'} && $collectdir eq "") {
+            $collectdir = $collectcfg->{'collectdir'};
+        }
+        if (defined $collectcfg->{'no_text'} && $no_text == 0) {
+            if ($collectcfg->{'no_text'} =~ /^true$/) {
+                $no_text = 1;
+            }
+        }
+        if (defined $collectcfg->{'allclassifications'} && $allclassifications == 0) {
+            if ($collectcfg->{'allclassifications'} =~ /^true$/) {
+                $allclassifications = 1;
+            }
+        }
-        if ($buildtype eq "mgpp" && defined $collectcfg->{'textcompress'}) {
-            $textindex = $collectcfg->{'textcompress'};
@@ -219 +238 @@
-    eval("\$builder = new $buildertype(\$collection, " .
-         "\$realarchivedir, \$realbuilddir, \$verbosity, " .
-
+
+
-    die "$@" if $@;
-

trunk/gsdl/perllib/colcfg.pm

@@ -66 +66 @@
-    return &cfgread::read_cfg_file ($filename, 
-                                    q/^(creator|public|beta|defaultindex|importdir|/ .
-
+
+
-                                    q/(maintainer|languages|indexsubcollections|/ .
-                                       q/indexes|dontbuild|dontgdbm|mirror|phind|levels)$/,
@@ -78 +79 @@
-    &cfgread::write_cfg_file($filename, $data,
-                             q/^(creator|public|beta|defaultindex|importdir|/ .
-
+
+
-                             q/^(maintainer|languages|indexsubcollections|/ .
-                                 q/indexes|dontbuild|dontgdbm|levels)$/,

trunk/gsdl/perllib/mgbuilder.pm

@@ -61 +61 @@
-sub new {
-    my ($class, $collection, $source_dir, $build_dir, $verbosity, 
-
+
+
-
-    $outhandle = STDERR unless defined $outhandle;
+    $no_text = 0 unless defined $no_text;
-
-    # create an mgbuilder object
@@ -75 +77 @@
-                      'allclassifications'=>$allclassifications,
-                      'outhandle'=>$outhandle,
+                      'no_text'=>$no_text,
-                      'notbuilt'=>[]    # indexes not built
-                      }, $class;
@@ -219 +222 @@
-    } else {
-        if (!-e "$mg_passes_exe" || 
-#           !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
-            !open (PIPEOUT, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
-            die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
@@ -230 +232 @@
-    $self->{'buildproc'}->set_index ($textindex);
-    $self->{'buildproc'}->set_indexing_text (0);
+    if ($self->{'no_text'}) {
+        $self->{'buildproc'}->set_store_text(0);
+    } else {
+        $self->{'buildproc'}->set_store_text(1);
+    }
-    $self->{'buildproc'}->reset();
-    &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
@@ -251 +258 @@
-            die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
-        }
-#       system ("\"$mg_compression_dict_exe\" -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
-        system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
-
-        # -b $maxdocsize sets the maximum document size to be 12 meg
-        if (!-e "$mg_passes_exe" || 
-#           !open ($handle, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
-            !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
-            die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
@@ -526 +531 @@
-    } else {
-        if (!-e "$mg_passes_exe" || 
-#           !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
-            !open (PIPEOUT, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
-                   "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
@@ -539 +543 @@
-    $self->{'buildproc'}->set_index ($index, $indexexparr);
-    $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
-
-    $self->{'buildproc'}->reset();
@@ -552 +557 @@
-            die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
-        }
-#       system ("\"$mg_perf_hash_build_exe\" -f \"$fullindexprefix\" $osextra");
-        system ("mg_perf_hash_build$exe -f \"$fullindexprefix\" $osextra");
-
-        if (!-e "$mg_passes_exe" || 
-#           !open ($handle, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
-            !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
-                   "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
@@ -581 +584 @@
-            die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
-        }
-#       system ("\"$mg_weights_build_exe\" -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
-        system ("mg_weights_build$exe -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
-
@@ -589 +591 @@
-            die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
-        }
-#       system ("\"$mg_invf_dict_exe\" -f \"$fullindexprefix\" $osextra");
-        system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");
-
@@ -598 +599 @@
-            die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
-        }
-#       system ("\"$mg_stem_idx_exe\" -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
-        system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
-#       system ("\"$mg_stem_idx_exe\" -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
-        system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
-#       system ("\"$mg_stem_idx_exe\" -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
-        system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
-
-    
-        # remove unwanted files
@@ -653 +650 @@
-        $handle = STDOUT;
-    } else {
-#       if (!-e "$txt2db_exe" || !open (PIPEOUT, "| \"$txt2db_exe\" \"$fulldbname\"")) {
-        if (!-e "$txt2db_exe" || !open (PIPEOUT, "| txt2db$exe \"$fulldbname\"")) {
-            die "mgbuilder::make_infodatabase - couldn't run $txt2db_exe\n";
@@ -666 +662 @@
-    $self->{'buildproc'}->set_classifiers ($self->{'classifiers'});
-    $self->{'buildproc'}->set_indexing_text (0);
+    $self->{'buildproc'}->set_store_text(1);
-    $self->{'buildproc'}->reset();
-
@@ -731 +728 @@
-    my $mgstat_exe = &util::filename_cat($exedir, "mgstat$exe");
-    my $input_file = &util::filename_cat ("text", $self->{'collection'});
-#    if (!-e "$mgstat_exe" || !open (PIPEIN, "\"$mgstat_exe\" -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
-    if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
-        print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
@@ -797 +793 @@
-    print $outhandle "Total bytes in $index: $num_processed_bytes\n";
-
-
+ && ($indexing_text || !$self->{'no_text'})
-        print $outhandle "***************\n";
-        print $outhandle "WARNING: There is very little or no text to process for $index\n";
-        if ($indexing_text) {
-
-
-
-
+
+
+
+
+
-        print $outhandle "***************\n";
-    }
@@ -810 +806 @@
-
-1;
-
-

trunk/gsdl/perllib/mgbuildproc.pm

@@ -65 +65 @@
-    $self->{'num_bytes'} = 0;
-    $self->{'num_processed_bytes'} = 0;
+    $self->{'store_text'} = 1;
-    $self->{'outhandle'} = $outhandle;
-
@@ -170 +171 @@
-    return $self->{'indexing_text'};
-}
+
+sub set_store_text {
+    my $self = shift (@_);
+    my ($store_text) = @_;
+
+    $self->{'store_text'} = $store_text;
+}
+
-
-sub process {
@@ -446 +455 @@
-                    my $new_text = "";
-                    if ($real_field eq "text") {
-
+ if $self->{'store_text'}
-                        $self->{'num_processed_bytes'} += length ($new_text);
-                        $new_text =~ s/[\cB\cC]//g;
@@ -457 +466 @@
-                            $self->{'num_processed_bytes'} += length ($meta);
-                            $new_text .= "\cC" unless $first;
-
+ if $self->{'store_text'}
-                            $first = 0;
-                        }