Changeset 2336

Timestamp:

2001-04-24T16:36:21+12:00 (23 years ago)

Author:

sjboddie

Message:

added a -no_text option to buildcol.pl to allow collections to be built
without storing compressed text (intended for use in collections where
original documents (PDFs or Word docs maybe) are returned instead of the
compressed text)

Location:

trunk/gsdl

Files:

• bin/script/buildcol.pl (modified) (5 diffs)
• perllib/colcfg.pm (modified) (2 diffs)
• perllib/mgbuilder.pm (modified) (16 diffs)
• perllib/mgbuildproc.pm (modified) (4 diffs)

trunk/gsdl/bin/script/buildcol.pl

@@ -58 +58 @@
     print STDERR "   -keepold              will not destroy the current contents of the\n";
     print STDERR "                         building directory\n";
+    print STDERR "   -no_text              Don't store compressed text. This option is\n";
+    print STDERR "                         useful for minimizing the size of the built\n";
+    print STDERR "                         indexes if you intend always to display the\n";
+    print STDERR "                         original documents at run time (i.e. you won't\n";
+    print STDERR "                         be able to retrieve the compressed text version)\n";
     print STDERR "   -allclassifications   Don't remove empty classifications\n";
     print STDERR "   -create_images        Attempt to create default images for new\n";
@@ -79 +84 @@
     $debug, $mode, $indexname, $keepold, $allclassifications,
     $create_images, $collectdir, $out, $buildtype, $textindex,
-    $no_strip_html);
+    $no_strip_html, $no_text);
     if (!parsargv::parse(\@ARGV, 
              'verbosity/\d+/2', \$verbosity,
@@ -89 +94 @@
              'mode/^(all|compress_text|build_index|infodb)$/all', \$mode,
              'index/.*/', \$indexname,
+             'no_text', \$no_text,
              'keepold', \$keepold,
              'allclassifications', \$allclassifications,
@@ -136 +142 @@
         $builddir = $collectcfg->{'builddir'};
     }
+    if (defined $collectcfg->{'collectdir'} && $collectdir eq "") {
+        $collectdir = $collectcfg->{'collectdir'};
+    }
+    if (defined $collectcfg->{'no_text'} && $no_text == 0) {
+        if ($collectcfg->{'no_text'} =~ /^true$/) {
+        $no_text = 1;
+        }
+    }
+    if (defined $collectcfg->{'allclassifications'} && $allclassifications == 0) {
+        if ($collectcfg->{'allclassifications'} =~ /^true$/) {
+        $allclassifications = 1;
+        }
+    }
     if ($buildtype eq "mgpp" && defined $collectcfg->{'textcompress'}) {
         $textindex = $collectcfg->{'textcompress'};
@@ -219 +238 @@
     eval("\$builder = new $buildertype(\$collection, " .
      "\$realarchivedir, \$realbuilddir, \$verbosity, " .
-     "\$maxdocs, \$debug, \$keepold, \$allclassifications, \$out)");
+     "\$maxdocs, \$debug, \$keepold, \$allclassifications, " .
+     "\$out, \$no_text)");
     die "$@" if $@;
 

trunk/gsdl/perllib/colcfg.pm

@@ -66 +66 @@
     return &cfgread::read_cfg_file ($filename, 
                     q/^(creator|public|beta|defaultindex|importdir|/ .
-                    q/archivedir|cachedir|builddir|removeold|textcompress|buildtype)$/,
+                    q/archivedir|cachedir|builddir|removeold|/ .
+                    q/textcompress|buildtype|collectdir|no_text|allclassifications)$/,
                     q/(maintainer|languages|indexsubcollections|/ .
                        q/indexes|dontbuild|dontgdbm|mirror|phind|levels)$/,
@@ -78 +79 @@
     &cfgread::write_cfg_file($filename, $data,
                  q/^(creator|public|beta|defaultindex|importdir|/ .
-                 q/archivedir|cachedir|builddir|removeold|textcompress|buildtype)$/,
+                 q/archivedir|cachedir|builddir|removeold|/ .
+                 q/textcompress|buildtype|collectdir|no_text|allclassifications)$/,
                  q/^(maintainer|languages|indexsubcollections|/ .
                  q/indexes|dontbuild|dontgdbm|levels)$/,

trunk/gsdl/perllib/mgbuilder.pm

@@ -61 +61 @@
 sub new {
     my ($class, $collection, $source_dir, $build_dir, $verbosity, 
-    $maxdocs, $debug, $keepold, $allclassifications, $outhandle) = @_;
+    $maxdocs, $debug, $keepold, $allclassifications, 
+    $outhandle, $no_text) = @_;
 
     $outhandle = STDERR unless defined $outhandle;
+    $no_text = 0 unless defined $no_text;
 
     # create an mgbuilder object
@@ -75 +77 @@
               'allclassifications'=>$allclassifications,
               'outhandle'=>$outhandle,
+              'no_text'=>$no_text,
               'notbuilt'=>[]    # indexes not built
               }, $class;
@@ -219 +222 @@
     } else {
     if (!-e "$mg_passes_exe" || 
-#       !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
         !open (PIPEOUT, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 $osextra")) {
         die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
@@ -230 +232 @@
     $self->{'buildproc'}->set_index ($textindex);
     $self->{'buildproc'}->set_indexing_text (0);
+    if ($self->{'no_text'}) {
+    $self->{'buildproc'}->set_store_text(0);
+    } else {
+    $self->{'buildproc'}->set_store_text(1);
+    }
     $self->{'buildproc'}->reset();
     &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
@@ -251 +258 @@
         die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
     }
-#   system ("\"$mg_compression_dict_exe\" -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
     system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
 
     # -b $maxdocsize sets the maximum document size to be 12 meg
     if (!-e "$mg_passes_exe" || 
-#       !open ($handle, "| \"$mg_passes_exe\" -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
         !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 $osextra")) {
         die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
@@ -526 +531 @@
     } else {
     if (!-e "$mg_passes_exe" || 
-#       !open (PIPEOUT, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
         !open (PIPEOUT, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
            "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
@@ -539 +543 @@
     $self->{'buildproc'}->set_index ($index, $indexexparr);
     $self->{'buildproc'}->set_indexing_text (1);
+    $self->{'buildproc'}->set_store_text(1);
 
     $self->{'buildproc'}->reset();
@@ -552 +557 @@
         die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
     }
-#   system ("\"$mg_perf_hash_build_exe\" -f \"$fullindexprefix\" $osextra");
     system ("mg_perf_hash_build$exe -f \"$fullindexprefix\" $osextra");
 
     if (!-e "$mg_passes_exe" || 
-#       !open ($handle, "| \"$mg_passes_exe\" -f \"$fullindexprefix\" -b $maxdocsize " .
         !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
            "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
@@ -581 +584 @@
         die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
     }
-#   system ("\"$mg_weights_build_exe\" -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
     system ("mg_weights_build$exe -f \"$fullindexprefix\" -t \"$fulltextprefix\" $osextra");
 
@@ -589 +591 @@
         die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
     }
-#   system ("\"$mg_invf_dict_exe\" -f \"$fullindexprefix\" $osextra");
     system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");
 
@@ -598 +599 @@
         die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
     }
-#   system ("\"$mg_stem_idx_exe\" -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
     system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
-#   system ("\"$mg_stem_idx_exe\" -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
     system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
-#   system ("\"$mg_stem_idx_exe\" -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
     system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
-
     
     # remove unwanted files
@@ -653 +650 @@
     $handle = STDOUT;
     } else {
-#   if (!-e "$txt2db_exe" || !open (PIPEOUT, "| \"$txt2db_exe\" \"$fulldbname\"")) {
     if (!-e "$txt2db_exe" || !open (PIPEOUT, "| txt2db$exe \"$fulldbname\"")) {
         die "mgbuilder::make_infodatabase - couldn't run $txt2db_exe\n";
@@ -666 +662 @@
     $self->{'buildproc'}->set_classifiers ($self->{'classifiers'});
     $self->{'buildproc'}->set_indexing_text (0);
+    $self->{'buildproc'}->set_store_text(1);
     $self->{'buildproc'}->reset();
 
@@ -731 +728 @@
     my $mgstat_exe = &util::filename_cat($exedir, "mgstat$exe");
     my $input_file = &util::filename_cat ("text", $self->{'collection'});
-#    if (!-e "$mgstat_exe" || !open (PIPEIN, "\"$mgstat_exe\" -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
     if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
     print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
@@ -797 +793 @@
     print $outhandle "Total bytes in $index: $num_processed_bytes\n";
 
-    if ($num_processed_bytes < 50) {
+    if ($num_processed_bytes < 50 && ($indexing_text || !$self->{'no_text'})) {
     print $outhandle "***************\n";
-    print $outhandle "WARNING: There is very little or no text to process for $index\n";
     if ($indexing_text) {
-        print $outhandle "This may cause an error while attempting to build the index\n";
-    } else {
-        print $outhandle "This may cause an error while attempting to compress the text\n";
-    }
+        print $outhandle "WARNING: There is very little or no text to process for $index\n";
+    } elsif (!$self->{'no_text'}) {
+        print $outhandle "WARNING: There is very little or no text to compress\n";
+    }
+    print $outhandle "         Was this your intention?\n";
     print $outhandle "***************\n";
     }
@@ -810 +806 @@
 
 1;
-
-

trunk/gsdl/perllib/mgbuildproc.pm

@@ -65 +65 @@
     $self->{'num_bytes'} = 0;
     $self->{'num_processed_bytes'} = 0;
+    $self->{'store_text'} = 1;
     $self->{'outhandle'} = $outhandle;
 
@@ -170 +171 @@
     return $self->{'indexing_text'};
 }
+
+sub set_store_text {
+    my $self = shift (@_);
+    my ($store_text) = @_;
+
+    $self->{'store_text'} = $store_text;
+}
+
 
 sub process {
@@ -446 +455 @@
             my $new_text = "";
             if ($real_field eq "text") {
-            $new_text = $doc_obj->get_text ($section);
+            $new_text = $doc_obj->get_text ($section) if $self->{'store_text'};
             $self->{'num_processed_bytes'} += length ($new_text);
             $new_text =~ s/[\cB\cC]//g;
@@ -457 +466 @@
                 $self->{'num_processed_bytes'} += length ($meta);
                 $new_text .= "\cC" unless $first;
-                $new_text .= $meta;
+                $new_text .= $meta if $self->{'store_text'};
                 $first = 0;
             }