Changeset 12910

Timestamp:

2006-09-28T16:45:30+12:00 (18 years ago)

Author:

kjdon

Message:

new indexoptions field in collect.cfg specifies which stem indexes should be built (stem, casefold, accentfold). mg and lucene ignore this, mg does stem and casefold, llucene does none. stemindexes is output to build.cfg so that the library knows what options are available for the collection - don't give stem option if stemming is not implemented, for example. added in accent fold stem indexes for mgpp (thanks to Juan Grigera). accent folding may be disabled in mgpp, so we check the first time we try to create one, and don't try to build the rest if it failed.

Location:

trunk/gsdl/perllib

Files:

• lucenebuilder.pm (modified) (1 diff)
• mgbuilder.pm (modified) (2 diffs)
• mgppbuilder.pm (modified) (4 diffs)

trunk/gsdl/perllib/lucenebuilder.pm

@@ -87 +87 @@
 }
 # /** new() **/
+
+# lucene has none of these options
+sub generate_index_options {
+    my $self = shift (@_);
+
+    $self->{'casefold'} = 0;
+    $self->{'stem'} = 0;
+    $self->{'accentfold'} = 0; 
+    $self->{'stemindexes'} = 0;
+}    
 
 sub default_buildproc {

trunk/gsdl/perllib/mgbuilder.pm

@@ -86 +86 @@
 }
 
+sub generate_index_options {
+    my $self = shift (@_);
+
+    $self->{'casefold'} = 0;
+    $self->{'stem'} = 0;
+    $self->{'accentfold'} = 0; #not yet implemented for mg
+    
+    if (!defined($self->{'collect_cfg'}->{'indexoptions'})) {
+    # just use default options
+    $self->{'casefold'} = 1;
+    $self->{'stem'} = 1;
+    } else {
+    foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {
+        if ($option =~ /stem/) {
+        $self->{'stem'} = 1;
+        } elsif ($option =~ /casefold/) {
+        $self->{'casefold'} = 1;
+        }
+    }
+    }
+    
+    # now we record this for the build cfg
+    $self->{'stemindexes'} = 0;
+    if ($self->{'casefold'}) {
+    $self->{'stemindexes'} += 1;
+    }
+    if ($self->{'stem'}) {
+    $self->{'stemindexes'} += 2;
+    }
+
+
+}
 
 sub compress_text {
@@ -456 +488 @@
         die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
     }
-    system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
-    system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
-    system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
-    
+    # currently mg wont work if we don't generate all the stem idexes
+    # so we generate them whatever, but don't advertise the fact
+    #if ($self->{'casefold'}) {
+        print STDERR "casefolding\n";
+        system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+    #}
+    #if ($self->{'stem'}) {
+        print STDERR "stemming\n";
+        system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+    #}
+    #if ($self->{'casefold'} && $self->{'stem'}) {
+        print STDERR "casefold and stem\n";
+        system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
+    #}
+
     # remove unwanted files
     my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);

trunk/gsdl/perllib/mgppbuilder.pm

@@ -59 +59 @@
                'ib2'=>1,
                'ib3'=>1,
+               'ib4'=>1,
+               'ib5'=>1,
+               'ib6'=>1,
+               'ib7'=>1,
                'i'=>1,
                'il'=>1,
@@ -151 +155 @@
 }
 
+sub generate_index_options {
+    my $self = shift (@_);
+
+    $self->{'casefold'} = 0;
+    $self->{'stem'} = 0;
+    $self->{'accentfold'} = 0; 
+    
+    if (!defined($self->{'collect_cfg'}->{'indexoptions'})) {
+    # just use default options
+    $self->{'casefold'} = 1;
+    $self->{'stem'} = 1;
+    $self->{'accentfold'} = 1;
+    } else {
+    foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {
+        if ($option =~ /stem/) {
+        $self->{'stem'} = 1;
+        } elsif ($option =~ /casefold/) {
+        $self->{'casefold'} = 1;
+        } elsif ($option =~ /accentfold/) {
+        $self->{'accentfold'} = 1;
+        }
+    }
+    }
+    
+    # now we record this for the build cfg
+    $self->{'stemindexes'} = 0;
+    if ($self->{'casefold'}) {
+    $self->{'stemindexes'} += 1;
+    }
+    if ($self->{'stem'}) {
+    $self->{'stemindexes'} += 2;
+    }
+    if ($self->{'accentfold'}) {
+    $self->{'stemindexes'} += 4;
+    }
+    print STDERR "temindexes = $self->{'stemindexes'}\n";
+}
+
 sub default_buildproc {
     my $self  = shift (@_);
@@ -574 +616 @@
         die "mgppbuilder::build_index - couldn't run $mgpp_stem_idx_exe\n";
     }
-    system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
-    system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
-    system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
-    
+    my $accent_folding_enabled = 1;
+    if ($self->{'accentfold'}) {
+        # the first time we do this, we test for accent folding enabled
+        if (system ("mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra") != 0) {
+        # accent folding has not been enabled in mgpp
+        $accent_folding_enabled = 0;
+        $self->{'stemindexes'} -= 4;
+        }
+    }
+    if ($self->{'casefold'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
+        if ($accent_folding_enabled && $self->{'accentfold'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra");
+        }
+    }
+    if ($self->{'stem'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
+        if ($accent_folding_enabled && $self->{'accentfold'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra");
+        }
+    }
+    if ($self->{'casefold'} && $self->{'stem'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
+        if ($accent_folding_enabled && $self->{'accentfold'}) {
+        system ("mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra");
+        }
+    }
+
     # remove unwanted files
     my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
@@ -800 +866 @@
 
 
-sub write_cfg_file {
-    my $self = shift(@_);
-    my ($build_cfg) = @_;
-
-    # write out the build information
-    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
-                 '^(builddate|buildtype|numdocs|numsections|numbytes|textlevel|indexstem|maxnumeric)$', 
-                             '^(indexmap|subcollectionmap|languagemap|indexfieldmap|notbuilt|indexfields|indexlevels|levelmap)$');
-
-}
-
 sub build_cfg_extra {
     my $self = shift (@_);