Changeset 10158

Timestamp:

2005-06-23T11:23:42+12:00 (19 years ago)

Author:

davidb

Message:

*builder.pm packages (principally lucenebuilder.pl which inherits from
mgppbuilder) upgraded to support incremental building.

Location:

trunk/gsdl/perllib

Files:

• lucenebuilder.pm (modified) (3 diffs)
• mgbuilder.pm (modified) (2 diffs)
• mgppbuilder.pm (modified) (16 diffs)

trunk/gsdl/perllib/lucenebuilder.pm

@@ -58 +58 @@
 
     eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
-     "\$source_dir, \$build_dir, \$verbosity, \$outhandle)");
+     "\$source_dir, \$build_dir, \$keepold, \$verbosity, \$outhandle)");
     die "$@" if $@;
 
@@ -217 +217 @@
     my $lucene_passes_sections = $llevel;
 
+    my $opt_create_index = ($self->{'keepold'}) ? "" : "-create";
+
     my $osextra = "";
     if ($ENV{'GSDLOS'} =~ /^windows$/i) {
     $build_dir =~ s@/@\\@g;
     } else {
-    $osextra = " -d /";
     if ($outhandle ne "STDERR") {
         # so lucene_passes doesn't print to stderr if we redirect output
@@ -275 +276 @@
     } else {
     if (!-e "$lucene_passes_exe" || 
-        !open (PIPEOUT, "| $full_lucene_passes_exe index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
+        !open (PIPEOUT, "| $full_lucene_passes_exe $opt_create_index index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
         print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
         die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";

trunk/gsdl/perllib/mgbuilder.pm

@@ -196 +196 @@
 
     eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
-     "\$source_dir, \$build_dir, \$verbosity, \$outhandle)");
+     "\$source_dir, \$build_dir, \$keepold, \$verbosity, \$outhandle)");
     die "$@" if $@;
 
@@ -922 +922 @@
 sub deinit {
     my $self = shift (@_);
+
+    &plugin::deinit($self->{'pluginfo'},$self->{'buildproc'});
 }
 

trunk/gsdl/perllib/mgppbuilder.pm

@@ -264 +264 @@
 
     eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
-     "\$source_dir, \$build_dir, \$verbosity, \$outhandle)");
+     "\$source_dir, \$build_dir, \$keepold, \$verbosity, \$outhandle)");
     die "$@" if $@;
 
@@ -864 +864 @@
     #$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
 
-    $self->{'buildproc'}->reset();
+    # make_infodatabase does not support incremental build 
+    # => full reset needed
+    $self->{'buildproc'}->zero_reset(); 
 
     # do the collection info
@@ -938 +940 @@
     }
     print $handle $subcoll_entry;
-     # now add language meta
+
+    # now add language meta
     my $lang_entry = "";
     foreach my $lang (@{$self->{'index_mapping'}->{'languagemaporder'}}) {
@@ -950 +953 @@
     }
     print $handle $lang_entry;
-   #end the collection entry
-    print $handle "\n" . ('-' x 70) . "\n";
-    
-    
+    # end the collection entry
+    print $handle "\n" . ('-' x 70) . "\n";        
     
     &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
@@ -1023 +1024 @@
 }
 
-# at the end of building, we have an indexfieldmap with all teh mappings, plus
-# some extras, and indexmap with any indexes in it that weren't specified in the index definition.
-# we want to make an ordered list of fields that are indexed, and a list of mappings that are used. this will be used for the build.cfg file, and for collection meta definition
-# we store these in a build.cfg bit
+# at the end of building, we have an indexfieldmap with all the mappings,
+# plus some extras, and indexmap with any indexes in it that weren't
+# specified in the index definition.  we want to make an ordered list of
+# fields that are indexed, and a list of mappings that are used. this will
+# be used for the build.cfg file, and for collection meta definition we
+# store these in a build.cfg bit
 sub make_final_field_list {
     my $self = shift (@_);
     
     $self->{'build_cfg'} = {};
-
+    
     # store the indexfieldmap information
     my @indexfieldmap = ();
@@ -1037 +1040 @@
     my $specifiedfields = {};
     my @specifiedfieldorder = ();
-    # go through the index definition and add each thing to a map, so we can easily check if it is already specified - when doing the metadata, we print out all the individual fields, but some may already be specified in the index definition, so we dont want to add those again.
+
+    # go through the index definition and add each thing to a map, so we
+    # can easily check if it is already specified - when doing the
+    # metadata, we print out all the individual fields, but some may
+    # already be specified in the index definition, so we dont want to add
+    # those again.
+
     foreach my $field (@{$self->{'collect_cfg'}->{'indexes'}}) {
     # remove subcoll stuff
@@ -1073 +1082 @@
     }
     }
+
     $self->{'build_cfg'}->{'indexfieldmap'} = \@indexfieldmap;
     $self->{'build_cfg'}->{'indexfields'} = \@indexfields;
-
-
-}
-
-
-# recreate the field list from the build.cfg file, look first in building, then in index to find it. if there is no build.cfg, we cant do the field list (there is unlikely to be any index anyway.)
+}
+
+
+# recreate the field list from the build.cfg file, look first in building,
+# then in index to find it. if there is no build.cfg, we can't do the field
+# list (there is unlikely to be any index anyway.)
 sub read_final_field_list {
     my $self = shift (@_);
@@ -1102 +1112 @@
     }
     } 
+
     my $buildcfg = &colcfg::read_build_cfg( $buildconfigfile);
+
     if (defined $buildcfg->{'indexfields'}) {
     foreach my $field (@{$buildcfg->{'indexfields'}}) {
@@ -1108 +1120 @@
     }
     }
+
     if (defined $buildcfg->{'indexfieldmap'}) {
     foreach my $field (@{$buildcfg->{'indexfieldmap'}}) {
@@ -1115 +1128 @@
     }
     }       
-    
+
     $self->{'build_cfg'}->{'indexfieldmap'} = \@indexfieldmap;
     $self->{'build_cfg'}->{'indexfields'} = \@indexfields;
-    
-}
+}
+
 sub make_auxiliary_files {
     my $self = shift (@_);
@@ -1129 +1142 @@
     $build_cfg = $self->{'build_cfg'};
     }
-    #my %build_cfg = ();
     
     my $outhandle =  $self->{'outhandle'};
@@ -1159 +1171 @@
     # store the number of documents and number of bytes
     $build_cfg->{'numdocs'} = $self->{'buildproc'}->get_num_docs();
+    $build_cfg->{'numsections'} = $self->{'buildproc'}->get_num_sections();
     $build_cfg->{'numbytes'} = $self->{'buildproc'}->get_num_bytes();
 
@@ -1184 +1197 @@
     $build_cfg->{'languagemap'} = \@languagemap if scalar (@languagemap);
 
-    #$build_cfg->{'notbuilt'} = $self->{'notbuilt'};
     my @notbuilt = ();
     foreach my $nb (keys %{$self->{'notbuilt'}}) {
@@ -1193 +1205 @@
     # write out the build information
     &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
-                 '^(builddate|buildtype|numdocs|numbytes|textlevel|indexstem)$', 
+                 '^(builddate|buildtype|numdocs|numsections|numbytes|textlevel|indexstem)$', 
                              '^(indexmap|subcollectionmap|languagemap|indexfieldmap|notbuilt|indexfields|indexlevels|levelmap)$');
 
@@ -1201 +1213 @@
 sub deinit {
     my $self = shift (@_);
+    
+    &plugin::deinit($self->{'pluginfo'},$self->{'buildproc'});
 }
 
@@ -1221 +1235 @@
 
     if ($num_processed_bytes < 50 && ($indexing_text || !$self->{'no_text'})) {
-    print $outhandle "***************\n";
-    if ($indexing_text) {
-        print $outhandle "WARNING: There is very little or no text to process for $index\n";
-    } elsif (!$self->{'no_text'}) {
-        print $outhandle "WARNING: There is very little or no text to compress\n";
-    }      
-    print $outhandle "         Was this your intention?\n";
-    print $outhandle "***************\n";
+    
+    if ($self->{'keepold'}) {
+        if ($num_processed_bytes == 0) {
+        if ($indexing_text) {
+            print $outhandle "No additional text was added to $index\n";
+        } elsif (!$self->{'no_text'}) {
+            print $outhandle "No additional text was compressed\n";
+        }   
+        }   
+    }
+    else {
+        print $outhandle "***************\n";
+        if ($indexing_text) {
+        print $outhandle "WARNING: There is very little or no text to process for $index\n";
+        } elsif (!$self->{'no_text'}) {
+        print $outhandle "WARNING: There is very little or no text to compress\n";
+        }      
+        print $outhandle "         Was this your intention?\n";
+        print $outhandle "***************\n";
+    }
+
     }