Changeset 38882 for main/trunk

Timestamp:

2024-03-27T11:19:44+13:00 (3 months ago)

Author:

kjdon

Message:

fine tuning on generating buckets. if one side (alpha or numeric) has made buckets, then the other side must - don't want to end up with HList with some buckets and some documents in it

File:

• main/trunk/greenstone2/perllib/classify/List.pm (modified) (8 diffs)

main/trunk/greenstone2/perllib/classify/List.pm

@@ -637 +637 @@
 }
 
-
+# when using approximate_size or constant_size buckets, if there are less items than partition_size, then we generally don't make a bucket.
+# However, if the alpha values have partitions and the numerica values don't, then this doesn't display nicely (get documents in an HList)
+# so if one side has partitions, then we force the other side to have them too.
+sub will_we_have_partitions
+{
+    my $self = shift(@_);
+    my $metadata_group = shift(@_);
+    my $metadata_value_to_OIDs_hash_ref = shift(@_);
+    my $numeric_metadata_value_to_OIDs_hash_ref = shift(@_);
+
+    my $partition_type = $self->{$metadata_group . ".partition_type_within_level"};
+    my $partition_size = $self->{$metadata_group . ".partition_size_within_level"};
+    my $bookshelf_type = $self->{$metadata_group. ".bookshelf_type"};
+    my $numeric_partition_type = $self->{$metadata_group . ".numeric_partition_type_within_level"};
+    my $numeric_partition_size = $self->{$metadata_group . ".numeric_partition_size_within_level"};
+
+    if ($partition_type eq "none") { #if this is none, then numeric will also be none, so only need to check one of them.
+        return 0;
+    }
+    # if either alpha or numeric has partitions, then we force partitions both sides
+    if ($partition_type =~ /^(per_letter|all_values)$/ || $numeric_partition_type =~ /^(single_partition|per_digit|all_values)/) {
+        return 1;
+    }
+    # we are using approximate/constant size buckets - lets count the items to see if either side will be making buckets
+    my $num_alpha = 0;
+    my $num_numeric = 0;
+    if ($bookshelf_type =~ /^(always|duplicate_only)$/) {
+        # each individual value will be a bookshelf or a single item
+        $num_alpha = scalar(keys %$metadata_value_to_OIDs_hash_ref);
+        $num_numeric = scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref);
+    } else {
+        # we have to count actual values to see how many entires there are
+        foreach my $metadata_value (keys %{$metadata_value_to_OIDs_hash_ref}) {
+            $num_alpha += scalar( @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}});
+        }
+        foreach my $metadata_value (keys %{$numeric_metadata_value_to_OIDs_hash_ref}) {
+            $num_numeric += scalar( @{$numeric_metadata_value_to_OIDs_hash_ref->{$metadata_value}});
+        }
+        
+    }
+
+    my $tolerance = 0;
+    my $numeric_tolerance = 0;
+    if ($partition_type eq "approximate_size") {
+        $tolerance = ($partition_size > 40 ? 10 : int($partition_size/4));
+    }
+    if ($numeric_partition_type eq "approximate_size" ) {
+        $numeric_tolerance = ($numeric_partition_size > 40 ? 10 : int($numeric_partition_size/4));
+    }
+    
+    # now, test to see if we have partition on either side - in which case we need to force partitions on the other side
+    if ($num_alpha > $partition_size+$tolerance || $num_numeric > $numeric_partition_size+$numeric_tolerance) {
+        return 1;
+    }
+    return 0;
+}
 sub add_level
 {
@@ -679 +734 @@
     my $bookshelf_type_within_level = $self->{$metadata_group. ".bookshelf_type"};
     
+    my ($has_partitions) = $self->will_we_have_partitions($metadata_group, \%metadata_value_to_OIDs_hash, \%numeric_metadata_value_to_OIDs_hash);
     
     #############################################
@@ -684 +740 @@
     #############################################
     if ($self->{'numbers_first'} && keys(%numeric_metadata_value_to_OIDs_hash)) {
-    $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash);
+    $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash, $has_partitions);
     }
 
@@ -694 +750 @@
         $self->split_per_letter_or_digit(\@metadata_groups, $classifier_node, $partition_sort_mode_within_level, \%metadata_value_to_OIDs_hash);
     }
-    elsif ($partition_type_within_level =~ /^approximate_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
+    elsif ($partition_type_within_level =~ /^approximate_size$/i && $has_partitions) { # scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
             $self->split_approximate_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, $bookshelf_type_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'}, $self->{'max_partition_name_length'});
         
     } 
-    elsif ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
+    elsif ($partition_type_within_level =~ /^constant_size$/i && $has_partitions) { #scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
         $self->split_constant_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, $bookshelf_type_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'}, $self->{'max_partition_name_length'});
     }
@@ -716 +772 @@
     ###########################################
     if (!$self->{'numbers_first'} && keys(%numeric_metadata_value_to_OIDs_hash)) {
-    $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash);
+    $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash, $has_partitions);
     }
 }
@@ -726 +782 @@
     my $classifier_node = shift(@_);
     my $numeric_metadata_value_to_OIDs_hash_ref = shift(@_);
+    my $has_partitions = shift(@_);
 
     my $metadata_group = $metadata_groups[0];
@@ -744 +801 @@
     $self->split_per_letter_or_digit(\@metadata_groups, $classifier_node, $numeric_partition_sort_mode_within_level, $numeric_metadata_value_to_OIDs_hash_ref, 1, $numeric_partition_name_length_within_level);
     }
-    elsif ($numeric_partition_type_within_level eq "constant_size" && scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
+    elsif ($numeric_partition_type_within_level eq "constant_size" && $has_partitions) { #scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
     # Generate hlists of a certain size
     
     $self->split_constant_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level,$numeric_partition_name_length_within_level, 1);
-    } elsif ($numeric_partition_type_within_level eq "approximate_size" && scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
+    } elsif ($numeric_partition_type_within_level eq "approximate_size" && $has_partitions) { #scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
     $self->split_approximate_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level, $numeric_partition_name_length_within_level, 1);
     }
@@ -779 +836 @@
     my $is_numeric = shift(@_);
 
-    my $tolerance = 10; # should this be an option??
+    my $tolerance = ($partition_size > 40 ? 10 : int($partition_size/4)); # should this be an option??
 
     # Generate hlist based on the first letter of the metadata value (like per_letter), or based on