Index: /main/trunk/greenstone2/perllib/classify/List.pm
===================================================================
--- /main/trunk/greenstone2/perllib/classify/List.pm	(revision 33462)
+++ /main/trunk/greenstone2/perllib/classify/List.pm	(revision 33463)
@@ -314,18 +314,8 @@
     # Default: duplicate_only, ie. leave leaf nodes ungrouped (equivalent to AZCompactList -mingroup 2)
     $self->{$last_metadata_group . ".bookshelf_type"} = $self->{'bookshelf_type'};
-        
-    # Whether to use an hlist or a vlist for each level in the hierarchy (default: vlist)
-    foreach my $metadata_group (@metadata_groups) {
-	$self->{$metadata_group . ".list_type"} = "VList";
-    }
-    foreach my $metadata_group (split(/\,/, $self->{'use_hlist_for'})) {
-	$self->{$metadata_group . ".list_type"} = "HList";
-    }
 
     # How the items are grouped into partitions (default: by letter) 
     # for each level (metadata group), separated by '/'
-
     $self->set_metadata_groups_info_per_level("partition_type_within_level", $partition_type_default, $valid_partition_types);
-
     $self->set_metadata_groups_info_per_level("numeric_partition_type_within_level", $numeric_partition_type_default, $valid_numeric_partition_types);
 
@@ -425,5 +415,4 @@
     # A hash for all the doc ids that we have seen, so we don't classify something twice
     $self->{'all_doc_OIDs'} = {};
-    #return bless $self, $class;
     return $self;
 }
@@ -443,5 +432,5 @@
 
     if (!defined $self->{$info_name}) {
-	print STDERR "**** undefined $info_name\n";
+	print STDERR "List Error: no values were set for option $info_name\n";
     }
     my @info_list = split(/\//, $self->{$info_name});
@@ -581,5 +570,4 @@
 		    my $lc_metadata_value = lc($metadata_value);
 		    $lc_metadata_value  = &sorttools::format_metadata_for_sorting($real_metadata_element, $lc_metadata_value, $doc_obj) unless $self->{'no_metadata_formatting'};
-		    #print STDERR "formatted value = $lc_metadata_value\n";
 	
 		    # Add the metadata value into the list for this combination of metadata group 
@@ -623,5 +611,5 @@
     # Create the root node of the classification hierarchy
     my %classifier_node = ( 'thistype' => "Invisible",
-			    'childtype' => $self->{$first_metadata_group . ".list_type"},
+			    #'childtype' => $self->{$first_metadata_group . ".list_type"},
 			    'Title' => $self->{'buttonname'},
 			    'contains' => [],
@@ -695,5 +683,5 @@
 	} 
 	elsif ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
-	    $self->split_constant_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'});
+	    $self->split_constant_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, $bookshelf_type_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'});
 	}
 	    
@@ -743,5 +731,5 @@
 	# Generate hlists of a certain size
 	
-	$self->split_constant_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level, 1);
+	$self->split_constant_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level, 1);
     } elsif ($numeric_partition_type_within_level eq "approximate_size" && scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
 	$self->split_approximate_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level, 1);
@@ -818,6 +806,6 @@
     for (my $i = 0; $i < scalar(@partition_buckets) - 1; $i++) {
     
-	my $this_bucket = $partition_buckets[$i]; #->{'name'};
-	my $next_bucket = $partition_buckets[$i+1]; #->{'name'};
+	my $this_bucket = $partition_buckets[$i]; 
+	my $next_bucket = $partition_buckets[$i+1]; 
 		    
 	my $items_in_partition = $this_bucket->{'size'};	    	    
@@ -848,4 +836,6 @@
     my $last_partition_end = "";
     my $partition_start = "";
+    my $partition_end = "";
+    my $partition_name = "";
     foreach my $partition (@new_partition_buckets) {
 	my @metadata_values = $self->sort_metadata_values_array($sort_mode, @{$partition->{'metadata_values'}});
@@ -853,5 +843,21 @@
 	$partition_start = $self->generate_partition_start($metadata_values[0], $last_partition_end, $partition_name_length, $is_numeric);
 	
-	if ($items_in_partition > $partition_size) {
+	if ($items_in_partition <= $partition_size) {
+	    # we can just add the partition as is
+	    my %metadata_values_to_OIDs_subhashes = ();
+	    for (my $i = 0; $i < scalar(@metadata_values); $i++) {
+		my $metadata_value = $metadata_values[$i];
+		$metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};		    
+	    }
+	    my $last_metadata_value = $metadata_values[scalar(@metadata_values)-1];
+	    $partition_end = $self->generate_partition_end($last_metadata_value, $partition_start, $partition_name_length, $is_numeric);
+	    $partition_name = $partition_start;
+	    if ($partition_end ne $partition_start) {
+		$partition_name = $partition_name . "-" . $partition_end;
+	    }
+	    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);			
+	    $last_partition_end = $partition_end;		
+	} else {
+	    # we have too many items, need to split the partition
 	    my $items_done = 0;
 	    my %metadata_values_to_OIDs_subhashes = ();
@@ -861,14 +867,8 @@
 		my $items_for_this_md_value = $bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1;
 		
-		my $partitionend = $self->generate_partition_end($metadata_value, $partition_start, $partition_name_length, $is_numeric);
-		my $partitionname = $partition_start;
-		if ($partitionend ne $partition_start) {
-		    $partitionname = $partitionname . "-" . $partitionend;
-		}
-		
 		if ($items_done + $items_for_this_md_value > $partition_size && $items_done != 0) {
-		    # Start a new partition
-		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_values_to_OIDs_subhashes);
-		    $last_partition_end = $partitionend;			
+		    # Save the stored items into a partition
+		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
+		    $last_partition_end = $partition_end;			
 		    $partition_start = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $is_numeric);
 		    $items_done = 0;
@@ -878,11 +878,10 @@
 		# If bookshelf_type is "never" and the current metadata value holds too many items, need to split into several partitions
 		if ($bookshelf_type eq "never" && $items_for_this_md_value > $partition_size) {
-		    
+
 		    my $partitionname_for_this_value = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $is_numeric);
 		    # Get the number of partitions needed for this value
 		    my $num_splits = int($items_for_this_md_value / $partition_size);
 		    $num_splits++ if ($items_for_this_md_value / $partition_size > $num_splits);
-		    
-		    my @OIDs_for_this_value = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
+		    my @OIDs_for_this_value = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
 		    for (my $i = 0; $i < $num_splits; $i++) {
 			my %OIDs_subhashes_for_this_value = ();
@@ -894,43 +893,37 @@
 			# The last bucket might have only a few items and need to be merged with buckets for subsequent metadata values 
 			if ($i == $num_splits - 1 && scalar(@OIDs_for_this_partition) < $partition_size) {
+			    $partition_start = $partitionname_for_this_value;
+			    $partition_name = $partition_start;
 			    $metadata_values_to_OIDs_subhashes{$metadata_value} = \@OIDs_for_this_partition;
 			    $items_done += scalar(@OIDs_for_this_partition);
-			    next;
+			    $last_partition_end = $partitionname_for_this_value
+			} else {
+			
+			    # Add an HList for this bucket
+			    $OIDs_subhashes_for_this_value{$metadata_value} = \@OIDs_for_this_partition;
+			    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname_for_this_value, \%OIDs_subhashes_for_this_value);
+			    $last_partition_end = $partitionname_for_this_value;
 			}
-			
-			# Add an HList for this bucket
-			$OIDs_subhashes_for_this_value{$metadata_value} = \@OIDs_for_this_partition;
-			$self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname_for_this_value, \%OIDs_subhashes_for_this_value);
-			$last_partition_end = $partitionname_for_this_value;
 		    }
-		    next;
+		} else {
+		  		    
+		    $metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
+		    $items_done += $bookshelf_type eq "never" ? scalar(@{$metadata_values_to_OIDs_subhashes{$metadata_value}}) : 1;
+		     $partition_end = $self->generate_partition_end($metadata_value, $partition_start, $partition_name_length, $is_numeric);
+		     $partition_name = $partition_start;
+		    if ($partition_end ne $partition_start) {
+			$partition_name = $partition_name . "-" . $partition_end;
+		    }
+		    
 		}
-		    		    
-		$metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
-		$items_done += $bookshelf_type eq "never" ? scalar(@{$metadata_values_to_OIDs_subhashes{$metadata_value}}) : 1;		    
 		
 		# The last partition
-		if($i == scalar(@metadata_values) - 1) {
-		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_values_to_OIDs_subhashes);
+		if($i == scalar(@metadata_values) - 1 && $items_done >0) {
+		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
 		}
-	    }		
+		
+	    }
 	} # end if items in partition > partition size
-	else {
-	    # The easier case, just add a partition
-	    my %metadata_values_to_OIDs_subhashes = ();
-	    for (my $i = 0; $i < scalar(@metadata_values); $i++) {
-		my $metadata_value = $metadata_values[$i];
-		$metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};		    
-	    }
-	    my $last_metadata_value = $metadata_values[scalar(@metadata_values)-1];
-	    my $partitionend = $self->generate_partition_end($last_metadata_value, $partition_start, $partition_name_length, $is_numeric);
-	    my $partitionname = $partition_start;
-	    if ($partitionend ne $partition_start) {
-		$partitionname = $partitionname . "-" . $partitionend;
-	    }
-	
-	    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_values_to_OIDs_subhashes);			
-	    $last_partition_end = $partitionend;		
-	}
+
     }	
 	
@@ -947,4 +940,5 @@
     my $partition_size = shift(@_);
     my $sort_mode = shift(@_);
+    my $bookshelf_type = shift(@_); 
     my $metadata_value_to_OIDs_hash_ref = shift(@_);
     my $partition_name_length = shift(@_);
@@ -952,21 +946,29 @@
 
     my @sortedmetadata_values = $self->sort_metadata_values_array($sort_mode, keys(%$metadata_value_to_OIDs_hash_ref));
-    my $itemsdone = 0;
-    my $numitems = scalar(@sortedmetadata_values);
+    my $items_in_partition = 0;
     my %metadata_value_to_OIDs_subhash = ();
     my $lastpartitionend = "";
     my $partitionstart;
+    
     foreach my $metadata_value (@sortedmetadata_values) {
-	$metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
-	$itemsdone++;
-	my $itemsinpartition = scalar(keys %metadata_value_to_OIDs_subhash);
-
-	# Is this the start of a new partition?
-	if ($itemsinpartition == 1) {
+	if ($items_in_partition == 0) {
+	    # a new partition, set the name
 	    $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $is_numeric);
 	}
-
-	# Is this the end of the partition?
-	if ($itemsinpartition == $partition_size || $itemsdone == $numitems) {
+	my $numitems_for_this_value = ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1);
+	if ($items_in_partition + $numitems_for_this_value <= $partition_size) {
+	    # add all the current values into the temporary list
+	    $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
+	    $items_in_partition += $numitems_for_this_value;
+	} elsif ($items_in_partition < $partition_size) {
+	    # only want to add some of the values into temporary list
+	    # note, we only get here if bookshelf type is never
+	    my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
+	    @OIDs = $self->sort_leaf_items(\@OIDs);
+	    my $num_items_needed = $partition_size - $items_in_partition;
+	    my @slice = splice(@OIDs, 0, $num_items_needed);
+	    $metadata_value_to_OIDs_subhash{$metadata_value} = \@slice;
+
+	    # now we have filled up the partition
 	    my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $is_numeric);
 	    my $partitionname = $partitionstart;
@@ -977,8 +979,58 @@
 	    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
 	    %metadata_value_to_OIDs_subhash = ();
+	    $items_in_partition = 0;
 	    $lastpartitionend = $partitionend;
-	}
-    }
-
+
+	    # can we get more partitions from this metadata value?
+	    while (scalar(@OIDs) >= $partition_size) {
+		my @slice = splice(@OIDs, 0, $partition_size);
+		$metadata_value_to_OIDs_subhash{$metadata_value} = \@slice;
+		$partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $is_numeric);
+		my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $is_numeric);
+		my $partitionname = $partitionstart;
+		if ($partitionend ne $partitionstart) {
+		    $partitionname = $partitionname . "-" . $partitionend;
+		}
+		$self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
+		%metadata_value_to_OIDs_subhash = ();
+		$items_in_partition = 0;
+		$lastpartitionend = $partitionend;
+
+	    }
+	    if (scalar(@OIDs) > 0) {
+		$metadata_value_to_OIDs_subhash{$metadata_value} = \@OIDs;
+		$items_in_partition = scalar(@OIDs);
+		$partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $is_numeric);
+	    }
+		
+		
+	}
+
+	if ($items_in_partition == $partition_size) {
+	    # its the end of a partition
+	    my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $is_numeric);
+	    my $partitionname = $partitionstart;
+	    if ($partitionend ne $partitionstart) {
+		$partitionname = $partitionname . "-" . $partitionend;
+	    }
+
+	    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
+	    %metadata_value_to_OIDs_subhash = ();
+	    $items_in_partition = 0;
+	    $lastpartitionend = $partitionend;
+	}
+    } # foreach metadata value
+
+    if ($items_in_partition > 0) {
+	# we have to add the last partition
+	my $partitionend = $self->generate_partition_end(@sortedmetadata_values[@sortedmetadata_values-1], $partitionstart, $partition_name_length, $is_numeric);
+	my $partitionname = $partitionstart;
+	if ($partitionend ne $partitionstart) {
+	    $partitionname = $partitionname . "-" . $partitionend;
+	}
+	
+	$self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
+    }
+    
     # The partitions are stored in an HList
     $classifier_node->{'childtype'} = "HList";
@@ -1134,5 +1186,4 @@
     my $classifier_node = shift(@_);
     my $metadata_value_to_OIDs_hash_ref = shift(@_);
-
     my $metadata_group = shift(@metadata_groups);
     $classifier_node->{'mdtype'} = $metadata_group;
@@ -1143,4 +1194,5 @@
     {
 	my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
+
 	# If there is only one item and 'bookshelf_type' is not always (ie. never or duplicate_only), add the item to the list
 	if (@OIDs == 1 && $self->{$metadata_group . ".bookshelf_type"} ne "always") {
@@ -1151,4 +1203,5 @@
 	# If 'bookshelf_type' is 'never', list all the items even if there are duplicated values
 	elsif ($self->{$metadata_group . ".bookshelf_type"} eq "never") {
+	    
 	    @OIDs = $self->sort_leaf_items(\@OIDs);
 	    foreach my $OID (@OIDs) {
@@ -1169,8 +1222,7 @@
 	    if (@metadata_groups > 0) {
 		my $next_metadata_group = $metadata_groups[0];		
-		$child_classifier_node{'childtype'} = $self->{$next_metadata_group . ".list_type"};
-
-		# separate metadata into those that below in the next/sub-metadata_group
-		# and those that below at the current level's metadata_group
+		
+		# separate metadata into those that belong in the next/sub-metadata_group
+		# and those that belong at the current level's metadata_group
 
 		my $OID_to_metadata_values_hash_ref = $self->{$next_metadata_group . ".list"};
@@ -1218,5 +1270,5 @@
     my $OID = shift(@_);
     my $metadata_value = shift(@_);
-    
+
     my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
     my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
