Index: gsdl/trunk/perllib/classify/GenericList.pm
===================================================================
--- gsdl/trunk/perllib/classify/GenericList.pm	(revision 18566)
+++ 	(revision )
@@ -1,582 +1,0 @@
-###########################################################################
-#
-# GenericList.pm -- A general and flexible list classifier with most of
-#                   the abilities of AZCompactList, and better Unicode,
-#                   metadata and sorting capabilities.
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
-#
-# Copyright (C) 2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-# TO DO: - Remove punctuation from metadata values before sorting.
-#        - Add an AZCompactList-style hlist option?
-#
-###########################################################################
-
-package GenericList;
-
-
-use BaseClassifier;
-
-use strict;
-
-
-sub BEGIN {
-    @GenericList::ISA = ('BaseClassifier');
-}
-
-
-my $arguments = 
-    [ { 'name' => "metadata",
-	'desc' => "{GenericList.metadata}",
-	'type' => "metadata",
-	'reqd' => "yes" },
-
-      # The interesting options
-      { 'name' => "always_bookshelf_last_level",
-	'desc' => "{GenericList.always_bookshelf_last_level}",
-	'type' => "flag" },
-      { 'name' => "classify_sections",
-	'desc' => "{GenericList.classify_sections}",
-	'type' => "flag" },
-      { 'name' => "partition_type_within_level",
-	'desc' => "{GenericList.partition_type_within_level}",
-	'type' => "string",
-	'deft' => "none" },
-      { 'name' => "partition_size_within_level",
-	'desc' => "{GenericList.partition_size_within_level}",
-	'type' => "string" },
-      { 'name' => "partition_name_length",
-	'desc' => "{GenericList.partition_name_length}",
-	'type' => "string" },
-      { 'name' => "sort_leaf_nodes_using",
-	'desc' => "{GenericList.sort_leaf_nodes_using}",
-	'type' => "metadata",
-	'deft' => "Title" },
-      { 'name' => "sort_using_unicode_collation",
-	'desc' => "{GenericList.sort_using_unicode_collation}",
-	'type' => "flag" },
-      { 'name' => "use_hlist_for",
-	'desc' => "{GenericList.use_hlist_for}",
-	'type' => "string" } ];
-
-my $options = { 'name'     => "GenericList",
-		'desc'     => "{GenericList.desc}",
-		'abstract' => "no",
-		'inherits' => "Yes",
-		'args'     => $arguments };
-
-
-sub new
-{
-    my ($class) = shift(@_);
-    my ($classifierslist, $inputargs, $hashArgOptLists) = @_;
-    push(@$classifierslist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); 
-    push(@{$hashArgOptLists->{"OptList"}}, $options); 
-
-    my $self = new BaseClassifier($classifierslist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    # The metadata elements to use (required)
-    if (!$self->{'metadata'}) {
-	die "Error: No metadata fields specified for GenericList.\n";
-    }
-    my @metadata_groups = split(/\//, $self->{'metadata'});
-    $self->{'metadata_groups'} = \@metadata_groups;
-
-    # The classifier button name (default: the first metadata element specified)
-    if (!$self->{'buttonname'}) {
-	my $first_metadata_group = $metadata_groups[0];
-	my $first_metadata_element = (split(/\;/, $first_metadata_group))[0];
-	$self->{'buttonname'} = $self->generate_title_from_metadata($first_metadata_element);
-    }
-
-    # Whether to group single items into a bookshelf (must be true for all metadata fields except the last)
-    foreach my $metadata_group (@metadata_groups) {
-	$self->{$metadata_group . ".always_bookshelf"} = "t";
-    }
-    if (!$self->{'always_bookshelf_last_level'}) {
-	# Default: leave leaf nodes ungrouped (equivalent to AZCompactList -mingroup 2)
-	my $last_metadata_group = $metadata_groups[$#metadata_groups];
-	$self->{$last_metadata_group . ".always_bookshelf"} = "f";
-    }
-
-    # Whether to use an hlist or a vlist for each level in the hierarchy (default: vlist)
-    foreach my $metadata_group (@metadata_groups) {
-	$self->{$metadata_group . ".list_type"} = "VList";
-    }
-    foreach my $metadata_group (split(/\,/, $self->{'use_hlist_for'})) {
-	$self->{$metadata_group . ".list_type"} = "HList";
-    }
-
-    # How the items are grouped into partitions (default: no partition)
-    if (!$self->{"partition_type_within_level"}) {
-	$self->{"partition_type_within_level"} = "none";
-    }
-
-    # The number of items in each partition
-    if (!$self->{'partition_size_within_level'}) {
-	# Default: 20
-	foreach my $metadata_group (@metadata_groups) {
-	    $self->{$metadata_group . ".partition_size_within_level"} = 20;
-	}
-    }
-    else {
-	my @partition_size_within_levellist = split(/\//, $self->{'partition_size_within_level'});
-
-	# Assign values based on the partition_size_within_level parameter
-	foreach my $metadata_group (@metadata_groups) {
-	    my $partition_size_within_levelelem = shift(@partition_size_within_levellist);
-	    if (defined($partition_size_within_levelelem)) {
-		$self->{$metadata_group . ".partition_size_within_level"} = $partition_size_within_levelelem;
-	    }
-	    else {
-		$self->{$metadata_group . ".partition_size_within_level"} = $self->{$metadata_groups[0] . ".partition_size_within_level"};
-	    }
-	}
-    }
-
-    # The metadata elements to use to sort the leaf nodes (default: Title)
-    my @sort_leaf_nodes_using_metadata_groups = ( "Title" );
-    if ($self->{'sort_leaf_nodes_using'}) {
-	@sort_leaf_nodes_using_metadata_groups = split(/\|/, $self->{'sort_leaf_nodes_using'});
-    }
-    $self->{'sort_leaf_nodes_using_metadata_groups'} = \@sort_leaf_nodes_using_metadata_groups;
-
-    # Create an instance of the Unicode::Collate object if better Unicode sorting is desired
-    if ($self->{'sort_using_unicode_collation'}) {
-	# To use this you first need to download the allkeys.txt file from 
-        # http://www.unicode.org/Public/UCA/latest/allkeys.txt and put it in the Perl
-        # Unicode/Collate directory.
-	require Unicode::Collate;
-	$self->{'unicode_collator'} = Unicode::Collate->new();
-    }
-
-    # An empty array for the document OIDs
-    $self->{'OIDs'} = [];
-
-    return bless $self, $class;
-}
-
-
-sub init
-{
-    # Nothing to do...
-}
-
-
-# Called for each document in the collection
-sub classify
-{
-    my $self = shift(@_);
-    my ($doc_obj,$edit_mode) = @_;
-
-    # If "-classify_sections" is set, classify every section of the document
-    if ($self->{'classify_sections'}) {
-	my $section = $doc_obj->get_next_section($doc_obj->get_top_section());
-	while (defined $section) {
-	    $self->classify_section($doc_obj, $doc_obj->get_OID() . ".$section", $section, $edit_mode);
-	    $section = $doc_obj->get_next_section($section);
-	}
-    }
-    # Otherwise just classify the top document section
-    else {
-	$self->classify_section($doc_obj, $doc_obj->get_OID(), $doc_obj->get_top_section(), $edit_mode);
-    }
-}
-
-
-sub classify_section
-{
-    my $self = shift(@_);
-    my ($doc_obj,$section_OID,$section,$edit_mode) = @_;
-
-    my @metadata_groups = @{$self->{'metadata_groups'}};
-
-    # Only classify the section if it has a value for one of the metadata elements in the first group
-    my $classify_section = 0;
-    my $first_metadata_group = $metadata_groups[0];
-    foreach my $first_metadata_group_element (split(/\;/, $first_metadata_group)) {
-	my $first_metadata_group_element_value = $doc_obj->get_metadata_element($section, $first_metadata_group_element);
-	if (defined($first_metadata_group_element_value) && $first_metadata_group_element_value ne "") {
-	    # This section must be included in the classifier
-	    $classify_section = 1;
-	    last;
-	}
-    }
-
-    # We're not classifying this section because it doesn't have the required metadata
-    return if (!$classify_section);
-
-
-    if (($edit_mode eq "delete") || ($edit_mode eq "update")) {
-	$self->oid_array_delete($section_OID,'OIDs');
-	if ($edit_mode eq "delete") {
-	    return;
-	}
-    }
-   
-    # Otherwise, include this section in the classifier
-    push(@{$self->{'OIDs'}}, $section_OID);
-
-    # Create a hash for the metadata values of each metadata element we're interested in
-    my %metadata_groups_done = ();
-    foreach my $metadata_group (@metadata_groups, @{$self->{'sort_leaf_nodes_using_metadata_groups'}}) {
-	# Take care not to do a metadata group more than once
-	unless ($metadata_groups_done{$metadata_group}) {
-	    foreach my $metadata_element (split(/\;/, $metadata_group)) {
-		my @metadata_values = @{$doc_obj->get_metadata($section, $metadata_element)};
-		foreach my $metadata_value (@metadata_values) {
-		    # Strip leading and trailing whitespace
-		    $metadata_value =~ s/^\s*//;
-		    $metadata_value =~ s/\s*$//;
-
-		    # Convert the metadata value from a UTF-8 string to a Unicode string
-		    # This means that length() and substr() work properly
-		    # We need to be careful to convert classifier node title values back to UTF-8, however
-		    my $metadata_value_unicode_string = $self->convert_utf8_string_to_unicode_string($metadata_value);
-
-		    # Add the metadata value into the list for this combination of metadata group and section
-		    push(@{$self->{$metadata_group . ".list"}->{$section_OID}}, $metadata_value_unicode_string);
-		}
-		last if (@metadata_values > 0);
-	    }
-
-	    $metadata_groups_done{$metadata_group} = 1;
-	}
-    }
-}
-
-
-sub get_classify_info
-{
-    my $self = shift(@_);
-
-    # The metadata groups to classify by
-    my @metadata_groups = @{$self->{'metadata_groups'}};
-    my $first_metadata_group = $metadata_groups[0];
-
-    # The OID values of the documents to include in the classifier
-    my @OIDs = @{$self->{'OIDs'}};
-
-    # Create the root node of the classification hierarchy
-    my %classifier_node = ( 'thistype' => "Invisible",
-			    'childtype' => $self->{$first_metadata_group . ".list_type"},
-			    'Title' => $self->{'buttonname'},
-			    'contains' => [],
-			    'mdtype' => $first_metadata_group );
-
-    # Recursively create the classification hierarchy, one level for each metadata group
-    $self->add_level(\@metadata_groups, \@OIDs, \%classifier_node);
-    return \%classifier_node;
-}
-
-
-sub add_level
-{
-    my $self = shift(@_);
-    my @metadata_groups = @{shift(@_)};
-    my @OIDs = @{shift(@_)};
-    my $classifier_node = shift(@_);
-    # print STDERR "\nAdding AZ list for " . $classifier_node->{'Title'} . "\n";
-
-    my $metadata_group = $metadata_groups[0];
-    # print STDERR "Processing metadata group: " . $metadata_group . "\n";
-    # print STDERR "Number of OID values: " . @OIDs . "\n";
-
-    if (!defined($self->{$metadata_group . ".list"})) {
-	print STDERR "Warning: No metadata values assigned to $metadata_group.\n";
-	return;
-    }
-
-    # Create a mapping from metadata value to OID
-    my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
-    my %metadata_value_to_OIDs_hash = ();
-    foreach my $OID (@OIDs)
-    {
-	if ($OID_to_metadata_values_hash_ref->{$OID})
-	{
-	    my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
-	    foreach my $metadata_value (@metadata_values)
-	    {
-		push(@{$metadata_value_to_OIDs_hash{$metadata_value}}, $OID);
-	    }
-	}
-    }
-    # print STDERR "Number of distinct values: " . scalar(keys %metadata_value_to_OIDs_hash) . "\n";
-
-    # Partition the values (if necessary)
-    my $partition_type_within_level = $self->{"partition_type_within_level"};
-    if ($partition_type_within_level =~ /^per_letter$/i) {
-	# Generate one hlist for each letter
-	my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash));
-	my %metadata_value_to_OIDs_subhash = ();
-
-	my $lastpartition = substr($sortedmetadata_values[0], 0, 1);
-	foreach my $metadata_value (@sortedmetadata_values) {
-	    my $metadata_valuepartition = substr($metadata_value, 0, 1);
-
-	    # Is this the start of a new partition?
-	    if ($metadata_valuepartition ne $lastpartition) {
-		$self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
-		%metadata_value_to_OIDs_subhash = ();
-		$lastpartition = $metadata_valuepartition;
-	    }
-
-	    $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value};	    
-	}
-
-	# Don't forget to add the last partition
-	$self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
-
-	# The partitions are stored in an HList
-	$classifier_node->{'childtype'} = "HList";
-    }
-
-    else {
-	# Generate hlists of a certain size
-	my $partition_size_within_level = $self->{$metadata_group . ".partition_size_within_level"};
-	if ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
-	    my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash));
-	    my $itemsdone = 0;
-	    my %metadata_value_to_OIDs_subhash = ();
-	    my $lastpartitionend = "";
-	    my $partitionstart;
-	    foreach my $metadata_value (@sortedmetadata_values) {
-		$metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value};
-		$itemsdone++;
-		my $itemsinpartition = scalar(keys %metadata_value_to_OIDs_subhash);
-
-		# Is this the start of a new partition?
-		if ($itemsinpartition == 1) {
-		    $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $self->{"partition_name_length"});
-		}
-
-		# Is this the end of the partition?
-		if ($itemsinpartition == $partition_size_within_level || $itemsdone == @sortedmetadata_values) {
-		    my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $self->{"partition_name_length"});
-		    my $partitionname = $partitionstart;
-		    if ($partitionend ne $partitionstart) {
-			$partitionname = $partitionname . "-" . $partitionend;
-		    }
-
-		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
-		    %metadata_value_to_OIDs_subhash = ();
-		    $lastpartitionend = $partitionend;
-		}
-	    }
-
-	    # The partitions are stored in an HList
-	    $classifier_node->{'childtype'} = "HList";
-	}
-
-	# Otherwise just add all the values to a VList
-	else {
-	    $self->add_vlist(\@metadata_groups, $classifier_node, \%metadata_value_to_OIDs_hash);
-	}
-    }
-}
-
-
-sub convert_utf8_string_to_unicode_string
-{
-    my $self = shift(@_);
-    my $utf8_string = shift(@_);
-
-    my $unicode_string = "";
-    foreach my $unicode_value (@{&unicode::utf82unicode($utf8_string)}) {
-	$unicode_string .= chr($unicode_value);
-    }
-    return $unicode_string;
-}
-
-
-sub convert_unicode_string_to_utf8_string
-{
-    my $self = shift(@_);
-    my $unicode_string = shift(@_);
-
-    my @unicode_array;
-    for (my $i = 0; $i < length($unicode_string); $i++) {
-	push(@unicode_array, ord(substr($unicode_string, $i, 1)));
-    }
-    return &unicode::unicode2utf8(\@unicode_array);
-}
-
-
-sub generate_partition_start
-{
-    my $self = shift(@_);
-    my $metadata_value = shift(@_);
-    my $lastpartitionend = shift(@_);
-    my $partition_name_length = shift(@_);
-
-    if ($partition_name_length) {
-	return substr($metadata_value, 0, $partition_name_length);
-    }
-
-    my $partitionstart = substr($metadata_value, 0, 1);
-    if ($partitionstart le $lastpartitionend) {
-	$partitionstart = substr($metadata_value, 0, 2);
-	# Give up after three characters
-	if ($partitionstart le $lastpartitionend) {
-	    $partitionstart = substr($metadata_value, 0, 3);
-	}
-    }
-
-    return $partitionstart;
-}
-
-
-sub generate_partition_end
-{
-    my $self = shift(@_);
-    my $metadata_value = shift(@_);
-    my $partitionstart = shift(@_);
-    my $partition_name_length = shift(@_);
-
-    if ($partition_name_length) {
-	return substr($metadata_value, 0, $partition_name_length);
-    }
-
-    my $partitionend = substr($metadata_value, 0, length($partitionstart));
-    if ($partitionend gt $partitionstart) {
-	$partitionend = substr($metadata_value, 0, 1);
-	if ($partitionend le $partitionstart) {
-	    $partitionend = substr($metadata_value, 0, 2);
-	    # Give up after three characters
-	    if ($partitionend le $partitionstart) {
-		$partitionend = substr($metadata_value, 0, 3);
-	    }
-	}
-    }
-
-    return $partitionend;
-}
-
-
-sub add_hlist_partition
-{
-    my $self = shift(@_);
-    my @metadata_groups = @{shift(@_)};
-    my $classifier_node = shift(@_);
-    my $partitionname = shift(@_);
-    my $metadata_value_to_OIDs_hash_ref = shift(@_);
-
-    # Create an hlist partition
-    my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($partitionname),
-				  'childtype' => "VList",
-				  'contains' => [] );
-
-    # Add the children to the hlist partition
-    $self->add_vlist(\@metadata_groups, \%child_classifier_node, $metadata_value_to_OIDs_hash_ref);
-    push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
-}
-
-
-sub add_vlist
-{
-    my $self = shift(@_);
-    my @metadata_groups = @{shift(@_)};
-    my $classifier_node = shift(@_);
-    my $metadata_value_to_OIDs_hash_ref = shift(@_);
-
-    my $metadata_group = shift(@metadata_groups);
-    $classifier_node->{'mdtype'} = $metadata_group;
-
-    # Create an entry in the vlist for each value
-    foreach my $metadata_value ($self->sort_metadata_values_array(keys(%{$metadata_value_to_OIDs_hash_ref})))
-    {
-	my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
-
-	# If there is only one item and 'always_bookshelf' is false, add the item to the list
-	if (@OIDs == 1 && $self->{$metadata_group . ".always_bookshelf"} eq "f") {
-	    my $OID = $OIDs[0];
-
-	    # Find the offset of this metadata value
-	    my $offset = 0;
-	    my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
-	    my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
-	    for (my $i = 0; $i < scalar(@metadata_values); $i++) {
-		if ($metadata_value eq $metadata_values[$i]) {
-		    $offset = $i;
-		    last;
-		}
-	    }
-	    push(@{$classifier_node->{'contains'}}, { 'OID' => $OID, 'offset' => $offset });
-	}
-
-	# Otherwise create a sublist (bookshelf) for the metadata value
-	else {
-	    my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($metadata_value),
-					  'childtype' => "VList",
-					  'contains' => [] );
-
-	    # If there are metadata elements remaining, recursively apply the process
-	    if (@metadata_groups > 0) {
-		my $next_metadata_group = $metadata_groups[0];
-		$child_classifier_node{'childtype'} = $self->{$next_metadata_group . ".list_type"};
-		$self->add_level(\@metadata_groups, \@OIDs, \%child_classifier_node);
-	    }
-	    # Otherwise just add the documents as children of this list
-	    else {
-		# Sort the leaf nodes by the metadata elements specified with -sort_leaf_nodes_using
-		my @sort_leaf_nodes_usingmetadata_groups = @{$self->{'sort_leaf_nodes_using_metadata_groups'}};
-		foreach my $sort_leaf_nodes_usingmetaelem (reverse @sort_leaf_nodes_usingmetadata_groups) {
-		    my $OID_to_metadata_values_hash_ref = $self->{$sort_leaf_nodes_usingmetaelem . ".list"};
-		    # Force a stable sort (Perl 5.6's sort isn't stable)
-		    # !! The [0] bits aren't ideal (multiple metadata values) !! 
-		    @OIDs = @OIDs[ sort { $OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0] cmp $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0] || $a <=> $b; } 0..$#OIDs ];
-		}
-
-  		foreach my $OID (@OIDs) {
-  		    push(@{$child_classifier_node{'contains'}}, { 'OID' => $OID });
-  		}
-	    }
-
-	    # Add the sublist to the list
-	    push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
-	}
-    }
-}
-
-
-sub sort_metadata_values_array
-{
-    my $self = shift(@_);
-    my @metadata_values = @_;
-
-    if ($self->{'unicode_collator'}) {
-	return $self->{'unicode_collator'}->sort(@metadata_values);
-    }
-    else {
-	return sort(@metadata_values);
-    }
-}
-
-
-1;
Index: gsdl/trunk/perllib/classify/List.pm
===================================================================
--- gsdl/trunk/perllib/classify/List.pm	(revision 18567)
+++ gsdl/trunk/perllib/classify/List.pm	(revision 18567)
@@ -0,0 +1,582 @@
+###########################################################################
+#
+# GenericList.pm -- A general and flexible list classifier with most of
+#                   the abilities of AZCompactList, and better Unicode,
+#                   metadata and sorting capabilities.
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ
+#
+# Copyright (C) 2005 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# TO DO: - Remove punctuation from metadata values before sorting.
+#        - Add an AZCompactList-style hlist option?
+#
+###########################################################################
+
+package GenericList;
+
+
+use BaseClassifier;
+
+use strict;
+
+
+sub BEGIN {
+    @GenericList::ISA = ('BaseClassifier');
+}
+
+
+my $arguments = 
+    [ { 'name' => "metadata",
+	'desc' => "{GenericList.metadata}",
+	'type' => "metadata",
+	'reqd' => "yes" },
+
+      # The interesting options
+      { 'name' => "always_bookshelf_last_level",
+	'desc' => "{GenericList.always_bookshelf_last_level}",
+	'type' => "flag" },
+      { 'name' => "classify_sections",
+	'desc' => "{GenericList.classify_sections}",
+	'type' => "flag" },
+      { 'name' => "partition_type_within_level",
+	'desc' => "{GenericList.partition_type_within_level}",
+	'type' => "string",
+	'deft' => "none" },
+      { 'name' => "partition_size_within_level",
+	'desc' => "{GenericList.partition_size_within_level}",
+	'type' => "string" },
+      { 'name' => "partition_name_length",
+	'desc' => "{GenericList.partition_name_length}",
+	'type' => "string" },
+      { 'name' => "sort_leaf_nodes_using",
+	'desc' => "{GenericList.sort_leaf_nodes_using}",
+	'type' => "metadata",
+	'deft' => "Title" },
+      { 'name' => "sort_using_unicode_collation",
+	'desc' => "{GenericList.sort_using_unicode_collation}",
+	'type' => "flag" },
+      { 'name' => "use_hlist_for",
+	'desc' => "{GenericList.use_hlist_for}",
+	'type' => "string" } ];
+
+my $options = { 'name'     => "GenericList",
+		'desc'     => "{GenericList.desc}",
+		'abstract' => "no",
+		'inherits' => "Yes",
+		'args'     => $arguments };
+
+
+sub new
+{
+    my ($class) = shift(@_);
+    my ($classifierslist, $inputargs, $hashArgOptLists) = @_;
+    push(@$classifierslist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); 
+    push(@{$hashArgOptLists->{"OptList"}}, $options); 
+
+    my $self = new BaseClassifier($classifierslist, $inputargs, $hashArgOptLists);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # The metadata elements to use (required)
+    if (!$self->{'metadata'}) {
+	die "Error: No metadata fields specified for GenericList.\n";
+    }
+    my @metadata_groups = split(/\//, $self->{'metadata'});
+    $self->{'metadata_groups'} = \@metadata_groups;
+
+    # The classifier button name (default: the first metadata element specified)
+    if (!$self->{'buttonname'}) {
+	my $first_metadata_group = $metadata_groups[0];
+	my $first_metadata_element = (split(/\;/, $first_metadata_group))[0];
+	$self->{'buttonname'} = $self->generate_title_from_metadata($first_metadata_element);
+    }
+
+    # Whether to group single items into a bookshelf (must be true for all metadata fields except the last)
+    foreach my $metadata_group (@metadata_groups) {
+	$self->{$metadata_group . ".always_bookshelf"} = "t";
+    }
+    if (!$self->{'always_bookshelf_last_level'}) {
+	# Default: leave leaf nodes ungrouped (equivalent to AZCompactList -mingroup 2)
+	my $last_metadata_group = $metadata_groups[$#metadata_groups];
+	$self->{$last_metadata_group . ".always_bookshelf"} = "f";
+    }
+
+    # Whether to use an hlist or a vlist for each level in the hierarchy (default: vlist)
+    foreach my $metadata_group (@metadata_groups) {
+	$self->{$metadata_group . ".list_type"} = "VList";
+    }
+    foreach my $metadata_group (split(/\,/, $self->{'use_hlist_for'})) {
+	$self->{$metadata_group . ".list_type"} = "HList";
+    }
+
+    # How the items are grouped into partitions (default: no partition)
+    if (!$self->{"partition_type_within_level"}) {
+	$self->{"partition_type_within_level"} = "none";
+    }
+
+    # The number of items in each partition
+    if (!$self->{'partition_size_within_level'}) {
+	# Default: 20
+	foreach my $metadata_group (@metadata_groups) {
+	    $self->{$metadata_group . ".partition_size_within_level"} = 20;
+	}
+    }
+    else {
+	my @partition_size_within_levellist = split(/\//, $self->{'partition_size_within_level'});
+
+	# Assign values based on the partition_size_within_level parameter
+	foreach my $metadata_group (@metadata_groups) {
+	    my $partition_size_within_levelelem = shift(@partition_size_within_levellist);
+	    if (defined($partition_size_within_levelelem)) {
+		$self->{$metadata_group . ".partition_size_within_level"} = $partition_size_within_levelelem;
+	    }
+	    else {
+		$self->{$metadata_group . ".partition_size_within_level"} = $self->{$metadata_groups[0] . ".partition_size_within_level"};
+	    }
+	}
+    }
+
+    # The metadata elements to use to sort the leaf nodes (default: Title)
+    my @sort_leaf_nodes_using_metadata_groups = ( "Title" );
+    if ($self->{'sort_leaf_nodes_using'}) {
+	@sort_leaf_nodes_using_metadata_groups = split(/\|/, $self->{'sort_leaf_nodes_using'});
+    }
+    $self->{'sort_leaf_nodes_using_metadata_groups'} = \@sort_leaf_nodes_using_metadata_groups;
+
+    # Create an instance of the Unicode::Collate object if better Unicode sorting is desired
+    if ($self->{'sort_using_unicode_collation'}) {
+	# To use this you first need to download the allkeys.txt file from 
+        # http://www.unicode.org/Public/UCA/latest/allkeys.txt and put it in the Perl
+        # Unicode/Collate directory.
+	require Unicode::Collate;
+	$self->{'unicode_collator'} = Unicode::Collate->new();
+    }
+
+    # An empty array for the document OIDs
+    $self->{'OIDs'} = [];
+
+    return bless $self, $class;
+}
+
+
+sub init
+{
+    # Nothing to do...
+}
+
+
+# Called for each document in the collection
+sub classify
+{
+    my $self = shift(@_);
+    my ($doc_obj,$edit_mode) = @_;
+
+    # If "-classify_sections" is set, classify every section of the document
+    if ($self->{'classify_sections'}) {
+	my $section = $doc_obj->get_next_section($doc_obj->get_top_section());
+	while (defined $section) {
+	    $self->classify_section($doc_obj, $doc_obj->get_OID() . ".$section", $section, $edit_mode);
+	    $section = $doc_obj->get_next_section($section);
+	}
+    }
+    # Otherwise just classify the top document section
+    else {
+	$self->classify_section($doc_obj, $doc_obj->get_OID(), $doc_obj->get_top_section(), $edit_mode);
+    }
+}
+
+
+sub classify_section
+{
+    my $self = shift(@_);
+    my ($doc_obj,$section_OID,$section,$edit_mode) = @_;
+
+    my @metadata_groups = @{$self->{'metadata_groups'}};
+
+    # Only classify the section if it has a value for one of the metadata elements in the first group
+    my $classify_section = 0;
+    my $first_metadata_group = $metadata_groups[0];
+    foreach my $first_metadata_group_element (split(/\;/, $first_metadata_group)) {
+	my $first_metadata_group_element_value = $doc_obj->get_metadata_element($section, $first_metadata_group_element);
+	if (defined($first_metadata_group_element_value) && $first_metadata_group_element_value ne "") {
+	    # This section must be included in the classifier
+	    $classify_section = 1;
+	    last;
+	}
+    }
+
+    # We're not classifying this section because it doesn't have the required metadata
+    return if (!$classify_section);
+
+
+    if (($edit_mode eq "delete") || ($edit_mode eq "update")) {
+	$self->oid_array_delete($section_OID,'OIDs');
+	if ($edit_mode eq "delete") {
+	    return;
+	}
+    }
+   
+    # Otherwise, include this section in the classifier
+    push(@{$self->{'OIDs'}}, $section_OID);
+
+    # Create a hash for the metadata values of each metadata element we're interested in
+    my %metadata_groups_done = ();
+    foreach my $metadata_group (@metadata_groups, @{$self->{'sort_leaf_nodes_using_metadata_groups'}}) {
+	# Take care not to do a metadata group more than once
+	unless ($metadata_groups_done{$metadata_group}) {
+	    foreach my $metadata_element (split(/\;/, $metadata_group)) {
+		my @metadata_values = @{$doc_obj->get_metadata($section, $metadata_element)};
+		foreach my $metadata_value (@metadata_values) {
+		    # Strip leading and trailing whitespace
+		    $metadata_value =~ s/^\s*//;
+		    $metadata_value =~ s/\s*$//;
+
+		    # Convert the metadata value from a UTF-8 string to a Unicode string
+		    # This means that length() and substr() work properly
+		    # We need to be careful to convert classifier node title values back to UTF-8, however
+		    my $metadata_value_unicode_string = $self->convert_utf8_string_to_unicode_string($metadata_value);
+
+		    # Add the metadata value into the list for this combination of metadata group and section
+		    push(@{$self->{$metadata_group . ".list"}->{$section_OID}}, $metadata_value_unicode_string);
+		}
+		last if (@metadata_values > 0);
+	    }
+
+	    $metadata_groups_done{$metadata_group} = 1;
+	}
+    }
+}
+
+
+sub get_classify_info
+{
+    my $self = shift(@_);
+
+    # The metadata groups to classify by
+    my @metadata_groups = @{$self->{'metadata_groups'}};
+    my $first_metadata_group = $metadata_groups[0];
+
+    # The OID values of the documents to include in the classifier
+    my @OIDs = @{$self->{'OIDs'}};
+
+    # Create the root node of the classification hierarchy
+    my %classifier_node = ( 'thistype' => "Invisible",
+			    'childtype' => $self->{$first_metadata_group . ".list_type"},
+			    'Title' => $self->{'buttonname'},
+			    'contains' => [],
+			    'mdtype' => $first_metadata_group );
+
+    # Recursively create the classification hierarchy, one level for each metadata group
+    $self->add_level(\@metadata_groups, \@OIDs, \%classifier_node);
+    return \%classifier_node;
+}
+
+
+sub add_level
+{
+    my $self = shift(@_);
+    my @metadata_groups = @{shift(@_)};
+    my @OIDs = @{shift(@_)};
+    my $classifier_node = shift(@_);
+    # print STDERR "\nAdding AZ list for " . $classifier_node->{'Title'} . "\n";
+
+    my $metadata_group = $metadata_groups[0];
+    # print STDERR "Processing metadata group: " . $metadata_group . "\n";
+    # print STDERR "Number of OID values: " . @OIDs . "\n";
+
+    if (!defined($self->{$metadata_group . ".list"})) {
+	print STDERR "Warning: No metadata values assigned to $metadata_group.\n";
+	return;
+    }
+
+    # Create a mapping from metadata value to OID
+    my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
+    my %metadata_value_to_OIDs_hash = ();
+    foreach my $OID (@OIDs)
+    {
+	if ($OID_to_metadata_values_hash_ref->{$OID})
+	{
+	    my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
+	    foreach my $metadata_value (@metadata_values)
+	    {
+		push(@{$metadata_value_to_OIDs_hash{$metadata_value}}, $OID);
+	    }
+	}
+    }
+    # print STDERR "Number of distinct values: " . scalar(keys %metadata_value_to_OIDs_hash) . "\n";
+
+    # Partition the values (if necessary)
+    my $partition_type_within_level = $self->{"partition_type_within_level"};
+    if ($partition_type_within_level =~ /^per_letter$/i) {
+	# Generate one hlist for each letter
+	my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash));
+	my %metadata_value_to_OIDs_subhash = ();
+
+	my $lastpartition = substr($sortedmetadata_values[0], 0, 1);
+	foreach my $metadata_value (@sortedmetadata_values) {
+	    my $metadata_valuepartition = substr($metadata_value, 0, 1);
+
+	    # Is this the start of a new partition?
+	    if ($metadata_valuepartition ne $lastpartition) {
+		$self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
+		%metadata_value_to_OIDs_subhash = ();
+		$lastpartition = $metadata_valuepartition;
+	    }
+
+	    $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value};	    
+	}
+
+	# Don't forget to add the last partition
+	$self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
+
+	# The partitions are stored in an HList
+	$classifier_node->{'childtype'} = "HList";
+    }
+
+    else {
+	# Generate hlists of a certain size
+	my $partition_size_within_level = $self->{$metadata_group . ".partition_size_within_level"};
+	if ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
+	    my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash));
+	    my $itemsdone = 0;
+	    my %metadata_value_to_OIDs_subhash = ();
+	    my $lastpartitionend = "";
+	    my $partitionstart;
+	    foreach my $metadata_value (@sortedmetadata_values) {
+		$metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value};
+		$itemsdone++;
+		my $itemsinpartition = scalar(keys %metadata_value_to_OIDs_subhash);
+
+		# Is this the start of a new partition?
+		if ($itemsinpartition == 1) {
+		    $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $self->{"partition_name_length"});
+		}
+
+		# Is this the end of the partition?
+		if ($itemsinpartition == $partition_size_within_level || $itemsdone == @sortedmetadata_values) {
+		    my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $self->{"partition_name_length"});
+		    my $partitionname = $partitionstart;
+		    if ($partitionend ne $partitionstart) {
+			$partitionname = $partitionname . "-" . $partitionend;
+		    }
+
+		    $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
+		    %metadata_value_to_OIDs_subhash = ();
+		    $lastpartitionend = $partitionend;
+		}
+	    }
+
+	    # The partitions are stored in an HList
+	    $classifier_node->{'childtype'} = "HList";
+	}
+
+	# Otherwise just add all the values to a VList
+	else {
+	    $self->add_vlist(\@metadata_groups, $classifier_node, \%metadata_value_to_OIDs_hash);
+	}
+    }
+}
+
+
+sub convert_utf8_string_to_unicode_string
+{
+    my $self = shift(@_);
+    my $utf8_string = shift(@_);
+
+    my $unicode_string = "";
+    foreach my $unicode_value (@{&unicode::utf82unicode($utf8_string)}) {
+	$unicode_string .= chr($unicode_value);
+    }
+    return $unicode_string;
+}
+
+
+sub convert_unicode_string_to_utf8_string
+{
+    my $self = shift(@_);
+    my $unicode_string = shift(@_);
+
+    my @unicode_array;
+    for (my $i = 0; $i < length($unicode_string); $i++) {
+	push(@unicode_array, ord(substr($unicode_string, $i, 1)));
+    }
+    return &unicode::unicode2utf8(\@unicode_array);
+}
+
+
+sub generate_partition_start
+{
+    my $self = shift(@_);
+    my $metadata_value = shift(@_);
+    my $lastpartitionend = shift(@_);
+    my $partition_name_length = shift(@_);
+
+    if ($partition_name_length) {
+	return substr($metadata_value, 0, $partition_name_length);
+    }
+
+    my $partitionstart = substr($metadata_value, 0, 1);
+    if ($partitionstart le $lastpartitionend) {
+	$partitionstart = substr($metadata_value, 0, 2);
+	# Give up after three characters
+	if ($partitionstart le $lastpartitionend) {
+	    $partitionstart = substr($metadata_value, 0, 3);
+	}
+    }
+
+    return $partitionstart;
+}
+
+
+sub generate_partition_end
+{
+    my $self = shift(@_);
+    my $metadata_value = shift(@_);
+    my $partitionstart = shift(@_);
+    my $partition_name_length = shift(@_);
+
+    if ($partition_name_length) {
+	return substr($metadata_value, 0, $partition_name_length);
+    }
+
+    my $partitionend = substr($metadata_value, 0, length($partitionstart));
+    if ($partitionend gt $partitionstart) {
+	$partitionend = substr($metadata_value, 0, 1);
+	if ($partitionend le $partitionstart) {
+	    $partitionend = substr($metadata_value, 0, 2);
+	    # Give up after three characters
+	    if ($partitionend le $partitionstart) {
+		$partitionend = substr($metadata_value, 0, 3);
+	    }
+	}
+    }
+
+    return $partitionend;
+}
+
+
+sub add_hlist_partition
+{
+    my $self = shift(@_);
+    my @metadata_groups = @{shift(@_)};
+    my $classifier_node = shift(@_);
+    my $partitionname = shift(@_);
+    my $metadata_value_to_OIDs_hash_ref = shift(@_);
+
+    # Create an hlist partition
+    my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($partitionname),
+				  'childtype' => "VList",
+				  'contains' => [] );
+
+    # Add the children to the hlist partition
+    $self->add_vlist(\@metadata_groups, \%child_classifier_node, $metadata_value_to_OIDs_hash_ref);
+    push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
+}
+
+
+sub add_vlist
+{
+    my $self = shift(@_);
+    my @metadata_groups = @{shift(@_)};
+    my $classifier_node = shift(@_);
+    my $metadata_value_to_OIDs_hash_ref = shift(@_);
+
+    my $metadata_group = shift(@metadata_groups);
+    $classifier_node->{'mdtype'} = $metadata_group;
+
+    # Create an entry in the vlist for each value
+    foreach my $metadata_value ($self->sort_metadata_values_array(keys(%{$metadata_value_to_OIDs_hash_ref})))
+    {
+	my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
+
+	# If there is only one item and 'always_bookshelf' is false, add the item to the list
+	if (@OIDs == 1 && $self->{$metadata_group . ".always_bookshelf"} eq "f") {
+	    my $OID = $OIDs[0];
+
+	    # Find the offset of this metadata value
+	    my $offset = 0;
+	    my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
+	    my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
+	    for (my $i = 0; $i < scalar(@metadata_values); $i++) {
+		if ($metadata_value eq $metadata_values[$i]) {
+		    $offset = $i;
+		    last;
+		}
+	    }
+	    push(@{$classifier_node->{'contains'}}, { 'OID' => $OID, 'offset' => $offset });
+	}
+
+	# Otherwise create a sublist (bookshelf) for the metadata value
+	else {
+	    my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($metadata_value),
+					  'childtype' => "VList",
+					  'contains' => [] );
+
+	    # If there are metadata elements remaining, recursively apply the process
+	    if (@metadata_groups > 0) {
+		my $next_metadata_group = $metadata_groups[0];
+		$child_classifier_node{'childtype'} = $self->{$next_metadata_group . ".list_type"};
+		$self->add_level(\@metadata_groups, \@OIDs, \%child_classifier_node);
+	    }
+	    # Otherwise just add the documents as children of this list
+	    else {
+		# Sort the leaf nodes by the metadata elements specified with -sort_leaf_nodes_using
+		my @sort_leaf_nodes_usingmetadata_groups = @{$self->{'sort_leaf_nodes_using_metadata_groups'}};
+		foreach my $sort_leaf_nodes_usingmetaelem (reverse @sort_leaf_nodes_usingmetadata_groups) {
+		    my $OID_to_metadata_values_hash_ref = $self->{$sort_leaf_nodes_usingmetaelem . ".list"};
+		    # Force a stable sort (Perl 5.6's sort isn't stable)
+		    # !! The [0] bits aren't ideal (multiple metadata values) !! 
+		    @OIDs = @OIDs[ sort { $OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0] cmp $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0] || $a <=> $b; } 0..$#OIDs ];
+		}
+
+  		foreach my $OID (@OIDs) {
+  		    push(@{$child_classifier_node{'contains'}}, { 'OID' => $OID });
+  		}
+	    }
+
+	    # Add the sublist to the list
+	    push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
+	}
+    }
+}
+
+
+sub sort_metadata_values_array
+{
+    my $self = shift(@_);
+    my @metadata_values = @_;
+
+    if ($self->{'unicode_collator'}) {
+	return $self->{'unicode_collator'}->sort(@metadata_values);
+    }
+    else {
+	return sort(@metadata_values);
+    }
+}
+
+
+1;
