########################################################################### # # List.pm -- A general and flexible list classifier with most of # the abilities of AZCompactList, and better Unicode, # metadata and sorting capabilities. # # A component of the Greenstone digital library software # from the New Zealand Digital Library Project at the # University of Waikato, New Zealand. # # Author: Michael Dewsnip, NZDL Project, University of Waikato, NZ # # Copyright (C) 2005 New Zealand Digital Library Project # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # TO DO: - Remove punctuation from metadata values before sorting. # - Add an AZCompactList-style hlist option? # ########################################################################### package List; use BaseClassifier; use strict; sub BEGIN { @List::ISA = ('BaseClassifier'); } my $arguments = [ { 'name' => "metadata", 'desc' => "{GenericList.metadata}", 'type' => "metadata", 'reqd' => "yes" }, # The interesting options { 'name' => "always_bookshelf_last_level", 'desc' => "{GenericList.always_bookshelf_last_level}", 'type' => "flag" }, { 'name' => "classify_sections", 'desc' => "{GenericList.classify_sections}", 'type' => "flag" }, { 'name' => "partition_type_within_level", 'desc' => "{GenericList.partition_type_within_level}", 'type' => "string", 'deft' => "none" }, { 'name' => "partition_size_within_level", 'desc' => "{GenericList.partition_size_within_level}", 'type' => "string" }, { 'name' => "partition_name_length", 'desc' => "{GenericList.partition_name_length}", 'type' => "string" }, { 'name' => "sort_leaf_nodes_using", 'desc' => "{GenericList.sort_leaf_nodes_using}", 'type' => "metadata", 'deft' => "Title" }, { 'name' => "sort_using_unicode_collation", 'desc' => "{GenericList.sort_using_unicode_collation}", 'type' => "flag" }, { 'name' => "use_hlist_for", 'desc' => "{GenericList.use_hlist_for}", 'type' => "string" } ]; my $options = { 'name' => "List", 'desc' => "{GenericList.desc}", 'abstract' => "no", 'inherits' => "Yes", 'args' => $arguments }; sub new { my ($class) = shift(@_); my ($classifierslist, $inputargs, $hashArgOptLists) = @_; push(@$classifierslist, $class); push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); push(@{$hashArgOptLists->{"OptList"}}, $options); my $self = new BaseClassifier($classifierslist, $inputargs, $hashArgOptLists); if ($self->{'info_only'}) { # don't worry about any options etc return bless $self, $class; } # The metadata elements to use (required) if (!$self->{'metadata'}) { die "Error: No metadata fields specified for List.\n"; } my @metadata_groups = split(/\//, $self->{'metadata'}); $self->{'metadata_groups'} = \@metadata_groups; # The classifier button name (default: the first metadata element specified) if (!$self->{'buttonname'}) { my $first_metadata_group = $metadata_groups[0]; my $first_metadata_element = (split(/\;/, $first_metadata_group))[0]; $self->{'buttonname'} = $self->generate_title_from_metadata($first_metadata_element); } # Whether to group single items into a bookshelf (must be true for all metadata fields except the last) foreach my $metadata_group (@metadata_groups) { $self->{$metadata_group . ".always_bookshelf"} = "t"; } if (!$self->{'always_bookshelf_last_level'}) { # Default: leave leaf nodes ungrouped (equivalent to AZCompactList -mingroup 2) my $last_metadata_group = $metadata_groups[$#metadata_groups]; $self->{$last_metadata_group . ".always_bookshelf"} = "f"; } # Whether to use an hlist or a vlist for each level in the hierarchy (default: vlist) foreach my $metadata_group (@metadata_groups) { $self->{$metadata_group . ".list_type"} = "VList"; } foreach my $metadata_group (split(/\,/, $self->{'use_hlist_for'})) { $self->{$metadata_group . ".list_type"} = "HList"; } # How the items are grouped into partitions (default: no partition) if (!$self->{"partition_type_within_level"}) { $self->{"partition_type_within_level"} = "none"; } # The number of items in each partition if (!$self->{'partition_size_within_level'}) { # Default: 20 foreach my $metadata_group (@metadata_groups) { $self->{$metadata_group . ".partition_size_within_level"} = 20; } } else { my @partition_size_within_levellist = split(/\//, $self->{'partition_size_within_level'}); # Assign values based on the partition_size_within_level parameter foreach my $metadata_group (@metadata_groups) { my $partition_size_within_levelelem = shift(@partition_size_within_levellist); if (defined($partition_size_within_levelelem)) { $self->{$metadata_group . ".partition_size_within_level"} = $partition_size_within_levelelem; } else { $self->{$metadata_group . ".partition_size_within_level"} = $self->{$metadata_groups[0] . ".partition_size_within_level"}; } } } # The metadata elements to use to sort the leaf nodes (default: Title) my @sort_leaf_nodes_using_metadata_groups = ( "Title" ); if ($self->{'sort_leaf_nodes_using'}) { @sort_leaf_nodes_using_metadata_groups = split(/\|/, $self->{'sort_leaf_nodes_using'}); } $self->{'sort_leaf_nodes_using_metadata_groups'} = \@sort_leaf_nodes_using_metadata_groups; # Create an instance of the Unicode::Collate object if better Unicode sorting is desired if ($self->{'sort_using_unicode_collation'}) { # To use this you first need to download the allkeys.txt file from # http://www.unicode.org/Public/UCA/latest/allkeys.txt and put it in the Perl # Unicode/Collate directory. require Unicode::Collate; $self->{'unicode_collator'} = Unicode::Collate->new(); } # An empty array for the document OIDs $self->{'OIDs'} = []; return bless $self, $class; } sub init { # Nothing to do... } # Called for each document in the collection sub classify { my $self = shift(@_); my ($doc_obj,$edit_mode) = @_; # If "-classify_sections" is set, classify every section of the document if ($self->{'classify_sections'}) { my $section = $doc_obj->get_next_section($doc_obj->get_top_section()); while (defined $section) { $self->classify_section($doc_obj, $doc_obj->get_OID() . ".$section", $section, $edit_mode); $section = $doc_obj->get_next_section($section); } } # Otherwise just classify the top document section else { $self->classify_section($doc_obj, $doc_obj->get_OID(), $doc_obj->get_top_section(), $edit_mode); } } sub classify_section { my $self = shift(@_); my ($doc_obj,$section_OID,$section,$edit_mode) = @_; my @metadata_groups = @{$self->{'metadata_groups'}}; # Only classify the section if it has a value for one of the metadata elements in the first group my $classify_section = 0; my $first_metadata_group = $metadata_groups[0]; foreach my $first_metadata_group_element (split(/\;/, $first_metadata_group)) { my $first_metadata_group_element_value = $doc_obj->get_metadata_element($section, $first_metadata_group_element); if (defined($first_metadata_group_element_value) && $first_metadata_group_element_value ne "") { # This section must be included in the classifier $classify_section = 1; last; } } # We're not classifying this section because it doesn't have the required metadata return if (!$classify_section); if (($edit_mode eq "delete") || ($edit_mode eq "update")) { $self->oid_array_delete($section_OID,'OIDs'); if ($edit_mode eq "delete") { return; } } # Otherwise, include this section in the classifier push(@{$self->{'OIDs'}}, $section_OID); # Create a hash for the metadata values of each metadata element we're interested in my %metadata_groups_done = (); foreach my $metadata_group (@metadata_groups, @{$self->{'sort_leaf_nodes_using_metadata_groups'}}) { # Take care not to do a metadata group more than once unless ($metadata_groups_done{$metadata_group}) { foreach my $metadata_element (split(/\;/, $metadata_group)) { my @metadata_values = @{$doc_obj->get_metadata($section, $metadata_element)}; foreach my $metadata_value (@metadata_values) { # Strip leading and trailing whitespace $metadata_value =~ s/^\s*//; $metadata_value =~ s/\s*$//; # Convert the metadata value from a UTF-8 string to a Unicode string # This means that length() and substr() work properly # We need to be careful to convert classifier node title values back to UTF-8, however my $metadata_value_unicode_string = $self->convert_utf8_string_to_unicode_string($metadata_value); # Add the metadata value into the list for this combination of metadata group and section push(@{$self->{$metadata_group . ".list"}->{$section_OID}}, $metadata_value_unicode_string); } last if (@metadata_values > 0); } $metadata_groups_done{$metadata_group} = 1; } } } sub get_classify_info { my $self = shift(@_); # The metadata groups to classify by my @metadata_groups = @{$self->{'metadata_groups'}}; my $first_metadata_group = $metadata_groups[0]; # The OID values of the documents to include in the classifier my @OIDs = @{$self->{'OIDs'}}; # Create the root node of the classification hierarchy my %classifier_node = ( 'thistype' => "Invisible", 'childtype' => $self->{$first_metadata_group . ".list_type"}, 'Title' => $self->{'buttonname'}, 'contains' => [], 'mdtype' => $first_metadata_group ); # Recursively create the classification hierarchy, one level for each metadata group $self->add_level(\@metadata_groups, \@OIDs, \%classifier_node); return \%classifier_node; } sub add_level { my $self = shift(@_); my @metadata_groups = @{shift(@_)}; my @OIDs = @{shift(@_)}; my $classifier_node = shift(@_); # print STDERR "\nAdding AZ list for " . $classifier_node->{'Title'} . "\n"; my $metadata_group = $metadata_groups[0]; # print STDERR "Processing metadata group: " . $metadata_group . "\n"; # print STDERR "Number of OID values: " . @OIDs . "\n"; if (!defined($self->{$metadata_group . ".list"})) { print STDERR "Warning: No metadata values assigned to $metadata_group.\n"; return; } # Create a mapping from metadata value to OID my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"}; my %metadata_value_to_OIDs_hash = (); foreach my $OID (@OIDs) { if ($OID_to_metadata_values_hash_ref->{$OID}) { my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}}; foreach my $metadata_value (@metadata_values) { push(@{$metadata_value_to_OIDs_hash{$metadata_value}}, $OID); } } } # print STDERR "Number of distinct values: " . scalar(keys %metadata_value_to_OIDs_hash) . "\n"; # Partition the values (if necessary) my $partition_type_within_level = $self->{"partition_type_within_level"}; if ($partition_type_within_level =~ /^per_letter$/i) { # Generate one hlist for each letter my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash)); my %metadata_value_to_OIDs_subhash = (); my $lastpartition = substr($sortedmetadata_values[0], 0, 1); foreach my $metadata_value (@sortedmetadata_values) { my $metadata_valuepartition = substr($metadata_value, 0, 1); # Is this the start of a new partition? if ($metadata_valuepartition ne $lastpartition) { $self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash); %metadata_value_to_OIDs_subhash = (); $lastpartition = $metadata_valuepartition; } $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value}; } # Don't forget to add the last partition $self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash); # The partitions are stored in an HList $classifier_node->{'childtype'} = "HList"; } else { # Generate hlists of a certain size my $partition_size_within_level = $self->{$metadata_group . ".partition_size_within_level"}; if ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) { my @sortedmetadata_values = $self->sort_metadata_values_array(keys(%metadata_value_to_OIDs_hash)); my $itemsdone = 0; my %metadata_value_to_OIDs_subhash = (); my $lastpartitionend = ""; my $partitionstart; foreach my $metadata_value (@sortedmetadata_values) { $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash{$metadata_value}; $itemsdone++; my $itemsinpartition = scalar(keys %metadata_value_to_OIDs_subhash); # Is this the start of a new partition? if ($itemsinpartition == 1) { $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $self->{"partition_name_length"}); } # Is this the end of the partition? if ($itemsinpartition == $partition_size_within_level || $itemsdone == @sortedmetadata_values) { my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $self->{"partition_name_length"}); my $partitionname = $partitionstart; if ($partitionend ne $partitionstart) { $partitionname = $partitionname . "-" . $partitionend; } $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash); %metadata_value_to_OIDs_subhash = (); $lastpartitionend = $partitionend; } } # The partitions are stored in an HList $classifier_node->{'childtype'} = "HList"; } # Otherwise just add all the values to a VList else { $self->add_vlist(\@metadata_groups, $classifier_node, \%metadata_value_to_OIDs_hash); } } } sub convert_utf8_string_to_unicode_string { my $self = shift(@_); my $utf8_string = shift(@_); my $unicode_string = ""; foreach my $unicode_value (@{&unicode::utf82unicode($utf8_string)}) { $unicode_string .= chr($unicode_value); } return $unicode_string; } sub convert_unicode_string_to_utf8_string { my $self = shift(@_); my $unicode_string = shift(@_); my @unicode_array; for (my $i = 0; $i < length($unicode_string); $i++) { push(@unicode_array, ord(substr($unicode_string, $i, 1))); } return &unicode::unicode2utf8(\@unicode_array); } sub generate_partition_start { my $self = shift(@_); my $metadata_value = shift(@_); my $lastpartitionend = shift(@_); my $partition_name_length = shift(@_); if ($partition_name_length) { return substr($metadata_value, 0, $partition_name_length); } my $partitionstart = substr($metadata_value, 0, 1); if ($partitionstart le $lastpartitionend) { $partitionstart = substr($metadata_value, 0, 2); # Give up after three characters if ($partitionstart le $lastpartitionend) { $partitionstart = substr($metadata_value, 0, 3); } } return $partitionstart; } sub generate_partition_end { my $self = shift(@_); my $metadata_value = shift(@_); my $partitionstart = shift(@_); my $partition_name_length = shift(@_); if ($partition_name_length) { return substr($metadata_value, 0, $partition_name_length); } my $partitionend = substr($metadata_value, 0, length($partitionstart)); if ($partitionend gt $partitionstart) { $partitionend = substr($metadata_value, 0, 1); if ($partitionend le $partitionstart) { $partitionend = substr($metadata_value, 0, 2); # Give up after three characters if ($partitionend le $partitionstart) { $partitionend = substr($metadata_value, 0, 3); } } } return $partitionend; } sub add_hlist_partition { my $self = shift(@_); my @metadata_groups = @{shift(@_)}; my $classifier_node = shift(@_); my $partitionname = shift(@_); my $metadata_value_to_OIDs_hash_ref = shift(@_); # Create an hlist partition my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($partitionname), 'childtype' => "VList", 'contains' => [] ); # Add the children to the hlist partition $self->add_vlist(\@metadata_groups, \%child_classifier_node, $metadata_value_to_OIDs_hash_ref); push(@{$classifier_node->{'contains'}}, \%child_classifier_node); } sub add_vlist { my $self = shift(@_); my @metadata_groups = @{shift(@_)}; my $classifier_node = shift(@_); my $metadata_value_to_OIDs_hash_ref = shift(@_); my $metadata_group = shift(@metadata_groups); $classifier_node->{'mdtype'} = $metadata_group; # Create an entry in the vlist for each value foreach my $metadata_value ($self->sort_metadata_values_array(keys(%{$metadata_value_to_OIDs_hash_ref}))) { my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}; # If there is only one item and 'always_bookshelf' is false, add the item to the list if (@OIDs == 1 && $self->{$metadata_group . ".always_bookshelf"} eq "f") { my $OID = $OIDs[0]; # Find the offset of this metadata value my $offset = 0; my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"}; my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}}; for (my $i = 0; $i < scalar(@metadata_values); $i++) { if ($metadata_value eq $metadata_values[$i]) { $offset = $i; last; } } push(@{$classifier_node->{'contains'}}, { 'OID' => $OID, 'offset' => $offset }); } # Otherwise create a sublist (bookshelf) for the metadata value else { my %child_classifier_node = ( 'Title' => $self->convert_unicode_string_to_utf8_string($metadata_value), 'childtype' => "VList", 'contains' => [] ); # If there are metadata elements remaining, recursively apply the process if (@metadata_groups > 0) { my $next_metadata_group = $metadata_groups[0]; $child_classifier_node{'childtype'} = $self->{$next_metadata_group . ".list_type"}; $self->add_level(\@metadata_groups, \@OIDs, \%child_classifier_node); } # Otherwise just add the documents as children of this list else { # Sort the leaf nodes by the metadata elements specified with -sort_leaf_nodes_using my @sort_leaf_nodes_usingmetadata_groups = @{$self->{'sort_leaf_nodes_using_metadata_groups'}}; foreach my $sort_leaf_nodes_usingmetaelem (reverse @sort_leaf_nodes_usingmetadata_groups) { my $OID_to_metadata_values_hash_ref = $self->{$sort_leaf_nodes_usingmetaelem . ".list"}; # Force a stable sort (Perl 5.6's sort isn't stable) # !! The [0] bits aren't ideal (multiple metadata values) !! @OIDs = @OIDs[ sort { $OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0] cmp $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0] || $a <=> $b; } 0..$#OIDs ]; } foreach my $OID (@OIDs) { push(@{$child_classifier_node{'contains'}}, { 'OID' => $OID }); } } # Add the sublist to the list push(@{$classifier_node->{'contains'}}, \%child_classifier_node); } } } sub sort_metadata_values_array { my $self = shift(@_); my @metadata_values = @_; if ($self->{'unicode_collator'}) { return $self->{'unicode_collator'}->sort(@metadata_values); } else { return sort(@metadata_values); } } 1;