source: main/trunk/greenstone2/perllib/classify/List.pm@ 36911

Last change on this file since 36911 was 36911, checked in by kjdon, 17 months ago

some changes I made for heritgae NZ. added a tolerance to merging/splitting buckets. added max_partition_name_length option - if a constant partition_name_length is not specified, then it used to try up to 3 characters to get a unique value - now it tries up to max_partition_name_length characters - you may only want 2 characters, not 3. added remove prefix/suffix options for sort leaf node values
  • Property svn:keywords set to Author Date Id Revision
File size: 67.6 KB
Line 
1###########################################################################
2#
3# List.pm -- A general and flexible list classifier with most of
4# the abilities of AZCompactList, and better Unicode,
5# metadata and sorting capabilities.
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11#
12# Copyright (C) 2005 New Zealand Digital Library Project
13#
14# This program is free software; you can redistribute it and/or modify
15# it under the terms of the GNU General Public License as published by
16# the Free Software Foundation; either version 2 of the License, or
17# (at your option) any later version.
18#
19# This program is distributed in the hope that it will be useful,
20# but WITHOUT ANY WARRANTY; without even the implied warranty of
21# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22# GNU General Public License for more details.
23#
24# You should have received a copy of the GNU General Public License
25# along with this program; if not, write to the Free Software
26# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27#
28# TO DO: - Remove punctuation from metadata values before sorting.
29# - Add an AZCompactList-style hlist option?
30#
31###########################################################################
32
33
34package List;
35
36
37use BaseClassifier;
38use Sort::Naturally;
39use strict;
40
41
42sub BEGIN {
43 @List::ISA = ('BaseClassifier');
44}
45
46
47my $metadata_selection_mode_list =
48 [
49 { 'name' => "firstvalue",
50 'desc' => "{List.metadata_selection.firstvalue}"},
51 { 'name' => "firstvalidmetadata",
52 'desc' => "{List.metadata_selection.firstvalidmetadata}"},
53 { 'name' => "allvalues",
54 'desc' => "{List.metadata_selection.allvalues}"} ];
55
56my $metadata_selection_mode_default = "firstvalidmetadata";
57
58my $valid_metadata_selection_modes = { 'firstvalue' => 1,
59 'firstvalidmetadata' => 1,
60 'allvalues' => 1 };
61
62my $metadata_sort_mode_list =
63 [
64 { 'name' => "unicode",
65 'desc' => "{List.metadata_sort.unicode}"},
66 { 'name' => "alphabetic",
67 'desc' => "{List.metadata_sort.alphabetic}"},
68 { 'name' => "alphanumeric",
69 'desc' => "{List.metadata_sort.alphanumeric}"} ];
70
71my $metadata_sort_mode_default = "alphanumeric";
72
73my $valid_metadata_sort_modes = { 'unicode' => 1,
74 'alphabetic' => 1,
75 'alphanumeric' => 1};
76my $partition_type_list =
77 [ { 'name' => "per_letter",
78 'desc' => "{List.level_partition.per_letter}" },
79 { 'name' => "approximate_size",
80 'desc' => "{List.level_partition.approximate_size}"},
81 { 'name' => "constant_size",
82 'desc' => "{List.level_partition.constant_size}" },
83 { 'name' => "all_values",
84 'desc' => "{List.level_partition.all_values}" },
85 { 'name' => "none",
86 'desc' => "{List.level_partition.none}" } ];
87
88my $partition_type_default = "per_letter";
89
90my $valid_partition_types = { 'per_letter' => 1,
91 'constant_size' => 1,
92 'approximate_size' => 1,
93 'all_values' => 1,
94 'none' => 1};
95
96my $partition_size_default = 20;
97
98my $numeric_partition_type_list =
99 [ { 'name' => "per_digit",
100 'desc' => "{List.level_partition.per_digit}" },
101 { 'name' => "per_number",
102 'desc' => "{List.level_partition.per_number}" },
103 { 'name' => "single_partition",
104 'desc' => "{List.level_partition.single}" },
105 { 'name' => "approximate_size",
106 'desc' => "{List.level_partition.approximate_size_numeric}"},
107 { 'name' => "constant_size",
108 'desc' => "{List.level_partition.constant_size}" },
109 { 'name' => "all_values",
110 'desc' => "{List.level_partition.all_values}" },
111 { 'name' => "none",
112 'desc' => "{List.level_partition.none}" } ];
113
114my $numeric_partition_type_default = "single_partition";
115
116my $valid_numeric_partition_types = { 'per_digit' => 1,
117 'per_number' => 1,
118 'constant_size' => 1,
119 'single_partition' => 1,
120 'approximate_size' => 1,
121 'all_values' => 1,
122 'none' =>1 };
123
124my $numeric_partition_size_default = 20;
125
126my $numeric_partition_name_length_default = "-1"; # use the full number
127
128my $bookshelf_type_list =
129 [ { 'name' => "always",
130 'desc' => "{List.bookshelf_type.always}" },
131 { 'name' => "duplicate_only",
132 'desc' => "{List.bookshelf_type.duplicate_only}" },
133 { 'name' => "never",
134 'desc' => "{List.bookshelf_type.never}" } ];
135
136my $bookshelf_type_default = "never";
137my $sort_leaf_nodes_using_default = "Title";
138my $arguments =
139 [ { 'name' => "metadata",
140 'desc' => "{List.metadata}",
141 'type' => "metadata",
142 'reqd' => "yes" },
143
144 { 'name' => "metadata_selection_mode_within_level",
145 'desc' => "{List.metadata_selection_mode_within_level}",
146 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
147 'list' => $metadata_selection_mode_list,
148 'deft' => $metadata_selection_mode_default },
149
150 { 'name' => "metadata_sort_mode_within_level",
151 'desc' => "{List.metadata_sort_mode_within_level}",
152 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
153 'list' => $metadata_sort_mode_list,
154 'deft' => $metadata_sort_mode_default },
155
156 { 'name' => "bookshelf_type",
157 'desc' => "{List.bookshelf_type}",
158 'type' => "enum",
159 'list' => $bookshelf_type_list,
160 'deft' => $bookshelf_type_default },
161
162 { 'name' => "classify_sections",
163 'desc' => "{List.classify_sections}",
164 'type' => "flag" },
165
166 { 'name' => "partition_type_within_level",
167 'desc' => "{List.partition_type_within_level}",
168 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
169 'list' => $partition_type_list,
170 'deft' => $partition_type_default },
171
172 { 'name' => "partition_size_within_level",
173 'desc' => "{List.partition_size_within_level}",
174 'type' => "string", # Must be string because multiple values can be specified (separated by '/')
175 'deft' => $partition_size_default},
176
177 { 'name' => "partition_name_length",
178 'desc' => "{List.partition_name_length}",
179 'type' => "string" },
180
181 { 'name' => "max_partition_name_length",
182 'desc' => "{List.max_partition_name_length}",
183 'type' => "string",
184 'deft' => "3" },
185
186 {'name' => "partition_sort_mode_within_level",
187 'desc' => "{List.partition_sort_mode_within_level}",
188 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
189 'list' => $metadata_sort_mode_list,
190 'deft' => $metadata_sort_mode_default },
191
192 { 'name' => "numeric_partition_type_within_level",
193 'desc' => "{List.numeric_partition_type_within_level}",
194 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
195 'list' => $numeric_partition_type_list,
196 'deft' => $numeric_partition_type_default },
197
198 { 'name' => "numeric_partition_size_within_level",
199 'desc' => "{List.numeric_partition_size_within_level}",
200 'type' => "string", # Must be string because multiple values can be specified (separated by '/')
201 'deft' => $numeric_partition_size_default},
202
203 { 'name' => "numeric_partition_name_length_within_level",
204 'desc' => "{List.numeric_partition_name_length_within_level}",
205 'type' => "string",
206 'deft' => $numeric_partition_name_length_default },
207
208 {'name' => "numeric_partition_sort_mode_within_level",
209 'desc' => "{List.numeric_partition_sort_mode_within_level}",
210 'type' => "enumstring", # Must be enumstring because multiple values can be specified (separated by '/')
211 'list' => $metadata_sort_mode_list,
212 'deft' => $metadata_sort_mode_default },
213
214 { 'name' => "numbers_first",
215 'desc' => "{List.numbers_first}",
216 'type' => 'flag'},
217
218 { 'name' => "sort_leaf_nodes_using",
219 'desc' => "{List.sort_leaf_nodes_using}",
220 'type' => "metadata",
221 'deft' => $sort_leaf_nodes_using_default },
222
223 { 'name' => "sort_leaf_nodes_sort_mode",
224 'desc' => "{List.sort_leaf_nodes_sort_mode}",
225 'type' => "enum",
226 'list' => $metadata_sort_mode_list,
227 'deft' => $metadata_sort_mode_default },
228
229 { 'name' => "reverse_sort_leaf_nodes",
230 'desc' => "{List.reverse_sort_leaf_nodes}",
231 'type' => "flag"},
232
233 { 'name' => "sort_leaf_nodes_removeprefix",
234 'desc' => "{List.sort_leaf_removeprefix}",
235 'type' => "regexp" },
236
237 { 'name' => "sort_leaf_nodes_removesuffix",
238 'desc' => "{List.sort_leaf_nodes_removesuffix}",
239 'type' => "regexp" },
240
241 { 'name' => "sort_using_unicode_collation",
242 'desc' => "{List.metadata_sort.unicode} {List.sort_using_unicode_collation}",
243 'type' => "flag" },
244
245 {'name' => "filter_metadata",
246 'desc' => "{List.filter_metadata}",
247 'type' => "metadata"},
248
249 {'name' => "filter_regex",
250 'desc' => "{List.filter_regex}",
251 'type' => "regexp"},
252
253 { 'name' => "use_formatted_metadata_for_bookshelf_display",
254 'desc' => "{List.use_formatted_metadata_for_bookshelf_display}",
255 'type' => "flag"},
256
257 { 'name' => "removeprefix",
258 'desc' => "{BasClas.removeprefix}",
259 'type' => "regexp" },
260
261 { 'name' => "removesuffix",
262 'desc' => "{BasClas.removesuffix}",
263 'type' => "regexp" } ];
264
265my $options = { 'name' => "List",
266 'desc' => "{List.desc}",
267 'abstract' => "no",
268 'inherits' => "yes",
269 'args' => $arguments };
270
271
272sub new
273{
274 my ($class) = shift(@_);
275 my ($classifierslist, $inputargs, $hashArgOptLists) = @_;
276 push(@$classifierslist, $class);
277
278 push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments});
279 push(@{$hashArgOptLists->{"OptList"}}, $options);
280
281 my $self = new BaseClassifier($classifierslist, $inputargs, $hashArgOptLists);
282
283 bless $self, $class;
284
285 if ($self->{'info_only'}) {
286 # don't worry about any options etc
287 return $self;
288 }
289
290 # The metadata elements to use (required)
291 if (!$self->{'metadata'}) {
292 die "Error: No metadata fields specified for List.\n";
293 }
294
295 my @metadata_groups = split(/[\/]/, $self->{'metadata'});
296 $self->{'metadata_groups'} = \@metadata_groups;
297
298 # The classifier button name (default: the first metadata element specified)
299 if (!$self->{'buttonname'}) {
300 my $first_metadata_group = $metadata_groups[0];
301 my $first_metadata_element = (split(/[\;|,\/]/, $first_metadata_group))[0];
302 $self->{'buttonname'} = $self->generate_title_from_metadata($first_metadata_element);
303 }
304
305 # meta selection mode for each level
306 $self->set_metadata_groups_info_per_level("metadata_selection_mode_within_level", $metadata_selection_mode_default, $valid_metadata_selection_modes);
307
308 # meta sort mode for each level
309 if ($self->{'sort_using_unicode_collation'}) {
310 print STDERR "WARNING: sort_using_unicode_collation is set, setting metadata_sort_mode_within_level to unicode for all levels, regardless of current setting\n";
311 $self->{'metadata_sort_mode_within_level'} = "unicode";
312 $metadata_sort_mode_default = "unicode";
313 } else {
314 if ($self->{'metadata_sort_mode_within_level'} =~ /unicode/) {
315 $self->{'sort_using_unicode_collation'} = 1;
316 }
317 }
318
319
320 $self->set_metadata_groups_info_per_level('metadata_sort_mode_within_level', $metadata_sort_mode_default, $valid_metadata_sort_modes);
321
322 # Whether to group items into a bookshelf, (must be 'always' for all metadata fields except the last)
323 foreach my $metadata_group (@metadata_groups) {
324 $self->{$metadata_group . ".bookshelf_type"} = "always";
325 }
326 my $last_metadata_group = $metadata_groups[$#metadata_groups];
327 # Default: duplicate_only, ie. leave leaf nodes ungrouped (equivalent to AZCompactList -mingroup 2)
328 $self->{$last_metadata_group . ".bookshelf_type"} = $self->{'bookshelf_type'};
329
330 # How the items are grouped into partitions (default: by letter)
331 # for each level (metadata group), separated by '/'
332 $self->set_metadata_groups_info_per_level("partition_type_within_level", $partition_type_default, $valid_partition_types);
333 $self->set_metadata_groups_info_per_level("numeric_partition_type_within_level", $numeric_partition_type_default, $valid_numeric_partition_types);
334
335 # now check whether a level was none - need to set the equivalent level in the other half also to none
336 foreach my $metadata_group (@metadata_groups) {
337 if ($self->{$metadata_group . ".partition_type_within_level"} eq "none" || $self->{$metadata_group . ".numeric_partition_type_within_level"} eq "none") {
338
339 print STDERR "WARNING: one of -partition_type_within_level or -numeric_partition_type_within_level was set to 'none' for level $metadata_group, overriding current value of both these options to 'none'\n";
340
341 $self->{$metadata_group . ".partition_type_within_level"} = "none";
342 $self->{$metadata_group . ".numeric_partition_type_within_level"} = "none";
343 }
344 }
345
346 $self->set_metadata_groups_info_per_level("partition_size_within_level", $partition_size_default);
347 $self->set_metadata_groups_info_per_level("numeric_partition_size_within_level", $numeric_partition_size_default);
348
349 $self->set_metadata_groups_info_per_level('partition_sort_mode_within_level', $metadata_sort_mode_default, $valid_metadata_sort_modes);
350
351 $self->set_metadata_groups_info_per_level('numeric_partition_sort_mode_within_level', $metadata_sort_mode_default, $valid_metadata_sort_modes);
352
353 $self->set_metadata_groups_info_per_level("numeric_partition_name_length_within_level", $numeric_partition_name_length_default);
354
355 # The removeprefix and removesuffix expressions
356 if ($self->{'removeprefix'}) {
357 # If there are more than one expressions, use '' to quote each experession and '/' to separate
358 my @removeprefix_exprs_within_levellist = split(/'\/'/, $self->{'removeprefix'});
359
360 foreach my $metadata_group (@metadata_groups) {
361 my $removeprefix_expr_within_levelelem = shift(@removeprefix_exprs_within_levellist);
362
363 if (defined($removeprefix_expr_within_levelelem) && $removeprefix_expr_within_levelelem ne "") {
364 # Remove the other ' at the beginning and the end if there is any
365 $removeprefix_expr_within_levelelem =~ s/^'//;
366 $removeprefix_expr_within_levelelem =~ s/'$//;
367 # Remove the extra ^ at the beginning
368 $removeprefix_expr_within_levelelem =~ s/^\^//;
369 $self->{$metadata_group . ".remove_prefix_expr"} = $removeprefix_expr_within_levelelem;
370 } else {
371 $self->{$metadata_group . ".remove_prefix_expr"} = $self->{$metadata_groups[0] . ".remove_prefix_expr"};
372 }
373 }
374 }
375 if ($self->{'removesuffix'}) {
376 my @removesuffix_exprs_within_levellist = split(/'\/'/, $self->{'removesuffix'});
377
378 foreach my $metadata_group (@metadata_groups) {
379 my $removesuffix_expr_within_levelelem = shift(@removesuffix_exprs_within_levellist);
380
381 if (defined($removesuffix_expr_within_levelelem) && $removesuffix_expr_within_levelelem ne "") {
382 $removesuffix_expr_within_levelelem =~ s/^'//;
383 $removesuffix_expr_within_levelelem =~ s/'$//;
384 # Remove the extra $ at the end
385 $removesuffix_expr_within_levelelem =~ s/\$$//;
386 $self->{$metadata_group . ".remove_suffix_expr"} = $removesuffix_expr_within_levelelem;
387 } else {
388 $self->{$metadata_group . ".remove_suffix_expr"} = $self->{$metadata_groups[0] . ".remove_suffix_expr"};
389 }
390 }
391 }
392
393 # The metadata elements to use to sort the leaf nodes (default: Title)
394 my @sort_leaf_nodes_using_metadata_groups = split(/\|/, $self->{'sort_leaf_nodes_using'});
395 $self->{'sort_leaf_nodes_using_metadata_groups'} = \@sort_leaf_nodes_using_metadata_groups;
396
397 foreach my $sort_group (@sort_leaf_nodes_using_metadata_groups) {
398 # set metadata_select_type, if not already set - might be already set if the same group was used in -metadata
399 if (!defined $self->{$sort_group . ".metadata_selection_mode_within_level"}) {
400 $self->{$sort_group . ".metadata_selection_mode_within_level"} = $metadata_selection_mode_default;
401 }
402
403 }
404
405 my @leaf_nodes_sort_modes = split (/\|/, $self->{'sort_leaf_nodes_sort_mode'});
406 foreach my $sort_group (@sort_leaf_nodes_using_metadata_groups) {
407 my $leaf_sort_mode = shift(@leaf_nodes_sort_modes);
408 if (!defined $self->{$sort_group . ".metadata_sort_mode_within_level"}) {
409 if (defined $leaf_sort_mode && defined $valid_metadata_sort_modes->{$leaf_sort_mode}) {
410 $self->{$sort_group . ".metadata_sort_mode_within_level"} = $leaf_sort_mode;
411 }
412 else {
413 $self->{$sort_group . ".metadata_sort_mode_within_level"} = $metadata_sort_mode_default;
414 }
415 }
416
417 }
418
419 # Create an instance of the Unicode::Collate object if better Unicode sorting is desired
420 if ($self->{'sort_using_unicode_collation'}) {
421 # To use this you first need to download the allkeys.txt file from
422 # http://www.unicode.org/Public/UCA/latest/allkeys.txt and put it in the Perl
423 # Unicode/Collate directory.
424 require Unicode::Collate;
425 $self->{'unicode_collator'} = Unicode::Collate->new();
426 }
427
428 # An empty array for the document/section OIDs that we are classifying
429 $self->{'OIDs'} = [];
430 # A hash for all the doc ids that we have seen, so we don't classify something twice
431 $self->{'all_doc_OIDs'} = {};
432 return $self;
433}
434
435
436sub init
437{
438 # Nothing to do...
439}
440
441sub set_metadata_groups_info_per_level
442{
443 my $self = shift(@_);
444 my $info_name = shift(@_);
445 my $info_default = shift(@_);
446 my $info_valid_types_hash_ref = shift(@_);
447
448 if (!defined $self->{$info_name}) {
449 print STDERR "List Error: no values were set for option $info_name\n";
450 }
451 my @info_list = split(/\//, $self->{$info_name});
452
453 my $first = 1;
454 foreach my $metadata_group (@{$self->{'metadata_groups'}}) {
455 my $info_elem = shift(@info_list);
456 if (defined ($info_elem) && (!defined $info_valid_types_hash_ref || defined $info_valid_types_hash_ref->{$info_elem})) {
457 $self->{$metadata_group .".$info_name"} = $info_elem;
458 } else {
459 # its empty or an invalid entry
460 my $new_info_elem;
461 if ($first) {
462 $new_info_elem = $info_default;
463 } else {
464 # get the value we had at first
465 $new_info_elem = $self->{@{$self->{'metadata_groups'}}[0] . ".$info_name"};
466 }
467 $self->{$metadata_group .".$info_name"} = $new_info_elem;
468 if (defined $info_elem) {
469 print STDERR "List Error: $info_elem is not a valid value for $info_name, changing it to $new_info_elem\n";
470
471 }
472 }
473 $first = 0;
474 }
475
476}
477
478# Called for each document in the collection
479sub classify
480{
481 my $self = shift(@_);
482 my ($doc_obj) = @_;
483
484 if (defined $self->{'all_doc_OIDs'}->{$doc_obj->get_OID()}) {
485 print STDERR "Warning, List classifier has already seen document ".$doc_obj->get_OID().", not classifying again\n";
486 return;
487 }
488
489 $self->{'all_doc_OIDs'}->{$doc_obj->get_OID()} = 1;
490
491 # check against filter here
492 if ($self->{'filter_metadata'}) {
493 my $meta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'filter_metadata'});
494 return unless defined $meta;
495 if ($self->{'filter_regex'} ne "" && $meta !~ /$self->{'filter_regex'}/) {
496 print STDERR "doc $doc_obj doesn't pass filtering requirement. Not classifying.\n" if ($self->{'verbosity'} > 3);
497 return;
498 }
499 }
500 # if we get here, we have passed the test for filtering
501 # If "-classify_sections" is set, classify every section of the document
502 if ($self->{'classify_sections'}) {
503 my $section = $doc_obj->get_next_section($doc_obj->get_top_section());
504 while (defined $section) {
505 $self->classify_section($doc_obj, $doc_obj->get_OID() . ".$section", $section);
506 $section = $doc_obj->get_next_section($section);
507 }
508 }
509 # Otherwise just classify the top document section
510 else {
511 $self->classify_section($doc_obj, $doc_obj->get_OID(), $doc_obj->get_top_section());
512 }
513
514}
515
516sub classify_section
517{
518 my $self = shift(@_);
519 my ($doc_obj,$section_OID,$section) = @_;
520
521 my @metadata_groups = @{$self->{'metadata_groups'}};
522
523 # Only classify the section if it has a value for one of the metadata elements in the first group
524 my $classify_section = 0;
525 my $first_metadata_group = $metadata_groups[0];
526 my $remove_prefix_expr = $self->{$first_metadata_group . ".remove_prefix_expr"};
527 my $remove_suffix_expr = $self->{$first_metadata_group . ".remove_suffix_expr"};
528 foreach my $first_metadata_group_element (split(/\;|,/, $first_metadata_group)) {
529 my $real_first_metadata_group_element = $self->strip_ex_from_metadata($first_metadata_group_element);
530 my $first_metadata_group_element_value = $doc_obj->get_metadata_element($section, $real_first_metadata_group_element);
531 # Remove prefix/suffix if requested
532 if (defined ($first_metadata_group_element_value)) {
533 if (defined $remove_prefix_expr && $remove_prefix_expr ne "") {
534 $first_metadata_group_element_value =~ s/^$remove_prefix_expr//;
535 }
536
537 if (defined $remove_suffix_expr && $remove_suffix_expr ne "") {
538 $first_metadata_group_element_value =~ s/$remove_suffix_expr$//;
539 }
540 }
541
542 $first_metadata_group_element_value = &sorttools::format_metadata_for_sorting($first_metadata_group, $first_metadata_group_element_value, $doc_obj, $self->{'casefold_metadata_for_sorting'}, $self->{'accentfold_metadata_for_sorting'}) unless $self->{'no_metadata_formatting'};
543 if (defined($first_metadata_group_element_value) && $first_metadata_group_element_value ne "") {
544 # This section must be included in the classifier as we have found a value
545 $classify_section = 1;
546 last;
547
548 }
549 }
550
551 # We're not classifying this section because it doesn't have the required metadata
552 return if (!$classify_section);
553
554 # Otherwise, include this section in the classifier
555
556 push(@{$self->{'OIDs'}}, $section_OID);
557
558 # Create a hash for the metadata values of each metadata element we're interested in
559 my %metadata_groups_done = ();
560 foreach my $metadata_group (@metadata_groups, @{$self->{'sort_leaf_nodes_using_metadata_groups'}}) {
561 # Take care not to do a metadata group more than once
562 unless ($metadata_groups_done{$metadata_group}) {
563 my $remove_prefix_expr = $self->{$metadata_group . ".remove_prefix_expr"};
564 my $remove_suffix_expr = $self->{$metadata_group . ".remove_suffix_expr"};
565 foreach my $metadata_element (split(/\;|,/, $metadata_group)) {
566 my $real_metadata_element = $self->strip_ex_from_metadata($metadata_element);
567
568 my @metadata_values = @{$doc_obj->get_metadata($section, $real_metadata_element)};
569 foreach my $metadata_value (@metadata_values) {
570 #print STDERR "working with value $metadata_value\n";
571 # Strip leading and trailing whitespace
572 $metadata_value =~ s/^\s*//;
573 $metadata_value =~ s/\s*$//;
574
575 # Remove prefix/suffix if requested
576 if (defined $remove_prefix_expr && $remove_prefix_expr ne "") {
577 $metadata_value =~ s/^$remove_prefix_expr//;
578 }
579 if (defined $remove_suffix_expr && $remove_suffix_expr ne "") {
580 $metadata_value =~ s/$remove_suffix_expr$//;
581 }
582
583 # lowercase metadata both for sorting meta (d/D under D), and to allow CSS to be able to
584 # text-transform the stored lowercase values as capitalize or uppercase (can't CSS
585 # text-transform if stored uppercase). 2 CSS text-transforms have been added to core.css
586 ### no longer do this, as lowercasing is now an option for the user, and is handled by format_metadata_for_sorting
587 my $lc_metadata_value = $metadata_value; #lc($metadata_value);
588 $lc_metadata_value = &sorttools::format_metadata_for_sorting($real_metadata_element, $lc_metadata_value, $doc_obj, $self->{'casefold_metadata_for_sorting'}, $self->{'accentfold_metadata_for_sorting'}) unless $self->{'no_metadata_formatting'};
589
590 # Add the metadata value into the list for this combination of metadata group
591 # and section - if we have some non-whitespace chars
592 # test that we have some non-whitespace chars
593 if ($lc_metadata_value =~ /\S/) {
594
595 push(@{$self->{$metadata_group . ".list"}->{$section_OID}}, $lc_metadata_value);
596
597 # add the actual value into the stored values so we can remember the case
598 if (!$self->{'use_formatted_metadata_for_bookshelf_display'}) {
599 if (defined $self->{$metadata_group . ".actualvalues"}->{$lc_metadata_value}->{$metadata_value}) {
600 $self->{$metadata_group . ".actualvalues"}->{$lc_metadata_value}->{$metadata_value}++;
601 } else {
602 $self->{$metadata_group . ".actualvalues"}->{$lc_metadata_value}->{$metadata_value} = 1;
603 }
604 }
605 last if ($self->{$metadata_group . ".metadata_selection_mode_within_level"} eq "firstvalue");
606 }
607 } # foreach metadatavalue
608 last if ((@metadata_values > 0) && $self->{$metadata_group . ".metadata_selection_mode_within_level"} =~ /^(firstvalue|firstvalidmetadata)$/ );
609 } # foreach metadata element
610
611 $metadata_groups_done{$metadata_group} = 1;
612 }
613 }
614}
615
616
617sub get_classify_info
618{
619 my $self = shift(@_);
620
621 # The metadata groups to classify by
622 my @metadata_groups = @{$self->{'metadata_groups'}};
623 my $first_metadata_group = $metadata_groups[0];
624
625 # The OID values of the documents to include in the classifier
626 my @OIDs = @{$self->{'OIDs'}};
627
628 # Create the root node of the classification hierarchy
629 my %classifier_node = ( 'thistype' => "Invisible",
630 'childtype' => "VList",
631 'Title' => $self->{'buttonname'},
632 'contains' => [],
633 'mdtype' => $first_metadata_group );
634
635 # Recursively create the classification hierarchy, one level for each metadata group
636 $self->add_level(\@metadata_groups, \@OIDs, \%classifier_node);
637 return \%classifier_node;
638}
639
640
641sub add_level
642{
643 my $self = shift(@_);
644 my @metadata_groups = @{shift(@_)};
645 my @OIDs = @{shift(@_)};
646 my $classifier_node = shift(@_);
647
648 my $metadata_group = $metadata_groups[0];
649 if (!defined($self->{$metadata_group . ".list"})) {
650 print STDERR "Warning: No metadata values assigned to $metadata_group.\n";
651 return;
652 }
653
654 # Create a mapping from metadata value to OID
655 my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
656 my %metadata_value_to_OIDs_hash = ();
657 my %numeric_metadata_value_to_OIDs_hash = ();
658 foreach my $OID (@OIDs)
659 {
660 if ($OID_to_metadata_values_hash_ref->{$OID})
661 {
662 my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
663 foreach my $metadata_value (@metadata_values)
664 {
665 if ($metadata_value =~ /^[0-9]/) {
666 push(@{$numeric_metadata_value_to_OIDs_hash{$metadata_value}}, $OID);
667 } else {
668 push(@{$metadata_value_to_OIDs_hash{$metadata_value}}, $OID);
669 }
670 }
671 }
672 }
673 #print STDERR "Number of distinct values: " . scalar(keys %metadata_value_to_OIDs_hash) . "\n";
674 #print STDERR "Number of distinct numeric values: " . scalar(keys %numeric_metadata_value_to_OIDs_hash) . "\n";
675
676 # Partition the values (if necessary)
677 my $partition_type_within_level = $self->{$metadata_group . ".partition_type_within_level"};
678 my $partition_size_within_level = $self->{$metadata_group . ".partition_size_within_level"};
679 my $partition_sort_mode_within_level = $self->{$metadata_group . ".partition_sort_mode_within_level"};
680 my $bookshelf_type_within_level = $self->{$metadata_group. ".bookshelf_type"};
681
682
683 #############################################
684 ### DO THE NUMBERS IF THEY ARE TO COME FIRST
685 #############################################
686 if ($self->{'numbers_first'} && keys(%numeric_metadata_value_to_OIDs_hash)) {
687 $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash);
688 }
689
690 ############################################
691 # DO THE LETTERS
692 ############################################
693 if (keys(%metadata_value_to_OIDs_hash)){ # make sure we have some values
694 if ($partition_type_within_level =~ /^per_letter$/i) {
695 $self->split_per_letter_or_digit(\@metadata_groups, $classifier_node, $partition_sort_mode_within_level, \%metadata_value_to_OIDs_hash);
696 }
697 elsif ($partition_type_within_level =~ /^approximate_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
698 $self->split_approximate_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, $bookshelf_type_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'}, $self->{'max_partition_name_length'});
699
700 }
701 elsif ($partition_type_within_level =~ /^constant_size$/i && scalar(keys %metadata_value_to_OIDs_hash) > $partition_size_within_level) {
702 $self->split_constant_size(\@metadata_groups, $classifier_node, $partition_size_within_level, $partition_sort_mode_within_level, $bookshelf_type_within_level, \%metadata_value_to_OIDs_hash, $self->{'partition_name_length'}, $self->{'max_partition_name_length'});
703 }
704
705 # Otherwise just add all the values to a VList
706 else {
707 if ($partition_type_within_level =~ /^all_values$/i) {
708 $classifier_node->{'childtype'} = "HList";
709 $self->{$metadata_group. ".bookshelf_type"} = "always";
710 }
711 $self->add_vlist(\@metadata_groups, $classifier_node, \%metadata_value_to_OIDs_hash);
712 }
713 }
714
715 ###########################################
716 ### DO THE NUMBERS IF THEY ARE TO COME LAST
717 ###########################################
718 if (!$self->{'numbers_first'} && keys(%numeric_metadata_value_to_OIDs_hash)) {
719 $self->partition_numeric_values(\@metadata_groups, $classifier_node, \%numeric_metadata_value_to_OIDs_hash);
720 }
721}
722
723sub partition_numeric_values
724{
725 my $self = shift(@_);
726 my @metadata_groups = @{shift(@_)};
727 my $classifier_node = shift(@_);
728 my $numeric_metadata_value_to_OIDs_hash_ref = shift(@_);
729
730 my $metadata_group = $metadata_groups[0];
731 my $numeric_partition_type_within_level = $self->{$metadata_group . ".numeric_partition_type_within_level"};
732 my $numeric_partition_size_within_level = $self->{$metadata_group . ".numeric_partition_size_within_level"};
733 my $numeric_partition_sort_mode_within_level = $self->{$metadata_group . ".numeric_partition_sort_mode_within_level"};
734 my $numeric_partition_name_length_within_level = $self->{$metadata_group . ".numeric_partition_name_length_within_level"};
735 my $bookshelf_type_within_level = $self->{$metadata_group. ".bookshelf_type"};
736
737 if ($numeric_partition_type_within_level eq "single_partition") {
738 $self->add_hlist_partition(\@metadata_groups, $classifier_node, "0-9", $numeric_metadata_value_to_OIDs_hash_ref);
739 }
740 elsif ($numeric_partition_type_within_level eq "per_digit") {
741 $self->split_per_letter_or_digit(\@metadata_groups, $classifier_node, $numeric_partition_sort_mode_within_level, $numeric_metadata_value_to_OIDs_hash_ref, 1, 1);
742 }
743 elsif ($numeric_partition_type_within_level eq "per_number") {
744 # each different number is a bucket
745 $self->split_per_letter_or_digit(\@metadata_groups, $classifier_node, $numeric_partition_sort_mode_within_level, $numeric_metadata_value_to_OIDs_hash_ref, 1, $numeric_partition_name_length_within_level);
746 }
747 elsif ($numeric_partition_type_within_level eq "constant_size" && scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
748 # Generate hlists of a certain size
749
750 $self->split_constant_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level,$numeric_partition_name_length_within_level, 1);
751 } elsif ($numeric_partition_type_within_level eq "approximate_size" && scalar(keys %$numeric_metadata_value_to_OIDs_hash_ref) > $numeric_partition_size_within_level) {
752 $self->split_approximate_size(\@metadata_groups, $classifier_node, $numeric_partition_size_within_level, $numeric_partition_sort_mode_within_level, $bookshelf_type_within_level, $numeric_metadata_value_to_OIDs_hash_ref, $numeric_partition_name_length_within_level, $numeric_partition_name_length_within_level, 1);
753 }
754 # Otherwise just add all the values to a VList
755 else {
756 if ($numeric_partition_type_within_level =~ /^all_values$/i) {
757 $classifier_node->{'childtype'} = "HList";
758 $self->{$metadata_group. ".bookshelf_type"} = "always";
759 }
760
761 $self->add_vlist(\@metadata_groups, $classifier_node, $numeric_metadata_value_to_OIDs_hash_ref);
762 }
763
764}
765
766
767sub split_approximate_size
768{
769
770 my $self = shift(@_);
771 my @metadata_groups = @{shift(@_)};
772 my $classifier_node = shift(@_);
773 my $partition_size = shift(@_);
774 my $sort_mode = shift(@_);
775 my $bookshelf_type = shift(@_);
776 my $metadata_value_to_OIDs_hash_ref = shift(@_);
777 my $partition_name_length = shift(@_);
778 my $max_partition_name_length = shift(@_);
779 my $is_numeric = shift(@_);
780
781 my $tolerance = 10; # should this be an option??
782
783 # Generate hlist based on the first letter of the metadata value (like per_letter), or based on
784 # numbers, but also with restriction on the partition size
785 # If a partition has fewer items than specified by the "partition_size_within_level", then group them together if possible
786 # If a partition has more items than specified, split into several hlists.
787 # Depends on the bookshelf_type, one item can be either a document (when bookshelf_type is "never") or a metadata value (otherwise)
788
789 my @sortedmetadata_values = $self->sort_metadata_values_array($sort_mode, keys(%$metadata_value_to_OIDs_hash_ref));
790
791 # Separate values by their first letter, each form a bucket, like the per_letter partition type
792 my $last_partition = $self->generate_partition_name($sortedmetadata_values[0], $partition_name_length, $is_numeric);
793
794 my @partition_buckets = ();
795 my @metadata_values_in_bucket = ();
796 my $num_items_in_bucket = 0;
797
798 foreach my $metadata_value (@sortedmetadata_values) {
799 my $metadata_valuepartition = $self->generate_partition_name($metadata_value, $partition_name_length, $is_numeric);
800 if ($metadata_valuepartition ne $last_partition) {
801 my @temp_array = @metadata_values_in_bucket;
802 # Cache the values that belong to this bucket, and the number of items in this bucket, not necessary to be the same number as the metadata values
803 my %partition_info = ();
804 $partition_info{'metadata_values'} = \@temp_array;
805 $partition_info{'size'} = $num_items_in_bucket;
806 $partition_info{'name'} = $last_partition;
807 push (@partition_buckets, \%partition_info);
808
809 @metadata_values_in_bucket = ($metadata_value);
810 $num_items_in_bucket = ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : scalar(@metadata_values_in_bucket));
811 $last_partition = $metadata_valuepartition;
812 } else {
813 $num_items_in_bucket += ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1); #scalar(@metadata_values_in_bucket);
814 push (@metadata_values_in_bucket, $metadata_value);
815 }
816 }
817 # Last one
818 my %partition_info = ();
819 $partition_info{'metadata_values'} = \@metadata_values_in_bucket;
820 $partition_info{'size'} = $num_items_in_bucket;
821 $partition_info{'name'} = $last_partition;
822
823 push (@partition_buckets, \%partition_info);
824
825 # now go through the array of A, B, C buckets, and merge small buckets
826 my @new_partition_buckets = ();
827 for (my $i = 0; $i < scalar(@partition_buckets) - 1; $i++) {
828
829 my $this_bucket = $partition_buckets[$i];
830 my $next_bucket = $partition_buckets[$i+1];
831
832 my $items_in_partition = $this_bucket->{'size'};
833
834 if ($items_in_partition < $partition_size ) {
835 my $items_in_next_partition = $next_bucket->{'size'};
836 if ($items_in_partition + $items_in_next_partition <= $partition_size+$tolerance ) {
837 # merge this bucket into the next bucket
838 foreach my $metadata_value_to_merge (@{$this_bucket->{'metadata_values'}}) {
839 push(@{$next_bucket->{'metadata_values'}}, $metadata_value_to_merge);
840 }
841 $next_bucket->{'size'} += $items_in_partition;
842
843 } else {
844 # remember this bucket
845 push (@new_partition_buckets, $this_bucket);
846 }
847 } else {
848 # remember this bucket
849 push (@new_partition_buckets, $this_bucket);
850 }
851 }
852 # add in the last bucket
853 my $last_bucket = $partition_buckets[scalar(@partition_buckets) - 1];
854 push (@new_partition_buckets, $last_bucket);
855
856 # Add partitions to the main list, but divide big bucket into several
857 my $last_partition_end = "";
858 my $partition_start = "";
859 my $partition_end = "";
860 my $partition_name = "";
861 foreach my $partition (@new_partition_buckets) {
862 my @metadata_values = $self->sort_metadata_values_array($sort_mode, @{$partition->{'metadata_values'}});
863 my $items_in_partition = $partition->{'size'};
864 $partition_start = $self->generate_partition_start($metadata_values[0], $last_partition_end, $partition_name_length, $max_partition_name_length, $is_numeric);
865
866 if ($items_in_partition <= $partition_size+$tolerance) {
867 # we can just add the partition as is
868 my %metadata_values_to_OIDs_subhashes = ();
869 for (my $i = 0; $i < scalar(@metadata_values); $i++) {
870 my $metadata_value = $metadata_values[$i];
871 $metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
872 }
873 my $last_metadata_value = $metadata_values[scalar(@metadata_values)-1];
874 $partition_end = $self->generate_partition_end($last_metadata_value, $partition_start, $partition_name_length, $max_partition_name_length, $is_numeric);
875 $partition_name = $partition_start;
876 if ($partition_end ne $partition_start) {
877 $partition_name = $partition_name . "-" . $partition_end;
878 }
879 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
880 $last_partition_end = $partition_end;
881 } else {
882 # we have too many items, need to split the partition
883 my $num_splits = int($items_in_partition/$partition_size);
884 $num_splits++ if (($items_in_partition - $partition_size*$num_splits) > $tolerance);
885 my $this_partition_size = $items_in_partition/$num_splits;
886 # regenerate-the start, noticing that we are a split
887 $partition_start = $self->generate_partition_start($metadata_values[0], $last_partition_end, $partition_name_length, $max_partition_name_length, $is_numeric, 1);
888 my $items_done = 0;
889 my $prev_done = 0;
890 my %metadata_values_to_OIDs_subhashes = ();
891 for (my $i = 0; $i < scalar(@metadata_values); $i++) {
892 my $metadata_value = $metadata_values[$i];
893 # If the bookshelf_type is "never", count the documents, otherwise count the distinct metadata values
894 my $items_for_this_md_value = $bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1;
895
896 if ($items_done + $items_for_this_md_value > $this_partition_size && $items_done != 0 && ($items_in_partition - ($prev_done+$items_done) >= $tolerance)) {
897 # Save the stored items into a partition
898 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
899 $last_partition_end = $partition_end;
900 $partition_start = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $max_partition_name_length, $is_numeric, 1);
901 $prev_done += $items_done;
902 $items_done = 0;
903 %metadata_values_to_OIDs_subhashes = ();
904 }
905
906 # If bookshelf_type is "never" and the current metadata value holds too many items, need to split into several partitions
907 if ($bookshelf_type eq "never" && $items_for_this_md_value > $this_partition_size) {
908 # if this is the case, we would have already stored what we have done in the previous step.
909 my $partitionname_for_this_value = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $max_partition_name_length, $is_numeric);
910 # Get the number of partitions needed for this value
911 my $num_splits = int($items_for_this_md_value / $this_partition_size);
912 $num_splits++ if ($items_for_this_md_value / $this_partition_size > $num_splits);
913 my @OIDs_for_this_value = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
914 for (my $i = 0; $i < $num_splits; $i++) {
915 my %OIDs_subhashes_for_this_value = ();
916 my @OIDs_for_this_partition = ();
917 for (my $d = $i * $this_partition_size; $d < (($i+1) * $this_partition_size > $items_for_this_md_value ? $items_for_this_md_value : ($i+1) * $partition_size); $d++) {
918 push (@OIDs_for_this_partition, $OIDs_for_this_value[$d]);
919 }
920
921 # The last bucket might have only a few items and need to be merged with buckets for subsequent metadata values
922 if ($i == $num_splits - 1 && scalar(@OIDs_for_this_partition) < $this_partition_size) {
923 $partition_start = $partitionname_for_this_value;
924 $partition_name = $partition_start;
925 $metadata_values_to_OIDs_subhashes{$metadata_value} = \@OIDs_for_this_partition;
926 $items_done += scalar(@OIDs_for_this_partition);
927 $last_partition_end = $partitionname_for_this_value
928 } else {
929
930 # Add an HList for this bucket
931 $OIDs_subhashes_for_this_value{$metadata_value} = \@OIDs_for_this_partition;
932 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname_for_this_value, \%OIDs_subhashes_for_this_value);
933 $last_partition_end = $partitionname_for_this_value;
934 }
935 }
936 } else {
937 $metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
938 $items_done += $bookshelf_type eq "never" ? scalar(@{$metadata_values_to_OIDs_subhashes{$metadata_value}}) : 1;
939 $partition_end = $self->generate_partition_end($metadata_value, $partition_start, $partition_name_length, $max_partition_name_length, $is_numeric);
940 $partition_name = $partition_start;
941 if ($partition_end ne $partition_start) {
942 $partition_name = $partition_name . "-" . $partition_end;
943 }
944
945 }
946
947 # The last partition
948 if($i == scalar(@metadata_values) - 1 && $items_done >0) {
949 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
950 }
951
952 }
953 } # end if items in partition > partition size
954
955 }
956
957 # The partitions are stored in an HList
958 $classifier_node->{'childtype'} = "HList";
959
960}
961sub split_approximate_size_ORIG
962{
963
964 my $self = shift(@_);
965 my @metadata_groups = @{shift(@_)};
966 my $classifier_node = shift(@_);
967 my $partition_size = shift(@_);
968 my $sort_mode = shift(@_);
969 my $bookshelf_type = shift(@_);
970 my $metadata_value_to_OIDs_hash_ref = shift(@_);
971 my $partition_name_length = shift(@_);
972 my $is_numeric = shift(@_);
973
974 # Generate hlist based on the first letter of the metadata value (like per_letter), or based on
975 # numbers, but also with restriction on the partition size
976 # If a partition has fewer items than specified by the "partition_size_within_level", then group them together if possible
977 # If a partition has more items than specified, split into several hlists.
978 # Depends on the bookshelf_type, one item can be either a document (when bookshelf_type is "never") or a metadata value (otherwise)
979
980 my @sortedmetadata_values = $self->sort_metadata_values_array($sort_mode, keys(%$metadata_value_to_OIDs_hash_ref));
981
982 # Separate values by their first letter, each form a bucket, like the per_letter partition type
983 my $last_partition = $self->generate_partition_name($sortedmetadata_values[0], $partition_name_length, $is_numeric);
984
985 my @partition_buckets = ();
986 my @metadata_values_in_bucket = ();
987 my $num_items_in_bucket = 0;
988
989 foreach my $metadata_value (@sortedmetadata_values) {
990 my $metadata_valuepartition = $self->generate_partition_name($metadata_value, $partition_name_length, $is_numeric);
991 if ($metadata_valuepartition ne $last_partition) {
992 my @temp_array = @metadata_values_in_bucket;
993 # Cache the values that belong to this bucket, and the number of items in this bucket, not necessary to be the same number as the metadata values
994 my %partition_info = ();
995 $partition_info{'metadata_values'} = \@temp_array;
996 $partition_info{'size'} = $num_items_in_bucket;
997 $partition_info{'name'} = $last_partition;
998 push (@partition_buckets, \%partition_info);
999
1000 @metadata_values_in_bucket = ($metadata_value);
1001 $num_items_in_bucket = ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : scalar(@metadata_values_in_bucket));
1002 $last_partition = $metadata_valuepartition;
1003 } else {
1004 $num_items_in_bucket += ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1); #scalar(@metadata_values_in_bucket);
1005 push (@metadata_values_in_bucket, $metadata_value);
1006 }
1007 }
1008 # Last one
1009 my %partition_info = ();
1010 $partition_info{'metadata_values'} = \@metadata_values_in_bucket;
1011 $partition_info{'size'} = $num_items_in_bucket;
1012 $partition_info{'name'} = $last_partition;
1013
1014 push (@partition_buckets, \%partition_info);
1015
1016 # now go through the array of buckets, and merge small buckets
1017 my @new_partition_buckets = ();
1018 for (my $i = 0; $i < scalar(@partition_buckets) - 1; $i++) {
1019
1020 my $this_bucket = $partition_buckets[$i];
1021 my $next_bucket = $partition_buckets[$i+1];
1022
1023 my $items_in_partition = $this_bucket->{'size'};
1024
1025 if ($items_in_partition < $partition_size ) {
1026 my $items_in_next_partition = $next_bucket->{'size'};
1027 if ($items_in_partition + $items_in_next_partition <= $partition_size ) {
1028 # merge this bucket into the next bucket
1029 foreach my $metadata_value_to_merge (@{$this_bucket->{'metadata_values'}}) {
1030 push(@{$next_bucket->{'metadata_values'}}, $metadata_value_to_merge);
1031 }
1032 $next_bucket->{'size'} += $items_in_partition;
1033
1034 } else {
1035 # remember this bucket
1036 push (@new_partition_buckets, $this_bucket);
1037 }
1038 } else {
1039 # remember this bucket
1040 push (@new_partition_buckets, $this_bucket);
1041 }
1042 }
1043 # add in the last bucket
1044 my $last_bucket = $partition_buckets[scalar(@partition_buckets) - 1];
1045 push (@new_partition_buckets, $last_bucket);
1046
1047 # Add partitions to the main list, but divide big bucket into several
1048 my $last_partition_end = "";
1049 my $partition_start = "";
1050 my $partition_end = "";
1051 my $partition_name = "";
1052 foreach my $partition (@new_partition_buckets) {
1053 my @metadata_values = $self->sort_metadata_values_array($sort_mode, @{$partition->{'metadata_values'}});
1054 my $items_in_partition = $partition->{'size'};
1055 $partition_start = $self->generate_partition_start($metadata_values[0], $last_partition_end, $partition_name_length, $is_numeric);
1056
1057 if ($items_in_partition <= $partition_size) {
1058 # we can just add the partition as is
1059 my %metadata_values_to_OIDs_subhashes = ();
1060 for (my $i = 0; $i < scalar(@metadata_values); $i++) {
1061 my $metadata_value = $metadata_values[$i];
1062 $metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
1063 }
1064 my $last_metadata_value = $metadata_values[scalar(@metadata_values)-1];
1065 $partition_end = $self->generate_partition_end($last_metadata_value, $partition_start, $partition_name_length, $is_numeric);
1066 $partition_name = $partition_start;
1067 if ($partition_end ne $partition_start) {
1068 $partition_name = $partition_name . "-" . $partition_end;
1069 }
1070 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
1071 $last_partition_end = $partition_end;
1072 } else {
1073 # we have too many items, need to split the partition
1074 my $items_done = 0;
1075 my %metadata_values_to_OIDs_subhashes = ();
1076 for (my $i = 0; $i < scalar(@metadata_values); $i++) {
1077 my $metadata_value = $metadata_values[$i];
1078 # If the bookshelf_type is "never", count the documents, otherwise count the distinct metadata values
1079 my $items_for_this_md_value = $bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1;
1080
1081 if ($items_done + $items_for_this_md_value > $partition_size && $items_done != 0) {
1082 # Save the stored items into a partition
1083 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
1084 $last_partition_end = $partition_end;
1085 $partition_start = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $is_numeric);
1086 $items_done = 0;
1087 %metadata_values_to_OIDs_subhashes = ();
1088 }
1089
1090 # If bookshelf_type is "never" and the current metadata value holds too many items, need to split into several partitions
1091 if ($bookshelf_type eq "never" && $items_for_this_md_value > $partition_size) {
1092
1093 my $partitionname_for_this_value = $self->generate_partition_start($metadata_value, $last_partition_end, $partition_name_length, $is_numeric);
1094 # Get the number of partitions needed for this value
1095 my $num_splits = int($items_for_this_md_value / $partition_size);
1096 $num_splits++ if ($items_for_this_md_value / $partition_size > $num_splits);
1097 my @OIDs_for_this_value = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
1098 for (my $i = 0; $i < $num_splits; $i++) {
1099 my %OIDs_subhashes_for_this_value = ();
1100 my @OIDs_for_this_partition = ();
1101 for (my $d = $i * $partition_size; $d < (($i+1) * $partition_size > $items_for_this_md_value ? $items_for_this_md_value : ($i+1) * $partition_size); $d++) {
1102 push (@OIDs_for_this_partition, $OIDs_for_this_value[$d]);
1103 }
1104
1105 # The last bucket might have only a few items and need to be merged with buckets for subsequent metadata values
1106 if ($i == $num_splits - 1 && scalar(@OIDs_for_this_partition) < $partition_size) {
1107 $partition_start = $partitionname_for_this_value;
1108 $partition_name = $partition_start;
1109 $metadata_values_to_OIDs_subhashes{$metadata_value} = \@OIDs_for_this_partition;
1110 $items_done += scalar(@OIDs_for_this_partition);
1111 $last_partition_end = $partitionname_for_this_value
1112 } else {
1113
1114 # Add an HList for this bucket
1115 $OIDs_subhashes_for_this_value{$metadata_value} = \@OIDs_for_this_partition;
1116 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname_for_this_value, \%OIDs_subhashes_for_this_value);
1117 $last_partition_end = $partitionname_for_this_value;
1118 }
1119 }
1120 } else {
1121
1122 $metadata_values_to_OIDs_subhashes{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
1123 $items_done += $bookshelf_type eq "never" ? scalar(@{$metadata_values_to_OIDs_subhashes{$metadata_value}}) : 1;
1124 $partition_end = $self->generate_partition_end($metadata_value, $partition_start, $partition_name_length, $is_numeric);
1125 $partition_name = $partition_start;
1126 if ($partition_end ne $partition_start) {
1127 $partition_name = $partition_name . "-" . $partition_end;
1128 }
1129
1130 }
1131
1132 # The last partition
1133 if($i == scalar(@metadata_values) - 1 && $items_done >0) {
1134 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partition_name, \%metadata_values_to_OIDs_subhashes);
1135 }
1136
1137 }
1138 } # end if items in partition > partition size
1139
1140 }
1141
1142 # The partitions are stored in an HList
1143 $classifier_node->{'childtype'} = "HList";
1144
1145}
1146
1147
1148sub split_constant_size
1149{
1150 my $self = shift(@_);
1151 my @metadata_groups = @{shift(@_)};
1152 my $classifier_node = shift(@_);
1153 my $partition_size = shift(@_);
1154 my $sort_mode = shift(@_);
1155 my $bookshelf_type = shift(@_);
1156 my $metadata_value_to_OIDs_hash_ref = shift(@_);
1157 my $partition_name_length = shift(@_);
1158 my $max_partition_name_length = shift(@_);
1159 my $is_numeric = shift(@_);
1160
1161 my @sortedmetadata_values = $self->sort_metadata_values_array($sort_mode, keys(%$metadata_value_to_OIDs_hash_ref));
1162 my $items_in_partition = 0;
1163 my %metadata_value_to_OIDs_subhash = ();
1164 my $lastpartitionend = "";
1165 my $partitionstart;
1166
1167 foreach my $metadata_value (@sortedmetadata_values) {
1168 if ($items_in_partition == 0) {
1169 # a new partition, set the name
1170 $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $max_partition_name_length, $is_numeric);
1171 }
1172 my $numitems_for_this_value = ($bookshelf_type eq "never" ? scalar(@{$metadata_value_to_OIDs_hash_ref->{$metadata_value}}) : 1);
1173 if ($items_in_partition + $numitems_for_this_value <= $partition_size) {
1174 # add all the current values into the temporary list
1175 $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
1176 $items_in_partition += $numitems_for_this_value;
1177 } elsif ($items_in_partition < $partition_size) {
1178 # only want to add some of the values into temporary list
1179 # note, we only get here if bookshelf type is never
1180 my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
1181 @OIDs = $self->sort_leaf_items(\@OIDs);
1182 my $num_items_needed = $partition_size - $items_in_partition;
1183 my @slice = splice(@OIDs, 0, $num_items_needed);
1184 $metadata_value_to_OIDs_subhash{$metadata_value} = \@slice;
1185
1186 # now we have filled up the partition
1187 my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $max_partition_name_length, $is_numeric);
1188 my $partitionname = $partitionstart;
1189 if ($partitionend ne $partitionstart) {
1190 $partitionname = $partitionname . "-" . $partitionend;
1191 }
1192
1193 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
1194 %metadata_value_to_OIDs_subhash = ();
1195 $items_in_partition = 0;
1196 $lastpartitionend = $partitionend;
1197
1198 # can we get more partitions from this metadata value?
1199 while (scalar(@OIDs) >= $partition_size) {
1200 my @slice = splice(@OIDs, 0, $partition_size);
1201 $metadata_value_to_OIDs_subhash{$metadata_value} = \@slice;
1202 $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $max_partition_name_length, $is_numeric);
1203 my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $max_partition_name_length, $is_numeric);
1204 my $partitionname = $partitionstart;
1205 if ($partitionend ne $partitionstart) {
1206 $partitionname = $partitionname . "-" . $partitionend;
1207 }
1208 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
1209 %metadata_value_to_OIDs_subhash = ();
1210 $items_in_partition = 0;
1211 $lastpartitionend = $partitionend;
1212
1213 }
1214 if (scalar(@OIDs) > 0) {
1215 $metadata_value_to_OIDs_subhash{$metadata_value} = \@OIDs;
1216 $items_in_partition = scalar(@OIDs);
1217 $partitionstart = $self->generate_partition_start($metadata_value, $lastpartitionend, $partition_name_length, $max_partition_name_length, $is_numeric);
1218 }
1219
1220
1221 }
1222
1223 if ($items_in_partition == $partition_size) {
1224 # its the end of a partition
1225 my $partitionend = $self->generate_partition_end($metadata_value, $partitionstart, $partition_name_length, $max_partition_name_length, $is_numeric);
1226 my $partitionname = $partitionstart;
1227 if ($partitionend ne $partitionstart) {
1228 $partitionname = $partitionname . "-" . $partitionend;
1229 }
1230
1231 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
1232 %metadata_value_to_OIDs_subhash = ();
1233 $items_in_partition = 0;
1234 $lastpartitionend = $partitionend;
1235 }
1236 } # foreach metadata value
1237
1238 if ($items_in_partition > 0) {
1239 # we have to add the last partition
1240 my $partitionend = $self->generate_partition_end(@sortedmetadata_values[@sortedmetadata_values-1], $partitionstart, $partition_name_length, $max_partition_name_length, $is_numeric);
1241 my $partitionname = $partitionstart;
1242 if ($partitionend ne $partitionstart) {
1243 $partitionname = $partitionname . "-" . $partitionend;
1244 }
1245
1246 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $partitionname, \%metadata_value_to_OIDs_subhash);
1247 }
1248
1249 # The partitions are stored in an HList
1250 $classifier_node->{'childtype'} = "HList";
1251
1252}
1253
1254sub split_per_letter_or_digit
1255{
1256 my $self = shift(@_);
1257 my @metadata_groups = @{shift(@_)};
1258 my $classifier_node = shift(@_);
1259 my $sort_mode = shift(@_);
1260 my $metadata_value_to_OIDs_hash_ref = shift(@_);
1261 my $is_numeric = shift(@_);
1262 my $numeric_partition_length = shift(@_);
1263
1264 if (not defined $is_numeric) {
1265 $is_numeric = 0;
1266 }
1267 if ($is_numeric && not defined($numeric_partition_length)) {
1268 $numeric_partition_length = 1;
1269 }
1270 # Generate one hlist for each letter
1271 my @sortedmetadata_values = $self->sort_metadata_values_array($sort_mode, keys(%$metadata_value_to_OIDs_hash_ref));
1272 my %metadata_value_to_OIDs_subhash = ();
1273
1274 my $lastpartition = $self->generate_partition_name($sortedmetadata_values[0], $numeric_partition_length, $is_numeric);
1275 foreach my $metadata_value (@sortedmetadata_values) {
1276 my $metadata_valuepartition = $self->generate_partition_name($metadata_value, $numeric_partition_length, $is_numeric);
1277 # Is this the start of a new partition?
1278 if ($metadata_valuepartition ne $lastpartition) {
1279 #print STDERR "new partition, old = $lastpartition, new=$metadata_valuepartition\n";
1280 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
1281 %metadata_value_to_OIDs_subhash = ();
1282 $lastpartition = $metadata_valuepartition;
1283 }
1284
1285 $metadata_value_to_OIDs_subhash{$metadata_value} = $metadata_value_to_OIDs_hash_ref->{$metadata_value};
1286 }
1287
1288 # Don't forget to add the last partition
1289 $self->add_hlist_partition(\@metadata_groups, $classifier_node, $lastpartition, \%metadata_value_to_OIDs_subhash);
1290
1291 # The partitions are stored in an HList
1292 $classifier_node->{'childtype'} = "HList";
1293
1294}
1295
1296sub generate_partition_name
1297{
1298 my $self = shift(@_);
1299 my $mvalue = shift(@_);
1300 my $numeric_partition_length = shift(@_);
1301 my $is_numeric = shift(@_);
1302
1303 if (!$is_numeric || $numeric_partition_length == 1 ) {
1304 return substr($mvalue, 0, 1);
1305 }
1306 if ($numeric_partition_length == -1) {
1307 my ($all_digits) = $mvalue =~ /^([0-9]+)/;
1308 return $all_digits;
1309 }
1310 my ($some_digits) = $mvalue =~ /^([0-9]{1,$numeric_partition_length})/;
1311 return $some_digits;
1312}
1313
1314sub generate_partition_start
1315{
1316 my $self = shift(@_);
1317 my $metadata_value = shift(@_);
1318 my $lastpartitionend = shift(@_);
1319 my $partition_name_length = shift(@_);
1320 my $max_partition_name_length = shift(@_);
1321 my $is_numeric = shift(@_);
1322 my $is_split = shift(@_);
1323
1324# print STDERR "generate-partitoin start, $metadata_value, $lastpartitionend, $max_partition_name_length\n";
1325 if ($is_numeric) {
1326 return $self->generate_partition_name($metadata_value, $partition_name_length, $is_numeric);
1327 }
1328
1329 $is_split = 0 unless defined $is_split;
1330 # print STDERR "is split = $is_split\n";
1331 if ($partition_name_length) {
1332 # print STDERR "partiton anme lenght = $partition_name_length, just using that length\n";
1333 return substr($metadata_value, 0, $partition_name_length);
1334 }
1335
1336 my $variable_partition_name_length = (1+$is_split <= $max_partition_name_length ? 1+$is_split : $max_partition_name_length);
1337 my $partitionstart = substr($metadata_value, 0, $variable_partition_name_length);
1338 # print STDERR "new start = $partitionstart, $variable_partition_name_length\n";
1339 while(($partitionstart le $lastpartitionend) && (($variable_partition_name_length += 1) <= $max_partition_name_length)) {
1340 # print STDERR "NEW###### $variable_partition_name_length\n";
1341 $partitionstart = substr($metadata_value, 0, $variable_partition_name_length);
1342 }
1343
1344 return $partitionstart;
1345}
1346
1347
1348sub generate_partition_end
1349{
1350 my $self = shift(@_);
1351 my $metadata_value = shift(@_);
1352 my $partitionstart = shift(@_);
1353 my $partition_name_length = shift(@_);
1354 my $max_partition_name_length = shift(@_);
1355 my $is_numeric = shift(@_);
1356 #print STDERR "geenrate end, $metadata_value: $partitionstart: $partition_name_length: $max_partition_name_length\n";
1357 if ($is_numeric) {
1358 return $self->generate_partition_name($metadata_value, $partition_name_length, $is_numeric);
1359 }
1360 if ($partition_name_length) {
1361 return substr($metadata_value, 0, $partition_name_length);
1362 }
1363
1364 my $partitionend = substr($metadata_value, 0, length($partitionstart));
1365 #if ($partitionend gt $partitionstart) {
1366# $partitionend = substr($metadata_value, 0, 1);
1367# if ($partitionend le $partitionstart) {
1368# $partitionend = substr($metadata_value, 0, 2);
1369# # Give up after three characters
1370# if ($partitionend le $partitionstart) {
1371# $partitionend = substr($metadata_value, 0, 3);
1372# }
1373# }
1374 # }
1375
1376 return $partitionend;
1377}
1378
1379
1380sub add_hlist_partition
1381{
1382 my $self = shift(@_);
1383 my @metadata_groups = @{shift(@_)};
1384 my $classifier_node = shift(@_);
1385 my $partitionname = shift(@_);
1386 my $metadata_value_to_OIDs_hash_ref = shift(@_);
1387
1388 # Create an hlist partition
1389 # Note that we don't need to convert from unicode-aware strings
1390 # to utf8 here, as that is handled elsewhere in the code
1391 my %child_classifier_node = ( 'Title' => $partitionname, #'Title' => $self->convert_unicode_string_to_utf8_string($partitionname),
1392 'childtype' => "VList",
1393 'contains' => [] );
1394
1395 # Add the children to the hlist partition
1396 $self->add_vlist(\@metadata_groups, \%child_classifier_node, $metadata_value_to_OIDs_hash_ref);
1397 push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
1398}
1399
1400
1401sub add_vlist
1402{
1403 my $self = shift(@_);
1404 my @metadata_groups = @{shift(@_)};
1405 my $classifier_node = shift(@_);
1406 my $metadata_value_to_OIDs_hash_ref = shift(@_);
1407 my $metadata_group = shift(@metadata_groups);
1408 $classifier_node->{'mdtype'} = $metadata_group;
1409
1410 my $sort_type = $self->{$metadata_group .".metadata_sort_mode_within_level"};
1411 # Create an entry in the vlist for each value
1412 foreach my $metadata_value ($self->sort_metadata_values_array($sort_type, keys(%{$metadata_value_to_OIDs_hash_ref})))
1413 {
1414 my @OIDs = @{$metadata_value_to_OIDs_hash_ref->{$metadata_value}};
1415
1416 # If there is only one item and 'bookshelf_type' is not always (ie. never or duplicate_only), add the item to the list
1417 if (@OIDs == 1 && $self->{$metadata_group . ".bookshelf_type"} ne "always") {
1418 my $OID = $OIDs[0];
1419 my $offset = $self->metadata_offset($metadata_group, $OID, $metadata_value);
1420 push(@{$classifier_node->{'contains'}}, { 'OID' => $OID, 'offset' => $offset });
1421 }
1422 # If 'bookshelf_type' is 'never', list all the items even if there are duplicated values
1423 elsif ($self->{$metadata_group . ".bookshelf_type"} eq "never") {
1424
1425 @OIDs = $self->sort_leaf_items(\@OIDs);
1426 foreach my $OID (@OIDs) {
1427 my $offset = $self->metadata_offset($metadata_group, $OID, $metadata_value);
1428 push(@{$classifier_node->{'contains'}}, { 'OID' => $OID , 'offset' => $offset });
1429 }
1430
1431 }
1432 # Otherwise create a sublist (bookshelf) for the metadata value
1433 else {
1434 my $metadata_value_display = $self->get_metadata_value_display($metadata_group, $metadata_value);
1435 my %child_classifier_node = ( 'Title' => $metadata_value_display,
1436 'childtype' => "VList",
1437 'mdtype' => $metadata_group,
1438 'contains' => [] );
1439
1440 # If there are metadata elements remaining, recursively apply the process
1441 if (@metadata_groups > 0) {
1442 my $next_metadata_group = $metadata_groups[0];
1443
1444 # separate metadata into those that belong in the next/sub-metadata_group
1445 # and those that belong at the current level's metadata_group
1446
1447 my $OID_to_metadata_values_hash_ref = $self->{$next_metadata_group . ".list"};
1448 my @current_level_OIDs = ();
1449 my @next_level_OIDs = ();
1450 foreach my $OID (@OIDs)
1451 {
1452 if ($OID_to_metadata_values_hash_ref->{$OID}) {
1453 push(@next_level_OIDs, $OID);
1454 } else {
1455 push(@current_level_OIDs, $OID);
1456 }
1457 }
1458 # recursively process those docs belonging to the sub-metadata_group
1459 $self->add_level(\@metadata_groups, \@next_level_OIDs, \%child_classifier_node);
1460
1461 # For those docs that don't belong in the sub/next_metadata_group, but which belong
1462 # at this level, just add the documents as children of this list at the current level
1463 @current_level_OIDs = $self->sort_leaf_items(\@current_level_OIDs);
1464 foreach my $current_level_OID (@current_level_OIDs) {
1465 my $offset = $self->metadata_offset($metadata_group, $current_level_OID, $metadata_value);
1466 push(@{$child_classifier_node{'contains'}}, { 'OID' => $current_level_OID , 'offset' => $offset });
1467 }
1468 }
1469 # Otherwise just add the documents as children of this list
1470 else {
1471 @OIDs = $self->sort_leaf_items(\@OIDs);
1472 foreach my $OID (@OIDs) {
1473 my $offset = $self->metadata_offset($metadata_group, $OID, $metadata_value);
1474 push(@{$child_classifier_node{'contains'}}, { 'OID' => $OID , 'offset' => $offset });
1475 }
1476
1477 }
1478
1479 # Add the sublist to the list
1480 push(@{$classifier_node->{'contains'}}, \%child_classifier_node);
1481 }
1482 }
1483}
1484
1485sub metadata_offset
1486{
1487 my $self = shift(@_);
1488 my $metadata_group = shift(@_);
1489 my $OID = shift(@_);
1490 my $metadata_value = shift(@_);
1491
1492 my $OID_to_metadata_values_hash_ref = $self->{$metadata_group . ".list"};
1493 my @metadata_values = @{$OID_to_metadata_values_hash_ref->{$OID}};
1494 for (my $i = 0; $i < scalar(@metadata_values); $i++) {
1495 if ($metadata_value eq $metadata_values[$i]) {
1496 return $i;
1497 }
1498 }
1499
1500 return 0;
1501}
1502
1503sub sort_leaf_items
1504{
1505 my $self = shift(@_);
1506 my @OIDs = @{shift(@_)};
1507
1508 # Sort leaf nodes and add to list
1509 my @sort_leaf_nodes_using_metadata_groups = @{$self->{'sort_leaf_nodes_using_metadata_groups'}};
1510 foreach my $sort_leaf_nodes_usingmetaelem (reverse @sort_leaf_nodes_using_metadata_groups) {
1511 my $OID_to_metadata_values_hash_ref = $self->{$sort_leaf_nodes_usingmetaelem . ".list"};
1512 my $sort_type = $self->{$sort_leaf_nodes_usingmetaelem . ".metadata_sort_mode_within_level"};
1513 # Force a stable sort (Perl 5.6's sort isn't stable)
1514 # !! The [0] bits aren't ideal (multiple metadata values) !!
1515 @OIDs = @OIDs[ sort {
1516 if (defined($OID_to_metadata_values_hash_ref->{$OIDs[$a]}) && defined($OID_to_metadata_values_hash_ref->{$OIDs[$b]}))
1517 {
1518 if ($sort_type eq "numeric") {
1519 $OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0] <=> $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0];
1520 } elsif ($sort_type eq "alphabetic") {
1521 $OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0] cmp $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0];
1522 } else {
1523 ncmp($OID_to_metadata_values_hash_ref->{$OIDs[$a]}[0], $OID_to_metadata_values_hash_ref->{$OIDs[$b]}[0]);
1524 }
1525 }
1526 else
1527 {
1528 $a <=> $b;
1529 }
1530 } 0..$#OIDs ];
1531 }
1532 if ($self->{'reverse_sort_leaf_nodes'}) {
1533 return reverse @OIDs;
1534 }
1535 return @OIDs;
1536}
1537
1538
1539sub sort_metadata_values_array
1540{
1541 my $self = shift(@_);
1542 my ($sort_mode) = shift(@_);
1543 my @metadata_values = @_;
1544
1545 if ($sort_mode eq "unicode") {
1546 if ($self->{'unicode_collator'}) {
1547 return $self->{'unicode_collator'}->sort(@metadata_values);
1548 }
1549 # the collator wasn't loaded, fall back on default
1550 $sort_mode = "alphanumeric";
1551 }
1552 if ($sort_mode eq "numeric") {
1553 return sort {$a <=> $b} @metadata_values;
1554 }
1555 if ($sort_mode eq "alphabetic") {
1556 return sort {$a cmp $b} @metadata_values;
1557 }
1558 # natural sort
1559 return nsort(@metadata_values);
1560}
1561
1562
1563# we are not using this any more. Using nsort instead
1564# $a and $b args automatically passed in and shouldn't be declared
1565sub alpha_numeric_cmp
1566{
1567 my $self = shift (@_);
1568 my ($aStr, $bStr) = @_;
1569 if ($aStr =~ m/^(\d+(\.\d+)?)/)
1570 {
1571 my $val_a = $1;
1572 if ($bStr =~ m/^(\d+(\.\d+)?)/)
1573 {
1574 my $val_b = $1;
1575 if ($val_a != $val_b)
1576 {
1577 return ($val_a <=> $val_b);
1578 }
1579 }
1580 }
1581
1582 return ($aStr cmp $bStr);
1583}
1584
1585
1586sub get_metadata_value_display {
1587 my $self = shift(@_);
1588 my ($metadata_group, $metadata_value) = @_;
1589 return $metadata_value if $self->{'use_formatted_metadata_for_bookshelf_display'};
1590 my $actual_values_hash = $self->{$metadata_group . ".actualvalues"}->{$metadata_value};
1591 my $display_value ="";
1592 my $max_count=0;
1593 foreach my $v (keys %$actual_values_hash) {
1594 if ($actual_values_hash->{$v} > $max_count) {
1595 $display_value = $v;
1596 $max_count = $actual_values_hash->{$v};
1597 }
1598 }
1599 return $display_value;
1600}
16011;
Note: See TracBrowser for help on using the repository browser.