| 1 | # This document processor outputs a document
|
|---|
| 2 | # for mg to process
|
|---|
| 3 |
|
|---|
| 4 |
|
|---|
| 5 | package mgbuildproc;
|
|---|
| 6 |
|
|---|
| 7 | use classify;
|
|---|
| 8 | use doc;
|
|---|
| 9 | use docproc;
|
|---|
| 10 | use util;
|
|---|
| 11 |
|
|---|
| 12 |
|
|---|
| 13 | BEGIN {
|
|---|
| 14 | @ISA = ('docproc');
|
|---|
| 15 | }
|
|---|
| 16 |
|
|---|
| 17 |
|
|---|
| 18 | sub new {
|
|---|
| 19 | my ($class, $collection, $source_dir, $build_dir, $verbosity) = @_;
|
|---|
| 20 | my $self = new docproc ();
|
|---|
| 21 |
|
|---|
| 22 | $self->{'collection'} = $collection;
|
|---|
| 23 | $self->{'source_dir'} = $source_dir;
|
|---|
| 24 | $self->{'build_dir'} = $build_dir;
|
|---|
| 25 | $self->{'verbosity'} = $verbosity;
|
|---|
| 26 | $self->{'classifiers'} = [];
|
|---|
| 27 | $self->{'mode'} = "text";
|
|---|
| 28 | $self->{'index'} = "section:text";
|
|---|
| 29 | $self->{'indexexparr'} = [];
|
|---|
| 30 | $self->{'output_handle'} = "STDOUT";
|
|---|
| 31 | $self->{'num_docs'} = 0;
|
|---|
| 32 | $self->{'num_sections'} = 0;
|
|---|
| 33 | $self->{'num_bytes'} = 0;
|
|---|
| 34 |
|
|---|
| 35 | return bless $self, $class;
|
|---|
| 36 | }
|
|---|
| 37 |
|
|---|
| 38 | sub reset {
|
|---|
| 39 | my $self = shift (@_);
|
|---|
| 40 |
|
|---|
| 41 | $self->{'num_docs'} = 0;
|
|---|
| 42 | $self->{'num_sections'} = 0;
|
|---|
| 43 | $self->{'num_bytes'} = 0;
|
|---|
| 44 | }
|
|---|
| 45 |
|
|---|
| 46 | sub get_num_docs {
|
|---|
| 47 | my $self = shift (@_);
|
|---|
| 48 |
|
|---|
| 49 | return $self->{'num_docs'};
|
|---|
| 50 | }
|
|---|
| 51 |
|
|---|
| 52 | sub get_num_sections {
|
|---|
| 53 | my $self = shift (@_);
|
|---|
| 54 |
|
|---|
| 55 | return $self->{'num_sections'};
|
|---|
| 56 | }
|
|---|
| 57 |
|
|---|
| 58 | sub get_num_bytes {
|
|---|
| 59 | my $self = shift (@_);
|
|---|
| 60 |
|
|---|
| 61 | return $self->{'num_bytes'};
|
|---|
| 62 | }
|
|---|
| 63 |
|
|---|
| 64 | sub set_output_handle {
|
|---|
| 65 | my $self = shift (@_);
|
|---|
| 66 | my ($handle) = @_;
|
|---|
| 67 |
|
|---|
| 68 | $self->{'output_handle'} = $handle;
|
|---|
| 69 | }
|
|---|
| 70 |
|
|---|
| 71 | sub set_mode {
|
|---|
| 72 | my $self = shift (@_);
|
|---|
| 73 | my ($mode) = @_;
|
|---|
| 74 |
|
|---|
| 75 | $self->{'mode'} = $mode;
|
|---|
| 76 | }
|
|---|
| 77 |
|
|---|
| 78 | sub set_index {
|
|---|
| 79 | my $self = shift (@_);
|
|---|
| 80 | my ($index, $indexexparr) = @_;
|
|---|
| 81 |
|
|---|
| 82 | $self->{'index'} = $index;
|
|---|
| 83 | $self->{'indexexparr'} = $indexexparr if defined $indexexparr;
|
|---|
| 84 | }
|
|---|
| 85 |
|
|---|
| 86 | sub set_classifiers {
|
|---|
| 87 | my $self = shift (@_);
|
|---|
| 88 | my ($classifiers) = @_;
|
|---|
| 89 |
|
|---|
| 90 | $self->{'classifiers'} = $classifiers;
|
|---|
| 91 | }
|
|---|
| 92 |
|
|---|
| 93 | sub process {
|
|---|
| 94 | my $self = shift (@_);
|
|---|
| 95 | my $method = $self->{'mode'};
|
|---|
| 96 |
|
|---|
| 97 | $self->$method(@_);
|
|---|
| 98 | }
|
|---|
| 99 |
|
|---|
| 100 | sub newinfodb {
|
|---|
| 101 | my $self = shift (@_);
|
|---|
| 102 | my ($doc_obj, $filename) = @_;
|
|---|
| 103 | my $handle = $self->{'output_handle'};
|
|---|
| 104 | # $handle = "main::STDOUT";
|
|---|
| 105 |
|
|---|
| 106 | # this was used in the old version
|
|---|
| 107 | return if ($doc_obj eq 'classifications');
|
|---|
| 108 |
|
|---|
| 109 | my $doctype = $doc_obj->get_doc_type();
|
|---|
| 110 |
|
|---|
| 111 | # only output this document if it is one to be indexed
|
|---|
| 112 | return if ($doctype ne "indexed_doc");
|
|---|
| 113 |
|
|---|
| 114 | # this is another document
|
|---|
| 115 | $self->{'num_docs'} += 1 unless ($doctype eq "classification");
|
|---|
| 116 |
|
|---|
| 117 | my $section = $doc_obj->get_top_section ();
|
|---|
| 118 | my $doc_OID = $doc_obj->get_OID();
|
|---|
| 119 | while (defined $section) {
|
|---|
| 120 | # update a few statistics
|
|---|
| 121 | $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
|
|---|
| 122 | $self->{'num_sections'} += 1 unless ($doctype eq "classification");
|
|---|
| 123 |
|
|---|
| 124 | # output the section name
|
|---|
| 125 | if ($section eq "") { print $handle "[$doc_OID]\n"; }
|
|---|
| 126 | else { print $handle "[$doc_OID.$section]\n"; }
|
|---|
| 127 |
|
|---|
| 128 | # output the fact that this document is a document
|
|---|
| 129 | print $handle "<doctype>doc\n";
|
|---|
| 130 |
|
|---|
| 131 | # output whether this node contains text
|
|---|
| 132 | if ($doc_obj->get_text_length($section) > 0) {
|
|---|
| 133 | print $handle "<hastxt>1\n";
|
|---|
| 134 | } else {
|
|---|
| 135 | print $handle "<hastxt>0\n";
|
|---|
| 136 | }
|
|---|
| 137 |
|
|---|
| 138 | # output all the section metadata
|
|---|
| 139 | my $metadata = $doc_obj->get_all_metadata ($section);
|
|---|
| 140 | foreach $pair (@$metadata) {
|
|---|
| 141 | my ($field, $value) = (@$pair);
|
|---|
| 142 |
|
|---|
| 143 | if ($field ne "Identifier" && $field !~ /^gsdl/ &&
|
|---|
| 144 | defined $value && $value ne "") {
|
|---|
| 145 | # escape problematic stuff
|
|---|
| 146 | $value =~ s/\\/\\\\/g;
|
|---|
| 147 | $value =~ s/\n/\\n/g;
|
|---|
| 148 | $value =~ s/\r/\\r/g;
|
|---|
| 149 |
|
|---|
| 150 | print $handle "<$field>$value\n";
|
|---|
| 151 | }
|
|---|
| 152 | }
|
|---|
| 153 |
|
|---|
| 154 | # output a list of children
|
|---|
| 155 | my $children = $doc_obj->get_children ();
|
|---|
| 156 | if (scalar(@$children) > 0) {
|
|---|
| 157 | print $handle "<contains>";
|
|---|
| 158 | my $firstchild = 1;
|
|---|
| 159 | foreach $child (@$children) {
|
|---|
| 160 | print $handle ";" unless $firstchild;
|
|---|
| 161 | $firstchild = 0;
|
|---|
| 162 | if ($child eq "") { print $handle "$doc_OID"; }
|
|---|
| 163 | else { print $handle "$doc_OID.$child"; }
|
|---|
| 164 | }
|
|---|
| 165 | print $handle "\n";
|
|---|
| 166 | }
|
|---|
| 167 |
|
|---|
| 168 | # output the matching document number
|
|---|
| 169 | print $handle "<docnum>$self->{'num_sections'}\n";
|
|---|
| 170 |
|
|---|
| 171 | print $handle '-' x 70, "\n";
|
|---|
| 172 |
|
|---|
| 173 |
|
|---|
| 174 | # output a database entry for the document number
|
|---|
| 175 | print $handle "[$self->{'num_sections'}]\n";
|
|---|
| 176 | if ($section eq "") { print $handle "<section>$doc_OID\n"; }
|
|---|
| 177 | else { print $handle "<section>$doc_OID.$section\n"; }
|
|---|
| 178 | print $handle '-' x 70, "\n";
|
|---|
| 179 |
|
|---|
| 180 |
|
|---|
| 181 | $section = $doc_obj->get_next_section($section);
|
|---|
| 182 | }
|
|---|
| 183 |
|
|---|
| 184 | # classify this document
|
|---|
| 185 | &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
|
|---|
| 186 |
|
|---|
| 187 | }
|
|---|
| 188 |
|
|---|
| 189 |
|
|---|
| 190 |
|
|---|
| 191 | sub infodb {
|
|---|
| 192 | my $self = shift (@_);
|
|---|
| 193 | my ($doc_obj, $filename) = @_;
|
|---|
| 194 | my $handle = $self->{'output_handle'};
|
|---|
| 195 |
|
|---|
| 196 | if ($doc_obj eq 'classifications') {
|
|---|
| 197 | # output classifications if all books have been processed
|
|---|
| 198 | foreach $key (keys %$saved_classifications) {
|
|---|
| 199 | $saved_classifications->{$key}->{'contains'} = undef if
|
|---|
| 200 | $saved_classifications->{$key}->{'contains'} eq "";
|
|---|
| 201 | $saved_classifications->{$key}->{'parent'} = undef if
|
|---|
| 202 | (!defined $saved_classifications->{$key}->{'parent'}) ||
|
|---|
| 203 | ($saved_classifications->{$key}->{'parent'} eq "");
|
|---|
| 204 | $self->write_to_gdbm ($handle, $key, $saved_classifications->{$key}->{'title'},
|
|---|
| 205 | undef, undef, undef, undef,
|
|---|
| 206 | $saved_classifications->{$key}->{'contains'}, undef,
|
|---|
| 207 | $saved_classifications->{$key}->{'parent'}, undef, undef);
|
|---|
| 208 | }
|
|---|
| 209 | return;
|
|---|
| 210 | }
|
|---|
| 211 |
|
|---|
| 212 | my $doctype = $doc_obj->get_doc_type();
|
|---|
| 213 |
|
|---|
| 214 | # only output this document if it is one to be indexed
|
|---|
| 215 | return if ($doctype ne "indexed_doc" &&
|
|---|
| 216 | $doctype ne "classification");
|
|---|
| 217 |
|
|---|
| 218 | # found classification document
|
|---|
| 219 | $saved_classifications = {} if ($doctype eq "classification");
|
|---|
| 220 | $sectionmap = {};
|
|---|
| 221 |
|
|---|
| 222 | # this is another document
|
|---|
| 223 | $self->{'num_docs'} += 1 unless ($doctype eq "classification");
|
|---|
| 224 |
|
|---|
| 225 | my $section = $doc_obj->get_top_section();
|
|---|
| 226 | while (defined $section) {
|
|---|
| 227 | # update a few statistics
|
|---|
| 228 | $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
|
|---|
| 229 | $self->{'num_sections'} += 1 unless ($doctype eq "classification");
|
|---|
| 230 |
|
|---|
| 231 | my $title = $doc_obj->get_metadata_element($section, "Title");
|
|---|
| 232 | my $source = $doc_obj->get_metadata_element($section, "Source");
|
|---|
| 233 | my $date = $doc_obj->get_metadata_element($section, "Date");
|
|---|
| 234 | my $jobnumber = $doc_obj->get_source_filename();
|
|---|
| 235 |
|
|---|
| 236 | my $mapped_section = $self->map_section($doctype, $section);
|
|---|
| 237 | if ($doctype eq "classification") {
|
|---|
| 238 | $mapped_section = $self->char_classification($mapped_section);
|
|---|
| 239 | } else {
|
|---|
| 240 | $mapped_section = "B.$self->{'num_docs'}.$mapped_section";
|
|---|
| 241 | }
|
|---|
| 242 | $mapped_section =~ s/\.+$//;
|
|---|
| 243 |
|
|---|
| 244 | my ($parent, $classification, $creator);
|
|---|
| 245 |
|
|---|
| 246 | $classification = $self->get_classifications($doc_obj, $section, $mapped_section)
|
|---|
| 247 | unless $doctype eq "classification";
|
|---|
| 248 |
|
|---|
| 249 | if ($section ne $doc_obj->get_top_section()) {
|
|---|
| 250 | $parent = $self->map_section($doctype, $doc_obj->get_parent_section($section));
|
|---|
| 251 | if ($doctype eq "classification") {
|
|---|
| 252 | $parent = $self->char_classification($parent);
|
|---|
| 253 | } else {
|
|---|
| 254 | $parent = "B.$self->{'num_docs'}.$parent";
|
|---|
| 255 | }
|
|---|
| 256 | $parent =~ s/\.+$//;
|
|---|
| 257 | } else {
|
|---|
| 258 | $creator = $doc_obj->get_metadata_element($section, "Creator");
|
|---|
| 259 |
|
|---|
| 260 | # need filename so we know what directory to look in for associated files
|
|---|
| 261 | $filename =~ s/^\/?(.*?\.dir).*$/$1/ if (defined $filename);
|
|---|
| 262 | }
|
|---|
| 263 |
|
|---|
| 264 | if ($doc_obj->is_leaf_section($section)) {
|
|---|
| 265 | if ($doctype eq "classification") {
|
|---|
| 266 | if (defined $saved_classifications->{$mapped_section}) {
|
|---|
| 267 | print STDERR "mgbuildproc:warning: classification $mapped_section " .
|
|---|
| 268 | "declared multiple times\n";
|
|---|
| 269 | } else {
|
|---|
| 270 | $saved_classifications->{$mapped_section} = {'title' => $title, 'contains' => "",
|
|---|
| 271 | 'parent' => $parent};
|
|---|
| 272 | }
|
|---|
| 273 | } else {
|
|---|
| 274 | $self->write_to_gdbm($handle, $mapped_section, $title, $creator, $source, $date, $jobnumber, undef,
|
|---|
| 275 | $self->{'num_sections'}, $parent, $classification, $filename);
|
|---|
| 276 | }
|
|---|
| 277 | } else {
|
|---|
| 278 |
|
|---|
| 279 | # add the introductory section if it exists
|
|---|
| 280 | my $contains = "";
|
|---|
| 281 | if ($doc_obj->get_text_length($section) > 0) {
|
|---|
| 282 | $contains .= "B.$self->{'num_docs'}." . $self->map_section ($doctype, "$section.0");
|
|---|
| 283 | }
|
|---|
| 284 |
|
|---|
| 285 | # add the rest of the children
|
|---|
| 286 | my @children = @{$doc_obj->get_children($section)};
|
|---|
| 287 | if ($doctype eq "classification") {
|
|---|
| 288 | map {$_ = $self->char_classification($_);} @children;
|
|---|
| 289 | if (defined $saved_classifications->{$mapped_section}) {
|
|---|
| 290 | print STDERR "mgbuildproc:warning: classification $mapped_section " .
|
|---|
| 291 | "declared multiple times\n";
|
|---|
| 292 | } else {
|
|---|
| 293 | $saved_classifications->{$mapped_section} = {'title' => $title, 'contains' => ""};
|
|---|
| 294 | $saved_classifications->{$mapped_section}->{'contains'} = join ":", @children;
|
|---|
| 295 | $saved_classifications->{$mapped_section}->{'parent'} = $parent;
|
|---|
| 296 | }
|
|---|
| 297 | } else {
|
|---|
| 298 | map {$_ = "B.$self->{'num_docs'}." . $self->map_section($doctype, $_);} @children;
|
|---|
| 299 | $contains .= ":" if $contains ne "";
|
|---|
| 300 | $contains .= join ":", @children;
|
|---|
| 301 | $self->write_to_gdbm ($handle, $mapped_section, $title, $creator, $source,
|
|---|
| 302 | $date, $jobnumber, $contains, $self->{'num_sections'},
|
|---|
| 303 | $parent, $classification, $filename);
|
|---|
| 304 |
|
|---|
| 305 | if ($doc_obj->get_text_length($section) > 0) {
|
|---|
| 306 | my $intromapsection = "B.$self->{'num_docs'}." .
|
|---|
| 307 | $self->map_section($doctype, "$section.0");
|
|---|
| 308 | $self->write_to_gdbm ($handle, $intromapsection, "<i>(introductory text)</i>", $creator,
|
|---|
| 309 | $source, $date, $jobnumber, undef, $self->{'num_sections'},
|
|---|
| 310 | $mapped_section, $classification, $filename);
|
|---|
| 311 | }
|
|---|
| 312 | }
|
|---|
| 313 | }
|
|---|
| 314 | $section = $doc_obj->get_next_section($section);
|
|---|
| 315 | }
|
|---|
| 316 |
|
|---|
| 317 | # update classification list with those books that
|
|---|
| 318 | # were processed before classification list
|
|---|
| 319 | if ($doctype eq "classification" && defined $temp_classifications) {
|
|---|
| 320 | foreach $key (keys(%$temp_classifications)) {
|
|---|
| 321 | if (!defined $saved_classifications->{$key}) {
|
|---|
| 322 | print STDERR "mgbuildproc:$temp_classifications->{$key} belong to " .
|
|---|
| 323 | "undefined classification $key\n";
|
|---|
| 324 | }
|
|---|
| 325 | $saved_classifications->{$key}->{'contains'} = $temp_classifications->{$key}->{'contains'};
|
|---|
| 326 | }
|
|---|
| 327 | $temp_classifications = undef;
|
|---|
| 328 | }
|
|---|
| 329 | }
|
|---|
| 330 |
|
|---|
| 331 |
|
|---|
| 332 | sub write_to_gdbm {
|
|---|
| 333 | my $self = shift (@_);
|
|---|
| 334 | my ($handle, $section, $title, $creator, $source, $date,
|
|---|
| 335 | $jobnumber, $contains, $docnum, $parent, $classification, $OID) = @_;
|
|---|
| 336 |
|
|---|
| 337 | print $handle "[$section]\n";
|
|---|
| 338 | print $handle "<t>$title\n" if (defined $title && $title ne "");
|
|---|
| 339 | print $handle "<a>$creator\n" if (defined $creator && $creator ne "");
|
|---|
| 340 | print $handle "<s>$source\n" if (defined $source && $source ne "");
|
|---|
| 341 | print $handle "<i>$date\n" if (defined $date && $date ne "");
|
|---|
| 342 | print $handle "<j>$jobnumber\n" if (defined $jobnumber && $jobnumber ne "");
|
|---|
| 343 | print $handle "<c>$contains\n" if (defined $contains && $contains ne "");
|
|---|
| 344 | print $handle "<d>$docnum\n" if (defined $docnum && $docnum ne "");
|
|---|
| 345 | print $handle "<p>$parent\n" if (defined $parent && $parent ne "");
|
|---|
| 346 | print $handle "<x>$classification\n" if (defined $classification && $classification ne "");
|
|---|
| 347 | print $handle "<o>$OID\n" if defined $OID;
|
|---|
| 348 | print $handle '-' x 70, "\n";
|
|---|
| 349 |
|
|---|
| 350 | if (defined $docnum) {
|
|---|
| 351 | print $handle "[$docnum]\n";
|
|---|
| 352 | print $handle "<x>$section\n";
|
|---|
| 353 | print $handle '-' x 70, "\n";
|
|---|
| 354 | }
|
|---|
| 355 | }
|
|---|
| 356 |
|
|---|
| 357 | sub get_classifications {
|
|---|
| 358 | my $self = shift (@_);
|
|---|
| 359 | my ($doc_obj, $section, $mapped_section) = @_;
|
|---|
| 360 |
|
|---|
| 361 | my ($classificationsref);
|
|---|
| 362 | if (defined $saved_classifications) {
|
|---|
| 363 | # classification list has been processed
|
|---|
| 364 | $classificationsref = $saved_classifications;
|
|---|
| 365 | } else {
|
|---|
| 366 | # classification list has yet to be processed, save
|
|---|
| 367 | # books in temp_classifications until list is processed
|
|---|
| 368 | # and they can be moved into saved_classifications
|
|---|
| 369 | $temp_classifications = {} unless defined $temp_classifications;
|
|---|
| 370 | $classificationsref = $temp_classifications;
|
|---|
| 371 | }
|
|---|
| 372 |
|
|---|
| 373 | my $classifications = $doc_obj->get_metadata($section, "Subject");
|
|---|
| 374 |
|
|---|
| 375 | # need to save which books belong in each classification
|
|---|
| 376 | # to output later
|
|---|
| 377 | foreach $classification (@$classifications) {
|
|---|
| 378 |
|
|---|
| 379 | if (!defined $classificationsref->{$classification}) {
|
|---|
| 380 | $classificationsref->{$classification}->{'parent'} = $doc_obj->get_parent_section($section);
|
|---|
| 381 | $classificationsref->{$classification}->{'contains'} = "";
|
|---|
| 382 | }
|
|---|
| 383 | $classificationsref->{$classification}->{'contains'} .= ":" unless
|
|---|
| 384 | $classificationsref->{$classification}->{'contains'} eq "";
|
|---|
| 385 | $classificationsref->{$classification}->{'contains'} .= $mapped_section;
|
|---|
| 386 | }
|
|---|
| 387 |
|
|---|
| 388 | return (join ":", @$classifications);
|
|---|
| 389 | }
|
|---|
| 390 |
|
|---|
| 391 | sub map_section {
|
|---|
| 392 | my $self = shift (@_);
|
|---|
| 393 | my ($doctype, $section) = @_;
|
|---|
| 394 |
|
|---|
| 395 | # classifications should never need to be mapped
|
|---|
| 396 | return $section if $doctype eq "classification";
|
|---|
| 397 |
|
|---|
| 398 | return "" unless (defined $section) && ($section ne "");
|
|---|
| 399 |
|
|---|
| 400 | $sectionmap = {} unless defined $sectionmap;
|
|---|
| 401 |
|
|---|
| 402 |
|
|---|
| 403 |
|
|---|
| 404 | # get the section into a standard format
|
|---|
| 405 | $section =~ s/^\.+|\.+$//g;
|
|---|
| 406 |
|
|---|
| 407 | # return the mapped section if it has been seen before
|
|---|
| 408 | if (defined $sectionmap->{$section}) {
|
|---|
| 409 | return $sectionmap->{$section};
|
|---|
| 410 | }
|
|---|
| 411 |
|
|---|
| 412 | # find out the parent section
|
|---|
| 413 | my ($parentsection, $num);
|
|---|
| 414 | if ($section =~ /^(.+)\.(\d+)$/) {
|
|---|
| 415 | $parentsection = $1;
|
|---|
| 416 | $num = $2;
|
|---|
| 417 | } elsif ($section =~ /^(\d+)$/) {
|
|---|
| 418 | $parentsection = "";
|
|---|
| 419 | $num = $1;
|
|---|
| 420 | } else {
|
|---|
| 421 | print STDERR "mgbuildproc:map_section - misformed section $section\n";
|
|---|
| 422 | }
|
|---|
| 423 | $parentsection = "" unless defined $parentsection;
|
|---|
| 424 |
|
|---|
| 425 | if ($parentsection eq "") {
|
|---|
| 426 | $num ++ if $num == 1 and defined $sectionmap->{'0'};
|
|---|
| 427 | } else {
|
|---|
| 428 | $num ++ if $num == 1 and defined $sectionmap->{"$parentsection.0"};
|
|---|
| 429 | }
|
|---|
| 430 |
|
|---|
| 431 | # find out the mapped parent section
|
|---|
| 432 | my $mappedparentsection = $self->map_section ($doctype, $parentsection);
|
|---|
| 433 |
|
|---|
| 434 | # find the next unused child section
|
|---|
| 435 | my $previousnum = $num - 1;
|
|---|
| 436 | my $previoussection = "";
|
|---|
| 437 | if ($parentsection eq "") {
|
|---|
| 438 | $previoussection = $previousnum;
|
|---|
| 439 | $previoussection = 0 if ($section == 1) && (defined $sectionmap->{'0'});
|
|---|
| 440 | } else{
|
|---|
| 441 | $previoussection = "$parentsection.$previousnum";
|
|---|
| 442 | $previoussection = "$parentsection.0" if ($section =~ /\.1$/) && (defined $sectionmap->{"$parentsection.0"});
|
|---|
| 443 | }
|
|---|
| 444 | while (($previousnum > 0) && (!defined $sectionmap->{$previoussection})) {
|
|---|
| 445 | $previousnum--;
|
|---|
| 446 | $previoussection = "$parentsection.$previousnum";
|
|---|
| 447 | $previoussection = $previousnum if $parentsection eq "";
|
|---|
| 448 | }
|
|---|
| 449 |
|
|---|
| 450 | # there has been no children under this parent, this section will be number 1
|
|---|
| 451 | if ($previousnum <= 0) {
|
|---|
| 452 | if ($mappedparentsection eq "") {
|
|---|
| 453 | $sectionmap->{$section} = "1";
|
|---|
| 454 | } else {
|
|---|
| 455 | $sectionmap->{$section} = "$mappedparentsection.1";
|
|---|
| 456 | }
|
|---|
| 457 |
|
|---|
| 458 | } else {
|
|---|
| 459 | # get the previous mapped number
|
|---|
| 460 | my $previousmapnum = 0;
|
|---|
| 461 | if ($sectionmap->{$previoussection} =~ /(^|\.)?(\d+)$/) {
|
|---|
| 462 | $previousmapnum = $2;
|
|---|
| 463 | }
|
|---|
| 464 |
|
|---|
| 465 | # increment it to get this mapped child
|
|---|
| 466 | my $mappednum = $previousmapnum+1;
|
|---|
| 467 |
|
|---|
| 468 | if ($mappedparentsection eq "") {
|
|---|
| 469 | $sectionmap->{$section} = $mappednum;
|
|---|
| 470 | } else {
|
|---|
| 471 | $sectionmap->{$section} = "$mappedparentsection.$mappednum";
|
|---|
| 472 | }
|
|---|
| 473 | }
|
|---|
| 474 |
|
|---|
| 475 | return $sectionmap->{$section};
|
|---|
| 476 | }
|
|---|
| 477 |
|
|---|
| 478 | sub text {
|
|---|
| 479 | my $self = shift (@_);
|
|---|
| 480 | my ($doc_obj) = @_;
|
|---|
| 481 | my $handle = $self->{'output_handle'};
|
|---|
| 482 | my $indexed_doc = 1;
|
|---|
| 483 |
|
|---|
| 484 | # only output this document if it is one to be indexed
|
|---|
| 485 | return if ($doc_obj->get_doc_type() ne "indexed_doc");
|
|---|
| 486 |
|
|---|
| 487 | # see if this document belongs to this subcollection
|
|---|
| 488 | foreach $indexexp (@{$self->{'indexexparr'}}) {
|
|---|
| 489 | $indexed_doc = 0;
|
|---|
| 490 | my ($field, $exp, $options) = split /\//, $indexexp;
|
|---|
| 491 | if (defined ($field) && defined ($exp)) {
|
|---|
| 492 | my ($bool) = $field =~ /^(.)/;
|
|---|
| 493 | $field =~ s/^.// if $bool eq '!';
|
|---|
| 494 | if ($field eq "filename") {
|
|---|
| 495 | $field = $doc_obj->get_source_filename();
|
|---|
| 496 | } else {
|
|---|
| 497 | $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
|
|---|
| 498 | }
|
|---|
| 499 | next unless defined $field;
|
|---|
| 500 | if ($bool eq '!') {
|
|---|
| 501 | if ($options =~ /^i$/i) {
|
|---|
| 502 | if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
|
|---|
| 503 | } else {
|
|---|
| 504 | if ($field !~ /$exp/) {$indexed_doc = 1; last;}
|
|---|
| 505 | }
|
|---|
| 506 | } else {
|
|---|
| 507 | if ($options =~ /^i$/i) {
|
|---|
| 508 | if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
|
|---|
| 509 | } else {
|
|---|
| 510 | if ($field =~ /$exp/) {$indexed_doc = 1; last;}
|
|---|
| 511 | }
|
|---|
| 512 | }
|
|---|
| 513 | }
|
|---|
| 514 | }
|
|---|
| 515 |
|
|---|
| 516 | # this is another document
|
|---|
| 517 | $self->{'num_docs'} += 1;
|
|---|
| 518 |
|
|---|
| 519 | # get the parameters for the output
|
|---|
| 520 | my ($level, $fields) = split (/:/, $self->{'index'});
|
|---|
| 521 | $fields =~ s/\ball\b/Title,Creator,text/;
|
|---|
| 522 | $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;
|
|---|
| 523 |
|
|---|
| 524 | my $doc_section = 0; # just for this document
|
|---|
| 525 | my $text = "";
|
|---|
| 526 | my $text_extra = "";
|
|---|
| 527 |
|
|---|
| 528 | # get the text for this document
|
|---|
| 529 | my $section = $doc_obj->get_top_section();
|
|---|
| 530 | while (defined $section) {
|
|---|
| 531 | # update a few statistics
|
|---|
| 532 | $doc_section++;
|
|---|
| 533 | $self->{'num_sections'} += 1;
|
|---|
| 534 |
|
|---|
| 535 | if ($indexed_doc) {
|
|---|
| 536 | $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
|
|---|
| 537 | foreach $field (split (/,/, $fields)) {
|
|---|
| 538 | # only deal with this field if it doesn't start with top or
|
|---|
| 539 | # this is the first section
|
|---|
| 540 | my $real_field = $field;
|
|---|
| 541 | if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
|
|---|
| 542 | my $new_text = "";
|
|---|
| 543 | if ($real_field eq "text") {
|
|---|
| 544 | $new_text = $doc_obj->get_text ($section);
|
|---|
| 545 | $new_text =~ s/[\cB\cC]//g;
|
|---|
| 546 | $new_text =~ s/(<p\b)/\cC$1/gi;
|
|---|
| 547 |
|
|---|
| 548 | } else {
|
|---|
| 549 | $new_text = join ("\cC", @{$doc_obj->get_metadata ($section, $real_field)});
|
|---|
| 550 | }
|
|---|
| 551 |
|
|---|
| 552 | $text .= "$new_text\cC";
|
|---|
| 553 | }
|
|---|
| 554 | }
|
|---|
| 555 | }
|
|---|
| 556 |
|
|---|
| 557 | if ($level eq "document") { $text_extra .= "\cB"; }
|
|---|
| 558 | else { $text .= "\cB"; }
|
|---|
| 559 |
|
|---|
| 560 | $section = $doc_obj->get_next_section($section);
|
|---|
| 561 | }
|
|---|
| 562 |
|
|---|
| 563 | print $handle "$text$text_extra";
|
|---|
| 564 | }
|
|---|
| 565 |
|
|---|
| 566 | # converts leading number in classification back
|
|---|
| 567 | # to letter it represents
|
|---|
| 568 | # i.e 67.2.4 becomes C.2.4
|
|---|
| 569 | sub char_classification {
|
|---|
| 570 | my $self = shift (@_);
|
|---|
| 571 | my ($classification) = @_;
|
|---|
| 572 |
|
|---|
| 573 | return $classification if $classification eq "";
|
|---|
| 574 | my ($c) = $classification =~ /^\.?(\d+)/;
|
|---|
| 575 | $c = chr($c);
|
|---|
| 576 | $classification =~ s/^\d+/$c/;
|
|---|
| 577 |
|
|---|
| 578 | return $classification;
|
|---|
| 579 | }
|
|---|
| 580 |
|
|---|
| 581 | # converts leading letter of a classification into its ascii equivalent
|
|---|
| 582 | # i.e C.2.4 becomes 67.2.4
|
|---|
| 583 | sub int_classification {
|
|---|
| 584 | my $self = shift (@_);
|
|---|
| 585 | my ($classification) = @_;
|
|---|
| 586 | my $c = ord($classification);
|
|---|
| 587 | $classification =~ s/^./$c/;
|
|---|
| 588 |
|
|---|
| 589 | return $classification;
|
|---|
| 590 | }
|
|---|
| 591 | 1;
|
|---|
| 592 |
|
|---|
