source: gs3-extensions/solr/trunk/src/perllib/solrbuildproc.pm@ 32441

Last change on this file since 32441 was 32441, checked in by Georgiy Litvinov, 3 years ago

Added create_shortname method to Solr as inherited method creates duplicate index field names (needed in case of using more than one analyzer on one field)

File size: 21.0 KB
Line 
1###########################################################################
2#
3# solrbuildproc.pm -- perl wrapper for building index with Solr
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package solrbuildproc;
27
28# This document processor outputs a document for solr to process
29
30# Rather then use the XML structure developed for mgppbuilder/mgppbuildproc
31# whose use was then extended to Lucene, Solr has its own XML syntax:
32#
33# http://wiki.apache.org/solr/UpdateXmlMessages
34#
35# Using this means we don't need to write SolrWrapper.jar, as had to be
36# done for Lucene, translating the XML syntax piped to it into appropriate
37# calls to the Lucene API
38
39
40use lucenebuildproc;
41use ghtml;
42use strict;
43no strict 'refs'; # allow filehandles to be variables and viceversa
44
45
46use IncrementalBuildUtils;
47
48sub BEGIN {
49 @solrbuildproc::ISA = ('lucenebuildproc');
50}
51
52
53sub new {
54 my $class = shift @_;
55 my $self = new lucenebuildproc (@_);
56
57 return bless $self, $class;
58}
59
60sub set_facetfields {
61 my $self = shift (@_);
62
63 my ($facetfields) = @_;
64 $self->{'facetfields'} = ();
65 # lets just go through and check for text, allfields, metadata which are only valid for indexes, not for facetfields
66 foreach my $s (@$facetfields) {
67 if ($s !~ /^(text|allfields|metadata)$/) {
68 push (@{$self->{'facetfields'}}, $s);
69 }
70 }
71}
72
73#----
74
75sub index_field_mapping_edit {
76 my $self = shift (@_);
77 my ($doc_obj,$file,$edit_mode) = @_;
78
79 # Only add/update gets to here
80 # Currently there is no need to distinguish between these edit modes
81
82 my $outhandle = $self->{'outhandle'};
83
84 # only study this document if it is one to be indexed
85 return if ($doc_obj->get_doc_type() ne "indexed_doc");
86
87 my $indexed_doc = $self->is_subcollection_doc($doc_obj);
88
89 # get the parameters for the output
90 # split on : just in case there is subcoll and lang stuff
91 my ($fields) = split (/:/, $self->{'index'});
92
93 my $doc_section = 0; # just for this document
94
95 # get the text for this document
96 my $section = $doc_obj->get_top_section();
97
98 while (defined $section)
99 {
100 $doc_section++;
101
102 # if we are doing subcollections, then some docs shouldn't be
103 # considered for indexing
104
105 my $indexed_section
106 = $doc_obj->get_metadata_element($section, "gsdldoctype")
107 || "indexed_section";
108
109 if (($indexed_doc == 0)
110 || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
111 $section = $doc_obj->get_next_section($section);
112 next;
113 }
114
115 # has the user added a 'metadata' index?
116 my $all_metadata_specified = 0;
117
118 # which fields have already been indexed?
119 # (same as fields, but in a map)
120 my $specified_fields = {};
121
122 # do we have an allfields index??
123 my $allfields_index = 0;
124
125 # collect up all the text for it in here
126 my $allfields_text = "";
127
128 foreach my $field (split (/;/, $fields)) {
129 if ($field eq "allfields") {
130 $allfields_index = 1;
131 } elsif ($field eq "metadata") {
132 $all_metadata_specified = 1;
133 }
134 }
135
136 foreach my $field (split (/;/, $fields)) {
137
138 # only deal with this field if it doesn't start with top or
139 # this is the first section
140 my $real_field = $field;
141 next if (($real_field =~ s/^top//) && ($doc_section != 1));
142
143 # process these two later
144 next if ($real_field eq "allfields" || $real_field eq "metadata");
145
146 # individual metadata and or text specified
147 # -- could be a comma separated list
148 $specified_fields->{$real_field} = 1;
149
150 if (!defined $self->{'indexfieldmap'}->{$real_field}) {
151 my $shortname = $self->create_shortname($real_field);
152 $self->{'indexfieldmap'}->{$real_field} = $shortname;
153 $self->{'indexfieldmap'}->{$shortname} = 1;
154 }
155 } # foreach field
156
157
158 if ($all_metadata_specified) {
159
160 my $new_text = "";
161 my $shortname = "";
162 my $metadata = $doc_obj->get_all_metadata ($section);
163
164 foreach my $pair (@$metadata) {
165 my ($mfield, $mvalue) = (@$pair);
166
167 # no value
168 next unless defined $mvalue && $mvalue ne "";
169
170 # we have already indexed this
171 next if defined ($specified_fields->{$mfield});
172
173 # check fields here, maybe others dont want - change to use dontindex!!
174 next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
175 next if ($mfield =~ /^gsdl/);
176
177 if (defined $self->{'indexfieldmap'}->{$mfield}) {
178 $shortname = $self->{'indexfieldmap'}->{$mfield};
179 }
180 else {
181 $shortname = $self->create_shortname($mfield);
182 $self->{'indexfieldmap'}->{$mfield} = $shortname;
183 $self->{'indexfieldmap'}->{$shortname} = 1;
184 }
185
186 if (!defined $self->{'indexfields'}->{$mfield}) {
187 $self->{'indexfields'}->{$mfield} = 1;
188 }
189 }
190 }
191
192 if ($allfields_index) {
193 # add the index name mapping
194 $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
195 $self->{'indexfieldmap'}->{"ZZ"} = 1;
196 }
197
198 $section = $doc_obj->get_next_section($section);
199
200 } # while defined section
201
202
203}
204sub create_shortname {
205 my $self = shift(@_);
206
207 my ($realname) = @_;
208 my @realnamelist = split(",", $realname);
209 map {$_=~ s/^[a-zA-Z]+\.//;} @realnamelist; #remove namespaces
210 my ($singlename) = $realnamelist[0];
211
212 # try our predefined static mapping
213 my $name;
214 # we can't use the quick map, so join all fields back together (without namespaces), and try sets of two characters.
215 $realname = join ("", @realnamelist);
216 #try the first two chars
217 my $shortname;
218 if ($realname =~ /^[^\w]*(\w)[^\w]*(\w)/) {
219 $shortname = "$1$2";
220 } else {
221 # there aren't two letdig's in the field - try arbitrary combinations
222 $realname = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
223 $shortname = "AB";
224 }
225 $shortname =~ tr/a-z/A-Z/;
226
227 #if already used, take the first and third letdigs and so on
228 my $count = 1;
229 while (defined $self->{'indexfieldmap'}->{$shortname}) {
230 if ($realname =~ /^[^\w]*(\w)([^\w]*\w){$count}[^\w]*(\w)/) {
231 $shortname = "$1$3";
232 $count++;
233 $shortname =~ tr/a-z/A-Z/;
234
235 }
236 else {
237 #remove up to and incl the first letdig
238 $realname =~ s/^[^\w]*\w//;
239 $count = 0;
240 }
241 }
242
243 return $shortname ;
244}
245
246
247sub index_field_mapping {
248 my $self = shift (@_);
249 my ($doc_obj,$file) = @_;
250
251 $self->index_field_mapping_edit($doc_obj,$file,"add");
252}
253
254sub index_field_mappingreindex
255{
256 my $self = shift (@_);
257 my ($doc_obj,$file) = @_;
258
259 $self->index_field_mapping_edit($doc_obj,$file,"update");
260}
261
262sub index_field_mappingdelete
263{
264 my $self = shift (@_);
265 my ($doc_obj,$file) = @_;
266
267 return; # nothing to be done
268}
269
270
271#----
272
273sub textedit {
274 my $self = shift (@_);
275 my ($doc_obj,$file,$edit_mode) = @_;
276
277
278 if (!$self->get_indexing_text()) {
279 # In text-compress mode:
280 # => want document to be output in the simple <Doc>..</Doc> as is
281 # done by its super-class
282 return $self->SUPER::textedit(@_);
283 }
284
285 # "update" for $edit_mode near identical to "add" as we use Solr in its
286 # default mode of replacing an existing document if the new document
287 # has the same doc id. Main area of difference between "add" and "update"
288 # is that we do not update our 'stats' for number of documents or number
289 # of bytes processed. The latter is inaccurate, but considered better
290 # than allowing the value to steadily climb.
291
292
293 my $solrhandle = $self->{'output_handle'};
294 my $outhandle = $self->{'outhandle'};
295
296 # only output this document if it is one to be indexed
297 return if ($doc_obj->get_doc_type() ne "indexed_doc");
298
299 # skip this document if in "compress-text" mode and asked to delete it
300 return if (!$self->get_indexing_text() && ($edit_mode eq "delete"));
301
302 my $indexed_doc = $self->is_subcollection_doc($doc_obj);
303
304 # this is another document
305 if ($edit_mode eq "add") {
306 $self->{'num_docs'} += 1;
307 }
308 elsif ($edit_mode eq "delete") {
309 $self->{'num_docs'} -= 1;
310 }
311
312 # get the parameters for the output
313 # split on : just in case there is subcoll and lang stuff
314 my ($fields) = split (/:/, $self->{'index'});
315
316 my $levels = $self->{'levels'};
317 my $ldoc_level = $levels->{'document'};
318 my $lsec_level = $levels->{'section'};
319
320 my $gs2_docOID = $doc_obj->get_OID();
321
322 my $start_doc;
323 my $end_doc;
324
325 if ($edit_mode eq "add") {
326 $start_doc = " <add>\n";
327 $start_doc .= " <doc>\n";
328 $start_doc .= " <field name=\"docOID\">$gs2_docOID</field>\n";
329
330 $end_doc = " </doc>\n";
331 $end_doc .= " </add>\n";
332 }
333 else {
334 $start_doc = " <delete>\n";
335 $start_doc .= " <id>$gs2_docOID</id>\n";
336
337 $end_doc = " </delete>\n";
338
339 # for delete mode, we need to specify just the docOID to delete and we're done
340 my $text = $start_doc;
341 $text .= $end_doc;
342 print $solrhandle $text;
343 return;
344 }
345
346 # add/update, delete
347
348 my $sec_tag_name = "";
349 if ($lsec_level)
350 {
351 $sec_tag_name = $mgppbuildproc::level_map{'section'};
352 }
353
354 my $doc_section = 0; # just for this document
355
356 # only output if working with doc level
357 # my $text = undef;
358
359 my $text = ($sec_tag_name eq "") ? $start_doc : "";
360
361# my $text = $start_doc if ($sec_tag_name eq "");
362
363 # get the text for this document
364 my $section = $doc_obj->get_top_section();
365
366 while (defined $section)
367 {
368 # update a few statistics
369 $doc_section++;
370 $self->{'num_sections'}++;
371
372 my $sec_gs2_id = $self->{'num_sections'};
373 my $sec_gs2_docOID = $gs2_docOID;
374 $sec_gs2_docOID .= ".$section" if ($section ne "");
375
376 my $start_sec;
377 my $end_sec;
378
379 if ($edit_mode eq "add") {
380 $start_sec = " <add>\n";
381 $start_sec .= " <doc>\n";
382 $start_sec .= " <field name=\"docOID\">$sec_gs2_docOID</field>\n";
383
384 $end_sec = " </doc>\n";
385 $end_sec .= " </add>\n";
386 }
387 else {
388 $start_sec = " <delete>\n";
389 $start_sec .= " <id>$sec_gs2_docOID</id>\n";
390
391 $end_sec = " </delete>\n";
392
393 # for delete mode, should specify only this section's docOID to delete, then move on to the next section
394 my $text = $start_sec;
395 $text .= $end_sec;
396 print $solrhandle $text;
397 $section = $doc_obj->get_next_section($section);
398 next;
399 }
400
401
402 # if we are doing subcollections, then some docs shouldn't be indexed.
403 # but we need to put the section tag placeholders in there so the
404 # sections match up with database
405 my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
406 if (($indexed_doc == 0) || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
407 if ($sec_tag_name ne "") {
408 $text .= $start_sec;
409 $text .= $end_sec;
410 }
411 $section = $doc_obj->get_next_section($section);
412 next;
413 }
414
415 # add in start section tag if indexing at the section level
416 $text .= $start_sec if ($sec_tag_name ne "");
417
418 if ($edit_mode eq "add") {
419 $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
420 }
421 elsif ($edit_mode eq "delete") {
422 $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
423 }
424
425
426 # has the user added a 'metadata' index?
427 my $all_metadata_specified = 0;
428 # which fields have already been indexed? (same as fields, but in a map)
429 my $specified_fields = {};
430
431 # do we have an allfields index??
432 my $allfields_index = 0;
433 # collect up all the text for it in here
434 my $allfields_text = "";
435 foreach my $field (split (/;/, $fields)) {
436 if ($field eq "allfields") {
437 $allfields_index = 1;
438 } elsif ($field eq "metadata") {
439 $all_metadata_specified = 1;
440 }
441 }
442
443 foreach my $field (split (/;/, $fields)) {
444
445 # only deal with this field if it doesn't start with top or
446 # this is the first section
447 my $real_field = $field;
448 next if (($real_field =~ s/^top//) && ($doc_section != 1));
449
450 # process these two later
451 next if ($real_field eq "allfields" || $real_field eq "metadata");
452
453 #individual metadata and or text specified - could be a comma separated list
454 $specified_fields->{$real_field} = 1;
455 my $shortname="";
456 my $new_field = 0; # have we found a new field name?
457 if (defined $self->{'indexfieldmap'}->{$real_field}) {
458 $shortname = $self->{'indexfieldmap'}->{$real_field};
459 }
460 else {
461 $shortname = $self->create_shortname($real_field);
462 $new_field = 1;
463 }
464
465 my @metadata_list = (); # put any metadata values in here
466 my $section_text = ""; # put the text in here
467 foreach my $submeta (split /,/, $real_field) {
468 if ($submeta eq "text") {
469 # no point in indexing text more than once
470 if ($section_text eq "") {
471 $section_text = $doc_obj->get_text($section);
472 if ($self->{'indexing_text'}) {
473 # we always strip html
474 &ghtml::htmlsafe($section_text);
475 #$section_text = $self->preprocess_text($section_text, 1, "");
476 }
477 else {
478 # leave html stuff in, but escape the tags
479 &ghtml::htmlsafe($section_text);
480 }
481 }
482 }
483 else {
484 $submeta =~ s/^ex\.//; #strip off ex.
485
486 # its a metadata element
487 my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
488 if ($section ne $doc_obj->get_top_section() && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
489 if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
490 push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
491 }
492 }
493 push (@metadata_list, @section_metadata);
494 }
495 } # for each field in this one index
496
497 # now we add the text and/or metadata into new_text
498 if ($section_text ne "" || scalar(@metadata_list)) {
499 my $new_text = "";
500
501 if ($section_text ne "") {
502 $new_text .= "$section_text ";
503 }
504
505 foreach my $item (@metadata_list) {
506 &ghtml::htmlsafe($item);
507 $new_text .= "$item ";
508 }
509
510 if ($allfields_index) {
511 $allfields_text .= $new_text;
512 }
513
514 # Remove any leading or trailing white space
515 $new_text =~ s/\s+$//;
516 $new_text =~ s/^\s+//;
517
518
519 if ($self->{'indexing_text'}) {
520 # add the tag
521 $new_text = "<field name=\"$shortname\" >$new_text</field>\n";
522 }
523 # filter the text
524 $new_text = $self->filter_text ($field, $new_text);
525
526 if ($edit_mode eq "add") {
527 $self->{'num_processed_bytes'} += length ($new_text);
528 $text .= "$new_text";
529 }
530 elsif ($edit_mode eq "update") {
531 $text .= "$new_text";
532 }
533 elsif ($edit_mode eq "delete") {
534 $self->{'num_processed_bytes'} -= length ($new_text);
535 }
536
537
538 if ($self->{'indexing_text'} && $new_field) {
539 # we need to add to the list in indexfields
540
541 $self->{'indexfieldmap'}->{$real_field} = $shortname;
542 $self->{'indexfieldmap'}->{$shortname} = 1;
543 }
544
545 }
546
547 } # foreach field
548
549
550 if ($all_metadata_specified) {
551
552 my $new_text = "";
553 my $shortname = "";
554 my $metadata = $doc_obj->get_all_metadata ($section);
555 foreach my $pair (@$metadata) {
556 my ($mfield, $mvalue) = (@$pair);
557
558 # no value
559 next unless defined $mvalue && $mvalue ne "";
560
561 # we have already indexed this
562 next if defined ($specified_fields->{$mfield});
563
564 # check fields here, maybe others dont want - change to use dontindex!!
565 next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
566 next if ($mfield =~ /^gsdl/);
567
568 &ghtml::htmlsafe($mvalue);
569
570 if (defined $self->{'indexfieldmap'}->{$mfield}) {
571 $shortname = $self->{'indexfieldmap'}->{$mfield};
572 }
573 else {
574 $shortname = $self->create_shortname($mfield);
575 $self->{'indexfieldmap'}->{$mfield} = $shortname;
576 $self->{'indexfieldmap'}->{$shortname} = 1;
577 }
578 $new_text .= "<field name=\"$shortname\">$mvalue</field>\n";
579 if ($allfields_index) {
580 $allfields_text .= "$mvalue ";
581 }
582
583 if (!defined $self->{'indexfields'}->{$mfield}) {
584 $self->{'indexfields'}->{$mfield} = 1;
585 }
586
587 }
588 # filter the text
589 $new_text = $self->filter_text ("metadata", $new_text);
590
591 if ($edit_mode eq "add") {
592 $self->{'num_processed_bytes'} += length ($new_text);
593 $text .= "$new_text";
594 }
595 elsif ($edit_mode eq "update") {
596 $text .= "$new_text";
597 }
598 elsif ($edit_mode eq "delete") {
599 $self->{'num_processed_bytes'} -= length ($new_text);
600 }
601 }
602
603 if ($allfields_index) {
604 # add the index name mapping
605 $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
606 $self->{'indexfieldmap'}->{"ZZ"} = 1;
607
608 my $new_text = "<field name=\"ZZ\">$allfields_text</field>\n";
609 # filter the text
610 $new_text = $self->filter_text ("allfields", $new_text);
611
612 if ($edit_mode eq "add") {
613 $self->{'num_processed_bytes'} += length ($new_text);
614 $text .= "$new_text";
615 }
616 elsif ($edit_mode eq "update") {
617 $text .= "$new_text";
618 }
619 elsif ($edit_mode eq "delete") {
620 $self->{'num_processed_bytes'} -= length ($new_text);
621 }
622 }
623
624 # only add sort fields for this section if we are indexing this section, we are doing section level indexing or this is the top section
625 if ($self->{'indexing_text'} && ($sec_tag_name ne "" || $doc_section == 1 )) {
626 # add sort fields if there are any
627 my $seenfields = {};
628 foreach my $sfield (@{$self->{'sortfields'}}, @{$self->{'facetfields'}}) {
629 # ignore special field rank/none
630 next if $sfield eq "rank" || $sfield eq "none";
631 # ignore any we have already done - we may have duplicates in the sort and facet lists
632 next if (defined $seenfields->{$sfield});
633 $seenfields->{$sfield} = 1;
634 my $sf_shortname;
635 if (defined $self->{'sortfieldnamemap'}->{$sfield}) {
636 $sf_shortname = $self->{'sortfieldnamemap'}->{$sfield};
637 }
638 else {
639 $sf_shortname = $self->create_sortfield_shortname($sfield);
640 $self->{'sortfieldnamemap'}->{$sfield} = $sf_shortname;
641 $self->{'sortfieldnamemap'}->{$sf_shortname} = 1;
642 }
643 my @metadata_list = (); # put any metadata values in here
644 foreach my $submeta (split /,/, $sfield) {
645 $submeta =~ s/^ex\.([^.]+)$/$1/; #strip off ex. iff it's the only metadata set prefix (will leave ex.dc.* intact)
646
647 my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
648 if ($section ne $doc_obj->get_top_section() && defined ($self->{'sections_sort_on_document_metadata'})) {
649 if ($self->{'sections_sort_on_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_sort_on_document_metadata'} eq "unless_section_metadata_exists")) {
650 push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
651 }
652 }
653 push (@metadata_list, @section_metadata);
654 }
655 # my $new_text = "";
656 # foreach my $item (@metadata_list) {
657 # &ghtml::htmlsafe($item);
658 # $new_text .= "$item ";
659 # }
660 # if ($new_text =~ /\S/) {
661 # $new_text = "<field name=\"$sf_shortname\">$new_text</field>\n";
662 # # filter the text???
663 # $text .= "$new_text"; # add it to the main text block
664 # print "#### new_text: $new_text\n";
665
666 # $self->{'actualsortfields'}->{$sfield} = 1;
667 # }
668 # print "#### TEXT: $text\n";
669
670 foreach my $item (@metadata_list) {
671 &ghtml::htmlsafe($item);
672
673 $item = "<field name=\"$sf_shortname\">$item</field>\n";
674 # filter the text???
675 $text .= "$item"; # add it to the main text block
676 #print "#### new_text: $item\n";
677 }
678 if(scalar @metadata_list > 0) {
679 $self->{'actualsortfields'}->{$sfield} = 1;
680 }
681
682 }
683 }
684
685 # add in end tag if at top-level doc root, or indexing at the section level
686 $text .= $end_sec if ($sec_tag_name ne "");
687
688 $section = $doc_obj->get_next_section($section);
689 } # while defined section
690
691
692 # only output if working with doc level
693 $text .= $end_doc if ($sec_tag_name eq "");
694
695## $text .= "<commit/>\n";
696
697# The following code looks like it's for debugging purposes, but
698# committed by accident. Commenting out for now ...
699
700# open(TEXTOUT, '>:utf8', "text.out");
701# print TEXTOUT "$text";
702# close TEXTOUT;
703
704 print $solrhandle $text;
705
706}
707
708
709
710
711sub textreindex
712{
713 my $self = shift (@_);
714 my ($doc_obj,$file) = @_;
715
716 # the update command does not exist in solrbuildproc
717 # reindexing consists of deleting and then adding the same file
718 #$self->textedit($doc_obj,$file,"update");
719 $self->textedit($doc_obj,$file,"delete");
720 $self->textedit($doc_obj,$file,"add");
721}
722
723
7241;
725
726
Note: See TracBrowser for help on using the repository browser.