root/gs3-extensions/solr/trunk/src/perllib/solrbuildproc.pm @ 32441

Revision 32441, 21.0 KB (checked in by litvinovg, 12 months ago)

Added create_shortname method to Solr as inherited method creates duplicate index field names (needed in case of using more than one analyzer on one field)

Line 
1###########################################################################
2#
3# solrbuildproc.pm -- perl wrapper for building index with Solr
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package solrbuildproc;
27
28# This document processor outputs a document for solr to process
29
30# Rather then use the XML structure developed for mgppbuilder/mgppbuildproc
31# whose use was then extended to Lucene, Solr has its own XML syntax:
32#
33#  http://wiki.apache.org/solr/UpdateXmlMessages
34#
35# Using this means we don't need to write SolrWrapper.jar, as had to be
36# done for Lucene, translating the XML syntax piped to it into appropriate
37# calls to the Lucene API
38
39
40use lucenebuildproc;
41use ghtml;
42use strict;
43no strict 'refs'; # allow filehandles to be variables and viceversa
44
45
46use IncrementalBuildUtils;
47
48sub BEGIN {
49    @solrbuildproc::ISA = ('lucenebuildproc');
50}
51
52
53sub new {
54    my $class = shift @_;
55    my $self = new lucenebuildproc (@_);
56
57    return bless $self, $class;
58}
59
60sub set_facetfields {
61    my $self = shift (@_);
62 
63    my ($facetfields) = @_;
64    $self->{'facetfields'} = ();
65    # lets just go through and check for text, allfields, metadata which are only valid for indexes, not for facetfields
66    foreach my $s (@$facetfields) {
67    if ($s !~ /^(text|allfields|metadata)$/) {
68        push (@{$self->{'facetfields'}}, $s);
69    }
70    }
71}
72
73#----
74
75sub index_field_mapping_edit {
76    my $self = shift (@_);
77    my ($doc_obj,$file,$edit_mode) = @_;
78
79    # Only add/update gets to here
80    # Currently there is no need to distinguish between these edit modes
81
82    my $outhandle = $self->{'outhandle'};
83
84    # only study this document if it is one to be indexed
85    return if ($doc_obj->get_doc_type() ne "indexed_doc");
86
87    my $indexed_doc = $self->is_subcollection_doc($doc_obj);
88
89    # get the parameters for the output
90    # split on : just in case there is subcoll and lang stuff
91    my ($fields) = split (/:/, $self->{'index'});
92
93    my $doc_section = 0; # just for this document
94
95    # get the text for this document
96    my $section = $doc_obj->get_top_section();
97
98    while (defined $section)
99    {
100    $doc_section++;
101
102    # if we are doing subcollections, then some docs shouldn't be
103    # considered for indexing
104
105    my $indexed_section
106        = $doc_obj->get_metadata_element($section, "gsdldoctype")
107          || "indexed_section";
108
109    if (($indexed_doc == 0)
110        || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
111            $section = $doc_obj->get_next_section($section);
112        next;
113          }
114
115    # has the user added a 'metadata' index?
116    my $all_metadata_specified = 0;
117
118    # which fields have already been indexed?
119    # (same as fields, but in a map)
120    my $specified_fields = {};
121   
122    # do we have an allfields index??
123    my $allfields_index = 0;
124
125    # collect up all the text for it in here
126    my $allfields_text = "";
127
128    foreach my $field (split (/;/, $fields)) {
129        if ($field eq "allfields") {
130        $allfields_index = 1;
131        } elsif ($field eq "metadata") {
132        $all_metadata_specified = 1;
133        }
134    }
135   
136    foreach my $field (split (/;/, $fields)) {
137       
138        # only deal with this field if it doesn't start with top or
139        # this is the first section
140        my $real_field = $field;
141        next if (($real_field =~ s/^top//) && ($doc_section != 1));
142       
143        # process these two later
144        next if ($real_field eq "allfields" || $real_field eq "metadata");
145       
146        # individual metadata and or text specified
147        # -- could be a comma separated list
148        $specified_fields->{$real_field} = 1;
149
150        if (!defined $self->{'indexfieldmap'}->{$real_field}) {
151        my $shortname = $self->create_shortname($real_field);
152        $self->{'indexfieldmap'}->{$real_field} = $shortname;
153        $self->{'indexfieldmap'}->{$shortname} = 1;
154        }       
155    } # foreach field
156
157
158    if ($all_metadata_specified) {
159       
160        my $new_text = "";
161        my $shortname = "";
162        my $metadata = $doc_obj->get_all_metadata ($section);
163
164        foreach my $pair (@$metadata) {
165        my ($mfield, $mvalue) = (@$pair);
166
167        # no value
168        next unless defined $mvalue && $mvalue ne "";
169
170        # we have already indexed this
171        next if defined ($specified_fields->{$mfield});
172
173        # check fields here, maybe others dont want - change to use dontindex!!
174        next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
175        next if ($mfield =~ /^gsdl/);
176       
177        if (defined $self->{'indexfieldmap'}->{$mfield}) {
178            $shortname = $self->{'indexfieldmap'}->{$mfield};
179        }
180        else {
181            $shortname = $self->create_shortname($mfield);
182            $self->{'indexfieldmap'}->{$mfield} = $shortname;
183            $self->{'indexfieldmap'}->{$shortname} = 1;
184        }     
185
186        if (!defined $self->{'indexfields'}->{$mfield}) {
187            $self->{'indexfields'}->{$mfield} = 1;
188        }                           
189        }
190    }
191
192    if ($allfields_index) {
193        # add the index name mapping
194        $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
195        $self->{'indexfieldmap'}->{"ZZ"} = 1;               
196    }
197       
198        $section = $doc_obj->get_next_section($section);
199
200    } # while defined section
201
202   
203}
204sub create_shortname {
205    my $self = shift(@_);
206
207    my ($realname) = @_;
208    my @realnamelist = split(",", $realname);
209    map {$_=~ s/^[a-zA-Z]+\.//;} @realnamelist; #remove namespaces
210    my ($singlename) = $realnamelist[0];
211
212    # try our predefined static mapping
213    my $name;
214    # we can't use the quick map, so join all fields back together (without namespaces), and try sets of two characters.
215    $realname = join ("", @realnamelist);
216    #try the first two chars
217    my $shortname;
218    if ($realname =~ /^[^\w]*(\w)[^\w]*(\w)/) {
219        $shortname = "$1$2";
220    } else {
221        # there aren't two letdig's in the field - try arbitrary combinations
222        $realname = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
223        $shortname = "AB";
224    }
225    $shortname =~ tr/a-z/A-Z/;
226
227    #if already used, take the first and third letdigs and so on
228    my $count = 1;
229    while (defined $self->{'indexfieldmap'}->{$shortname}) {
230        if ($realname =~ /^[^\w]*(\w)([^\w]*\w){$count}[^\w]*(\w)/) {
231            $shortname = "$1$3";
232            $count++;
233            $shortname =~ tr/a-z/A-Z/;
234
235        }
236        else {
237            #remove up to and incl the first letdig
238            $realname =~ s/^[^\w]*\w//;
239            $count = 0;
240        }
241    }
242
243    return $shortname ;
244}
245
246
247sub index_field_mapping {
248    my $self = shift (@_);
249    my ($doc_obj,$file) = @_;
250
251    $self->index_field_mapping_edit($doc_obj,$file,"add");
252}
253
254sub index_field_mappingreindex
255{
256    my $self = shift (@_);
257    my ($doc_obj,$file) = @_;
258
259    $self->index_field_mapping_edit($doc_obj,$file,"update");
260}
261
262sub index_field_mappingdelete
263{
264    my $self = shift (@_);
265    my ($doc_obj,$file) = @_;
266
267    return; # nothing to be done
268}
269
270
271#----
272
273sub textedit {
274    my $self = shift (@_);
275    my ($doc_obj,$file,$edit_mode) = @_;
276
277
278    if (!$self->get_indexing_text()) {
279    # In text-compress mode:
280    # => want document to be output in the simple <Doc>..</Doc> as is
281    # done by its super-class
282    return $self->SUPER::textedit(@_);
283    }
284
285    # "update" for $edit_mode near identical to "add" as we use Solr in its
286    # default mode of replacing an existing document if the new document
287    # has the same doc id.  Main area of difference between "add" and "update"
288    # is that we do not update our 'stats' for number of documents or number
289    # of bytes processed.  The latter is inaccurate, but considered better
290    # than allowing the value to steadily climb.
291
292
293    my $solrhandle = $self->{'output_handle'};
294    my $outhandle = $self->{'outhandle'};
295
296    # only output this document if it is one to be indexed
297    return if ($doc_obj->get_doc_type() ne "indexed_doc");
298
299    # skip this document if in "compress-text" mode and asked to delete it
300    return if (!$self->get_indexing_text() && ($edit_mode eq "delete"));
301
302    my $indexed_doc = $self->is_subcollection_doc($doc_obj);
303
304    # this is another document
305    if ($edit_mode eq "add") {
306    $self->{'num_docs'} += 1;
307    }
308    elsif ($edit_mode eq "delete") {
309    $self->{'num_docs'} -= 1;
310    }
311
312    # get the parameters for the output
313    # split on : just in case there is subcoll and lang stuff
314    my ($fields) = split (/:/, $self->{'index'});
315
316    my $levels = $self->{'levels'};
317    my $ldoc_level = $levels->{'document'};
318    my $lsec_level = $levels->{'section'};
319
320    my $gs2_docOID = $doc_obj->get_OID();
321
322    my $start_doc;
323    my $end_doc;
324
325    if ($edit_mode eq "add") {
326    $start_doc  = "  <add>\n";
327    $start_doc .= "    <doc>\n";
328    $start_doc .= "      <field name=\"docOID\">$gs2_docOID</field>\n";
329   
330    $end_doc    = "    </doc>\n";
331    $end_doc   .= "  </add>\n";
332    }
333    else {
334    $start_doc  = "  <delete>\n";
335    $start_doc .= "    <id>$gs2_docOID</id>\n";
336
337    $end_doc    = "  </delete>\n";
338
339    # for delete mode, we need to specify just the docOID to delete and we're done
340    my $text = $start_doc;
341    $text .= $end_doc;
342    print $solrhandle $text;
343    return;
344    }
345
346    # add/update, delete
347
348    my $sec_tag_name = "";
349    if ($lsec_level)
350    {
351    $sec_tag_name = $mgppbuildproc::level_map{'section'};
352    }
353
354    my $doc_section = 0; # just for this document
355
356    # only output if working with doc level
357    # my $text = undef;
358   
359    my $text = ($sec_tag_name eq "") ? $start_doc : "";
360
361#     my $text = $start_doc if ($sec_tag_name eq "");
362     
363    # get the text for this document
364    my $section = $doc_obj->get_top_section();
365
366    while (defined $section)
367    {
368    # update a few statistics
369    $doc_section++;
370    $self->{'num_sections'}++;
371
372    my $sec_gs2_id = $self->{'num_sections'};
373    my $sec_gs2_docOID = $gs2_docOID;
374    $sec_gs2_docOID .= ".$section" if ($section ne "");
375   
376    my $start_sec;
377    my $end_sec;
378
379    if ($edit_mode eq "add") {
380        $start_sec  = "  <add>\n";
381        $start_sec .= "    <doc>\n";
382        $start_sec .= "      <field name=\"docOID\">$sec_gs2_docOID</field>\n";
383   
384        $end_sec    = "    </doc>\n";
385        $end_sec   .= "  </add>\n";
386    }
387    else {
388        $start_sec  = "  <delete>\n";
389        $start_sec .= "    <id>$sec_gs2_docOID</id>\n";
390
391        $end_sec    = "  </delete>\n";
392
393        # for delete mode, should specify only this section's docOID to delete, then move on to the next section
394        my $text = $start_sec;
395        $text .= $end_sec;
396        print $solrhandle $text;
397        $section = $doc_obj->get_next_section($section);
398        next;
399    }
400
401
402    # if we are doing subcollections, then some docs shouldn't be indexed.
403    # but we need to put the section tag placeholders in there so the
404    # sections match up with database
405    my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
406    if (($indexed_doc == 0) || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
407        if ($sec_tag_name ne "") {
408        $text .= $start_sec;
409        $text .= $end_sec;
410        }
411            $section = $doc_obj->get_next_section($section);
412        next;
413          }
414
415    # add in start section tag if indexing at the section level
416    $text .= $start_sec if ($sec_tag_name ne "");
417
418    if ($edit_mode eq "add") {
419        $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
420    }
421    elsif ($edit_mode eq "delete") {
422        $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
423    }
424
425
426    # has the user added a 'metadata' index?
427    my $all_metadata_specified = 0;
428    # which fields have already been indexed? (same as fields, but in a map)
429    my $specified_fields = {};
430   
431    # do we have an allfields index??
432    my $allfields_index = 0;
433    # collect up all the text for it in here
434    my $allfields_text = "";
435    foreach my $field (split (/;/, $fields)) {
436        if ($field eq "allfields") {
437        $allfields_index = 1;
438        } elsif ($field eq "metadata") {
439        $all_metadata_specified = 1;
440        }
441    }
442   
443    foreach my $field (split (/;/, $fields)) {
444       
445        # only deal with this field if it doesn't start with top or
446        # this is the first section
447        my $real_field = $field;
448        next if (($real_field =~ s/^top//) && ($doc_section != 1));
449       
450        # process these two later
451        next if ($real_field eq "allfields" || $real_field eq "metadata");
452       
453        #individual metadata and or text specified - could be a comma separated list
454        $specified_fields->{$real_field} = 1;
455        my $shortname="";
456        my $new_field = 0; # have we found a new field name?
457        if (defined $self->{'indexfieldmap'}->{$real_field}) {
458        $shortname = $self->{'indexfieldmap'}->{$real_field};
459        }
460        else {
461        $shortname = $self->create_shortname($real_field);
462        $new_field = 1;
463        }
464
465        my @metadata_list = (); # put any metadata values in here
466        my $section_text = ""; # put the text in here
467        foreach my $submeta (split /,/, $real_field) {
468        if ($submeta eq "text") {
469            # no point in indexing text more than once
470            if ($section_text eq "") {
471            $section_text = $doc_obj->get_text($section);
472            if ($self->{'indexing_text'}) {
473                # we always strip html
474                &ghtml::htmlsafe($section_text);
475                #$section_text = $self->preprocess_text($section_text, 1, "");
476            }
477            else {
478                # leave html stuff in, but escape the tags
479                &ghtml::htmlsafe($section_text);
480            }
481            }
482        }
483        else {
484            $submeta =~ s/^ex\.//; #strip off ex.
485
486            # its a metadata element
487            my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
488            if ($section ne $doc_obj->get_top_section() && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
489            if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
490                push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
491            }
492            }
493            push (@metadata_list, @section_metadata);
494        }
495        } # for each field in this one index
496       
497        # now we add the text and/or metadata into new_text
498        if ($section_text ne "" || scalar(@metadata_list)) {
499        my $new_text = "";
500       
501        if ($section_text ne "") {
502            $new_text .= "$section_text ";
503        }
504       
505        foreach my $item (@metadata_list) {
506            &ghtml::htmlsafe($item);
507            $new_text .= "$item ";
508        }
509
510        if ($allfields_index) {
511            $allfields_text .= $new_text;
512        }
513
514        # Remove any leading or trailing white space
515        $new_text =~ s/\s+$//;
516        $new_text =~ s/^\s+//;
517   
518       
519        if ($self->{'indexing_text'}) {
520            # add the tag
521            $new_text = "<field name=\"$shortname\" >$new_text</field>\n";
522        }
523        # filter the text
524        $new_text = $self->filter_text ($field, $new_text);
525
526        if ($edit_mode eq "add") {
527            $self->{'num_processed_bytes'} += length ($new_text);
528            $text .= "$new_text";
529        }
530        elsif ($edit_mode eq "update") {
531            $text .= "$new_text";
532        }
533        elsif ($edit_mode eq "delete") {
534            $self->{'num_processed_bytes'} -= length ($new_text);
535        }
536       
537
538        if ($self->{'indexing_text'} && $new_field) {
539            # we need to add to the list in indexfields
540           
541            $self->{'indexfieldmap'}->{$real_field} = $shortname;
542            $self->{'indexfieldmap'}->{$shortname} = 1;
543        }
544       
545        }
546       
547    } # foreach field
548
549
550    if ($all_metadata_specified) {
551       
552        my $new_text = "";
553        my $shortname = "";
554        my $metadata = $doc_obj->get_all_metadata ($section);
555        foreach my $pair (@$metadata) {
556        my ($mfield, $mvalue) = (@$pair);
557
558        # no value
559        next unless defined $mvalue && $mvalue ne "";
560
561        # we have already indexed this
562        next if defined ($specified_fields->{$mfield});
563
564        # check fields here, maybe others dont want - change to use dontindex!!
565        next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
566        next if ($mfield =~ /^gsdl/);
567       
568        &ghtml::htmlsafe($mvalue);
569       
570        if (defined $self->{'indexfieldmap'}->{$mfield}) {
571            $shortname = $self->{'indexfieldmap'}->{$mfield};
572        }
573        else {
574            $shortname = $self->create_shortname($mfield);
575            $self->{'indexfieldmap'}->{$mfield} = $shortname;
576            $self->{'indexfieldmap'}->{$shortname} = 1;
577        }     
578        $new_text .= "<field name=\"$shortname\">$mvalue</field>\n";
579        if ($allfields_index) {
580            $allfields_text .= "$mvalue ";
581        }
582
583        if (!defined $self->{'indexfields'}->{$mfield}) {
584            $self->{'indexfields'}->{$mfield} = 1;
585        }                   
586       
587        }
588        # filter the text
589        $new_text = $self->filter_text ("metadata", $new_text);
590       
591        if ($edit_mode eq "add") {
592        $self->{'num_processed_bytes'} += length ($new_text);
593        $text .= "$new_text";
594        }
595        elsif ($edit_mode eq "update") {
596        $text .= "$new_text";
597        }
598        elsif ($edit_mode eq "delete") {
599        $self->{'num_processed_bytes'} -= length ($new_text);
600        }       
601    }
602
603    if ($allfields_index) {
604        # add the index name mapping
605        $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
606        $self->{'indexfieldmap'}->{"ZZ"} = 1;
607       
608        my $new_text = "<field name=\"ZZ\">$allfields_text</field>\n";
609        # filter the text
610        $new_text = $self->filter_text ("allfields", $new_text);
611       
612        if ($edit_mode eq "add") {
613        $self->{'num_processed_bytes'} += length ($new_text);
614        $text .= "$new_text";
615        }
616        elsif ($edit_mode eq "update") {
617        $text .= "$new_text";
618        }
619        elsif ($edit_mode eq "delete") {
620        $self->{'num_processed_bytes'} -= length ($new_text);
621        }
622    }
623       
624    # only add sort fields for this section if we are indexing this section, we are doing section level indexing or this is the top section
625    if ($self->{'indexing_text'} && ($sec_tag_name ne "" || $doc_section == 1 )) {
626    # add sort fields if there are any
627        my $seenfields = {};
628    foreach my $sfield (@{$self->{'sortfields'}}, @{$self->{'facetfields'}}) {
629        # ignore special field rank/none
630        next if $sfield eq "rank" || $sfield eq "none";
631        # ignore any we have already done - we may have duplicates in the sort and facet lists
632        next if (defined $seenfields->{$sfield});
633        $seenfields->{$sfield} = 1;
634        my $sf_shortname;
635        if (defined $self->{'sortfieldnamemap'}->{$sfield}) {
636        $sf_shortname = $self->{'sortfieldnamemap'}->{$sfield};
637        }
638        else {
639        $sf_shortname = $self->create_sortfield_shortname($sfield);
640        $self->{'sortfieldnamemap'}->{$sfield} = $sf_shortname;
641        $self->{'sortfieldnamemap'}->{$sf_shortname} = 1;
642        }
643        my @metadata_list = (); # put any metadata values in here
644        foreach my $submeta (split /,/, $sfield) {
645        $submeta =~ s/^ex\.([^.]+)$/$1/; #strip off ex. iff it's the only metadata set prefix (will leave ex.dc.* intact)
646       
647        my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
648            if ($section ne $doc_obj->get_top_section() && defined ($self->{'sections_sort_on_document_metadata'})) {
649            if ($self->{'sections_sort_on_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_sort_on_document_metadata'} eq "unless_section_metadata_exists")) {
650                push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
651            }
652            }
653        push (@metadata_list, @section_metadata);
654        }
655        # my $new_text = "";
656        # foreach my $item (@metadata_list) {
657        #   &ghtml::htmlsafe($item);
658        #   $new_text .= "$item ";
659        # }
660        # if ($new_text =~ /\S/) {
661        #   $new_text = "<field name=\"$sf_shortname\">$new_text</field>\n";
662        #   # filter the text???
663        #   $text .= "$new_text"; # add it to the main text block
664        #   print "#### new_text: $new_text\n";
665
666        #   $self->{'actualsortfields'}->{$sfield} = 1;
667        # }
668        # print "#### TEXT: $text\n";
669
670        foreach my $item (@metadata_list) {
671        &ghtml::htmlsafe($item);
672       
673        $item = "<field name=\"$sf_shortname\">$item</field>\n";
674        # filter the text???
675        $text .= "$item"; # add it to the main text block
676        #print "#### new_text: $item\n";
677        }
678        if(scalar @metadata_list > 0) {
679            $self->{'actualsortfields'}->{$sfield} = 1;
680        }
681       
682    }
683    }
684
685    # add in end tag if at top-level doc root, or indexing at the section level
686    $text .= $end_sec if ($sec_tag_name ne "");
687
688        $section = $doc_obj->get_next_section($section);
689    } # while defined section
690
691   
692    # only output if working with doc level
693    $text .= $end_doc if ($sec_tag_name eq "");
694
695##    $text .= "<commit/>\n";
696
697# The following code looks like it's for debugging purposes, but
698# committed by accident.  Commenting out for now ...
699
700#    open(TEXTOUT, '>:utf8', "text.out");
701#    print TEXTOUT "$text";
702#    close TEXTOUT;
703
704    print $solrhandle $text;
705
706}
707
708
709
710
711sub textreindex
712{
713    my $self = shift (@_);
714    my ($doc_obj,$file) = @_;
715
716    # the update command does not exist in solrbuildproc
717    # reindexing consists of deleting and then adding the same file
718    #$self->textedit($doc_obj,$file,"update");
719    $self->textedit($doc_obj,$file,"delete");
720    $self->textedit($doc_obj,$file,"add");
721}
722
723
7241;
725
726
Note: See TracBrowser for help on using the browser.