Changeset 27358 for main

Timestamp:

2013-05-16T15:30:15+12:00 (11 years ago)

Author:

kjdon

Message:

indexing sortfields separately. tidy up the parsing of the indexes list - only work out shortnames etc once, not every document and every section. Note, need to do this for sort fields too

File:

• main/trunk/greenstone2/perllib/lucenebuildproc.pm (modified) (12 diffs)

main/trunk/greenstone2/perllib/lucenebuildproc.pm

@@ -49 +49 @@
 
     $self->{'numincdocs'} = 0;
-
+    $self->{'specified_fields'} = (); # list of fields actually specified in the index, in a map
+    $self->{'allfields_index'} = 0; # do we need allfields index?
+    $self->{'all_metadata_specified'} = 0; # are we indexing all metadata?
+    $self->{'actualsortfields'} = {}; # sort fields that have actually been used
+    $self->{'sortfieldnamemap'} = {}; # mapping between field name and field shortname, eg dc.Title->byTI
     return bless $self, $class;
 }
 
+sub set_index {
+    my $self = shift (@_);
+    my ($index, $indexexparr) = @_;
+
+    $self->mgppbuildproc::set_index($index, $indexexparr);
+    
+    # just get the list of index fields without any subcoll stuff
+    my ($fields) = split (/:/, $self->{'index'});
+
+    foreach my $field (split (/;/, $fields)) {
+    if ($field eq "allfields") {
+        $self->{'allfields_index'} = 1;
+    } elsif ($field eq "metadata") {
+        $self->{'all_metadata_specified'} = 1;
+    } else {
+        $field =~ s/^top//;
+        $self->{'specified_fields'} ->{$field} = 1;
+    }
+    }   
+}
+
+sub set_sortfields {
+    my $self = shift (@_);
+ 
+    ($self->{'sortfields'}) = @_;
+}
 
 sub is_incremental_capable
@@ -76 +106 @@
     return if (!$self->get_indexing_text() && ($edit_mode eq "delete"));
 
+    # 0/1 to indicate whether this doc is part of the specified subcollection
     my $indexed_doc = $self->is_subcollection_doc($doc_obj);
 
@@ -85 +116 @@
     $self->{'num_docs'} -= 1;
     }
+
 
     # get the parameters for the output
@@ -152 +184 @@
 
 
-    # has the user added a 'metadata' index?
-    my $all_metadata_specified = 0;
-    # which fields have already been indexed? (same as fields, but in a map)
-    my $specified_fields = {};
-    
-    # do we have an allfields index??
-    my $allfields_index = 0;
-    # collect up all the text for it in here
+    # collect up all the text for allfields index in here (if there is one)
     my $allfields_text = "";
-    foreach my $field (split (/;/, $fields)) {
-        if ($field eq "allfields") {
-        $allfields_index = 1;
-        } elsif ($field eq "metadata") {
-        $all_metadata_specified = 1;
-        }
-    }
-    
+
     foreach my $field (split (/;/, $fields)) {
         
@@ -180 +198 @@
         
         #individual metadata and or text specified - could be a comma separated list
-        $specified_fields->{$real_field} = 1;
+        #$specified_fields->{$real_field} = 1;
         my $shortname="";
         my $new_field = 0; # have we found a new field name?
@@ -235 +253 @@
         }
 
-        if ($allfields_index) {
+        if ($self->{'allfields_index'}) {
             $allfields_text .= $new_text;
         }
@@ -259 +277 @@
     } # foreach field
 
-    if ($all_metadata_specified) {
+    if ($self->{'all_metadata_specified'}) {
         
         my $new_text = "";
@@ -269 +287 @@
         next unless defined $mvalue && $mvalue ne "";
         # we have already indexed this
-        next if defined ($specified_fields->{$mfield});
+        next if defined ($self->{'specified_fields'}->{$mfield});
         # check fields here, maybe others dont want - change to use dontindex!!
         next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
@@ -286 +304 @@
         $self->{'allindexfields'}->{$mfield} = 1;
         $new_text .= "<$shortname index=\"1\">$mvalue</$shortname>\n";
-        if ($allfields_index) {
+        if ($self->{'allfields_index'}) {
             $allfields_text .= "$mvalue ";
         }
@@ -308 +326 @@
     }
 
-    if ($allfields_index) {
+    if ($self->{'allfields_index'}) {
         
         my $new_text = "<ZZ index=\"1\">$allfields_text</ZZ>\n";
@@ -323 +341 @@
         }
     }
-        
+    # only add sort fields for this section if we are indexing this section, we are doing section level indexing or this is the top section
+    if ($self->{'indexing_text'} && ($sec_tag_name ne "" || $doc_section == 1 )) {
+    # add sort fields if there are any
+        
+    foreach my $sfield (@{$self->{'sortfields'}}) {
+        my $sf_shortname;
+        if (defined $self->{'sortfieldnamemap'}->{$sfield}) {
+        $sf_shortname = $self->{'sortfieldnamemap'}->{$sfield};
+        }
+        else {
+        $sf_shortname = $self->create_sortfield_shortname($sfield);
+        $self->{'sortfieldnamemap'}->{$sfield} = $sf_shortname;
+        $self->{'sortfieldnamemap'}->{$sf_shortname} = 1;
+        }
+        my @metadata_list = (); # put any metadata values in here
+        foreach my $submeta (split /,/, $sfield) {
+        $submeta =~ s/^ex\.([^.]+)$/$1/; #strip off ex. iff it's the only metadata set prefix (will leave ex.dc.* intact)
+        
+        my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
+        push (@metadata_list, @section_metadata);
+        }
+        my $new_text = "";
+        foreach my $item (@metadata_list) {
+        &ghtml::htmlsafe($item);
+        $new_text .= "$item ";
+        }
+        if ($new_text =~ /\S/) {
+        $new_text = "<$sf_shortname index=\"1\" tokenize=\"0\">$new_text</$sf_shortname>";
+        # filter the text???
+        $text .= "$new_text"; # add it to the main text block
+        $self->{'actualsortfields'}->{$sfield} = 1;
+        }
+    }
+    }
     $text .= "\n</$sec_tag_name>\n" if ($sec_tag_name ne "");
 
         $section = $doc_obj->get_next_section($section);
-    } # while defined section
+    } # for each section
+    
+    #open (TEXTOUT, ">text.out");
+    #print TEXTOUT "$text\n$documentendtag";
+    #close TEXTOUT;
 
     print $lucenehandle "$text\n$documentendtag";
@@ -556 +611 @@
     }
 }
+
+sub create_sortfield_shortname {
+    my $self = shift(@_);
+
+    my ($realname) = @_;
+
+    my $index_shortname;
+    # if we have created a shortname for an index on this field, then use it.
+    if (defined $self->{'fieldnamemap'}->{$realname}) {
+    $index_shortname = $self->{'fieldnamemap'}->{$realname};
+    } else {
+    $index_shortname = $self->create_shortname($realname);
+    }
+    return "by".$index_shortname;
+}
+  
+
 1;