Changeset 27328

Timestamp:

2013-05-14T11:09:58+12:00 (11 years ago)

Author:

kjdon

Message:

changed the way we store the list of fields that has been indexed, and the mapping between index and shortname. They are separated now, to avoid calculating a shortname for a field each time a new document is indexed - previously if there was no value, then the shortname was not remembered as it wasn't indexed, so each new document saw the shortname being calculated again. remove namespaces from meta fields before calculating shortnames, to make them more sensible. eg dc.Title->TI instead of DC.

Location:

main/trunk/greenstone2/perllib

Files:

• mgppbuilder.pm (modified) (4 diffs)
• mgppbuildproc.pm (modified) (7 diffs)

main/trunk/greenstone2/perllib/mgppbuilder.pm

@@ -489 +489 @@
     $self->{'buildproc'}->set_index_languages ($languagemetadata, $langarr) if (defined $language);
     $self->{'buildproc'}->set_indexing_text (1);
-    #$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
     $self->{'buildproc'}->set_levels ($self->{'levels'}); 
     $self->{'buildproc'}->set_db_level ($db_level);   
@@ -637 +636 @@
     # these now come from collection meta. if that is not defined, uses the metadata name
     my $collmeta = "";
-    if (defined $self->{'build_cfg'}->{'indexfields'}) {
-    foreach my $longfield (@{$self->{'build_cfg'}->{'indexfields'}}){
-        my $shortfield = $self->{'buildproc'}->{'indexfieldmap'}->{$longfield};
+    if (defined $self->{'build_cfg'}->{'extraindexfields'}) {
+    foreach my $longfield (@{$self->{'build_cfg'}->{'extraindexfields'}}){
+        my $shortfield = $self->{'buildproc'}->{'fieldnamemap'}->{$longfield};
         next if $shortfield eq 1;
         
@@ -743 +742 @@
     
     #add all fields bit 
-    my $ifm = $self->{'buildproc'}->{'indexfieldmap'};
+    my $fnm = $self->{'buildproc'}->{'fieldnamemap'};
     
     foreach $field (@specifiedfieldorder) {
     if ($field eq "metadata") {
-        foreach my $newfield (keys %{$self->{'buildproc'}->{'indexfields'}}) {
+        foreach my $newfield (keys %{$self->{'buildproc'}->{'extraindexfields'}}) {
         if (!defined $specifiedfields->{$newfield}) {
-            push (@indexfieldmap, "$newfield\-\>$self->{'buildproc'}->{'indexfieldmap'}->{$newfield}");
+            push (@indexfieldmap, "$newfield\-\>$fnm->{$newfield}");
             push (@indexfields, "$newfield");
         }
@@ -762 +761 @@
     } else {
         # we only add in the ones that have been processed
-        if (defined $ifm->{$field}) {
-        push (@indexfieldmap, "$field\-\>$ifm->{$field}");
+        if (defined $self->{'buildproc'}->{'allindexfields'}->{$field}) {
+        push (@indexfieldmap, "$field\-\>$fnm->{$field}");
         push (@indexfields, "$field");
         }

main/trunk/greenstone2/perllib/mgppbuildproc.pm

@@ -90 +90 @@
 
     $self->{'dontindex'} = {};
-    $self->{'indexfieldmap'} = {};
-    $self->{'indexfields'} = {}; # only put in the ones that are not specified directly in the index
+    $self->{'allindexfields'} = {}; # list of all actually indexed fields
+    $self->{'extraindexfields'} = {}; # indexed fields not specfied in original index list - ie if 'metadata' was specified.
+    $self->{'fieldnamemap'} = {'allfields'=>'ZZ',
+              'ZZ'=>1,
+              'text'=>'TX',
+              'TX'=>1}; # mapping between index full names and short names. Once we have decided on a mapping it goes in here, whether we have indexed something or not.
     $self->{'strip_html'}=1;
     
     return bless $self, $class;
-}
-
-
-#sub set_indexfieldmap {
-#    my $self = shift (@_);
-#    my ($indexmap) = @_;
-
-#    $self->{'default_index_field_mapping'} = $indexmap;
-    #$self->{'indexfieldmap'} = $indexmap;
-#}
-
-sub get_indexfieldmap {
-    my $self = shift (@_);
-
-    return $self->{'indexfieldmap'};
 }
 
@@ -279 +268 @@
         my $new_text = ""; 
 
-        # we get allfields by default - do nothing except add into the map
-        if ($real_field eq "allfields") {
-        $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
-        $self->{'indexfieldmap'}->{"ZZ"} = 1;
-        }
+        # we get allfields by default 
+        next if ($real_field eq "allfields"); 
         
         # metadata - output all metadata we know about except gsdl stuff
@@ -290 +276 @@
         # we will process this later, so we are not reindexing metadata already indexed
         $all_metadata_specified = 1;
+        next;
         }
         
-        else {
-        
         #individual metadata and or text specified - could be 
         # a comma separated list
         $specified_fields->{$real_field} = 1;
         my $shortname="";
-        my $new_field = 0; # have we found a new field name?
-
-        if (defined $self->{'indexfieldmap'}->{$real_field}) {
-            $shortname = $self->{'indexfieldmap'}->{$real_field};
-        }
-        else {
-            $shortname = $self->create_shortname($real_field);
-            $new_field = 1; # we want to record this shortname, but only if we have actually found some metadata values
-        }
+
+        if (defined $self->{'fieldnamemap'}->{$real_field}) {
+        $shortname = $self->{'fieldnamemap'}->{$real_field};
+        } else {
+        $shortname = $self->create_shortname($real_field);
+        $self->{'fieldnamemap'}->{$real_field} = $shortname;
+        $self->{'fieldnamemap'}->{$shortname} = 1;
+        }
+
         my @metadata_list = (); # put any meta values in here
         my $section_text = ""; # put any text in here
@@ -357 +342 @@
             # only add tags in if indexing
             $new_text .= "</$shortname>";
-            }
-            if ($self->{'indexing_text'} && $new_field) {
-            # we need to add to the list in indexfields
-
-            $self->{'indexfieldmap'}->{$real_field} = $shortname;
-            $self->{'indexfieldmap'}->{$shortname} = 1;
+            $self->{'allindexfields'}->{$real_field} = 1;
             }
         }
-        }
 
         # filter the text
@@ -388 +367 @@
         next if ($mfield =~ /^gsdl/);
         
-            
-        if (defined $self->{'indexfieldmap'}->{$mfield}) {
-            $shortname = $self->{'indexfieldmap'}->{$mfield};
+        if (defined $self->{'fieldnamemap'}->{$mfield}) {
+            $shortname = $self->{'fieldnamemap'}->{$mfield};
+        } else {
+            $shortname = $self->create_shortname($mfield);
+            $self->{'fieldnamemap'}->{$mfield} = $shortname;
+            $self->{'fieldnamemap'}->{$shortname} = 1;
         }
-        else {
-            $shortname = $self->create_shortname($mfield);
-            $self->{'indexfieldmap'}->{$mfield} = $shortname;
-            $self->{'indexfieldmap'}->{$shortname} = 1;
-        }      
+        $self->{'allindexfields'}->{$mfield} = 1;
         $new_text .= "$paratag<$shortname>$mvalue</$shortname>\n";
-        if (!defined $self->{'indexfields'}->{$mfield}) {
-            $self->{'indexfields'}->{$mfield} = 1;
+        if (!defined $self->{'extraindexfields'}->{$mfield}) {
+            $self->{'extraindexfields'}->{$mfield} = 1;
         }                   
         
@@ -426 +404 @@
     
     my ($realname) = @_;
+    my @realnamelist = split(",", $realname);
+    map {$_=~ s/^[a-zA-Z]+\.//;} @realnamelist; #remove namespaces
+    my ($singlename) = $realnamelist[0];
+
     # try our predefined static mapping
-    if (defined $static_indexfield_map{$realname}) {
-    return $static_indexfield_map{$realname};
-    }
+    my $name;
+    if (defined ($name = $static_indexfield_map{$singlename})) {
+    if (! defined $self->{'fieldnamemap'}->{$name}) {
+        # has this shortname already been used??
+        return $static_indexfield_map{$singlename};
+    }
+    }
+    # we can't use the quick map, so join all fields back together (without namespaces), and try sets of two characters.
+    $realname = join ("", @realnamelist);
     #try the first two chars
     my $shortname;
@@ -443 +431 @@
     #if already used, take the first and third letdigs and so on
     my $count = 1;
-    while (defined $self->{'indexfieldmap'}->{$shortname} || defined $static_indexfield_map{$shortname}) {
+    while (defined $self->{'fieldnamemap'}->{$shortname} || defined $static_indexfield_map{$shortname}) {
     if ($realname =~ /^[^\w]*(\w)([^\w]*\w){$count}[^\w]*(\w)/) {
         $shortname = "$1$3";