Changeset 33302 for main/trunk

Timestamp:

2019-07-05T22:23:48+12:00 (5 years ago)

Author:

ak19

Message:

1. Adding GPSMapOverlayLabel extracted from GPS.mapOverlay meta to text indexes for searching, as with Coordinte and CoordShort. 2. Added a shortname for this index, ML for MapLabel. 3. On testing the indexing of the GPSMapOverlayLabel text, the old problem of increasingly duplicated Coordinate/CoordShort and now also GPSMapOverlayLabel meta in the infodb reappeared. Dr Bainbridge explained why this was (documented as comments in this commit) and fixed the problem by not processing GPS.mapOverlay meta into Coordinate and Label meta during the infodb pass (and dummy pass, so specifically specifically non-text passes) of buildcol. A natural consequence is that to check whether Coord and Label meta have been indexed, can no longer check the index/text/col.jdb but need to use Luke (if a lucene collection ) to check contents of index/sidx and index/didx. 4. An important change needed for the bugfix in 3 is reordering call to &classify::reconstruct_doc_objs_metadata() in basebuilder.pm to take place AFTER build_proc->set_mode(infodb) has taken place. 5. Changed cross-files global variables declared in doc.pm from our to my variables and tested this works.

Location:

main/trunk/greenstone2/perllib

Files:

• basebuilder.pm (modified) (2 diffs)
• basebuildproc.pm (modified) (1 diff)
• doc.pm (modified) (7 diffs)
• mgppbuildproc.pm (modified) (1 diff)

main/trunk/greenstone2/perllib/basebuilder.pm

@@ -480 +480 @@
     }
 
-    if ($self->{'incremental'}) {
-    # reconstruct doc_obj metadata from database for all docs
-    $reconstructed_docs 
-        = &classify::reconstruct_doc_objs_metadata($infodb_type, 
-                               $infodb_file_path,
-                               $database_recs);
-    }
-
     # set up the document processor
 
@@ -498 +490 @@
     $self->{'buildproc'}->set_store_text(1);
 
+    if ($self->{'incremental'}) {
+    # reconstruct doc_obj metadata from database for all docs
+    $reconstructed_docs 
+        = &classify::reconstruct_doc_objs_metadata($infodb_type, 
+                               $infodb_file_path,
+                               $database_recs);
+    }
+    
     # make_infodatabase needs full reset even for incremental build
     # as incremental works by reconstructing all docs from the database and

main/trunk/greenstone2/perllib/basebuildproc.pm

@@ -212 +212 @@
 
     $self->{'mode'} = $mode;
+    $doc::processor_mode = $mode; # doc.pm needs to know what buildcol pass we're at
 }
 

main/trunk/greenstone2/perllib/doc.pm

@@ -62 +62 @@
 # Decided to follow the usage of $OIDcount above to declare the necessary package-level variables 'globally' accessible
 # actually accessible only for those packages that import this one (with 'use doc')
-our $cmd_line_mode = undef;
+my $cmd_line_mode = undef;
+
+# processor_mode keeps track of which buildcol pass we're at: dummy, text (sidx/didx passes) or infodb
+my $processor_mode = undef;
 
 # rename_method can be 'url', 'none', 'base64'
@@ -1160 +1163 @@
         }
     }
-
-    elsif($field eq "GPS.mapOverlay") { # then the value is a JSON string
-
-      if($cmd_line_mode eq "buildcol") {
-       #my $metaMap = $self->get_metadata_hashmap($section); ## TODO: Check if necessary to avoid duplication of <Coordinate> meta in index\text\<coll>.jdb
-      
-       #if(!$metaMap->{'Coordinate'}) {
-        #print STDERR "@@@@@@@@@@@@@@ cmd line mode (build phase) is now: $doc::cmd_line_mode\n";
-    
+    
+    elsif($field eq "GPS.mapOverlay") { # then the $value is a JSON string
+
+        # In order to allow searching map data enriched documents by map shape descriptions,
+        # and to run rawquery searches for other docs by proximity based on their map data,
+        # need to store the shape descriptions and Coordinate info for shapes into the text index.
+        # We add the description for each shape in the mapoverlay into the text index as GPSMapOverlayLabel
+        # And we add Coordinate (CD) and CoordShort (CS) info for each shape in the mapoverlay in the format (Lat, Lng) as ("37S339 175E342")
+        # where the digits before the N/S/E/W direction represents the whole number. And the digits after the direction are the
+        # decimal places which can range from 0 and 2-4 digits.
+        
+        # However, we only want to process GPS.mapOverlay only during buildcol and only in the text indexing passes (e.g. sidx and didx for lucene)
+        # and certainly never during the infodb pass of buildcol. The latter can end up duplicating Coordinate/CoordShort/GPSMapOverlayLabel for
+        # when rebuilding with the online doc editor as that runs incremental-rebuild which then calls basebuilder::reconstruct_doc_objs_metadata()
+        # on all docs NOT being incrementally rebuilt. That call would get meta from the infodb and use it to reconstruct the doc objects of docs
+        # NOT being incrementally built. If the Coord and Label meta were written to the infodb, they would then be loaded back in when the collection
+        # is incrementally rebuilt for those docs that don't need incremental processing. Then this function would once again add the same meta into
+        # the infodb, thus duplicating what goes into the infodb. Hence, don't do all the following if doc::processor_mode eq "infodb".
+
+        # Note that for incremental rebuilding, the text pass can be called textreindex for instance (and infodb pass can be incinfodb).
+        # So don't check for exact string match
+
+        if($doc::cmd_line_mode eq "buildcol" && $doc::processor_mode =~ m/text/) {# && $doc::processor_mode !~ m/infodb/) # if dummy pass important
         
-        print STDERR "GPS.mapOverlay before decoding, val = " . $value . "\n";
+        ###print STDERR "GPS.mapOverlay before decoding, val = " . $value . "\n";
         
-        # TODO html decode?
+        # TODO: html decode?
         $value =~ s@&#091;@[@g;
         $value =~ s@&#093;@]@g;
         $value =~ s@&quot;@"@g;
-        print STDERR "GPS.mapOverlay after decoding, val = " . $value . "\n";
+        ###print STDERR "GPS.mapOverlay after decoding, val = " . $value . "\n";
 
         my $json_array = decode_json $value;
-        #my $json = JSON->new->allow_nonref;
-        #&printAllShapes($json, $json_array);
-
-        foreach my $shape (@$json_array) {      
-
+        
+        foreach my $shape (@$json_array) {
+
+            # Put each available shape description/label into this section's metadata with GPSMapOverlayLabel as metaname.
+            # Just as for Coordinate meta, don't need to know which shape a label belongs too. This is just so each label
+            # will be indexed, and therefore can be searched.
+            
+            my $description = $shape->{"description"};
+            if($description) {
+            push (@{$section_ptr->{'metadata'}}, ["GPSMapOverlayLabel", $description]);
+            ###print STDERR "@@@@############################################ Just added description meta: " . $description . "\n";
+            }
+            
             my $type = $shape->{"type"};
-            print STDERR "Type : " . $type . "\n";
+            ###print STDERR "Shape type : " . $type . "\n";
         
             if($type eq "circle") {
-                #print STDERR "Found a circle:\n" . &printShape($json, $shape);
+                ###print STDERR "Found a circle:\n" . &printShape($json, $shape);
         
                 # work out bounding box
@@ -1206 +1231 @@
                 my $radius = $shape->{"radius"}; # in metres!
 
-                print STDERR "@@@ circle centre: ($centre_lat, $centre_lng), radius: $radius\n";
+                ###print STDERR "@@@ circle centre: ($centre_lat, $centre_lng), radius: $radius\n";
 
                 my $lat_north = $centre_lat + ($radius/111111);
                 my $lat_south = $centre_lat - ($radius/111111);
                 
-                print STDERR "### lat_north:  $lat_north\n"; 
-                print STDERR "### lat_south:  $lat_south\n";
+                ###print STDERR "### lat_north:  $lat_north\n"; 
+                ###print STDERR "### lat_south:  $lat_south\n";
 
                 # our latitude and longitude values are in degrees. But cos and sin etc in perl and generally all prog languages
@@ -1218 +1243 @@
                 my $centre_lat_radians = $self->degreesToRadians($centre_lat);
                 my $cos_in_radians = cos($centre_lat_radians);              
-                print STDERR "cos $centre_lat_radians " . cos($centre_lat_radians) . "\n";
+                ###print STDERR "cos $centre_lat_radians " . cos($centre_lat_radians) . "\n";
                 my $lng_east = $centre_lng + ($radius/(111111 * $cos_in_radians));
                 my $lng_west = $centre_lng - ($radius/(111111 * $cos_in_radians));
-                print STDERR "### lng_east  $lng_east\n"; 
-                print STDERR "### lng_west  $lng_west\n";
+                ###print STDERR "### lng_east  $lng_east\n"; 
+                ###print STDERR "### lng_west  $lng_west\n";
 
                 my $cos_lat = cos($centre_lat);             
-                print STDERR "cos $centre_lat is $cos_lat\n";
+                ###print STDERR "cos $centre_lat is $cos_lat\n";
 
                 $self->processCoordinate($section, $lat_north, $lng_east);
@@ -1234 +1259 @@
             }
             elsif ($type eq "marker") {
-                print STDERR "@@ MARKER FOUND WITH LAT: " . $shape->{"position"}->{"lat"} . "\n";
-                print STDERR "@@ MARKER FOUND WITH LNG: " . $shape->{"position"}->{"lng"} . "\n";
+                ###print STDERR "@@ MARKER FOUND WITH LAT: " . $shape->{"position"}->{"lat"} . "\n";
+                ###print STDERR "@@ MARKER FOUND WITH LNG: " . $shape->{"position"}->{"lng"} . "\n";
                 $self->processCoordinate($section, $shape->{"position"}->{"lat"}, $shape->{"position"}->{"lng"});               
             }
@@ -1254 +1279 @@
         
         } # end for on each shape in GPS.mapOverlay
-       #}
-      }
+        } # end if(buildcol and text pass)
     } # end GPS.mapOverlay meta
 
@@ -1278 +1302 @@
 }
 
+# Call as:
+# my $json = JSON->new->allow_nonref;
+# &printAllShapes($json, $json_array);
 sub printAllShapes {
     my ($json, $json_array) = @_; 

main/trunk/greenstone2/perllib/mgppbuildproc.pm

@@ -73 +73 @@
               'text'=>'TX',
               'TX'=>1,
+              'GPSMapOverlayLabel'=> 'ML',
+              'ML'=>1,
               'Coordinate'=>'CD',
               'CD'=>1,