Changeset 34840

Timestamp:

2021-02-13T23:37:22+13:00 (3 years ago)

Author:

davidb

Message:

Changed to apply extra-metadata before trying to work out doc-id. This trial change is realted to JSONSparqlResultPlugin (which inherits from SplitTextFile), but want to use metadata fields in the JSON file (country+year combined) to form the ID of the document, so the extra-metadata needs to be processed before the add_OID call is made

File:

• main/trunk/greenstone2/perllib/plugins/SplitTextFile.pm--for-gs311 (copied) (copied from main/trunk/greenstone2/perllib/plugins/SplitTextFile.pm ) (4 diffs)

main/trunk/greenstone2/perllib/plugins/SplitTextFile.pm--for-gs311

@@ -114 +114 @@
 sub get_default_split_exp {
     return q^\n\s*\n^;
+}
+
+sub split_text_into_segments {
+    my $self = shift (@_);
+    my ($textref) = @_;
+    
+        
+    # Split the text into several smaller segments
+    my $split_exp = $self->{'split_exp'};
+    my @tmp  = split(/$split_exp/i, $$textref);
+    
+    my @segments =();
+    ## get rid of empty segments
+    foreach my $seg (@tmp){
+    if ($seg ne ""){
+        push @segments, $seg;
+    }
+    }
+
+    return \@segments;
 }
 
@@ -160 +180 @@
     
     # Split the text into several smaller segments
-    my $split_exp = $self->{'split_exp'};
-        my @tmp  = split(/$split_exp/i, $text);
-    my @segments =();
-    ## get rid of empty segments
-    foreach my $seg (@tmp){
-        if ($seg ne ""){
-        push @segments, $seg;
-        }
-    }
-
-    print $outhandle "SplitTextFile found " . (scalar @segments) . " documents in $filename\n" 
+#   my $split_exp = $self->{'split_exp'};
+#        my @tmp  = split(/$split_exp/i, $text);
+#   my @segments =();
+#   ## get rid of empty segments
+#   foreach my $seg (@tmp){
+#       if ($seg ne ""){
+#       push @segments, $seg;
+#       }
+#   }
+#
+#       print $outhandle "SplitTextFile found " . (scalar @segments) . " documents in $filename\n" 
+#       if $self->{'verbosity'};
+#   
+#   $self->{'split_segments'}->{$file} = \@segments;
+#       return  scalar(@segments);
+    
+        my $segments = $self->split_text_into_segments(\$text);
+
+        my $num_segments = scalar(@$segments);
+    
+        print $outhandle "SplitTextFile found $num_segments documents in $filename\n" 
         if $self->{'verbosity'};
     
-    $self->{'split_segments'}->{$file} = \@segments;
-    
-    return  scalar(@segments);
+    $self->{'split_segments'}->{$file} = $segments;
+
+        return  $num_segments;
 }
 
@@ -243 +273 @@
     #$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "Split");
 
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    
     # Calculate a "base" document ID.
     if (!defined $id) {
@@ -248 +282 @@
     }
     
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+#   # include any metadata passed in from previous plugins 
+#   # note that this metadata is associated with the top level section
+#   $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
 
     # do plugin specific processing of doc_obj