Changeset 34878

Timestamp:

2021-02-16T01:09:13+13:00 (3 years ago)

Author:

davidb

Message:

Further changes to work more smoothly with JSONSparqlResultsPlugin, which makes use of OIDtype metadata for each record to provide nice IDs

File:

• main/trunk/greenstone2/perllib/plugins/SplitTextFile.pm--for-gs311 (modified) (5 diffs)

main/trunk/greenstone2/perllib/plugins/SplitTextFile.pm--for-gs311

@@ -144 +144 @@
     # can we process this file??
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
     return undef unless $self->can_process_this_file($filename_full_path);
 
@@ -273 +274 @@
     #$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "Split");
 
+#   # include any metadata passed in from previous plugins 
+#   # note that this metadata is associated with the top level section
+#   $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+
+#   # Calculate a "base" document ID.
+#   if (!defined $id) {
+#       $id = $self->get_base_OID($doc_obj);
+#   }
+
     # include any metadata passed in from previous plugins 
     # note that this metadata is associated with the top level section
     $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
-    
-    # Calculate a "base" document ID.
-    if (!defined $id) {
-        $id = $self->get_base_OID($doc_obj);
-    }
-    
-#   # include any metadata passed in from previous plugins 
-#   # note that this metadata is associated with the top level section
-#   $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
 
     # do plugin specific processing of doc_obj
@@ -305 +306 @@
     $self->auto_extract_metadata ($doc_obj);
 
+    # This used to be done earlier on in routine, however $id generated
+    # isn't used until here!
+    # Calculate a "base" document ID.
+    if (!defined $id) {
+        $id = $self->get_base_OID($doc_obj);
+    }
+    
     # add an OID
-    $self->add_OID($doc_obj, $id, $segment);
+    $self->add_segment_OID($doc_obj, $id, $segment);
 
     # process the document
@@ -312 +320 @@
 
     $self->{'num_processed'} ++;
+
+    if ($self->{'num_processed'} >= $maxdocs) {
+        last;
+    }
     }
 
@@ -324 +336 @@
     my ($doc_obj) = @_;
 
+    my $identifier = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), $self->{'OIDmetadata'});
+    # print STDERR "**** get_baseOID identifier = $identifier\n";
+    
     $self->SUPER::add_OID($doc_obj);
     return $doc_obj->get_OID();
 }
 
-sub add_OID {
+sub add_segment_OID {
     my $self = shift (@_);
     my ($doc_obj, $id, $segment) = @_;