Changeset 38851 for main/trunk/greenstone2/perllib/plugouts/OpenAIGPTsPlugout.pm

Timestamp:

2024-03-19T16:02:07+13:00 (3 months ago)

Author:

davidb

Message:

General upgrade to support exporting files suitable for inclusion in OpenAI's Assistants API

File:

• main/trunk/greenstone2/perllib/plugouts/OpenAIGPTsPlugout.pm (modified) (8 diffs)

main/trunk/greenstone2/perllib/plugouts/OpenAIGPTsPlugout.pm

@@ -79 +79 @@
     }
 
-    # For now, don't allow groups
-    if ($self->{'group_size'} > 1) {
-    print STDERR "OpenAIGPTsPlugout: Has not been tested with groups\n";
-    print STDERR "=> Reverting group_size back to 1\n";
-    
-    $self->{'group_size'} = 1;
-    }
-    
     return bless $self, $class;
 }
@@ -102 +94 @@
 }
 
+# Note: This type of grouping (into dirs) is different to the base
+# classes group_size. 'group_size' is about concatenating multiple
+# Greenstone documents into one file (useful for small docs/records
+# such as MARC).  The 'grouped_into_dirs' variables provide
+# an ability to multiple, separate documents, saved into the same
+# archives/export folder.
+
+my $grouped_into_dirs_root = "group-";
+my $grouped_into_dirs_doc_count = 0;
+
+my $max_docs_per_grouped_dir = 10;
+my $grouped_into_dirs_group_count = 0;
+    
+    
 sub get_new_doc_dir
 {
@@ -107 +113 @@
   my($working_info,$working_dir,$OID) = @_;
 
-  # A slimmed down version of BasePlugout::get_new_doc_dir()
-  # which creates a flat file structure, rather then nested
+  my $doc_dir;
   
-  my $doc_dir_rest = $OID;
-
-  # remove any \ and / from the OID
-  $doc_dir_rest =~ s/[\\\/]//g;
-
-  # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
-  if ($ENV{'GSDLOS'} =~ /^windows$/i)
-  {
-    $doc_dir_rest =~ s/\://g;
-  }
-
-  my $doc_dir = $doc_dir_rest;
-
+  if (defined $grouped_into_dirs_root) {
+      $grouped_into_dirs_doc_count++;
+
+      if (($grouped_into_dirs_doc_count % $max_docs_per_grouped_dir) == 0) {
+      $grouped_into_dirs_group_count++;
+      }
+      $doc_dir = sprintf("${grouped_into_dirs_root}%04d", $grouped_into_dirs_group_count);
+  }
+  else {
+
+      # A slimmed down version of BasePlugout::get_new_doc_dir()
+      # which creates a flat file structure, rather then nested
+      
+      my $doc_dir_rest = $OID;
+      
+      # remove any \ and / from the OID
+      $doc_dir_rest =~ s/[\\\/]//g;
+
+      # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
+      if ($ENV{'GSDLOS'} =~ /^windows$/i)
+      {
+      $doc_dir_rest =~ s/\://g;
+      }
+      
+      $doc_dir = $doc_dir_rest;
+  }
+  
   my $created_directory = 0;
   
@@ -131 +150 @@
       $created_directory = 1;
   }
-
+  else {
+      $created_directory = 1;
+  }
   
-  # in theory this should never happen
   if (!$created_directory)
   {
@@ -144 +164 @@
 
 
+sub get_group_doc_dir {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    # If this Plugout is being used to with grouped_into_dirs, then
+    # how get_group_dor_dir() needs to operate is different.  In fact
+    # it is simpler than the super-class implementation, because (due
+    # to the prefix manipulation of gsdlassocfiles) it is safe for
+    # associated files to be saved in the same directory as other
+    # documents.
+
+    my $doc_dir = undef;
+    
+    if (defined $grouped_into_dirs_root) {
+
+    my $outhandle = $self->{'output_handle'};
+    my $OID = $doc_obj->get_OID(); 
+    $OID = "NULL" unless defined $OID;
+
+    my $groupsize = $self->{'group_size'};
+    my $gs_count = $self->{'gs_count'};
+    
+    my $open_new_file = (($gs_count % $groupsize)==0);
+    
+    # opening a new file
+    if (($open_new_file)  || !defined($self->{'gs_doc_dir'})) {
+        # first we close off the old output
+        if ($gs_count>0)
+        {
+        return if (!$self->close_group_output());
+        }
+        
+        # this will create the directory
+        $doc_dir = $self->get_doc_dir ($doc_obj); 
+        $self->{'new_doc_dir'} = 1;
+        $self->{'gs_doc_dir'} = $doc_dir;
+        $self->{'group_position'} = 1;
+    }
+    else {
+        $doc_dir = $self->{'gs_doc_dir'};
+        $self->{'new_doc_dir'} = 0;
+    }
+        
+    }
+    else {
+    $doc_dir = $self->SUPER::get_group_doc_dir();
+    }
+    
+    return $doc_dir;
+}
+
+
+
+
 sub recursive_process_section_content
 {
@@ -158 +232 @@
     $section_ptr->{'text'} = $text;
 
+    # Turn into text
+    $text =~ s/<style[^>]*>.*?<\/style>//si;
+    $text =~ s/<[^>]*>/ /g;
+    $text =~ s/\s+ / /mg;
+
+    my $gsdoc_marker = "<span fromGSDocId=\"$oid\"></span>";
+    $text =~ s/((?:[^\s]+\s*){10})/$1$gsdoc_marker/sg;
+    
+    
+    $section_ptr->{'text'} = "<div gsdocid=\"$oid\">$gsdoc_marker$text$gsdoc_marker</div>";
+
+    # print STDERR "*** text = $text\n";
+    
     # work through all the sub-sections
     foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
@@ -215 +302 @@
     # print $outhandle "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/strict.dtd\">\n";
     #print $outhandle "<html xmlns=\"http://www.w3.org/TR/xhtml1/strict\">\n";
+    
     print $outhandle "<Archive>\n";
 }
@@ -222 +310 @@
     my ($outhandle) = @_;
 
-    #print $outhandle "</html>\n";
     print $outhandle "</Archive>\n";    
 }