Changeset 38783

Timestamp:

2024-02-25T00:15:29+13:00 (2 months ago)

Author:

davidb

Message:

Additional functionality introduced to offset the fact the the OpenAI file-store is quite a flat structure, with our needs, where the site and collection (normally handled through directories/sub-directories) needs to represented/tracked

File:

• gs3-installations/thewillow/trunk/sites/thewillow/dlcol-chatgpt/create-assistant/main.js (modified) (3 diffs)

gs3-installations/thewillow/trunk/sites/thewillow/dlcol-chatgpt/create-assistant/main.js

@@ -37 +37 @@
 
 
-function listFilesRec(input_full_dirname,output_full_filenames)
+function listFilesRec(input_full_dirname,output_full_filenames,opt_regex)
 {
     try {
-    let filenames_and_dirnames = fs.readdirSync(input_full_dirname);
+    let filenames_and_dirnames = fs.readdirSync(input_full_dirname,opt_regex);
 
     filenames_and_dirnames.forEach(filename_or_dirname => {
         
         const full_filename_or_dirname = path.join(input_full_dirname, filename_or_dirname);
-        
+        
         if (fs.statSync(full_filename_or_dirname).isDirectory()) {
         const inner_full_dirname = full_filename_or_dirname;
-        listFilesRec(inner_full_dirname,output_full_filenames);
+        listFilesRec(inner_full_dirname,output_full_filenames,opt_regex);
         }
         else {
-        const inner_full_filename = full_filename_or_dirname;
-        output_full_filenames.push(inner_full_filename);
+        if ((opt_regex == null) || opt_regex.test(filename_or_dirname)) {
+
+            const inner_full_filename = full_filename_or_dirname;
+            output_full_filenames.push(inner_full_filename);
+        }
         }
     });
@@ -63 +66 @@
 // upgrade to include a regex to filter dirs/filenames
 
-function listAllFiles(full_dirname)
+function listAllFiles(full_dirname,opt_regex)
 {
     let all_files  = [];
 
-    listFilesRec(full_dirname,all_files);
-
+    
+    //const opt_regex = (opt_regex_str != null) ? new RegExp(opt_regex_str) : null;
+
+    listFilesRec(full_dirname,all_files,opt_regex);
+    
     return all_files;
 }
 
-
-
-
-// check for GSDL3SRCHOME
-const gsdl3srchome = process.env.GSDL3SRCHOME
-
-if (!gsdl3srchome) {
-    console.error("Environment variable GSDL3SRCHOME is not set");
-    process.exit(1);
-}
-
-const progname = process.argv[1];
-const cmdline_args = process.argv.slice(2);
-
-
-let site_name = "localsite";
-let coll_name  = null;
-
-if (cmdline_args.length == 3) {
-    // looks like we're in the form, -site xxx col
-    if (cmdline_args[0] == "-site") {
-    site_name = cmdline_args[1];
+function parseCommandLineArgs(progname,cmdline_args)
+{
+
+    let site_name = "localsite";
+    let coll_name  = null;
+    
+    if (cmdline_args.length == 3) {
+    // looks like we're in the form, -site xxx col
+    if (cmdline_args[0] == "-site") {
+        site_name = cmdline_args[1];
+    }
+    else {
+        printUsage(progname);
+    }
+    
+    coll_name = cmdline_args[2];
+    }
+    else if (cmdline_args.length == 1) {
+    // no site given, default to localsite
+    coll_name = cmdline_args[0];
     }
     else {
@@ -99 +103 @@
     }
 
-    coll_name = cmdline_args[2];
-}
-else if (cmdline_args.length == 1) {
-    // no site given, default to localsite
-    coll_name = cmdline_args[0];
-}
-else {
-    printUsage(progname);
-}
-
-const full_export_dirname = path.join(gsdl3srchome,"web","sites",site_name,"collect",coll_name,"export");
-
-
-let all_exported_files = listAllFiles(full_export_dirname);
-console.log(all_exported_files);
-
-
-
-
-/*
-const file = await openai.files.create({
-  file: fs.createReadStream("mydata.csv"),
-  purpose: "assistants",
-});
-*/
+    return [site_name,coll_name];
+}
+
+function getExportedFiles(coll_info)
+{
+    const full_collect_dirname = coll_info.collect_dir;
+    
+    const full_export_dirname = path.join(full_collect_dirname,"export");
+    let all_exported_files = listAllFiles(full_export_dirname,/\.html$/);
+
+    return all_exported_files
+}
+
+function createSiteCollectTmpDir(coll_info)
+{
+    const full_collect_dirname = coll_info.collect_dir;
+
+    const full_collect_tmp_dirname = path.join(full_collect_dirname,"tmp");
+
+    if (!fs.existsSync(full_collect_tmp_dirname)) {
+    console.log("Creating collection 'tmp' directory");
+    fs.mkdir(full_collect_tmp_dirname);
+    }
+}
+
+function copyToSiteCollectTmp(src_full_filename, coll_info)
+{
+    // While OpenAI generates unique IDs for every file stored, its
+    // associated (metadata) filename is only the last part (basename)
+    // of the full filename
+
+    // This function exists make a copy of the file that is being
+    // imported storing it in the collection's 'tmp' folder.  In
+    // making the copy, it changes the filename in tmp to be one that
+    // encode the site and collect names
+
+    const site_name = coll_info.site_name;
+    const coll_name = coll_info.coll_name;
+    const full_collect_dirname = coll_info.collect_dir;
+    
+    const src_filename = path.basename(src_full_filename);
+
+    // In encoding the site and collect name into the destination file,
+    // we have chosen to do this with '|' as a separator character,
+    // suitable escaped, of course, where needed!
+    //
+    // As this is a more unusual character to include in a file, we
+    // document the discision here.  It is done so this way so when we
+    // look through all the files returned by openai form its
+    // file-store (which, recall, uses only the tail-part of a
+    // filename) we can can track which site/collect a given file-id
+    // entry was fore.
+
+    //const dst_filename = site_name + "\\/" + coll_name + "\\/" + src_filename;
+    const dst_filename = site_name + "|" + coll_name + "|" + src_filename;
+    
+    const dst_full_filename = path.join(full_collect_dirname,"tmp",dst_filename);
+    
+    fs.copyFile(src_full_filename, dst_full_filename, (err) => {
+    if (err) throw err;
+    console.log(`${src_filename} was copied as ${dst_filename} to collection's tmp directory`);
+    });
+
+    return dst_full_filename;
+}
+
+async function purgeOpenaiSiteCollectFiles(coll_info)
+{
+    
+    const site_name = coll_info.site_name;
+    const coll_name = coll_info.coll_name;
+
+    //const site_coll_regex = new RegExp(`^${site_name}\/${coll_name}\/`);
+    const site_coll_regex = new RegExp(`^${site_name}\|${coll_name}\|`);
+    
+    const stored_openai_files = await myopenai.files.list();
+    console.log(stored_openai_files);
+    
+    for await (const openai_file of stored_openai_files) {
+    
+    const openai_filename = openai_file.filename;
+    
+    if (site_coll_regex.test(openai_filename)) {
+        console.log(`  Deleting OpenAI Stored File ID: ${openai_file.id}`);
+        const file = await myopenai.files.del(openai_file.id);
+    }
+    }
+}    
+
+    
+    
+async function main()
+{
+    // check for GSDL3SRCHOME
+    const gsdl3srchome = process.env.GSDL3SRCHOME
+
+    if (!gsdl3srchome) {
+    console.error("Environment variable GSDL3SRCHOME is not set");
+    process.exit(1);
+    }
+
+    const progname = process.argv[1];
+    const cmdline_args = process.argv.slice(2);
+
+    const [site_name,coll_name] = parseCommandLineArgs(progname,cmdline_args);
+
+    const full_collect_dirname = path.join(gsdl3srchome,"web","sites",site_name,"collect",coll_name);
+    
+    const coll_info = {
+    "gsdl3srchome": gsdl3srchome,
+    "site_name"   : site_name,
+    "coll_name"   : coll_name,
+    "collect_dir" : full_collect_dirname    
+    };
+    
+    createSiteCollectTmpDir(coll_info);
+    
+    let all_exported_files = getExportedFiles(coll_info)
+    //all_exported_files = [ all_exported_files[0] ];
+
+    await purgeOpenaiSiteCollectFiles(coll_info);
+
+    const regex_replace = new RegExp('^'+gsdl3srchome);
+    let openai_files = [];
+    
+    console.log(`Uploading exported ${site_name}/collect/${coll_name} files to OpenAI:`)
+    all_exported_files.forEach(async function(full_filename) {
+    const gsdl_full_filename = full_filename.replace(regex_replace,'$GSDL3SRCHOME');
+    
+    console.log(`  ${gsdl_full_filename}`); 
+
+    const tmp_site_collect_full_filename = copyToSiteCollectTmp(full_filename, coll_info)
+    
+    
+    const openai_file = await myopenai.files.create({
+        file: fs.createReadStream(tmp_site_collect_full_filename),
+        purpose: "assistants"
+    });
+    
+    
+    openai_files.push(openai_file);
+    });
+}
+
+
+main();
+