source: gs3-installations/thewillow/trunk/sites/thewillow/dlcol-chatgpt/create-assistant/main.js@ 38783

#!/usr/bin/env node

// Make sure you have created a '.env' field in the top-level of
// this NodeJS project, and set the following lines accordingly
// with the relevant values from your OpenAI account:
//
//   OPENAI_API_KEY=sk-????????????????????????????????????????????????
//   ORGANIZATION_ID=org-????????????????????????
//   ASSISTANT_ID=asst_????????????????????????


// For an (alternative) approach that looks like it retrieve the already created Assistant directly, see:
//    https://gist.github.com/drifterz13/0cbe93ced5dc7958d7841a29c1721d1c



const dotenv = require('dotenv').config();
const path   = require("path");
const fs     = require("fs");

const openai = require('openai');

const myopenai = new openai.OpenAI({
    apiKey:       process.env.OPENAI_API_KEY,
    organization: process.env.ORGANIZATION_ID
});


function printUsage(progname)
{
    console.error("Usage: " + progname + " [-site site-name] collect-name");
    process.exit(1);
}

// Based off the standalone solution given on StackOverflow
//   https://stackoverflow.com/questions/41462606/get-all-files-recursively-in-directories-nodejs


function listFilesRec(input_full_dirname,output_full_filenames,opt_regex)
{
    try {
    let filenames_and_dirnames = fs.readdirSync(input_full_dirname,opt_regex);

    filenames_and_dirnames.forEach(filename_or_dirname => {
        
        const full_filename_or_dirname = path.join(input_full_dirname, filename_or_dirname);
        
        if (fs.statSync(full_filename_or_dirname).isDirectory()) {
        const inner_full_dirname = full_filename_or_dirname;
        listFilesRec(inner_full_dirname,output_full_filenames,opt_regex);
        }
        else {
        if ((opt_regex == null) || opt_regex.test(filename_or_dirname)) {

            const inner_full_filename = full_filename_or_dirname;
            output_full_filenames.push(inner_full_filename);
        }
        }
    });
    }
    catch (e) {
    console.error("Unable to read directory: " + input_full_dirname);
    }
}

// upgrade to include a regex to filter dirs/filenames

function listAllFiles(full_dirname,opt_regex)
{
    let all_files  = [];

    
    //const opt_regex = (opt_regex_str != null) ? new RegExp(opt_regex_str) : null;

    listFilesRec(full_dirname,all_files,opt_regex);
    
    return all_files;
}

function parseCommandLineArgs(progname,cmdline_args)
{

    let site_name = "localsite";
    let coll_name  = null;
    
    if (cmdline_args.length == 3) {
    // looks like we're in the form, -site xxx col
    if (cmdline_args[0] == "-site") {
        site_name = cmdline_args[1];
    }
    else {
        printUsage(progname);
    }
    
    coll_name = cmdline_args[2];
    }
    else if (cmdline_args.length == 1) {
    // no site given, default to localsite
    coll_name = cmdline_args[0];
    }
    else {
    printUsage(progname);
    }

    return [site_name,coll_name];
}

function getExportedFiles(coll_info)
{
    const full_collect_dirname = coll_info.collect_dir;
    
    const full_export_dirname = path.join(full_collect_dirname,"export");
    let all_exported_files = listAllFiles(full_export_dirname,/\.html$/);

    return all_exported_files
}

function createSiteCollectTmpDir(coll_info)
{
    const full_collect_dirname = coll_info.collect_dir;

    const full_collect_tmp_dirname = path.join(full_collect_dirname,"tmp");

    if (!fs.existsSync(full_collect_tmp_dirname)) {
    console.log("Creating collection 'tmp' directory");
    fs.mkdir(full_collect_tmp_dirname);
    }
}

function copyToSiteCollectTmp(src_full_filename, coll_info)
{
    // While OpenAI generates unique IDs for every file stored, its
    // associated (metadata) filename is only the last part (basename)
    // of the full filename

    // This function exists make a copy of the file that is being
    // imported storing it in the collection's 'tmp' folder.  In
    // making the copy, it changes the filename in tmp to be one that
    // encode the site and collect names

    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;
    const full_collect_dirname = coll_info.collect_dir;
    
    const src_filename = path.basename(src_full_filename);

    // In encoding the site and collect name into the destination file,
    // we have chosen to do this with '|' as a separator character,
    // suitable escaped, of course, where needed!
    //
    // As this is a more unusual character to include in a file, we
    // document the discision here.  It is done so this way so when we
    // look through all the files returned by openai form its
    // file-store (which, recall, uses only the tail-part of a
    // filename) we can can track which site/collect a given file-id
    // entry was fore.

    //const dst_filename = site_name + "\\/" + coll_name + "\\/" + src_filename;
    const dst_filename = site_name + "|" + coll_name + "|" + src_filename;
    
    const dst_full_filename = path.join(full_collect_dirname,"tmp",dst_filename);
    
    fs.copyFile(src_full_filename, dst_full_filename, (err) => {
    if (err) throw err;
    console.log(`${src_filename} was copied as ${dst_filename} to collection's tmp directory`);
    });

    return dst_full_filename;
}

async function purgeOpenaiSiteCollectFiles(coll_info)
{
    
    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;

    //const site_coll_regex = new RegExp(`^${site_name}\/${coll_name}\/`);
    const site_coll_regex = new RegExp(`^${site_name}\|${coll_name}\|`);
    
    const stored_openai_files = await myopenai.files.list();
    console.log(stored_openai_files);
    
    for await (const openai_file of stored_openai_files) {
    
    const openai_filename = openai_file.filename;
    
    if (site_coll_regex.test(openai_filename)) {
        console.log(`  Deleting OpenAI Stored File ID: ${openai_file.id}`);
        const file = await myopenai.files.del(openai_file.id);
    }
    }
}    

    
    
async function main()
{
    // check for GSDL3SRCHOME
    const gsdl3srchome = process.env.GSDL3SRCHOME

    if (!gsdl3srchome) {
    console.error("Environment variable GSDL3SRCHOME is not set");
    process.exit(1);
    }

    const progname = process.argv[1];
    const cmdline_args = process.argv.slice(2);

    const [site_name,coll_name] = parseCommandLineArgs(progname,cmdline_args);

    const full_collect_dirname = path.join(gsdl3srchome,"web","sites",site_name,"collect",coll_name);
    
    const coll_info = {
    "gsdl3srchome": gsdl3srchome,
    "site_name"   : site_name,
    "coll_name"   : coll_name,
    "collect_dir" : full_collect_dirname    
    };
    
    createSiteCollectTmpDir(coll_info);
    
    let all_exported_files = getExportedFiles(coll_info)
    //all_exported_files = [ all_exported_files[0] ];

    await purgeOpenaiSiteCollectFiles(coll_info);

    const regex_replace = new RegExp('^'+gsdl3srchome);
    let openai_files = [];
    
    console.log(`Uploading exported ${site_name}/collect/${coll_name} files to OpenAI:`)
    all_exported_files.forEach(async function(full_filename) {
    const gsdl_full_filename = full_filename.replace(regex_replace,'$GSDL3SRCHOME');
    
    console.log(`  ${gsdl_full_filename}`); 

    const tmp_site_collect_full_filename = copyToSiteCollectTmp(full_filename, coll_info)
    
    
    const openai_file = await myopenai.files.create({
        file: fs.createReadStream(tmp_site_collect_full_filename),
        purpose: "assistants"
    });
    
    
    openai_files.push(openai_file);
    });
}


main();