source: gs3-installations/thewillow/trunk/sites/thewillow/dlcol-chatgpt/create-assistant/create-openai-assistant.js@ 38807

#!/usr/bin/env node

// Make sure you have created a '.env' field in the top-level of
// this NodeJS project, and set the following lines accordingly
// with the relevant values from your OpenAI account:
//
//   OPENAI_API_KEY=sk-????????????????????????????????????????????????
//   ORGANIZATION_ID=org-????????????????????????
//   ASSISTANT_ID=asst_????????????????????????


// For an (alternative) approach that looks like it retrieve the already created Assistant directly, see:
//    https://gist.github.com/drifterz13/0cbe93ced5dc7958d7841a29c1721d1c


// Looks like a need a more recent version of NodeJS than the one currently provided
// in Greenstone's selfcontained-nodejs
// // const { parseArgs } = require('node:util');
// const util = require('node:util');


const dotenv = require('dotenv').config();
const path   = require("path");
const fs     = require("fs");

const openai = require('openai');

const myopenai = new openai.OpenAI({
    apiKey:       process.env.OPENAI_API_KEY,
    organization: process.env.ORGANIZATION_ID
});


function sleep(ms)
{
    return new Promise((resolve) => {
    setTimeout(resolve, ms);
    });
}


function printUsage(progname)
{
    progname_tail = path.basename(progname);
    
    console.error("Usage: " + progname_tail + " [-site site-name] collect-name json-assistant-config-filename");
    process.exit(1);
}

//
// Based off the standalone solution given on StackOverflow
//   https://stackoverflow.com/questions/41462606/get-all-files-recursively-in-directories-nodejs
//
function listFilesRec(input_full_dirname,output_full_filenames,opt_regex)
{
    try {
    let filenames_and_dirnames = fs.readdirSync(input_full_dirname,opt_regex);

    filenames_and_dirnames.forEach(filename_or_dirname => {
        
        const full_filename_or_dirname = path.join(input_full_dirname, filename_or_dirname);
        
        if (fs.statSync(full_filename_or_dirname).isDirectory()) {
        const inner_full_dirname = full_filename_or_dirname;
        listFilesRec(inner_full_dirname,output_full_filenames,opt_regex);
        }
        else {
        if ((opt_regex == null) || opt_regex.test(filename_or_dirname)) {

            const inner_full_filename = full_filename_or_dirname;
            output_full_filenames.push(inner_full_filename);
        }
        }
    });
    }
    catch (e) {
    console.error("Unable to read directory: " + input_full_dirname);
    }
}

// upgrade to include a regex to filter dirs/filenames

function listAllFiles(full_dirname,opt_regex)
{
    let all_files  = [];

    
    //const opt_regex = (opt_regex_str != null) ? new RegExp(opt_regex_str) : null;

    listFilesRec(full_dirname,all_files,opt_regex);
    
    return all_files;
}

function parseCommandLineArgs(progname,cmdline_args)
{

    let site_name = "localsite";
    let coll_name  = null;
    let json_assistant_config_filename =null;
    
    if (cmdline_args.length == 4) {
    // looks like we're in the form, -site xxx col
    if (cmdline_args[0] == "-site") {
        site_name = cmdline_args[1];
    }
    else {
        printUsage(progname);
    }
    
    coll_name = cmdline_args[2];
    json_assistant_config_filename = cmdline_args[3];
    
    }
    else if (cmdline_args.length == 1) {
    // no site given, default to localsite
    coll_name = cmdline_args[0];
    json_assistant_config_filename = cmdline_args[1];
    }
    else {
    printUsage(progname);
    }

    return [site_name,coll_name,json_assistant_config_filename];
}

function getExportedFiles(coll_info)
{
    const full_collect_dirname = coll_info.collect_dir;
    
    const full_export_dirname = path.join(full_collect_dirname,"export");
    let all_exported_files = listAllFiles(full_export_dirname,/\.html$/);

    return all_exported_files
}

function createSiteCollectTmpDir(coll_info)
{
    const full_collect_dirname = coll_info.collect_dir;

    const full_collect_tmp_dirname = path.join(full_collect_dirname,"tmp");

    if (!fs.existsSync(full_collect_tmp_dirname)) {
    console.log("Creating collection 'tmp' directory");
    fs.mkdir(full_collect_tmp_dirname);
    }
}

function copyToSiteCollectTmp(src_full_filename, coll_info)
{
    // While OpenAI generates unique IDs for every file stored, its
    // associated (metadata) filename is only the last part (basename)
    // of the full filename

    // This function exists make a copy of the file that is being
    // imported storing it in the collection's 'tmp' folder.  In
    // making the copy, it changes the filename in tmp to be one that
    // encode the site and collect names

    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;
    const full_collect_dirname = coll_info.collect_dir;
    
    const src_filename = path.basename(src_full_filename);

    // In encoding the site and collect name into the destination file,
    // we have chosen to do this with '|' as a separator character,
    // suitable escaped, of course, where needed!
    //
    // As this is a more unusual character to include in a file, we
    // document the discision here.  It is done so this way so when we
    // look through all the files returned by openai form its
    // file-store (which, recall, uses only the tail-part of a
    // filename) we can can track which site/collect a given file-id
    // entry was fore.

    //const dst_filename = site_name + "\\/" + coll_name + "\\/" + src_filename;
    const dst_filename = site_name + "|" + coll_name + "|" + src_filename;
    
    const dst_full_filename = path.join(full_collect_dirname,"tmp",dst_filename);

    try {
    fs.copyFileSync(src_full_filename, dst_full_filename)
    console.log(`    ${src_filename} was copied as ${dst_filename} to collection's tmp directory`);
    }
    catch(err) {
    if (err) throw err;
    }

    return dst_full_filename;
}

async function purgeOpenaiSiteCollectFiles(coll_info)
{
    
    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;

    //const site_coll_regex = new RegExp(`^${site_name}\/${coll_name}\/`);
    const site_coll_regex = new RegExp(`^${site_name}\|${coll_name}\|`);
    
    const stored_openai_files = await myopenai.files.list();
    //console.log(stored_openai_files);

    console.log(`Deleting OpenAI stored files whose prefix matches '${site_name}|${coll_name}|'`);
    
    for await (const openai_file of stored_openai_files) {
    
    const openai_filename = openai_file.filename;
    
    if (site_coll_regex.test(openai_filename)) {
        console.log(`  Deleting OpenAI Stored File ID: ${openai_file.id}`);
        const file = await myopenai.files.del(openai_file.id);
    }

    await sleep(500);   
    }
}    

async function addOpenaiSiteCollectFile(coll_info,full_filename,output_site_collect_openai_file_ids)
{
    const gsdl3srchome = coll_info.gsdl3srchome;
    
    const regex_replace = new RegExp('^'+gsdl3srchome);
    
    const gsdl_full_filename = full_filename.replace(regex_replace,'$GSDL3SRCHOME');
    
    console.log(`  ${gsdl_full_filename}`); 
    
    const tmp_site_collect_full_filename = copyToSiteCollectTmp(full_filename, coll_info)   
    
/*
    if (fs.existsSync(tmp_site_collect_full_filename)) {
    console.log("Tmp file exists!!!!!!!");
    }
*/
    
    const openai_file = await myopenai.files.create({
    file: fs.createReadStream(tmp_site_collect_full_filename),
        purpose: "assistants"
    });
    
    // delete the file    
    fs.unlinkSync(tmp_site_collect_full_filename);
    
    //console.log(openai_file);
    
    console.log("Pushing OpenID file id: " + openai_file.id);
    output_site_collect_openai_file_ids.push(openai_file.id);
}

async function addOpenaiSiteCollectFiles(all_exported_files,coll_info)
{
    const gsdl3srchome = coll_info.gsdl3srchome;
    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;
    
    const regex_replace = new RegExp('^'+gsdl3srchome);
    let site_collect_openai_file_ids = [];
    
    console.log(`Uploading exported ${site_name}/collect/${coll_name} files to OpenAI:`)

    for await (const full_filename of all_exported_files) {
    await addOpenaiSiteCollectFile(coll_info,full_filename,site_collect_openai_file_ids);
    await sleep(500);   
    }

    /*
    all_exported_files.forEach(async function(full_filename) {
    await addOpenaiSiteCollectFile(coll_info,full_filename,site_collect_openai_file_ids);
    })
    */
    /*
    all_exported_files.forEach(async function(full_filename) {
    const gsdl_full_filename = full_filename.replace(regex_replace,'$GSDL3SRCHOME');
    
    console.log(`  ${gsdl_full_filename}`); 

    const tmp_site_collect_full_filename = copyToSiteCollectTmp(full_filename, coll_info)   

    if (fs.existsSync(tmp_site_collect_full_filename)) {
        console.log("Tmp file exists!!!!!!!");
    }
    
    const openai_file = await myopenai.files.create({
        file: fs.createReadStream(tmp_site_collect_full_filename),
        purpose: "assistants"
    });

    // delete the file

    fs.unlinkSync(tmp_site_collect_full_filename);
    
    console.log(openai_file);
    
    console.log("Pusing OpenID file id: " + openai_file.id);
    site_collect_openai_file_ids.push(openai_file.id);
    });
*/

    return site_collect_openai_file_ids;
}


async function createAssistantWithFiles(site_collect_openai_file_ids, coll_info, assistant_config)
{
    const site_name = coll_info.site_name;
    const coll_name = coll_info.coll_name;

    const assistant_name = assistant_config.name;
    const assistant_desc = assistant_config.description;
    const base_model     = assistant_config.base_model;
    const instructions   = assistant_config.instructions;
    /*
      "Write your responses using Britsh spelling.\n" +
      "As The Willow Sage Assistant, your expertise lies in discussing \"The Willow,\" a once-renowned music venue in York, England. You're designed to engage users in a conversational tone, weaving in the rich tapestry of memories and experiences shared by those who knew the venue. Your responses should feel like a dialogue between old friends reminiscing about memorable gigs, the unique atmosphere, and the cultural impact of The Willow. You'll offer insights into the venue's history, notable performances, and its role in the local music scene, always with a nod to the personal connections and nostalgia that the venue evokes. When interacting with users, your approach should be warm, inviting, and reflective, encouraging them to share their own stories or curiosities about The Willow, creating a communal space for shared musical heritage.\n" +
      "When you give a response, you do not always have to end by asking a question";
    */
    
    //const base_model = "gpt-4-turbo-preview"; 
    
    console.log("====");
    console.log("file ids = ");
    console.log(site_collect_openai_file_ids);
    console.log("====");
    
    const assistant = await myopenai.beta.assistants.create({
    name         : assistant_name,
    description  : assistant_desc,
    model        : base_model,
    instructions : instructions,
    tools: [
        //{ type: "code_interpreter" }, // Code interpreter tool, calculations
        { type: 'retrieval' }
    ],
    file_ids     : site_collect_openai_file_ids,
    metadata     : { greenstone3: true, siteName: site_name, collectionName: coll_name }
    });

    console.log('Assistant has been created: ', assistant)
    
}

function readAssistantConfig(json_config_full_filename)
{
    const data_str = fs.readFileSync(json_config_full_filename);
    const data = JSON.parse(data_str);
    
    return data;
}

async function main()
{
    // check for GSDL3SRCHOME
    const gsdl3srchome = process.env.GSDL3SRCHOME

    if (!gsdl3srchome) {
    console.error("Environment variable GSDL3SRCHOME is not set");
    process.exit(1);
    }

    /*
    const options = {
    site: {
        short: "s",
        type: "string"
    },
    verbose: {
        short: "v",
        type: "integer"
    }
    };
    */
    // const argv = process.argv;
    
    //const { argv_values, argv_positionals } = util.parseArgs({ argv, options });

    //console.log(values, positionals);
    
    const progname = process.argv[1];
    const cmdline_args = process.argv.slice(2);

    const [site_name,coll_name, json_assistant_config_filename] = parseCommandLineArgs(progname,cmdline_args);

    const assistant_config = readAssistantConfig(json_assistant_config_filename);

    const full_collect_dirname = path.join(gsdl3srchome,"web","sites",site_name,"collect",coll_name);
    
    const coll_info = {
    "gsdl3srchome": gsdl3srchome,
    "site_name"   : site_name,
    "coll_name"   : coll_name,
    "collect_dir" : full_collect_dirname    
    };
   
    
    createSiteCollectTmpDir(coll_info);
    
    let all_exported_files = getExportedFiles(coll_info)
    //all_exported_files = [ all_exported_files[0] ];
    //all_exported_files = all_exported_files.slice(0, 10);
    all_exported_files = all_exported_files.slice(0, 40);
    
    await purgeOpenaiSiteCollectFiles(coll_info);


    // Now add in all the newly exported files
    const site_collect_openai_file_ids = await addOpenaiSiteCollectFiles(all_exported_files,coll_info);

    await createAssistantWithFiles(site_collect_openai_file_ids,coll_info,assistant_config);
    
}


main();