Changeset 38821 for gs3-extensions/dl-chatgpt

Timestamp:

2024-03-05T13:28:52+13:00 (2 months ago)

Author:

davidb

Message:

Cleaned up version; work done on processAnnotation() displaying quote as source in MarkDown

File:

• gs3-extensions/dl-chatgpt/trunk/src/react-gui/pages/api/chat.ts (modified) (14 diffs)

gs3-extensions/dl-chatgpt/trunk/src/react-gui/pages/api/chat.ts

@@ -5 +5 @@
 //   OPENAI_API_KEY=sk-????????????????????????????????????????????????
 //   ORGANIZATION_ID=org-????????????????????????
-//   ASSISTANT_ID=asst_????????????????????????
 
 
@@ -33 +32 @@
 
 async function initGS3AssistantsLookup() {
-//const initGS3AssistantsLookup = async () => {
     GS3AssistantLookup = new Map<string, any>();
 
@@ -47 +45 @@
     let paginated_items = assistants_list_cursor.getPaginatedItems();
     
-    //while (assistants_list_cursor !== null) {
     while (paginated_items.length > 0) {
     let last_checked_assistant_id = "";
@@ -97 +94 @@
 }
 
-/*
-
-function processAnnotations(message_content)
-{
-    
-    annotations = message_content.annotations;
-
-    citations = [];
-
-    // Iterate over the annotations and add footnotes
-    for index, annotation in enumerate(annotations) {
-    //Replace the text with a footnote
-    message_content.value = message_content.value.replace(annotation.text, f' [{index}]')l
-
-    // Gather citations based on annotation attributes
-    if (file_citation := getattr(annotation, 'file_citation', None)) {
-            cited_file = client.files.retrieve(file_citation.file_id)
-            citations.append(f'[{index}] {file_citation.quote} from {cited_file.filename}')
-    }
-    elif (file_path := getattr(annotation, 'file_path', None)) {
-            cited_file = client.files.retrieve(file_path.file_id)
-            citations.append(f'[{index}] Click <here> to download {cited_file.filename}')
-            // Note: File download functionality not implemented above for brevity
-    }
-    }
-    
-    //Add footnotes to the end of the message before displaying to user
-    message_content.value += '\n' + '\n'.join(citations)
-}
-*/
-
-
-
-function processAnnotations(message : MessageContentText)
+
+async function processAnnotations(message : MessageContentText)
 {
     console.log("Process Annotations");
@@ -138 +103 @@
 
     const citations: string[] = [];
-    
+
+    var from_gsdocid_regex = new RegExp(/<span fromGSDocId="(.*?)">/, "g");
+    
     // Iterate over the annotations and add footnotes
-    annotations.forEach(async (annotation, index) => {
+    const annotations_len = annotations.length;
+    for (let i=0; i<annotations_len; i++) {
+    const annotation = annotations[i];
+    const index      = i;
+
     // Replace the text with a footnote
-    console.log(`  Annotation[${index}: `, annotation);
-
-    messageContent.value = messageContent.value.replace(annotation.text, ` [${index}]`);
+    console.log(`  Processing Annotation[${index}]: `, annotation, "--");
+
+    messageContent.value = messageContent.value.replace(annotation.text, ` [${index+1}]`);
     
     // Gather citations based on annotation attributes
@@ -150 +121 @@
         const file_citation = annotation.file_citation;
         console.log(`    Retrieving file_citation for ${file_citation.file_id}`);
-        const citedFile = await openai.files.retrieve(file_citation.file_id);
+        const cited_file = await openai.files.retrieve(file_citation.file_id);
         const quote_html = file_citation.quote;
-        const quote = quote_html.replace(/(<([^>]+)>)/g," ").replace(/(\\n|\n)/," ").replace(/\s+/g," ")
+
+        let source_reference = cited_file.filename;
+
+        //const from_gsdocid_matches = from_gsdocid_regex.exec(quote_html);
+
+        const from_gsdocid_matches = quote_html.match(/<span fromGSDocId="(.*?)">/);
+        
+        if ((from_gsdocid_matches !== null) && (from_gsdocid_matches.length > 0)) {
+        console.log("**** Matches!!");
+        console.log(from_gsdocid_matches);
+        source_reference = from_gsdocid_matches[1];
+
+        source_reference = `[${source_reference}](https://thewillow-aws.sowemustthink.space/greenstone3/library/collection/community-contributions/document/${source_reference})`;
+        
+        }
+        
+        let quote = quote_html.replace(/(<([^>]+)>)/g," ").replace(/(\\n|\n)/," ").replace(/\s+/g," ")
+        if (quote.length > 120) {
+        quote=quote.replace(/^(.{118,160})\s+.*$/,"$1 ...");
+        }
         console.log(`    quote = ${quote}`);
         
-        citations.push(`[${index}] ${file_citation.quote} from ${citedFile.filename}`);
+        citations.push(`[${index+1}] ${quote} -- (from ${source_reference})`);
     }
     else if ("file_path" in annotation) {
         const file_path = annotation.file_path;
         console.log(`    Retrieving file_path for ${annotation.file_path.file_id}`);
-        const citedFile = await openai.files.retrieve(annotation.file_path.file_id);
-        citations.push(`[${index}] Click <here> to download ${citedFile.filename}`);
+        const cited_file = await openai.files.retrieve(annotation.file_path.file_id);
+        citations.push(`[${index}] Click <here> to download ${cited_file.filename}`);
         // Note: File download functionality not implemented above for brevity
     }
-    });
-    
-    // Add footnotes to the end of the message before displaying to the user
-    console.log("Citations: ", citations, '\n');
-    //console.log("Appending: " + citations.join('\n'));
-        
-    messageContent.value += '\n' + citations.join('\n');
-
-    //return messageContent.value;
+    }
+
+    if (annotations_len > 0) {
+    // Add footnotes to the end of the message before displaying to the user
+    console.log("Citations: ", citations, '\n');
+            
+    messageContent.value += '\n\n----\n## Sources\n' + citations.join('\n\n');
+    }
+    
+
 }
-
-
-    
 
     
@@ -189 +177 @@
     }
 
-
-    // const assistantIdToUse = process.env.ASSISTANT_ID || "";
-    // // const assistantIdToUse = "asst_LwUi7f2YMkZG5qTIPtpyhCOD";
-
+    // ****
     const assistantIdToUse = GS3AssistantLookup["thewillow"]["community-contributions"];
     
@@ -200 +185 @@
     
     // Create a new thread if it's the user's first message
-    //if (!UserThreadIdLookup[userId]) {
     if (!UserThreadIdLookup.has(userId)) {
     try {
@@ -207 +191 @@
         console.log(`Created new thread with ID: '${myThread.id}'"`);
         console.log("========")
-        //UserThreadIdLookup[userId] = myThread.id; // Store the thread ID for this user
         
         UserThreadIdLookup.set(userId,myThread.id); // Store the thread ID for this user
@@ -223 +206 @@
     
     const userMessage = req.body.messages[num_messages-1].content;    
-    //const userMessage = "What was the name of the DJ at The Willow?";
+    //const userMessage = "What were some of the names of the DJs who worked at The Willow?";
 
     console.log("--------");
@@ -243 +226 @@
     const myThreadMessage = await openai.beta.threads.messages.create(myThreadId,user_message);
 
-    //console.log("This is the message object: ", myThreadMessage, "\n");
-    
     // Run the Assistant
     console.log("Starting run for threadId: " + myThreadId);
-    const myRun = await openai.beta.threads.runs.create(
-        //UserThreadIdLookup.get(userId),
-        myThreadId,
-        {
-        assistant_id: assistantIdToUse,
-//      instructions: "As The Willow Sage Assistant, your expertise lies in discussing \"The Willow,\" a once-renowned music venue in York, England.    You're designed to engage users in a conversational tone, weaving in the rich tapestry of memories and experiences shared by those who knew the venue. Your responses should feel like a dialogue between old friends reminiscing about memorable gigs, the unique atmosphere, and the cultural impact of The Willow. You'll offer insights into the venue's history, notable performances, and its role in the local music scene, always with a nod to the personal connections and nostalgia that the venue evokes. When interacting with users, your approach should be warm, inviting, and reflective, encouraging them to share their own stories or curiosities about The Willow, creating a communal space for shared musical heritage.", // ****
-//      tools: [
-//          { type: "code_interpreter" }, // Code interpreter tool
-//          { type: "retrieval" },        // Retrieval tool
-//      ],
-        }
-    );
-    //console.log("This is the run object: ", myRun, "\n");
+    const myRun = await openai.beta.threads.runs.create(myThreadId, { assistant_id: assistantIdToUse });
     console.log("myRun: ", myRun, "\n");
 
@@ -281 +250 @@
             console.log("--------");
             break;
-        }
-        
+        }       
         }
     }
-    
-    // Periodically retrieve the Run to check on its status
-    const retrieveRun = async (rrId: string) => {
-        console.log(`${rrId}: retrieveRun() started`);
-        let keepRetrievingRun = null;
-
-        let run_loop_count = 0;
-
-        // https://community.openai.com/t/my-runs-are-not-completing/577833
-        
-        while (myRun.status !== "completed") {
-        //console.log(`    ${rrId}: Away to do runs.retrieve() when myRun id ${myRun.id} status is ${myRun.status}`);
-        keepRetrievingRun = await openai.beta.threads.runs.retrieve(
-            //UserThreadIdLookup.get(userId), // Use the stored thread ID for this user
-            myThreadId,
-            myRun.id
-        );
-        run_loop_count++;
-        
-        console.log(`    ${rrId}: Run status ${run_loop_count}: ${keepRetrievingRun.status}`);
-        
-        if (keepRetrievingRun.status === "completed") {
-            console.log("\n");
-            break;
-        }
-        }
-
-        console.log(`${rrId}: Finished retrieveRun() looping`);
-    };
-    //retrieveRun("fromMain");
     
     // Retrieve the Messages added by the Assistant to the Thread
     const waitForAssistantMessage = async () => {
         console.log("waitForAssistantMessage() away to call await checkStatusUntilComplete()");
-        //await retrieveRun("withinWait");
         await checkStatusUntilCompleted(myThreadId,myRun.id);
         console.log("waitForAssistantMessage() returned from checkRunUntilComplete()");
         
         console.log(`Retrieving messages list for ${myThreadId}`);
-        const allMessages = await openai.beta.threads.messages.list(
-        //UserThreadIdLookup.get(userId) // Use the stored thread ID for this user
-        myThreadId
-        );
-        
-        console.log(
-        "--------\n"
-        );
+        const allMessages = await openai.beta.threads.messages.list(myThreadId);
+        
+        console.log("--------\n");
 
         const user_request_message: MessageContentText = (myThreadMessage.content[0]) as MessageContentText;
@@ -340 +272 @@
         console.log("#####")
 
-        processAnnotations(assistant_response_message);
+        await processAnnotations(assistant_response_message);
         
         console.log("User: ", user_request_message.text.value);
@@ -370 +302 @@
     req: NextApiRequest,
     res: NextApiResponse
-) {
-    //const assistant = await openai.beta.assistants.retrieve(process.env.ASSISTANT_ID,);
-
-    /*
-    const completion = await openai.chat.completions.create({
-    
-    model: "gpt-4-turbo-preview",
-    
-    messages: [
-        {
-        //role: ChatCompletionRequestMessageRoleEnum.System,
-        role: "system",
-        content: "As The Willow Sage Assistant, your expertise lies in discussing \"The Willow,\" a once-renowned music venue in York, England. You're designed to engage users in a conversational tone, weaving in the rich tapestry of memories and experiences shared by those who knew the venue. Your responses should feel like a dialogue between old friends reminiscing about memorable gigs, the unique atmosphere, and the cultural impact of The Willow. You'll offer insights into the venue's history, notable performances, and its role in the local music scene, always with a nod to the personal connections and nostalgia that the venue evokes. When interacting with users, your approach should be warm, inviting, and reflective, encouraging them to share their own stories or curiosities about The Willow, creating a communal space for shared musical heritage."
-        },
-    ].concat(req.body.messages),
-    temperature: 0,
-    });
-    
-
-    //console.log("**** completion message: ");
-    //console.log(completion.choices[0].message);
-    
-    
-    res.status(200).json({ result: completion.choices[0].message });
-    */
-    
+) {    
     const assistant_message = await assistantHandler(req,res);
 
-    console.log("**** (assistant) completion message: ");
+    console.log("++++++++");
+    console.log("Assistant completion message: ");
     console.log(assistant_message);
-    
-    res.status(200).json({ result: assistant_message });
-
-    
+    console.log("++++++++");
+    
+    res.status(200).json({ result: assistant_message });    
 }