From 080d2ffc97b4a16ee59e9c67e59e4580b8bb21fe Mon Sep 17 00:00:00 2001 From: Lluis Date: Tue, 12 Nov 2024 20:53:02 +0100 Subject: [PATCH] upload chat to openai --- handlers/openAI.js | 22 +++++++++++++++++++++- helpers/dbmanager.js | 10 +++++++++- lib/inputParser.js | 4 ++++ lib/messenger.js | 7 +++++-- lib/notifier.js | 4 ++++ services/logger.js | 9 ++++++++- services/openAI.js | 31 ++++++++++++++++++++++++++++++- 7 files changed, 81 insertions(+), 6 deletions(-) diff --git a/handlers/openAI.js b/handlers/openAI.js index 89d1bdf..0759886 100644 --- a/handlers/openAI.js +++ b/handlers/openAI.js @@ -1,5 +1,6 @@ const OpenAIService = require('../services/openAI') - +const LoggerService = require('../services/logger') +const moment = require("moment/moment"); class OpenAI { async askOpenAI (target, text, username, bot) { const response = await OpenAIService.askAssistant(text) @@ -7,6 +8,25 @@ class OpenAI { bot.say(target, `@${username} ${response}`) } } + + async createAndUploadToChat (target, bot, isToday = false) { + const today = moment().tz('Europe/Madrid') + const date = isToday ? today : today.subtract(1, 'days') + const startOfDay = date.startOf('day').toDate(); + const endOfDay = date.endOf('day').toDate(); + + const response = await LoggerService.getLogChatMessagesBetweenDays(config.twitch.roomId,startOfDay, endOfDay) + const json = JSON.stringify(response) + + const formattedDate = date.format('YYYY-MM-DD'); + const result = await OpenAIService.uploadFileToVectorStore(json, formattedDate) + if (result.success) { + bot.say(target, `IA actualizada con el chat de ${formattedDate}`) + console.log('Chat uploaded to openai ' + result.filename) + } else { + console.log('Error uploading chat to openai of date ' + formattedDate) + } + } } module.exports = OpenAI diff --git a/helpers/dbmanager.js b/helpers/dbmanager.js index e36b1a6..ec4b548 100644 --- a/helpers/dbmanager.js +++ b/helpers/dbmanager.js @@ -84,6 +84,13 @@ async function addChatLogLine (roomId, nick, text, date) { return conn.model('chatLog').insertMany({roomId: roomId, nick: nick, text: text, date: date}) } +async function getChatLogLines (roomId, startOfDay, endOfDay) { + const conn = await logConn.getConnection() + return conn.model('chatLog') + .find({roomId: roomId, date: { $gte: startOfDay, $lt: endOfDay }}) + .select('nick text date -_id').lean() +} + async function addTitleLogLine (roomId, title, date) { const conn = await logConn.getConnection() return conn.model('titleLog').insertMany({roomId: roomId, title: title, date: date}) @@ -139,5 +146,6 @@ module.exports = { addBans, getPermanentBans, getTimeouts, - removeBan + removeBan, + getChatLogLines } diff --git a/lib/inputParser.js b/lib/inputParser.js index 856b059..4ab5bf8 100644 --- a/lib/inputParser.js +++ b/lib/inputParser.js @@ -134,6 +134,10 @@ class InputParser { isAskingOpenAI (text) { return text.toLowerCase().startsWith('!chat') } + + isAskingToUpdateChatToOpenAI (text) { + return text.toLowerCase().startsWith('!updatechat') + } } module.exports = InputParser diff --git a/lib/messenger.js b/lib/messenger.js index 6411781..20208f8 100644 --- a/lib/messenger.js +++ b/lib/messenger.js @@ -182,15 +182,18 @@ class Messenger { return handlers.events.sendTarracoMangaEvent() // OPEN AI - if (textSplit.length > 1 && inputParser.isAskingOpenAI(textSplit[0]) && this._isNotCooldown('openai',15)) { + if (textSplit.length > 1 && inputParser.isAskingOpenAI(textSplit[0]) && this._isNotCooldown('openai',10)) { return handlers.openAI.askOpenAI(target, textSplit.slice(1).join(' '), context['display-name'], this.bot) } - if (textSplit.length > 1 && inputParser.isAskingBotOpenAI(text) && this._isNotCooldown('openai',15)) { + if (textSplit.length > 1 && inputParser.isAskingBotOpenAI(text) && this._isNotCooldown('openai',10)) { const regex = new RegExp(`@?${config.twitch.username}`, 'gi') const textWithoutMention = text.replace(regex, '').trim() return handlers.openAI.askOpenAI(target, textWithoutMention, context['display-name'], this.bot) } + + if (textSplit.length > 0 && inputParser.isAskingToUpdateChatToOpenAI(textSplit[0]) && isAdmin(context)) + return await handlers.openAI.createAndUploadToChat(target, this.bot, true) } handleConnect (addr, port) { diff --git a/lib/notifier.js b/lib/notifier.js index 8c8974b..3c1bc97 100644 --- a/lib/notifier.js +++ b/lib/notifier.js @@ -32,6 +32,10 @@ class Notifier { await handlers.ban.updateBansList() }) + cron.schedule('15 4 * * *', async () => { + await handlers.openAI.createAndUploadToChat(this.twitchBot, this.target) + }) + return Promise.resolve() } diff --git a/services/logger.js b/services/logger.js index 0c246f2..332e445 100644 --- a/services/logger.js +++ b/services/logger.js @@ -13,8 +13,15 @@ async function logStreamTitle(roomId, title) { return result } +async function getLogChatMessagesBetweenDays(roomId, startOfDay, endOfDay) { + let result = null + result = await dbManager.getChatLogLines(roomId, startOfDay, endOfDay) + return result +} + module.exports = { logChatMessage, - logStreamTitle + logStreamTitle, + getLogChatMessagesBetweenDays } diff --git a/services/openAI.js b/services/openAI.js index 40e7322..54ed944 100644 --- a/services/openAI.js +++ b/services/openAI.js @@ -51,6 +51,32 @@ async function askOpenAI(message) { let assistantThread = null + +async function uploadFileToVectorStore(json, formattedDate) { + try { + const filename = `chat_${formattedDate}.json` + const buffer = Buffer.from(json, 'utf-8'); + const newFile = new File([buffer], filename, { + type: 'application/json', + }); + + const files = []; + for await (const file of assistantsClient.files.list()) { + files.push({id: file.id, name: file.filename }); + } + const matchedFiles = files.filter((file) => file.name === filename); + for (const file of matchedFiles) { + await assistantsClient.files.del(file.id); + } + + await assistantsClient.beta.vectorStores.files.uploadAndPoll(config.openAI.vectorStoreId, newFile); + return { success: true, filename: filename } + + } catch (error) { + return { success: false } + } +} + async function askAssistant(message) { let result try { @@ -106,6 +132,8 @@ async function askAssistant(message) { return result } + + function cleanAssistantText(text) { return text.replaceAll(/【.*?】/g, "") } @@ -123,5 +151,6 @@ async function _getHeaders () { module.exports = { askOpenAI, - askAssistant + askAssistant, + uploadFileToVectorStore }