feat: add voice

package.json

@@ -14,11 +14,13 @@
   "license": "ISC",
   "dependencies": {
     "@builderbot-plugins/telegram": "^1.0.0",
-    "@builderbot/bot": "1.0.23-alpha.0",
-    "@builderbot/provider-baileys": "1.0.23-alpha.0",
+    "@builderbot/bot": "1.0.27-alpha.0",
+    "@builderbot/provider-baileys": "1.0.27-alpha.0",
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "date-fns": "^3.3.1",
     "date-fns-tz": "3.0.0-beta.2",
     "dotenv": "^16.4.2",
+    "fluent-ffmpeg": "^2.1.2",
     "multer": "1.4.5-lts.1",
     "openai": "^4.27.0"
   },

src/flows/index.ts

@@ -4,11 +4,13 @@ import { welcomeFlow } from "./welcome.flow";
 import { flowSeller } from "./seller.flow";
 import { flowSchedule } from "./schedule.flow";
 import { flowConfirm } from "./confirm.flow";
+import { voiceFlow } from "./voice.flow";
 
 
 export default createFlow([
     welcomeFlow,
     flowSeller,
     flowSchedule,
-    flowConfirm
+    flowConfirm,
+    voiceFlow
 ])

src/flows/seller.flow.ts

@@ -1,6 +1,6 @@
 import { addKeyword, EVENTS } from "@builderbot/bot";
 import { generateTimer } from "../utils/generateTimer";
-import { getHistoryParse, handleHistory } from "../utils/handleHistory";
+import { getHistory, getHistoryParse, handleHistory } from "../utils/handleHistory";
 import AIClass from "../services/ai";
 import { getFullCurrentDate } from "src/utils/currentDate";
 import { pdfQuery } from "src/services/pdf";
@@ -43,13 +43,14 @@ export const generatePromptSeller = (history: string, database: string) => {
 
 const flowSeller = addKeyword(EVENTS.ACTION)
     .addAnswer(`⏱️`)
-    .addAction(async (ctx, { state, flowDynamic, extensions }) => {
+    .addAction(async (_, { state, flowDynamic, extensions }) => {
         try {
 
             const ai = extensions.ai as AIClass
+            const lastMessage = getHistory(state).at(-1)
             const history = getHistoryParse(state)
 
-            const dataBase = await pdfQuery(ctx.body)
+            const dataBase = await pdfQuery(lastMessage.content)
             console.log({ dataBase })
             const promptInfo = generatePromptSeller(history, dataBase)
 

src/flows/voice.flow.ts

@@ -0,0 +1,26 @@
+import { EVENTS, addKeyword } from "@builderbot/bot";
+import { BaileysProvider } from "@builderbot/provider-baileys";
+import AIClass from "src/services/ai";
+import { processAudio } from "src/utils/process";
+import { welcomeFlow } from "./welcome.flow";
+import mainLayer from "src/layers/main.layer";
+import { handleHistory } from "src/utils/handleHistory";
+
+const voiceFlow = addKeyword<BaileysProvider>(EVENTS.VOICE_NOTE)
+    .addAction(async (_, {  flowDynamic }) => {
+        await flowDynamic(`dame un momento para esucharte...`)
+    })
+    .addAction(async (ctx: any, { provider, extensions, gotoFlow, state }) => {
+        try{
+            const ai = extensions.ai as AIClass
+            const pathVoice = await provider.saveFile(ctx)
+            const mp3Path = await processAudio(pathVoice)
+            const text = await ai.voiceToText(mp3Path)
+            await handleHistory({ content: text, role: 'user' }, state)
+            return gotoFlow(welcomeFlow)
+        }catch(e){
+        console.log(`Err`,e)}
+    })
+    .addAction(mainLayer)
+
+export { voiceFlow }

src/layers/conversational.layer.ts

@@ -5,5 +5,6 @@ import { handleHistory } from "../utils/handleHistory";
  * Su funcion es almancenar en el state todos los mensajes que el usuario  escriba
  */
 export default async ({ body }: BotContext, { state, }: BotMethods) => {
+    if(body.includes('_event_')) return
     await handleHistory({ content: body, role: 'user' }, state)
 }

src/layers/main.layer.ts

@@ -19,15 +19,15 @@ Por favor, analiza la siguiente conversación y determina la intención del usua
 export default async (_: BotContext, { state, gotoFlow, extensions }: BotMethods) => {
     const ai = extensions.ai as AIClass
     const history = getHistoryParse(state)
-    const prompt = PROMPT_DISCRIMINATOR
+    const prompt = PROMPT_DISCRIMINATOR.replace('{HISTORY}', history)
 
 
-    console.log(prompt.replace('{HISTORY}', history))
+    console.log(prompt)
 
     const { prediction } = await ai.determineChatFn([
         {
             role: 'system',
-            content: prompt.replace('{HISTORY}', history)
+            content: prompt
         }
     ])
 

src/utils/process.ts

@@ -0,0 +1,27 @@
+import ffmpeg from "fluent-ffmpeg";
+import {join} from "path";
+import {tmpdir} from "os";
+import { path as ffmpegPath } from '@ffmpeg-installer/ffmpeg';
+ffmpeg.setFfmpegPath(ffmpegPath);
+
+const convertOggMp3 = async (inputStream: string, outStream: string) => {
+    return new Promise((resolve) => {
+        ffmpeg(inputStream)
+            .audioQuality(96)
+            .toFormat("mp3")
+            .save(outStream)
+            .on("progress", () => null)
+            .on("end", () => {
+                resolve(true);
+            });
+    });
+};
+
+/**
+ * Return mp3 path
+ */
+export const processAudio =  async (pathOgg: string ) => {
+    const pathTmpMp3 = join(tmpdir(),`voice-note-${Date.now()}.mp3`);
+    await convertOggMp3(pathOgg, pathTmpMp3);
+    return pathTmpMp3
+}