@elimeleth/vct-layers
Version:
Crea un archivo app.ts, coloca el codigo de abajo alli y luego puedes correrlo con `npx tsx src/app.ts`
658 lines (649 loc) • 27.6 kB
JavaScript
;
var vctFlow = require('@elimeleth/vct-flow');
var axios = require('axios');
var fs = require('fs');
var openai = require('openai');
var path = require('path');
var vctAssistants = require('@elimeleth/vct-assistants');
const composing = async (ctx, provider) => {
try {
if (provider && provider?.vendor && provider.vendor?.sendPresenceUpdate) {
const id = ctx.key.remoteJid;
await provider.vendor.sendPresenceUpdate('composing', id);
}
}
catch (errorComposing) { }
};
function imageToBase64(imagePath) {
try {
// Leer el archivo como un buffer
const imageBuffer = fs.readFileSync(imagePath);
// Obtener el MIME type basándote en la extensión del archivo
const mimeType = getMimeType(imagePath);
// Convertir el buffer a Base64
const base64Data = imageBuffer.toString('base64');
// Combinar MIME type y Base64 data
const base64String = base64Data; //`data:${mimeType};base64,${base64Data}`;
return {
mimeType,
base64: base64String
};
}
catch (error) {
vctFlow.logging.error('Error al convertir la imagen:', error);
return null;
}
}
function getMimeType(imagePath) {
const extension = path.extname(imagePath).toLowerCase();
const mimeTypes = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.webp': 'image/webp'
};
return mimeTypes[extension] || 'application/octet-stream';
}
class Assistant$1 {
static async fn() {
throw new Error("Assistant only has layer method");
}
static layer(config) {
async function _layer(ctx, { state, send, end, extensions: { database, assistant, provider } }) {
let asst = assistant;
const bd = database;
const controller = new AbortController();
let file_id = '';
let client = await bd.getClient(ctx.from);
if (vctFlow.EVENTS.MEDIA.test(ctx.body) || ctx?.body?.includes("_event_media_")) {
if (["jpeg", "jpg", "png", "webp"].includes(ctx?.file_dir_path?.split('.')?.at(-1))) {
await send(config?.image_msg || "Dame un momento para ver la imagen 👁️...");
const data = await asst.openai.files.create({
file: await openai.toFile(fs.createReadStream(ctx.file_dir_path)),
purpose: "vision",
});
ctx.body = ctx?.caption || null;
file_id = data.id;
}
}
if (vctFlow.EVENTS.VOICE_NOTE.test(ctx.body) || ctx?.body?.includes("_event_voice_note_")) {
await send(config?.listen_msg || "Dame un momento para escuchar el audio 👂🏼...");
const path = await vctFlow.helpers.convertOpusToMp3(ctx.file_dir_path);
ctx.body = await vctFlow.helpers.sendToOpenaiWhisper(path);
vctFlow.logging.info(`🤖 Fin voz a texto....[TEXT]: ${ctx.body}`);
}
try {
const messages = [];
let combinedMessage = ctx.messages.filter(e => !e?.includes("_event"));
combinedMessage.push(ctx?.body);
combinedMessage = [...new Set(combinedMessage)].join(" ");
if (file_id.length) {
messages.push({
image_file: {
file_id,
detail: "high"
},
type: "image_file"
});
}
if (combinedMessage.length) {
messages.push({
type: "text",
text: combinedMessage
});
}
let thread_id = await state.get("thread_id");
if (!thread_id) {
const thread = await assistant.create_thread();
thread_id = thread.id;
await state.update("thread_id", thread_id);
}
if (!messages.length) {
messages.push({
type: "text",
text: ctx?.body || ""
});
}
asst.events.on("runCompleted", async ({ run, output }) => {
try {
const params = {
message: {
from: ctx.from,
body: combinedMessage,
type: "plain",
metadata: {
name: process.env.IMAGE,
create_at: new Date()
}
},
thread_id: "thread_id",
output: {
completed_at: run.completed_at,
body: output,
total_tokens: run.usage.total_tokens,
thread_id: thread_id
},
model: run.model
};
if (process.env.ACCOUNT_ID && process.env.SECRET_ROLE) {
await axios.post("https://vapi.flippoapp.com/api/v1/account/message", params, {
headers: {
"Content-Type": "application/json",
"x-account-id": process.env.ACCOUNT_ID,
"x-role-user": process.env.SECRET_ROLE
}
});
}
}
catch (_) { }
});
let answer = await assistant.invoke({
signal: controller.signal,
thread_id: thread_id,
message: messages,
metadata: {
assistant_name: process.env.IMAGE
},
tool_choice: "auto",
extra: {
truncation_strategy: {
type: "last_messages",
last_messages: 6
},
additional_instructions: `
Estas son metadata de la información del cliente con quien interactuas:
Nombre: ${ctx?.name || "Nombre no asignado aún"}
Telefono: ${ctx.from}
`,
},
...config?.invoke_params
}, {
timeout: 300 * 1000,
...config?.retry_config
});
file_id && await state.delete("file_id");
client = await bd.getClient(ctx.from);
if (client.is_paused)
return await end();
let ints = "";
let svcs = "";
try {
if (config?.functions?.intentions) {
const { intentions, services } = await config.functions.intentions(combinedMessage);
ints = intentions;
svcs = services;
}
}
catch (_) { }
if (config?.functions?.send) {
await composing(ctx, provider);
await config.functions.send(answer, {
usage: {
total_tokens: answer.usage.total_tokens,
cost: (answer.usage.total_tokens * .75) / 1_000_000,
model: "gpt-4o-mini"
},
intentions: ints,
services: svcs,
});
}
else {
await composing(ctx, provider);
let chunks = answer.output.split("\n\n").filter(a => Boolean(a));
await send([{
body: chunks, save_metadata: {
usage: {
total_tokens: answer.usage.total_tokens,
cost: (answer.usage.total_tokens * .75) / 1_000_000,
model: "gpt-4o-mini"
},
intentions: ints,
services: svcs,
}
}]);
}
}
catch (error) {
vctFlow.logging.error(error);
controller.abort();
await end();
throw error;
}
}
return _layer;
}
}
class Assistant {
static async fn() {
throw new Error("Assistant only has layer method");
}
static layer(config) {
async function _layer(ctx, { state, send, end, extensions: { database, assistant, provider } }) {
const bd = database;
const controller = new AbortController();
let file_id = '';
let client = await bd.getClient(ctx.from);
let last_messages = await database._query(`SELECT
user_message,
assistant_message
FROM
message
WHERE
phone = $1
AND created_at >= CURRENT_DATE
ORDER BY
created_at DESC
LIMIT $2`, [ctx.from, config?.claude_params?.history_length || 4]);
let combinedMessage = ctx.messages.filter(e => !e?.includes("_event"));
combinedMessage.push(ctx?.body);
combinedMessage = [...new Set(combinedMessage)].join(" ");
last_messages = last_messages.map((m) => {
let answer = [];
if (m.user_message) {
answer.push({ role: "user", content: m.user_message });
}
if (m.assistant_message) {
answer.push({ role: "assistant", content: m.assistant_message });
}
return answer;
}).reverse().flat();
let messages = [];
if (vctFlow.EVENTS.MEDIA.test(ctx.body) || ctx?.body?.includes("_event_media_")) {
// NOTE: haiku does'nt support image
if (["jpeg", "jpg", "png", "webp"].includes(ctx?.file_dir_path?.split('.')?.at(-1))) {
await send(config?.image_msg || "Dame un momento para ver la imagen 👁️...");
const { mimeType, base64 } = imageToBase64(ctx.file_dir_path);
messages.push({
"role": "user",
"content": [
{
"type": "image",
"source": {
"type": "base64",
"media_type": mimeType,
"data": base64,
},
}
]
});
ctx.body = ctx?.caption || null;
}
else {
return await end();
}
return await end();
}
if (vctFlow.EVENTS.VOICE_NOTE.test(ctx.body) || ctx?.body?.includes("_event_voice_note_")) {
await send(config?.listen_msg || "Dame un momento para escuchar el audio 👂🏼...");
const path = await vctFlow.helpers.convertOpusToMp3(ctx.file_dir_path);
const audio = await vctFlow.helpers.sendToOpenaiWhisper(path);
ctx.body = null;
messages.push({
"role": "user",
"content": audio
});
vctFlow.logging.info(`🤖 Fin voz a texto....[TEXT]: ${ctx.body}`);
}
if (ctx.body) {
messages.push({
role: "user",
content: combinedMessage
});
}
try {
const answer = await new vctAssistants.ClaudeAssistant({
apiKey: config?.claude_params?.api_key,
functions: config?.claude_params?.functions || []
}).invoke({
message: [
...last_messages,
...messages
],
instructions: config?.claude_params.instructions,
extra: {
params: {
...config?.claude_params?.api_params
},
additional_instructions: `
Estas son metadata de la información del cliente con quien interactuas:
Nombre: ${ctx?.name || "Nombre no asignado aún"}
Telefono: ${ctx.from}
`,
...config?.invoke_params?.extra
}
});
file_id && await state.delete("file_id");
client = await bd.getClient(ctx.from);
if (client.is_paused)
return await end();
let ints = "";
let svcs = "";
try {
if (config?.functions?.intentions) {
const { intentions, services } = await config.functions.intentions(combinedMessage);
ints = intentions;
svcs = services;
}
}
catch (_) { }
const total_tokens = answer?.usage?.input_tokens && answer?.usage?.output_tokens ? (answer.usage.input_tokens + answer.usage.output_tokens) : 0;
if (config?.functions?.send) {
await composing(ctx, provider);
let output = answer.content.filter(c => c.type === "text").map(c => c.text).join("\n\n");
output = vctAssistants.utils.parser(output);
await config.functions.send({ output, usage: {
total_tokens,
cost: (total_tokens * 1.50) / 1_000_000,
model: config?.invoke_params?.extra?.params?.model || "claude-3-5-haiku-latest"
} }, {
usage: {
total_tokens,
cost: (total_tokens * 1.50) / 1_000_000,
model: config?.invoke_params?.extra?.params?.model || "claude-3-5-haiku-latest"
},
intentions: ints,
services: svcs,
});
}
else {
await composing(ctx, provider);
let output = answer.content.filter(c => c.type === "text").map(c => c.text).join("\n\n");
output = vctAssistants.utils.parser(output);
let chunks = output.split("\n\n").filter(a => Boolean(a));
await send([{
body: chunks, save_metadata: {
usage: {
total_tokens,
cost: (total_tokens * 1.50) / 1_000_000,
model: config?.invoke_params?.extra?.params?.model || "claude-3-5-haiku-latest"
},
intentions: ints,
services: svcs,
}
}]);
}
}
catch (error) {
controller.abort();
await end();
throw error;
}
}
return _layer;
}
}
class STT {
static async fn(path, provider, confs) {
const output_path = await vctFlow.helpers.convertOpusToMp3(path);
return provider === "groq"
? await vctFlow.helpers.sendToGroqWhisper(output_path, confs?.model, { prompt: confs?.prompt })
: await vctFlow.helpers.sendToOpenaiWhisper(output_path, confs?.model, { prompt: confs?.prompt });
}
static layer(listen_msg = "Dame un momento para escuchar el audio 👂🏼...", provider = "openai", confs) {
async function _layer(ctx, { send, state }) {
if (vctFlow.EVENTS.VOICE_NOTE) {
await send(listen_msg);
const path = await vctFlow.helpers.convertOpusToMp3(ctx.file_dir_path);
const audio = provider === "groq"
? await vctFlow.helpers.sendToGroqWhisper(path, confs?.model, { prompt: confs?.prompt })
: await vctFlow.helpers.sendToOpenaiWhisper(path, confs?.model, { prompt: confs?.prompt });
ctx.body = audio;
await state.update("audio", audio);
}
}
return _layer;
}
}
const map_messages = (type, m) => {
switch (type) {
case "gemini": {
return m.map((m) => {
let answer = [];
if (m.user_message) {
answer.push({
role: "user", parts: [{
text: m.user_message
}]
});
}
if (m.assistant_message) {
answer.push({
role: "assistant", parts: [{
text: m.assistant_message
}]
});
}
return answer;
}).reverse().flat();
}
case "openai_legacy": {
return m.map((m) => {
let answer = [];
if (m.user_message) {
answer.push({ role: "user", content: m.user_message });
}
if (m.assistant_message) {
answer.push({ role: "assistant", content: m.assistant_message });
}
return answer;
}).reverse().flat();
}
case "claude": {
return m.map((m) => {
let answer = [];
if (m.user_message) {
answer.push({ role: "user", content: m.user_message });
}
if (m.assistant_message) {
answer.push({ role: "assistant", content: m.assistant_message });
}
return answer;
}).reverse().flat();
}
}
};
class AsstLayer {
static async fn() {
throw new Error("Assistant only has layer method");
}
static layer(config) {
async function _layer(ctx, { state, send, end, extensions: { database, assistant, provider } }) {
const bd = database;
const controller = new AbortController();
let asst = assistant;
let client = await bd.getClient(ctx.from);
let last_messages = await database._query(`SELECT
user_message,
assistant_message
FROM
message
WHERE
phone = $1
AND created_at >= CURRENT_DATE
ORDER BY
created_at DESC
LIMIT $2`, [ctx.from, 6]);
let messages = map_messages(asst.type, last_messages);
let combinedMessage = ctx.messages.filter(e => !e?.includes("_event"));
combinedMessage.push(ctx?.body);
combinedMessage = [...new Set(combinedMessage)].join(" ");
if (vctFlow.EVENTS.MEDIA.test(ctx.body) || ctx?.body?.includes("_event_media_")) {
if (config.image_config) {
await send(config.image_config.message);
messages.push(...await config.image_config.callback(ctx));
}
else {
return await end();
}
}
if (vctFlow.EVENTS.VOICE_NOTE.test(ctx.body) || ctx?.body?.includes("_event_voice_note_")) {
if (config.audio_config) {
await send(config.audio_config.message);
messages.push(...await config.audio_config.callback(ctx));
}
else {
return await end();
}
}
if (ctx.body) {
let m = map_messages(asst.type, [{ user_message: combinedMessage, assistant_message: null }]);
messages.push(...m);
}
try {
const answer = await asst.invoke({
message: messages,
model: config.request?.model,
instructions: config.request.instructions,
extra: {
params: {
...config?.request?.api_params
},
additional_instructions: `
Estas son metadata de la información del cliente con quien interactuas:
Nombre: ${ctx?.name || "Nombre no asignado aún"}
Telefono: ${ctx.from}
`,
...config?.invoke_params?.extra
}
});
client = await bd.getClient(ctx.from);
if (client.is_paused)
return await end();
let ints = "";
let svcs = "";
try {
if (config?.functions?.intentions) {
const { intentions, services } = await config.functions.intentions(combinedMessage);
ints = intentions;
svcs = services;
}
}
catch (_) { }
const total_tokens = answer.usage.total_tokens;
if (config?.functions?.send) {
await composing(ctx, provider);
await config.functions.send(answer, {
usage: {
total_tokens: answer.usage.total_tokens,
cost: (answer.usage.total_tokens * .75) / 1_000_000,
model: ""
},
intentions: ints,
services: svcs,
});
}
else {
await composing(ctx, provider);
let output = answer.output;
output = vctAssistants.utils.parser(output);
let chunks = output.split("\n\n").filter(a => Boolean(a));
await send([{
body: chunks, save_metadata: {
usage: {
total_tokens,
cost: (total_tokens * 1.50) / 1_000_000,
model: config?.request.model
},
intentions: ints,
services: svcs,
}
}]);
}
}
catch (error) {
controller.abort();
await end();
vctFlow.logging.error(error?.message);
}
}
return _layer;
}
}
// Función para generar un nombre de archivo único basado en la fecha y hora
function generateUniqueFileName(baseName, extension) {
const timestamp = new Date().toISOString().replace(/[:.-]/g, "_");
return `${baseName}_${timestamp}.${extension}`;
}
// Función para sintetizar texto a voz con Eleven Labs
async function elevenlabs_tts(text, outputDir = "tmp") {
await vctFlow.helpers.createDirectoryIfNotExists(outputDir);
const axios = (await import('axios')).default;
const outputPath = path.resolve(outputDir, generateUniqueFileName("eleven", "mp3"));
try {
const response = await axios.post(`https://api.elevenlabs.io/v1/text-to-speech/${process.env.ELEVENLABS_MODEL_ID}`, { text }, {
headers: {
"xi-api-key": process.env.ELEVENLABS_API_KEY,
"Content-Type": "application/json",
},
responseType: "arraybuffer",
});
await fs.promises.mkdir(outputDir, { recursive: true });
await fs.promises.writeFile(outputPath, response.data);
return outputPath;
}
catch (error) {
vctFlow.logging.error("Error en Eleven Labs TTS:", error);
throw error;
}
}
// Función para sintetizar texto a voz con OpenAI
async function openai_tts(input, outputDir = "tmp", voice = "alloy", model = "tts-1") {
await vctFlow.helpers.createDirectoryIfNotExists(outputDir);
const { OpenAIClient } = await import('@langchain/openai');
const openai = new OpenAIClient();
const outputPath = path.resolve(outputDir, generateUniqueFileName("openai", "mp3"));
try {
const mp3 = await openai.audio.speech.create({
model,
voice,
input,
});
const buffer = Buffer.from(await mp3.arrayBuffer());
await fs.promises.mkdir(outputDir, { recursive: true });
await fs.promises.writeFile(outputPath, buffer);
return outputPath;
}
catch (error) {
vctFlow.logging.error("Error en OpenAI TTS:", error);
throw error;
}
}
class OpenAITTS {
static async fn(input, outputDir = "tmp", voice = "alloy", model = "tts-1") {
const audio = await openai_tts(input, outputDir, voice, model);
return audio;
}
static layer(text) {
async function _layer(ctx, { extensions: { provider } }) {
const audio = await openai_tts(text);
if (audio) {
// @ts-ignore
provider.sendAudio(ctx.from, audio);
}
}
return _layer;
}
}
class ElebenLabsAITTS {
static async fn(input, outputDir = "tmp") {
const audio = await elevenlabs_tts(input, outputDir);
return audio;
}
static layer(text) {
async function _layer(ctx, { extensions: { provider } }) {
const audio = await elevenlabs_tts(text);
if (audio) {
// @ts-ignore
provider.sendAudio(ctx.from, audio);
}
}
return _layer;
}
}
exports.Assistant = Assistant$1;
exports.ClaudeAssistant = Assistant;
exports.ElebenLabsAITTS = ElebenLabsAITTS;
exports.FactoryLayer = AsstLayer;
exports.OpenAITTS = OpenAITTS;
exports.Stt = STT;