@websolutespa/payload-plugin-bowl-llm
Version:
LLM plugin for Bowl PayloadCms plugin
1,395 lines (1,377 loc) • 218 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var src_exports = {};
__export(src_exports, {
KbFileNoToolOption: () => KbFileNoToolOption,
LlmKnowledgeBaseCreationMode: () => LlmKnowledgeBaseCreationMode,
appHandler: () => appHandler,
default: () => src_default,
defaultGroup: () => defaultGroup,
defaultSlug: () => defaultSlug,
downloadVectorDbFile: () => downloadVectorDbFile,
errorHandler: () => errorHandler,
fineTuningHandler: () => fineTuningHandler,
fineTuningJobsHandler: () => fineTuningJobsHandler,
getRulesPath: () => getRulesPath,
infoHandler: () => infoHandler,
internalSlugs: () => internalSlugs,
isChunkString: () => isChunkString,
isChunkText: () => isChunkText,
isChunkTextOrString: () => isChunkTextOrString,
kbEndpointHandler: () => kbEndpointHandler,
kbPollEndpointHandler: () => kbPollEndpointHandler,
llm: () => llm,
llmAppsHandler: () => llmAppsHandler,
llmKnowledgeBase: () => llmKnowledgeBase,
llmVectorDbsCleaningHandler: () => llmVectorDbsCleaningHandler,
messageHandler: () => messageHandler,
options: () => options,
pythonStreamHandler: () => pythonStreamHandler,
rulesEndpointHandler: () => rulesEndpointHandler,
rulesHandler: () => rulesHandler,
textToChunksLastUnparsed: () => textToChunksLastUnparsed,
vectorDbTypesHandler: () => vectorDbTypesHandler
});
module.exports = __toCommonJS(src_exports);
// src/translations.ts
var translations = {
en: {
group: {
llm: "LLM"
},
collection: {
singular: {
llmApp: "Application",
llmTool: "Tool",
llmAppTool: "Tool",
llmThread: "Thread",
llmFineTuning: "Fine Tuning",
llmKbAzure: "Azure",
llmKbConfluence: "Confluence",
llmKbDropbox: "Dropbox",
llmKbGcs: "Google cloud storage",
llmKbGithub: "Github",
llmKbGoogledrive: "Google drive",
llmKbFile: "Knowledgebase file",
llmKbJira: "Jira",
llmPrompt: "Mental model",
llmKbS3: "S3",
llmKbSftp: "SFTP",
llmKbSharepoint: "Sharepoint",
llmKbSitemap: "Sitemap",
llmKbSlack: "Slack",
llmRule: "Rule",
llmVectorDb: "Vector DB",
card: "Card",
cta: "Call to action"
},
plural: {
llmApp: "Applications",
llmTool: "Tools",
llmAppTool: "Tools",
llmThread: "Threads",
llmFineTuning: "Fine Tuning",
llmKbAzure: "Azure",
llmKbConfluence: "Confluence",
llmKbDropbox: "Dropbox",
llmKbGcs: "Google cloud storage",
llmKbGithub: "Github",
llmKbGoogledrive: "Google drive",
llmKbFile: "Knowledgebase files",
llmKbJira: "Jira",
llmPrompt: "Mental models",
llmKbS3: "S3",
llmKbSftp: "SFTP",
llmKbSharepoint: "Sharepoint",
llmKbSitemap: "Sitemaps",
llmKbSlack: "Slack",
llmRule: "Rules",
llmVectorDb: "Vector DBs",
card: "Cards",
cta: "Call to action"
}
},
global: {},
description: {
__from_env: "Inserisci la variabile d'ambiente (es. JIRA_CONFIG).",
__from_file: "Inserisci il percorso del file JSON (es. /opt/configs/jira_config.json).",
confluence_password: "Confluence password or Cloud API token, if filled, set the access_token to None and vice versa.",
confluence_accessToken: "The personal access token for authenticating with Confluence, e.g., 'AT....'",
confluence_url: "The URL of the Confluence instance, e.g., 'https://example.atlassian.net'.",
confluence_userEmail: "The email address or username of the Confluence user",
confluence_spaces: "A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].",
confluence_extension: "A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx']."
},
field: {
__from_env: "Variabile d'ambiente (__from_env)",
__from_file: "File di configurazione (__from_file)",
accessToken: "Access token",
accountKey: "Account key",
accountName: "Account name",
answer: "Answer",
apiKey: "API key",
appKey: "App key",
appTool: "App tool",
boards: "Boards",
body: "Body",
branch: "Branch",
channels: "Channels",
clientId: "Client ID",
clientSecret: "Client secret",
columnName: "Column name",
columnValues: "Column values",
configSource: "Modalit\xE0 configurazione",
connectionString: "DB connection string",
credentials: "Credentials",
dataSource: "Data source",
dbSettings: "DB settings",
deletedFields: "Deleted fields",
destName: "Dest Name",
driveId: "Drive ID",
endpointUrl: "Endpoint URL",
excludeClass: "Exclude class",
excludeId: "Exclude ID",
excludeTag: "Exclude tag",
extension: "Extension",
externalEndpoints: "External endpoints",
feedback: "Feedback",
fieldsMapping: "Fields mapping",
filterUrls: "Filter URLs",
functionId: "LLM functionId",
functionName: "LLM functionName",
functionDescription: "LLM functionDescription",
functionSettings: "Function settings",
files: "Files",
host: "Host",
httpReferrers: "HTTP referrers",
key: "Key",
icon: "Icon",
includeOnlySelector: "Include only selector",
issues: "Issues",
knowledgeBase: "Knowledge base",
knowledgeBaseEndpoints: "Endpoints import knowledge base",
knowledgebaseSettings: "Knowledge base settings",
llmApp: "LLM App",
llmConfig: "LLM Config",
llmChainSettings: "LLM chain settings",
message: "Message",
messageId: "Message ID",
mode: "Mode",
model: "Model",
nightlyKbGeneration: "Nightly KB generation",
notes: "Notes",
newFields: "New fields",
numDays: "Number of days",
openAIApiKey: "OpenAI ApiKey",
outputFormat: "Output format",
port: "Port",
projects: "Projects",
prompt: "Mental model",
question: "Question",
rating: "Rating",
recursive: "Recursive",
remoteUrl: "Remote URL",
replacedFields: "Replaced fields",
repo: "Repository",
role: "Role",
sasToken: "SAS token",
secret: "Secret",
secretId: "Secret ID",
secrets: "secrets",
secretValue: "Secret Value",
serviceAccountKey: "Service account key",
siteId: "Site ID",
sitemapUrl: "Sitemap URL",
restrictDomain: "Restrict domain",
spaces: "Spaces",
srcName: "Source name",
systemMessage: "systemMessage",
tenantId: "Tenant ID",
token: "Token",
tools: "Tools",
userEmail: "User email",
username: "Username",
value: "Value",
vectorDbFile: "Vector DB file",
waitingMessage: "Waiting Message",
previewEnabled: "Enable Preview",
sharepointSettings: "Sharepoint settings",
tagsToInclude: "Tags to include",
tagsToExclude: "Tags to exclude",
fileExtToExclude: "File extensions to exclude",
fileExt: "File extensions to exclude",
metaFields: "Metadata filters",
threshold: "Threshold"
},
general: {
emptyLlmAppLinks: "No links configured for the app",
generateKb: "Generate knowledgebase",
generatingKb: "Generating knowledgebase...",
generateKbSuccess: "Knowledgebase generated successfully",
generateKbError: "There was an error while generating the knowledgebase",
generateRules: "Generate rules",
generatingRules: "Generating rules...",
generateRulesSuccess: "Rules generated successfully",
generateRulesError: "Error generating rules",
kbFileNoToolOption: "None: file is associated with app",
generateKbAppOnly: "App only",
generateKbToolsOnly: "Tools only",
generateKbAppAndTools: "Both app and tools",
generateKbSpecificTool: "Specific tool",
generateKbModalTitle: "Generate knowledgebase",
generateRulesModalTitle: "Generate rules",
selectLlmApp: "Select the app for which to generate the knowledgebase",
selectKbGenerationMode: "Select the knowledgebase generation mode",
selectApp: "Select app",
selectTool: "Select tool"
}
},
it: {
group: {
llm: "LLM"
},
collection: {
singular: {
llmApp: "Applicazione",
llmTool: "Tool",
llmAppTool: "Tool",
llmThread: "Thread",
llmFineTuning: "Fine Tuning",
llmKbAzure: "Azure",
llmKbConfluence: "Confluence",
llmKbDropbox: "Dropbox",
llmKbGcs: "Google cloud storage",
llmKbFile: "Knowledgebase file",
llmKbGithub: "Github",
llmKbGoogledrive: "Google drive",
llmKbJira: "Jira",
llmPrompt: "Mental model",
llmKbS3: "S3",
llmKbSftp: "SFTP",
llmKbSharepoint: "Sharepoint",
llmKbSitemap: "Sitemap",
llmKbSlack: "Slack",
llmRule: "Rule",
llmVectorDb: "Vector DB",
card: "Card",
cta: "Call to action"
},
plural: {
llmApp: "Applicazioni",
llmTool: "Tools",
llmAppTool: "Tools",
llmThread: "Threads",
llmFineTuning: "Fine Tuning",
llmKbAzure: "Azure",
llmKbConfluence: "Confluence",
llmKbDropbox: "Dropbox",
llmKbGcs: "Google cloud storage",
llmKbFile: "Knowledgebase file",
llmKbGithub: "Github",
llmKbGoogledrive: "Google drive",
llmKbJira: "Jira",
llmPrompt: "Mental models",
llmKbS3: "S3",
llmKbSftp: "SFTP",
llmKbSharepoint: "Sharepoint",
llmKbSitemap: "Sitemap",
llmKbSlack: "Slack",
llmRule: "Rules",
llmVectorDb: "Vector DBs",
card: "Cards",
cta: "Call to action"
}
},
global: {},
description: {
__from_env: "Inserisci la variabile d'ambiente (es. JIRA_CONFIG).",
__from_file: "Inserisci il percorso del file JSON (es. /opt/configs/jira_config.json).",
confluence_password: "Password di Confluence o token API Cloud, se compilato, impostare l'access_token su Nessuno e viceversa",
confluence_accessToken: "Il token di accesso personale per l'autenticazione con Confluence, ad esempio, 'AT....'",
confluence_url: "L'URL dell'istanza di Confluence, ad esempio, 'https://example.atlassian.net'",
confluence_userEmail: "L'indirizzo e-mail o il nome utente dell'utente di Confluence",
confluence_spaces: "Un elenco di spazi Confluence con cui interagire, ad esempio ['SPACE1', 'SPACE2']",
confluence_extension: "Un elenco di estensioni di file da filtrare. Si imposta come nessuno, ad esempio ['.pdf', '.docx']."
},
field: {
__from_env: "Variabile d'ambiente (__from_env)",
__from_file: "File di configurazione (__from_file)",
accessToken: "Access token",
accountKey: "Account key",
accountName: "Account name",
answer: "Risposta",
apiKey: "API key",
appKey: "App key",
appTool: "App tool",
boards: "Boards",
body: "Corpo",
branch: "Branch",
channels: "Channels",
clientId: "Client ID",
clientSecret: "Client secret",
columnName: "Nome colonna",
columnValues: "Valori colonna",
"description.config__from_env_description": "Inserisci la variabile d'ambiente (es. JIRA_CONFIG).",
config__from_file_description: "Inserisci il percorso del file JSON (es. /opt/configs/jira_config.json).",
configSource: "Modalit\xE0 configurazione",
connectionString: "Stringa di connessione al DB",
credentials: "Credenziali",
dataSource: "Sorgente dati",
dbSettings: "DB Settings",
deletedFields: "Campi eliminati",
destName: "Nome destinazione",
driveId: "Drive ID",
endpointUrl: "Endpoint URL",
excludeClass: "Escludi classi",
excludeId: "Escludi ID",
excludeTag: "Escludi tag",
extension: "Estensioni",
externalEndpoints: "Endpoint esterni",
feedback: "Feedback",
fieldsMapping: "Fields mapping",
files: "Files",
filterUrls: "Filtro URLs",
functionId: "LLM functionId",
functionName: "LLM functionName",
functionDescription: "LLM functionDescription",
functionSettings: "Function settings",
host: "Host",
httpReferrers: "HTTP referrers",
key: "Key",
icon: "Icona",
includeOnlySelector: "Includi solo selettori",
issues: "Issues",
knowledgeBase: "Knowledge base",
knowledgeBaseEndpoints: "Endpoints import knowledge base",
knowledgebaseSettings: "Knowledge base settings",
llmApp: "LLM App",
llmConfig: "LLM Config",
llmChainSettings: "Impostazioni LLM chain",
message: "Message",
messageId: "ID messaggio",
mode: "Modalit\xE0",
model: "Model",
nightlyKbGeneration: "Generazione KB notturna",
notes: "Note",
newFields: "Nuovi campi",
numDays: "Numero di giorni",
openAIApiKey: "OpenAI ApiKey",
outputFormat: "Formato output",
port: "Port",
projects: "Progetti",
prompt: "Mental model",
question: "Domanda",
rating: "Rating",
recursive: "Ricorsivo",
remoteUrl: "URL remoto",
replacedFields: "Campi sostituiti",
repo: "Repository",
role: "Role",
sasToken: "SAS token",
secret: "Secret",
secretId: "Secret ID",
secrets: "secrets",
secretValue: "Secret Value",
serviceAccountKey: "Service account key",
siteId: "Site ID",
sitemapUrl: "Sitemap URL",
restrictDomain: "Limita dominio",
spaces: "Spaces",
srcName: "Nome sorgente",
systemMessage: "systemMessage",
tenantId: "Tenant ID",
token: "Token",
tools: "Tools",
userEmail: "User email",
username: "Username",
value: "Valore",
vectorDbFile: "Vector DB File",
waitingMessage: "Waiting Message",
previewEnabled: "Abilita Preview",
sharepointSettings: "Impostazioni Sharepoint",
tagsToInclude: "Tags da includere",
tagsToExclude: "Tags da escludere",
fileExtToExclude: "Estensioni file da escludere",
fileExt: "File extensions to exclude",
metaFields: "Filtro metadati",
threshold: "Threshold"
},
general: {
emptyLlmAppLinks: "Nessun link configurato per l'app",
generateKb: "Genera knowledgebase",
generatingKb: "Generazione knowledgebase in corso...",
generateKbSuccess: "Knowledgebase generata con successo",
generateKbError: "Errore durante la generazione della knowledgebase",
generateRules: "Genera rules",
generatingRules: "Generazione rules in corso...",
generateRulesSuccess: "Rules generate con successo",
generateRulesError: "Errore durante la generazione delle rules",
kbFileNoToolOption: "Nessuno: il file \xE8 associato all'app",
generateKbAppOnly: "Solo app",
generateKbToolsOnly: "Solo tools",
generateKbAppAndTools: "App e tools",
generateKbSpecificTool: "Tool specifico",
generateKbModalTitle: "Genera knowledgebase",
generateRulesModalTitle: "Genera rules",
selectLlmApp: "Seleziona l'app per la quale generare la knowledgebase",
selectKbGenerationMode: "Seleziona la modalit\xE0 di generazione della knowledgebase",
selectApp: "Seleziona app",
selectTool: "Seleziona tool"
}
}
};
// src/options.ts
var defaultSlug = {
llmApp: "llmApp",
llmTool: "llmTool",
llmAppTool: "llmAppTool",
llmThread: "llmThread",
llmFineTuning: "llmFineTuning",
llmPrompt: "llmPrompt",
llmRule: "llmRule",
llmKbAzure: "llmKbAzure",
llmKbConfluence: "llmKbConfluence",
llmKbDropbox: "llmKbDropbox",
llmKbFile: "llmKbFile",
llmKbGcs: "llmKbGcs",
llmKbGithub: "llmKbGithub",
llmKbGoogledrive: "llmKbGoogledrive",
llmKbJira: "llmKbJira",
llmKbS3: "llmKbS3",
llmKbSftp: "llmKbSftp",
llmKbSharepoint: "llmKbSharepoint",
llmKbSitemap: "llmKbSitemap",
llmKbSlack: "llmKbSlack",
llmVectorDb: "llmVectorDb"
};
var defaultGroup = {
llm: "llm"
};
var options = {
group: defaultGroup,
slug: defaultSlug,
translations,
roles: {
LlmEditor: "llm_editor",
LlmContributor: "llm_contributor"
}
};
var internalSlugs = Object.entries(defaultSlug).map((x) => x[1]);
// src/api/error.handler.ts
var errorHandler = /* @__PURE__ */ __name((error, res) => {
const status = error && error.status ? error.status : 500;
const message = error && error.message ? error.message : "Unknown error";
const exception = {
...typeof error === "object" ? error : {},
status,
message
};
res.status(exception.status).send(exception);
}, "errorHandler");
// src/api/app.handler.ts
var appHandler = /* @__PURE__ */ __name(async (req) => {
const { payload: payload4 } = req;
if (!payload4) {
throw { status: 500, message: "Server Error: Cannot resolve payload" };
}
const { query } = req;
if (!query) {
throw { status: 400, message: "Bad Request: locale is missing" };
}
const { locale } = query;
if (!locale) {
throw { status: 400, message: "Bad Request: locale is missing" };
}
const { appKey, apiKey, preview } = req.body;
if (!appKey) {
throw { status: 401, message: "Unauthorized: appKey is missing" };
}
if (!apiKey) {
throw { status: 401, message: "Unauthorized: apiKey is missing" };
}
const defaultLocale = payload4.config.localization ? payload4.config.localization.defaultLocale : "en";
const response = await payload4.find({
collection: options.slug.llmApp,
where: {
"settings.credentials.appKey": {
equals: appKey
},
"settings.credentials.apiKey": {
equals: apiKey
},
...preview ? {
"settings.credentials.previewEnabled": {
equals: true
}
} : {
isActive: {
equals: true
}
}
},
depth: 3,
locale,
fallbackLocale: defaultLocale,
showHiddenFields: true,
pagination: false,
overrideAccess: true
});
if (!response.docs.length) {
throw { status: 404, message: "Not Found: application not found" };
}
const app = response.docs[0];
const referrer = req.headers.referrer || req.headers.referer || "";
const isReferrefAllowed = !referrer || app.settings.credentials.httpReferrers.reduce((p, c) => {
return p || referrer.indexOf(c.referrer) !== -1;
}, false);
if (!isReferrefAllowed) {
console.log("referrer", `[${referrer}]`, app.settings.credentials.httpReferrers);
throw { status: 401, message: `Unauthorized: referrer is not allowed [${referrer}]` };
}
return app;
}, "appHandler");
var llmAppsHandler = /* @__PURE__ */ __name(async (req, res, next) => {
try {
const { payload: payload4 } = req;
const response = await payload4.find({
collection: options.slug.llmApp,
pagination: false,
req,
overrideAccess: false
});
res.status(200).send(response.docs);
} catch (error) {
return errorHandler(error, res);
}
}, "llmAppsHandler");
// src/api/fineTuning.handler.ts
var import_fs = __toESM(require("fs"));
var import_openai = __toESM(require("openai"));
var import_path = __toESM(require("path"));
var setLlmAppCurrentJobId = /* @__PURE__ */ __name(async (payload4, app, jobId) => {
await payload4.update({
collection: options.slug.llmApp,
id: app.id,
data: {
settings: {
fineTuning: {
currentJobId: jobId
}
}
},
overrideAccess: true
});
}, "setLlmAppCurrentJobId");
var createFineTuningJob = /* @__PURE__ */ __name(async (payload4, app, jsonlFilePath) => {
const openai = new import_openai.default({ apiKey: app.settings.llmConfig.secrets.openAIApiKey });
const file = await openai.files.create({ file: import_fs.default.createReadStream(jsonlFilePath), purpose: "fine-tune" });
const job = await openai.fineTuning.jobs.create({ training_file: file.id, model: "gpt-3.5-turbo-1106", suffix: app.settings?.fineTuning?.modelNameSuffix ?? "" });
return job;
}, "createFineTuningJob");
var writeFineTuningJsonlFile = /* @__PURE__ */ __name(async (payload4, app) => {
const response = await payload4.find({
collection: options.slug.llmFineTuning,
where: {
llmApp: {
equals: app.id
}
},
overrideAccess: true
});
const systemMessage = app.settings.llmConfig.prompt.prompt?.systemMessage || app.settings.llmConfig.prompt.systemMessage || "";
let jsonl = "";
for (const doc of response.docs) {
const json = {
messages: [
{
role: "system",
content: systemMessage
},
{
role: "user",
content: doc.question
},
{
role: "assistant",
content: doc.answer
}
]
};
jsonl = jsonl.concat(JSON.stringify(json) + "\n");
}
const fineTuningPath = import_path.default.join(process.cwd(), "lib", "fine-tuning");
import_fs.default.mkdirSync(fineTuningPath, { recursive: true });
const dest = import_path.default.join(fineTuningPath, "finetuning.jsonl");
import_fs.default.writeFileSync(dest, jsonl);
return dest;
}, "writeFineTuningJsonlFile");
async function fineTuningJobsHandler(payload4) {
const result = await payload4.find({
collection: options.slug.llmApp,
where: {
"settings.fineTuning.currentJobId": {
not_equals: null
}
},
overrideAccess: true
});
for (const app of result.docs) {
const jobId = app.settings.fineTuning.currentJobId;
if (jobId) {
const openai = new import_openai.default({ apiKey: app.settings.llmConfig.secrets.openAIApiKey });
const fineTune = await openai.fineTuning.jobs.retrieve(jobId);
if (fineTune.status === "succeeded") {
await payload4.update({
collection: options.slug.llmApp,
id: app.id,
data: {
settings: {
fineTuning: {
currentJobId: null,
fineTunedModelName: fineTune.fine_tuned_model
}
}
},
overrideAccess: true
});
}
}
}
}
__name(fineTuningJobsHandler, "fineTuningJobsHandler");
var fineTuningHandler = /* @__PURE__ */ __name(async (req, res, next) => {
try {
const { payload: payload4 } = req;
const app = await appHandler(req);
const jsonlFilePath = await writeFineTuningJsonlFile(payload4, app);
const job = await createFineTuningJob(payload4, app, jsonlFilePath);
await setLlmAppCurrentJobId(payload4, app, job.id);
res.status(200).send(job);
} catch (error) {
return errorHandler(error, res);
}
}, "fineTuningHandler");
// src/api/thread.handler.ts
var threadHandler = /* @__PURE__ */ __name(async (req, appId) => {
const { payload: payload4 } = req;
if (!payload4) {
throw { status: 500, message: "Server Error: Cannot resolve payload" };
}
const { query } = req;
if (!query) {
throw { status: 400, message: "Bad Request: locale is missing" };
}
const { locale } = query;
if (!locale) {
throw { status: 400, message: "Bad Request: locale is missing" };
}
const { appKey, apiKey, threadId } = req.body;
if (!appKey) {
throw { status: 401, message: "Unauthorized: appKey is missing" };
}
if (!apiKey) {
throw { status: 401, message: "Unauthorized: apiKey is missing" };
}
if (!threadId) {
throw { status: 401, message: "Unauthorized: threadId is missing" };
}
const defaultLocale = payload4.config.localization ? payload4.config.localization.defaultLocale : "en";
const response = await payload4.find({
collection: options.slug.llmThread,
where: {
id: {
equals: threadId
},
llmApp: {
equals: appId
}
},
depth: 3,
locale,
fallbackLocale: defaultLocale,
showHiddenFields: true,
pagination: false,
overrideAccess: true
});
if (!response.docs.length) {
throw { status: 404, message: "Not Found: thread not found" };
}
const thread = response.docs[0];
return thread;
}, "threadHandler");
// src/api/info.handler.ts
var infoHandler = /* @__PURE__ */ __name(async (req, res, next) => {
try {
const app = await appHandler(req);
const info = {
contents: app.contents
};
const { threadId } = req.body;
if (app && threadId) {
const thread = await threadHandler(req, app.id);
info.thread = {
messages: thread.message.map((x) => ({
content: x.content,
role: x.role
})),
threadId
};
}
res.status(200).send(info);
} catch (error) {
return errorHandler(error, res);
}
}, "infoHandler");
// src/api/knowledgeBase.handler.ts
var import_payload_plugin_cron_job2 = require("@websolutespa/payload-plugin-cron-job");
var import_fs5 = __toESM(require("fs"));
var import_path5 = __toESM(require("path"));
var import_uuid2 = require("uuid");
// src/utils/knowledgebase.ts
var import_fs2 = __toESM(require("fs"));
var import_http = __toESM(require("http"));
var import_https = __toESM(require("https"));
var import_path2 = __toESM(require("path"));
function getKnowledgebasePath(basePath2, appName, toolName = "") {
let kbPath = import_path2.default.join(basePath2, "knowledge-base", appName);
if (toolName !== "") {
kbPath = import_path2.default.join(kbPath, toolName);
}
return kbPath;
}
__name(getKnowledgebasePath, "getKnowledgebasePath");
async function downloadFile(url, dest, requestOptions = {}) {
const file = import_fs2.default.createWriteStream(dest);
return new Promise((resolve, reject) => {
const client = url.startsWith("https") ? import_https.default : import_http.default;
const request = client.get(url, requestOptions, (response) => {
if (response.statusCode !== 200) {
reject(new Error(`Failed to download file from '${url}' (${response.statusCode})`));
return;
}
response.pipe(file);
file.on("finish", () => {
file.close();
resolve();
});
});
request.on("error", (err) => {
import_fs2.default.unlink(dest, () => reject(err));
});
file.on("error", (err) => {
import_fs2.default.unlink(dest, () => reject(err));
});
});
}
__name(downloadFile, "downloadFile");
function deleteFolderRecursive(path8) {
if (!import_fs2.default.existsSync(path8))
return;
try {
import_fs2.default.rmSync(path8, { recursive: true, force: true });
} catch (err) {
if (err) {
throw err;
}
}
}
__name(deleteFolderRecursive, "deleteFolderRecursive");
// src/api/llmKbWebhook.handler.ts
var import_bom_core = require("@websolutespa/bom-core");
var import_payload_plugin_cron_job = require("@websolutespa/payload-plugin-cron-job");
var import_fs4 = __toESM(require("fs"));
var import_mime_types = __toESM(require("mime-types"));
var import_path4 = __toESM(require("path"));
var import_payload = __toESM(require("payload"));
var import_raw_body = __toESM(require("raw-body"));
var import_standardwebhooks = require("standardwebhooks");
var import_uuid = require("uuid");
// src/utils/robot.ts
var import_fs3 = __toESM(require("fs"));
var import_path3 = __toESM(require("path"));
var import_stream = require("stream");
async function downloadRobot(file, downloadPath) {
const result = await fetch(`${process.env.ROBOT_HOST}/api/llm/kb/file/${file}`, {
headers: {
"Content-Type": "application/json",
"Authorization": `Basic ${btoa(`${process.env.ROBOT_USER}:${process.env.ROBOT_PASSWORD}`)}`
}
});
if (!result.ok) {
throw new Error(`Error while downloading file ${file} from ROBOT. Status: ${result.status} - ${result.statusText}`);
}
const contentDisposition = result.headers.get("content-disposition");
const match = contentDisposition?.match(/filename="?(.+?)"?$/);
if (!match) {
throw new Error(`Error while parsing filename from Content-Disposition header: ${contentDisposition}`);
}
const filename = match[1];
const filePath = import_path3.default.join(downloadPath, filename);
const fileStream = import_fs3.default.createWriteStream(filePath);
const resultBody = result.body;
if (resultBody) {
await new Promise((resolve, reject) => {
import_stream.Readable.fromWeb(resultBody).pipe(fileStream);
fileStream.on("finish", resolve);
fileStream.on("error", reject);
});
}
return filename;
}
__name(downloadRobot, "downloadRobot");
async function fetchRobot(endpoint, method, body, headers = {}) {
const result = await fetch(`${process.env.ROBOT_HOST}/${endpoint}`, {
method,
headers: {
"Content-Type": "application/json",
"Authorization": `Basic ${btoa(`${process.env.ROBOT_USER}:${process.env.ROBOT_PASSWORD}`)}`,
...headers
},
body: JSON.stringify(body)
});
if (!result.ok) {
const error = new Error(`Error while fetching from ROBOT endpoint ${endpoint}. Status: ${result.status} - ${result.statusText}`);
error.details = await result.text();
throw error;
}
return await result.json();
}
__name(fetchRobot, "fetchRobot");
// src/api/llmKbWebhook.handler.ts
var createTaskLog = /* @__PURE__ */ __name(async (job, traceId, logData) => {
const log = await (0, import_payload_plugin_cron_job.createCronLog)({
job,
message: JSON.stringify(logData),
traceId
});
return log;
}, "createTaskLog");
var updateTaskLog = /* @__PURE__ */ __name(async (logId, logData, newStatus) => {
const result = await import_payload.default.find({
collection: "cronLog",
where: {
id: { equals: logId }
},
limit: 1
});
const log = result.docs.length ? result.docs[0] : void 0;
if (log) {
(0, import_payload_plugin_cron_job.updateCronLog)(log.id, {
status: newStatus || log.status,
message: JSON.stringify(logData)
});
}
}, "updateTaskLog");
var getTaskLog = /* @__PURE__ */ __name(async (taskId) => {
const result = await import_payload.default.find({
collection: "cronLog",
where: {
id: { equals: taskId }
},
limit: 1
});
return result.docs.length ? result.docs[0] : void 0;
}, "getTaskLog");
var verifyWebhook = /* @__PURE__ */ __name(async (req) => {
const wh = new import_standardwebhooks.Webhook(btoa(`${process.env.ROBOT_USER}:${process.env.ROBOT_PASSWORD}`));
const headers = {
"webhook-id": req.headers["webhook-id"],
"webhook-timestamp": req.headers["webhook-timestamp"],
"webhook-signature": req.headers["webhook-signature"]
};
wh.verify(req.body, headers);
}, "verifyWebhook");
async function saveAppVectorDb(appId, robotDbVectorFilename) {
const app = await import_payload.default.findByID({
collection: options.slug.llmApp,
id: appId
});
const tempKnowledgebasePath = import_path4.default.join(process.cwd(), "lib", "knowledge-base", (0, import_uuid.v4)());
const appPath = getKnowledgebasePath(tempKnowledgebasePath, app.name);
import_fs4.default.mkdirSync(appPath, { recursive: true });
const dbVectorFilename = await downloadRobot(robotDbVectorFilename, appPath);
const dbVectorPath = import_path4.default.join(appPath, dbVectorFilename);
const dbVectorDoc = await import_payload.default.create({
collection: options.slug.llmVectorDb,
data: {
notes: `Knowledgebase app "${app.name}"`,
llmApp: app.id
},
file: {
data: import_fs4.default.readFileSync(dbVectorPath),
mimetype: import_mime_types.default.lookup(dbVectorPath),
name: dbVectorFilename,
size: import_fs4.default.statSync(dbVectorPath).size
},
overrideAccess: true
});
await import_payload.default.update({
collection: options.slug.llmApp,
id: app.id,
data: {
settings: {
knowledgeBase: {
vectorDbFile: dbVectorDoc.id
}
}
},
overrideAccess: true
});
const knowledgebasePath = getKnowledgebasePath(process.cwd(), app.name);
deleteFolderRecursive(knowledgebasePath);
return dbVectorDoc;
}
__name(saveAppVectorDb, "saveAppVectorDb");
async function saveToolVectorDb(appId, toolId, robotDbVectorFilename) {
const app = await import_payload.default.findByID({
collection: options.slug.llmApp,
id: appId
});
const appTools = app.settings.appTools;
const tool = appTools.find((x) => x.id === toolId);
if (tool === void 0) {
throw new Error(`Tool with id ${toolId} not found in app ${app.name}`);
}
const tempKnowledgebasePath = import_path4.default.join(process.cwd(), "lib", "knowledge-base", (0, import_uuid.v4)());
const appPath = getKnowledgebasePath(tempKnowledgebasePath, app.name, tool.name);
import_fs4.default.mkdirSync(appPath, { recursive: true });
const dbVectorFilename = await downloadRobot(robotDbVectorFilename, appPath);
const dbVectorPath = import_path4.default.join(appPath, dbVectorFilename);
const dbVectorDoc = await import_payload.default.create({
collection: options.slug.llmVectorDb,
data: {
notes: `Knowledgebase tool "${tool.functionId} - ${app.name}"`,
llmApp: app.id,
appTool: tool.functionId
},
file: {
data: import_fs4.default.readFileSync(dbVectorPath),
mimetype: import_mime_types.default.lookup(dbVectorPath),
name: dbVectorFilename,
size: import_fs4.default.statSync(dbVectorPath).size
},
overrideAccess: true
});
const appToolIndex = appTools.findIndex((x) => x.id === tool.id);
if (appToolIndex !== -1) {
const appTool = appTools[appToolIndex];
appTool.knowledgeBase.vectorDbFile = dbVectorDoc.id;
delete appTool.knowledgeBase.files;
}
const knowledgebasePath = getKnowledgebasePath(process.cwd(), app.name, tool.name);
deleteFolderRecursive(knowledgebasePath);
for (const tool2 of appTools) {
if ((0, import_bom_core.isEntity)(tool2?.knowledgeBase?.vectorDbFile)) {
tool2.knowledgeBase.vectorDbFile = tool2.knowledgeBase.vectorDbFile.id;
}
for (const file of tool2?.knowledgeBase?.files || []) {
if ((0, import_bom_core.isEntity)(file?.media)) {
file.media = file.media.id;
}
}
}
await import_payload.default.update({
collection: options.slug.llmApp,
id: app.id,
data: {
settings: {
appTools
}
},
overrideAccess: true
});
return dbVectorDoc;
}
__name(saveToolVectorDb, "saveToolVectorDb");
var llmKbWebhookHandler = /* @__PURE__ */ __name(async (req, res, next) => {
let taskResponse = null;
let logId;
try {
if (req.headers["content-type"] === void 0 || req.headers["content-type"] === "text/plain") {
req.body = (await (0, import_raw_body.default)(req)).toString("utf8");
} else if (req.headers["content-type"] === "application/json") {
req.body = JSON.stringify(req.body);
}
verifyWebhook(req);
taskResponse = JSON.parse(req.body);
if (!taskResponse || typeof taskResponse !== "object") {
throw new Error("Invalid task response format");
}
logId = taskResponse.id;
if (taskResponse.status === "failed" || !taskResponse.result.success || !taskResponse.result.file) {
updateTaskLog(logId, {
taskStatus: "webhook verified, but robot generation failed",
taskResponse
}, import_payload_plugin_cron_job.CronLogStatus.Failed);
} else {
const log = await getTaskLog(logId);
if (log === void 0) {
throw new Error(`No processing log found for task ${logId}`);
} else if (log.status === import_payload_plugin_cron_job.CronLogStatus.Succeeded) {
return res.status(200).send({ "success": true });
} else if (log.status === import_payload_plugin_cron_job.CronLogStatus.Failed) {
}
const appId = taskResponse.metadata.extra?.appId;
const toolId = taskResponse.metadata.extra?.toolId;
let vectorDb;
if (appId && toolId) {
vectorDb = await saveToolVectorDb(appId, toolId, taskResponse.result.file);
} else if (appId) {
vectorDb = await saveAppVectorDb(appId, taskResponse.result.file);
} else {
throw new Error("No appId or toolId found in task metadata");
}
updateTaskLog(log.id, {
taskStatus: "vectorDb successfully saved",
savedVectorDb: vectorDb,
taskResponse
}, import_payload_plugin_cron_job.CronLogStatus.Succeeded);
}
res.status(200).send({ "success": true });
} catch (error) {
if (logId) {
updateTaskLog(logId, {
taskStatus: "an exception occurred while processing the webhook",
taskResponse: taskResponse ?? void 0,
error: error.message
}, import_payload_plugin_cron_job.CronLogStatus.Failed);
}
return errorHandler(error, res);
}
}, "llmKbWebhookHandler");
// src/api/python/pythonKnowledgeBase.handler.ts
async function pythonKnowledgeBaseHandler(request) {
for (const endpoint of request.endpoints ?? []) {
endpoint.authentication = endpoint.authentication || "none";
endpoint.authSecret = endpoint.authSecret || "";
}
const serverURL2 = process.env.PAYLOAD_PUBLIC_LLM_WEBHOOK_URL || process.env.PAYLOAD_PUBLIC_SERVER_URL || "http://localhost:4000";
const basePath2 = process.env.PAYLOAD_PUBLIC_BASE_PATH || "";
const metadataExtraHeader = Object.entries(request.metadata).map(([key, value]) => `${key}=${value}`).join(",");
const response = await fetchRobot("api/llm/kb/task", "POST", request, {
// todo: better define type
"X-ws-bom-msg-type": "create.kb",
"X-ws-bom-msg-id": request.taskId,
"X-ws-bom-webhooks": `${serverURL2}${basePath2}/api/llm/kb/webhook`,
"X-ws-bom-msg-extra": metadataExtraHeader
});
return response.id;
}
__name(pythonKnowledgeBaseHandler, "pythonKnowledgeBaseHandler");
// src/api/knowledgeBase.handler.ts
var KbFileNoToolOption = "none";
var kbPollEndpointHandler = /* @__PURE__ */ __name(async (req, res, next) => {
try {
const traceId = req.params.traceId;
const logs = await req.payload.find({
collection: import_payload_plugin_cron_job2.options.slug.cronLog,
where: {
traceId: { equals: traceId }
},
overrideAccess: true
});
let response;
const processing = logs.docs.some((x) => x.status === import_payload_plugin_cron_job2.CronLogStatus.Processing);
if (processing) {
response = { processing: true };
} else {
const errorLogs = logs.docs.filter((x) => x.status !== import_payload_plugin_cron_job2.CronLogStatus.Succeeded);
if (errorLogs.length === 0) {
response = { processing: false, success: true };
} else {
response = {
processing: false,
success: false,
errors: errorLogs.map((log) => {
const taskData = JSON.parse(log.message);
let error;
if (taskData.taskResponse?.status === "completed") {
error = taskData.taskResponse.result.error || "Unknown error";
} else {
error = taskData.taskResponse?.error || taskData.error || "Unknown error";
}
return {
job: log.job,
taskStatus: taskData.taskStatus || "Unknown taskStatus",
error
};
})
};
}
}
res.status(200).send(response);
} catch (error) {
return errorHandler(error, res);
}
}, "kbPollEndpointHandler");
var kbEndpointHandler = /* @__PURE__ */ __name(async (req, res, next) => {
try {
const { appId, mode, toolFunctionId } = req.body;
const { payload: payload4 } = req;
const app = await payload4.findByID({
collection: options.slug.llmApp,
id: appId,
overrideAccess: true
});
const traceId = await llmKnowledgeBase(payload4, app, { mode, toolFunctionId, forceKbGeneration: true });
res.status(200).send({
traceId
});
} catch (error) {
return errorHandler(error, res);
}
}, "kbEndpointHandler");
var createAppVectorDb = /* @__PURE__ */ __name(async (payload4, app, creationOptions, tempKnowledgebasePath, traceId) => {
if (app.settings.nightlyKbGeneration === false && !creationOptions.forceKbGeneration) {
return;
}
const llmKbFiles = await payload4.find({
collection: options.slug.llmKbFile,
pagination: false,
where: {
and: [
{ "isActive": { equals: true } },
{ llmApp: { equals: app.id } },
{
or: [
{ appTool: { equals: KbFileNoToolOption } },
{ appTool: { equals: null } },
{ appTool: { exists: false } }
]
}
]
},
overrideAccess: true
});
const filenames = llmKbFiles.docs.filter((x) => x.url != null).map((x) => import_path5.default.basename(x.url)) || [];
const endpoints = app?.settings?.knowledgeBase?.externalEndpoints?.filter((x) => x.endpointUrl !== "") || [];
if (endpoints.length === 0 && filenames.length === 0) {
return;
}
if (endpoints.length === 0 && !creationOptions.forceKbGeneration) {
const vectorDbs = await payload4.find({
collection: options.slug.llmVectorDb,
limit: 1,
pagination: false,
sort: "-createdAt",
where: {
and: [
{ llmApp: { equals: app.id } },
{
or: [
{ appTool: { equals: KbFileNoToolOption } },
{ appTool: { equals: null } },
{ appTool: { exists: false } }
]
}
]
},
overrideAccess: true
});
if (vectorDbs.docs.length > 0) {
const latestVectorDb = vectorDbs.docs[0];
const latestFile = llmKbFiles.docs.reduce((prev, current) => prev.updatedAt > current.updatedAt ? prev : current);
if (latestVectorDb.createdAt > latestFile.updatedAt) {
return;
}
}
}
const jobName = `KB generation of app "${app.name}"`;
const log = await createTaskLog(jobName, traceId, {
taskStatus: "Preparing to generate knowledge base"
});
try {
const lastMinute = /* @__PURE__ */ new Date();
lastMinute.setMinutes(lastMinute.getMinutes() - 1);
const existingLogs = await payload4.find({
collection: import_payload_plugin_cron_job2.options.slug.cronLog,
where: {
id: { not_equals: log.id },
job: { equals: jobName },
status: { equals: import_payload_plugin_cron_job2.CronLogStatus.Processing },
createdAt: { greater_than_equal: lastMinute }
},
overrideAccess: true
});
if (existingLogs.docs.length > 0) {
throw new Error("Job execution aborted: same job was started (and is still processing) less than a minute ago");
}
const appPath = getKnowledgebasePath(tempKnowledgebasePath, app.name);
import_fs5.default.mkdirSync(appPath, { recursive: true });
const taskRequest = {
taskId: log.id,
model: app.settings.llmConfig.model,
provider: app.settings.llmConfig.provider,
files: filenames,
secrets: app.settings.llmConfig.secrets,
endpoints,
downloadPath: appPath,
metadata: {
traceId,
appId: app.id
}
};
await pythonKnowledgeBaseHandler(taskRequest);
await updateTaskLog(log.id, {
taskStatus: "ROBOT task started, waiting for webhook",
taskRequest
});
} catch (error) {
await updateTaskLog(log.id, {
error: error.message
}, import_payload_plugin_cron_job2.CronLogStatus.Failed);
}
}, "createAppVectorDb");
var createAppToolsVectorDb = /* @__PURE__ */ __name(async (payload4, app, creationOptions, tempKnowledgebasePath, traceId) => {
for (const tool of app.settings.appTools.filter((x) => x.isActive && (!creationOptions.toolFunctionId || x.functionId === creationOptions.toolFunctionId))) {
if ((app.settings.nightlyKbGeneration === false || tool.nightlyKbGeneration === false) && !creationOptions.forceKbGeneration) {
continue;
}
const llmKbFiles = await payload4.find({
collection: options.slug.llmKbFile,
pagination: false,
where: {
"isActive": { equals: true },
"llmApp": { equals: app.id },
"appTool": { equals: tool.functionId }
},
overrideAccess: true
});
const filenames = llmKbFiles.docs.filter((x) => x.url != null).map((x) => import_path5.default.basename(x.url)) || [];
const endpoints = tool.knowledgeBase?.externalEndpoints?.filter((x) => x.endpointUrl !== "") || [];
const integrations = tool.knowledgeBase?.integrations || [];
if (endpoints.length === 0 && filenames.length === 0 && integrations.length === 0) {
continue;
}
const jobName = `KB generation of tool "${tool.functionName} - ${tool.name}" [app "${app.name}"]"`;
const log = await createTaskLog(jobName, traceId, {
taskStatus: "Preparing to generate knowledge base"
});
try {
const lastMinute = /* @__PURE__ */ new Date();
lastMinute.setMinutes(lastMinute.getMinutes() - 1);
const existingLogs = await payload4.find({
collection: import_payload_plugin_cron_job2.options.slug.cronLog,
where: {
id: { not_equals: log.id },
job: { equals: jobName },
status: { equals: import_payload_plugin_cron_job2.CronLogStatus.Processing },
createdAt: { greater_than_equal: lastMinute }
},
overrideAccess: true
});
if (existingLogs.docs.length > 0) {
throw new Error("Job execution aborted: same job was started (and is still processing) less than a minute ago");
}
const appPath = getKnowledgebasePath(tempKnowledgebasePath, app.name, tool.name);
import_fs5.default.mkdirSync(appPath, { recursive: true });
const taskRequest = {
taskId: log.id,
model: app.settings.llmConfig.model,
provider: app.settings.llmConfig.provider,
files: filenames,
vectorDbType: tool.knowledgeBase?.vectorDbType,
dataSource: tool.dataSource,
integrations: tool.knowledgeBase?.integrations,
endpoints,
secrets: app.settings.llmConfig.secrets,
downloadPath: appPath,
metadata: {
traceId,
appId: app.id,
toolId: tool.id
}
};
await pythonKnowledgeBaseHandler(taskRequest);
await updateTaskLog(log.id, {
taskStatus: "ROBOT task started, waiting for webhook",
taskRequest
});
} catch (error) {
await updateTaskLog(log.id, {
error: error.message
}, import_payload_plugin_cron_job2.CronLogStatus.Failed);
}
}
}, "createAppToolsVectorDb");
var LlmKnowledgeBaseCreationMode = {
appAndTools: "appAndTools",
appOnly: "appOnly",
toolsOnly: "toolsOnly",
specificTool: "specificTool"
};
async function llmKnowledgeBase(payload4, app, creationOptions = { mode: LlmKnowledgeBaseCreationMode.appAndTools, forceKbGeneration: false }) {
let apps;
if (app) {
apps = [app];
} else {
const response = await payload4.find({
collection: options.slug.llmApp,
where: {
"isActive": { equals: true }
},
pagination: false,
overrideAccess: true
});
apps = response.docs;
}
const traceId = (0, import_uuid2.v4)();
const tempKnowledgebasePath = import_path5.default.join(process.cwd(), "lib", "knowledge-base", (0, import_uuid2.v4)());
for (const app2 of apps) {
switch (creationOptions.mode) {
case LlmKnowledgeBaseCreationMode.appAndTools:
createAppToolsVectorDb(payload4, app2, creationOptions, tempKnowledgebasePath, traceId);
createAppVectorDb(payload4, app2, creationOptions, tempKnowledgebasePath, traceId);
break;
case LlmKnowledgeBaseCreationMode.appOnly:
createAppVectorDb(payload4, app2, creationOptions, tempKnowledgebasePath, traceId);
break;
case LlmKnowledgeBaseCreationMode.toolsOnly:
case LlmKnowledgeBaseCreationMode.specificTool:
createAppToolsVectorDb(payload4, app2, creationOptions, tempKnowledgebasePath, traceId);
break;
}
}
deleteFolderRecursive(tempKnowledgebasePath);
return traceId;
}
__name(llmKnowledgeBase, "llmKnowledgeBase");
// src/api/message.handler.ts
var import_fs6 = __toESM(require("fs"));
var import_path6 = __toESM(require("path"));
var import_uuid3 = require("uuid");
// src/api/message.service.ts
var import_bom_core2 = require("@websolutespa/bom-core");
// src/api/types.ts
function isChunkText(chunk) {
return chunk.type === "text";
}
__name