@rcb-plugins/llm-connector
Version:
A generic LLM connector for integrating Large Language Models (LLMs) in React ChatBotify!
457 lines (456 loc) • 17.4 kB
JavaScript
import { useCallback as A, useRef as S, useEffect as G } from "react";
import { useOnRcbEvent as P, RcbEvent as T, useFlow as L, useAudio as z, useMessages as K, usePaths as N, useTextArea as J, useChatWindow as H } from "react-chatbotify";
const Y = {
autoConfig: !0
}, V = (i, e) => {
const t = A(
(s) => {
const o = i()[s.data.nextPath];
e(o);
},
[i, e]
);
P(T.CHANGE_PATH, t);
}, q = (i, e) => {
const { outputTypeRef: t } = i, {
toggleTextAreaDisabled: s,
toggleIsBotTyping: n,
focusTextArea: o,
injectMessage: r,
simulateStreamMessage: a,
getIsChatBotVisible: c
} = e, l = A(
(d) => {
var p;
const h = d.data.block;
h.llmConnector && (d.preventDefault(), d.type === "rcb-pre-process-block" && ((p = h.llmConnector) != null && p.initialMessage && (t.current === "full" ? r(i.initialMessageRef.current) : a(i.initialMessageRef.current)), n(!1), s(!1), setTimeout(() => {
c() && o();
})));
},
[n, s, o, c]
);
P(T.PRE_PROCESS_BLOCK, l), P(T.POST_PROCESS_BLOCK, l);
}, Q = async function* (i, e) {
for await (const t of i)
for (const s of t)
yield s, await new Promise((n) => setTimeout(n, e));
}, X = async function* (i, e) {
for await (const t of i)
yield t, await new Promise((s) => setTimeout(s, e));
}, Z = async function* (i, e, t) {
e === "character" ? yield* Q(i, t) : yield* X(i, t);
}, ee = async function* (i, e) {
for await (const t of i)
e(t), yield t;
}, te = async (i, e, t, s = {}) => {
var M, y;
if (!e.providerRef.current)
return;
const {
speakAudio: n,
toggleIsBotTyping: o,
toggleTextAreaDisabled: r,
focusTextArea: a,
injectMessage: c,
streamMessage: l,
endStreamMessage: d,
getIsChatBotVisible: h
} = t, p = e.providerRef.current.sendMessages(i), f = e.outputTypeRef.current, g = e.outputSpeedRef.current;
if (f === "full") {
let u = "";
for await (const m of p) {
if ((M = s.signal) != null && M.aborted) break;
u += m;
}
o(!1), c(u), setTimeout(() => {
r(!1), h() && a();
});
} else {
const u = Z(ee(p, n), f, g);
let m = "", b = !1;
for await (const E of u) {
if ((y = s.signal) != null && y.aborted)
break;
b || (o(!1), b = !0), m += E, l(m);
}
d(), setTimeout(() => {
r(!1), h() && a();
});
}
}, se = 500, oe = (i, e) => {
const { messagesRef: t, outputTypeRef: s, onUserMessageRef: n, onKeyDownRef: o, errorMessageRef: r } = i, {
injectMessage: a,
simulateStreamMessage: c,
toggleTextAreaDisabled: l,
toggleIsBotTyping: d,
goToPath: h,
focusTextArea: p,
getIsChatBotVisible: f
} = e, g = S(null), M = A(
(y) => {
if (!i.providerRef.current)
return;
const u = y.data.message, m = u.sender.toUpperCase();
u.tags = u.tags ?? [], u.tags.push(`rcb-llm-connector-plugin:${m}`), m === "USER" && (d(!0), l(!0), setTimeout(async () => {
var v;
if (n.current) {
const R = await n.current(u);
if (R)
return (v = g.current) == null || v.abort(), g.current = null, h(R);
}
const b = i.historySizeRef.current, E = t.current, x = b ? [...E.slice(-(b - 1)), u] : [u], C = new AbortController();
g.current = C, te(x, i, e, { signal: C.signal }).catch((R) => {
d(!1), l(!1), setTimeout(() => {
f() && p();
}), console.error("LLM prompt failed", R), s.current === "full" ? a(r.current) : c(r.current);
});
}, se));
},
[i, e]
);
P(T.POST_INJECT_MESSAGE, M), P(T.STOP_SIMULATE_STREAM_MESSAGE, M), P(T.STOP_STREAM_MESSAGE, M), G(() => {
const y = async (u) => {
var m;
if (o.current) {
const b = await o.current(u);
b && ((m = g.current) == null || m.abort(), g.current = null, h(b));
}
};
return window.addEventListener("keydown", y), () => window.removeEventListener("keydown", y);
}, []);
}, re = (i) => {
const e = S([]), t = S(null), s = S("chunk"), n = S(30), o = S(0), r = S(""), a = S("Unable to get response, please try again."), c = S(null), l = S(null), { getFlow: d } = L(), { speakAudio: h } = z(), { messages: p, injectMessage: f, simulateStreamMessage: g, streamMessage: M, endStreamMessage: y } = K(), { goToPath: u } = N(), { toggleTextAreaDisabled: m, focusTextArea: b } = J(), { toggleIsBotTyping: E, getIsChatBotVisible: x } = H(), C = { ...Y, ...i ?? {} };
G(() => {
e.current = p;
}, [p]), V(d, (w) => {
var k, B, U, F, I, D, W, _, $, j;
t.current = ((k = w.llmConnector) == null ? void 0 : k.provider) ?? null, s.current = ((B = w.llmConnector) == null ? void 0 : B.outputType) ?? "chunk", n.current = ((U = w.llmConnector) == null ? void 0 : U.outputSpeed) ?? 30, o.current = ((F = w.llmConnector) == null ? void 0 : F.historySize) ?? 0, r.current = ((I = w.llmConnector) == null ? void 0 : I.initialMessage) ?? "", a.current = ((D = w.llmConnector) == null ? void 0 : D.errorMessage) ?? "Unable to get response, please try again.", c.current = ((_ = (W = w.llmConnector) == null ? void 0 : W.stopConditions) == null ? void 0 : _.onUserMessage) ?? null, l.current = ((j = ($ = w.llmConnector) == null ? void 0 : $.stopConditions) == null ? void 0 : j.onKeyDown) ?? null;
});
const v = {
providerRef: t,
messagesRef: e,
outputTypeRef: s,
outputSpeedRef: n,
historySizeRef: o,
initialMessageRef: r,
errorMessageRef: a,
onUserMessageRef: c,
onKeyDownRef: l
}, R = {
speakAudio: h,
injectMessage: f,
simulateStreamMessage: g,
streamMessage: M,
endStreamMessage: y,
toggleTextAreaDisabled: m,
toggleIsBotTyping: E,
focusTextArea: b,
goToPath: u,
getIsChatBotVisible: x
};
q(v, R), oe(v, R);
const O = { name: "@rcb-plugins/llm-connector" };
return C != null && C.autoConfig && (O.settings = {
event: {
rcbChangePath: !0,
rcbPostInjectMessage: !0,
rcbStopSimulateStreamMessage: !0,
rcbStopStreamMessage: !0,
rcbPreProcessBlock: !0,
rcbPostProcessBlock: !0
}
}), O;
}, ie = (i) => () => re(i);
class ce {
/**
* Sets default values for the provider based on given configuration. Configuration guide here:
* https://github.com/React-ChatBotify-Plugins/llm-connector/blob/main/docs/providers/Gemini.md
*
* @param config configuration for setup
*/
constructor(e) {
this.debug = !1, this.roleMap = (s) => {
switch (s) {
case "USER":
return "user";
default:
return "model";
}
}, this.constructBodyWithMessages = (s) => {
let n;
return this.messageParser ? n = this.messageParser(s) : n = s.filter(
(r) => typeof r.content == "string" && r.sender.toUpperCase() !== "SYSTEM"
).map((r) => {
const a = this.roleMap(r.sender.toUpperCase()), c = r.content;
return {
role: a,
parts: [{ text: c }]
};
}), this.systemMessage && (n = [{ role: "user", parts: [{ text: this.systemMessage }] }, ...n]), {
contents: n,
...this.body
};
}, this.handleStreamResponse = async function* (s) {
var r, a, c, l, d;
const n = new TextDecoder("utf-8");
let o = "";
for (; ; ) {
const { value: h, done: p } = await s.read();
if (p) break;
o += n.decode(h, { stream: !0 });
const f = o.split(`
`);
o = f.pop();
for (const g of f) {
const M = g.trim();
if (!M.startsWith("data: ")) continue;
const y = M.slice(6);
try {
const m = (d = (l = (c = (a = (r = JSON.parse(y).candidates) == null ? void 0 : r[0]) == null ? void 0 : a.content) == null ? void 0 : c.parts) == null ? void 0 : l[0]) == null ? void 0 : d.text;
m && (yield m);
} catch (u) {
console.error("SSE JSON parse error:", y, u);
}
}
}
}, this.method = e.method ?? "POST", this.body = e.body ?? {}, this.systemMessage = e.systemMessage, this.responseFormat = e.responseFormat ?? "stream", this.messageParser = e.messageParser, this.debug = e.debug ?? !1, this.headers = {
"Content-Type": "application/json",
Accept: this.responseFormat === "stream" ? "text/event-stream" : "application/json",
...e.headers
};
const t = e.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta";
if (e.mode === "direct")
this.endpoint = this.responseFormat === "stream" ? `${t}/models/${e.model}:streamGenerateContent?alt=sse&key=${e.apiKey || ""}` : `${t}/models/${e.model}:generateContent?key=${e.apiKey || ""}`;
else if (e.mode === "proxy")
this.endpoint = `${t}/${e.model}`;
else
throw Error("Invalid mode specified for Gemini provider ('direct' or 'proxy').");
}
/**
* Calls Gemini and yields each chunk (or the full text).
*
* @param messages messages to include in the request
*/
async *sendMessages(e) {
var s, n, o, r, a;
if (this.debug) {
const c = this.endpoint.replace(/\?key=([^&]+)/, "?key=[REDACTED]"), l = { ...this.headers };
console.log("[GeminiProvider] Request:", {
method: this.method,
endpoint: c,
headers: l,
body: this.constructBodyWithMessages(e)
});
}
const t = await fetch(this.endpoint, {
method: this.method,
headers: this.headers,
body: JSON.stringify(this.constructBodyWithMessages(e))
});
if (this.debug && console.log("[GeminiProvider] Response status:", t.status), !t.ok)
throw new Error(`Gemini API error ${t.status}: ${await t.text()}`);
if (this.responseFormat === "stream") {
if (!t.body)
throw new Error("Response body is empty – cannot stream");
const c = t.body.getReader();
for await (const l of this.handleStreamResponse(c))
yield l;
} else {
const c = await t.json();
this.debug && console.log("[GeminiProvider] Response body:", c);
const l = (a = (r = (o = (n = (s = c.candidates) == null ? void 0 : s[0]) == null ? void 0 : n.content) == null ? void 0 : o.parts) == null ? void 0 : r[0]) == null ? void 0 : a.text;
if (typeof l == "string")
yield l;
else
throw new Error("Unexpected response shape – no text candidate");
}
}
}
class le {
/**
* Sets default values for the provider based on given configuration. Configuration guide here:
* https://github.com/React-ChatBotify-Plugins/llm-connector/blob/main/docs/providers/OpenAI.md
*
* @param config configuration for setup
*/
constructor(e) {
if (this.debug = !1, this.roleMap = (t) => {
switch (t) {
case "USER":
return "user";
case "SYSTEM":
return "system";
default:
return "assistant";
}
}, this.constructBodyWithMessages = (t) => {
let s;
return this.messageParser ? s = this.messageParser(t) : s = t.filter(
(o) => typeof o.content == "string" && o.sender.toUpperCase() !== "SYSTEM"
).map((o) => {
const r = this.roleMap(o.sender.toUpperCase()), a = o.content;
return {
role: r,
content: a
};
}), this.systemMessage && (s = [{ role: "system", content: this.systemMessage }, ...s]), {
messages: s,
...this.body
};
}, this.handleStreamResponse = async function* (t) {
var o, r, a;
const s = new TextDecoder("utf-8");
let n = "";
for (; ; ) {
const { value: c, done: l } = await t.read();
if (l) break;
n += s.decode(c, { stream: !0 });
const d = n.split(/\r?\n/);
n = d.pop();
for (const h of d) {
if (!h.startsWith("data: ")) continue;
const p = h.slice(6).trim();
if (p === "[DONE]") return;
try {
const g = (a = (r = (o = JSON.parse(p).choices) == null ? void 0 : o[0]) == null ? void 0 : r.delta) == null ? void 0 : a.content;
g && (yield g);
} catch (f) {
console.error("Stream parse error", f);
}
}
}
}, this.method = e.method ?? "POST", this.endpoint = e.baseUrl ?? "https://api.openai.com/v1/chat/completions", this.systemMessage = e.systemMessage, this.responseFormat = e.responseFormat ?? "stream", this.messageParser = e.messageParser, this.debug = e.debug ?? !1, this.headers = {
"Content-Type": "application/json",
Accept: this.responseFormat === "stream" ? "text/event-stream" : "application/json",
...e.headers
}, this.body = {
model: e.model,
stream: this.responseFormat === "stream",
...e.body
}, e.mode === "direct") {
this.headers = { ...this.headers, Authorization: `Bearer ${e.apiKey}` };
return;
}
if (e.mode !== "proxy")
throw Error("Invalid mode specified for OpenAI provider ('direct' or 'proxy').");
}
/**
* Calls Openai and yields each chunk (or the full text).
*
* @param messages messages to include in the request
*/
async *sendMessages(e) {
var s, n, o;
if (this.debug) {
const r = { ...this.headers };
delete r.Authorization, console.log("[OpenaiProvider] Request:", {
method: this.method,
endpoint: this.endpoint,
headers: r,
body: this.constructBodyWithMessages(e)
});
}
const t = await fetch(this.endpoint, {
method: this.method,
headers: this.headers,
body: JSON.stringify(this.constructBodyWithMessages(e))
});
if (this.debug && console.log("[OpenaiProvider] Response status:", t.status), !t.ok)
throw new Error(`Openai API error ${t.status}: ${await t.text()}`);
if (this.responseFormat === "stream") {
if (!t.body)
throw new Error("Response body is empty – cannot stream");
const r = t.body.getReader();
for await (const a of this.handleStreamResponse(r))
yield a;
} else {
const r = await t.json();
this.debug && console.log("[OpenaiProvider] Response body:", r);
const a = (o = (n = (s = r.choices) == null ? void 0 : s[0]) == null ? void 0 : n.message) == null ? void 0 : o.content;
if (typeof a == "string")
yield a;
else
throw new Error("Unexpected response shape – no text candidate");
}
}
}
class de {
/**
* Sets default values for the provider based on given configuration. Configuration guide here:
* https://github.com/React-ChatBotify-Plugins/llm-connector/blob/main/docs/providers/WebLlm.md
*
* @param config configuration for setup
*/
constructor(e) {
this.debug = !1, this.roleMap = (t) => {
switch (t) {
case "USER":
return "user";
case "SYSTEM":
return "system";
default:
return "assistant";
}
}, this.constructBodyWithMessages = (t) => {
let s;
return this.messageParser ? s = this.messageParser(t) : s = t.filter(
(o) => typeof o.content == "string" && o.sender.toUpperCase() !== "SYSTEM"
).map((o) => {
const r = this.roleMap(o.sender.toUpperCase()), a = o.content;
return {
role: r,
content: a
};
}), this.systemMessage && (s = [
{
role: "system",
content: this.systemMessage
},
...s
]), {
messages: s,
stream: this.responseFormat === "stream",
...this.chatCompletionOptions
};
}, this.model = e.model, this.systemMessage = e.systemMessage, this.responseFormat = e.responseFormat ?? "stream", this.messageParser = e.messageParser, this.engineConfig = e.engineConfig ?? {}, this.chatCompletionOptions = e.chatCompletionOptions ?? {}, this.debug = e.debug ?? !1, this.createEngine();
}
/**
* Creates MLC Engine for inferencing.
*/
async createEngine() {
const { CreateMLCEngine: e } = await import("@mlc-ai/web-llm");
this.engine = await e(this.model, {
...this.engineConfig
});
}
/**
* Calls WebLlm and yields each chunk (or the full text).
*
* @param messages messages to include in the request
*/
async *sendMessages(e) {
var s, n, o, r, a, c;
this.engine || await this.createEngine(), this.debug && console.log("[WebLlmProvider] Request:", {
model: this.model,
systemMessage: this.systemMessage,
responseFormat: this.responseFormat,
engineConfig: this.engineConfig,
chatCompletionOptions: this.chatCompletionOptions,
messages: this.constructBodyWithMessages(e).messages
// Log messages being sent
});
const t = await ((s = this.engine) == null ? void 0 : s.chat.completions.create(this.constructBodyWithMessages(e)));
if (this.debug && console.log("[WebLlmProvider] Response:", t), t && Symbol.asyncIterator in t)
for await (const l of t) {
const d = (o = (n = l.choices[0]) == null ? void 0 : n.delta) == null ? void 0 : o.content;
d && (yield d);
}
else (c = (a = (r = t == null ? void 0 : t.choices) == null ? void 0 : r[0]) == null ? void 0 : a.message) != null && c.content && (yield t.choices[0].message.content);
}
}
export {
ce as GeminiProvider,
le as OpenaiProvider,
de as WebLlmProvider,
ie as default
};