@d-id/client-sdk
Version:
d-id client sdk
429 lines (428 loc) • 16.5 kB
JavaScript
import { c as De, C as O, S as p, a as Fe, b as Ne, A as m, d as Ue, l as de, e as v, f as N, g as je, h as xe, i as A, n as Oe } from "./index-BPiQZpON.js";
function Je(T, L, g, i) {
const M = De(T, `${L}/v2/agents/${g}`, i);
return {
async createStream(l) {
return M.post("/sessions", l);
}
};
}
const Be = {
[p.ChatAnswer]: O.Answer,
[p.ChatPartial]: O.Partial
}, ue = 2e4;
async function Qe() {
try {
return await import("./livekit-client.esm-RGXA_gNu.js");
} catch {
throw new Error(
"LiveKit client is required for this streaming manager. Please install it using: npm install livekit-client"
);
}
}
const qe = {
excellent: N.Strong,
good: N.Strong,
poor: N.Weak,
lost: N.Unknown,
unknown: N.Unknown
}, U = JSON.stringify({
kind: "InternalServerError",
description: "Stream Error"
});
function pe(T, L, g) {
var i, M;
throw L("Failed to connect to LiveKit room:", T), (i = g.onConnectionStateChange) == null || i.call(g, v.Fail, "internal:init-error"), (M = g.onError) == null || M.call(g, T, { sessionId: "" }), T;
}
async function We(T, L, g) {
var re;
const i = Fe(g.debug || !1, "LiveKitStreamingManager"), { Room: M, RoomEvent: l, ConnectionState: k, Track: H } = await Qe(), { callbacks: n, auth: le, baseURL: he, analytics: G } = g;
let a = null, w = !1;
const X = Ne.Fluent;
let R = null;
const P = { isPublishing: !1, publication: null }, J = { isPublishing: !1, publication: null };
let h = null, f = null, j = null, _ = !1;
a = new M({
adaptiveStream: !1,
// Must be false to use mediaStreamTrack directly
dynacast: !0
});
let E = null, b = m.Idle, I = !0;
const ge = Je(le, he || Ue, T, n.onError);
let C, $, K, Y = !0;
try {
const e = await ge.createStream({
transport: L.transport,
chat_persist: L.chat_persist ?? !0
}), { id: t, session_token: r, session_url: o, interrupt_enabled: c } = e;
(re = n.onStreamCreated) == null || re.call(n, { session_id: t, stream_id: t, agent_id: T }), C = t, $ = r, K = o, Y = c ?? !0, await a.prepareConnection(K, $);
} catch (e) {
pe(e, i, n);
}
if (!K || !$ || !C)
return Promise.reject(new Error("Failed to initialize LiveKit stream"));
a.on(l.ConnectionStateChanged, me).on(l.ConnectionQualityChanged, Ce).on(l.ParticipantConnected, Se).on(l.ParticipantDisconnected, Te).on(l.TrackSubscribed, ve).on(l.TrackUnsubscribed, we).on(l.DataReceived, Pe).on(l.MediaDevicesError, Le).on(l.TranscriptionReceived, fe).on(l.EncryptionError, Me).on(l.TrackSubscriptionFailed, ke);
function fe(e, t) {
t != null && t.isLocal && (de.update(), b === m.Talking && (b = m.Idle));
}
try {
await a.connect(K, $), i("LiveKit room joined successfully"), E = setTimeout(() => {
var e;
i(
`Track subscription timeout - no track subscribed within ${ue / 1e3} seconds after connect`
), E = null, G.track("connectivity-error", {
error: "Track subscription timeout",
sessionId: C
}), (e = n.onError) == null || e.call(n, new Error("Track subscription timeout"), { sessionId: C }), W("internal:track-subscription-timeout");
}, ue);
} catch (e) {
pe(e, i, n);
}
G.enrich({
"stream-type": X
});
function me(e) {
var t, r, o, c;
switch (i("Connection state changed:", e), e) {
case k.Connecting:
i("CALLBACK: onConnectionStateChange(Connecting)"), (t = n.onConnectionStateChange) == null || t.call(n, v.Connecting, "livekit:connecting");
break;
case k.Connected:
i("LiveKit room connected successfully"), w = !0;
break;
case k.Disconnected:
i("LiveKit room disconnected"), w = !1, _ = !1, P.publication = null, J.publication = null, (r = n.onConnectionStateChange) == null || r.call(n, v.Disconnected, "livekit:disconnected");
break;
case k.Reconnecting:
i("LiveKit room reconnecting..."), (o = n.onConnectionStateChange) == null || o.call(n, v.Connecting, "livekit:reconnecting");
break;
case k.SignalReconnecting:
i("LiveKit room signal reconnecting..."), (c = n.onConnectionStateChange) == null || c.call(n, v.Connecting, "livekit:signal-reconnecting");
break;
}
}
function Ce(e, t) {
var r;
i("Connection quality:", e), t != null && t.isLocal && ((r = n.onConnectivityStateChange) == null || r.call(n, qe[e]));
}
function Se(e) {
i("Participant connected:", e.identity);
}
function Te(e) {
i("Participant disconnected:", e.identity), W("livekit:participant-disconnected");
}
function ye() {
var e;
j !== A.Start && (i("CALLBACK: onVideoStateChange(Start)"), j = A.Start, (e = n.onVideoStateChange) == null || e.call(n, A.Start));
}
function Z(e) {
var t;
j !== A.Stop && (i("CALLBACK: onVideoStateChange(Stop)"), j = A.Stop, (t = n.onVideoStateChange) == null || t.call(n, A.Stop, e));
}
function ve(e, t, r) {
var c, s, u;
i(`Track subscribed: ${e.kind} from ${r.identity}`);
const o = e.mediaStreamTrack;
if (!o) {
i(`No mediaStreamTrack available for ${e.kind}`);
return;
}
R ? (R.addTrack(o), i(`Added ${e.kind} track to shared MediaStream`)) : (R = new MediaStream([o]), i(`Created shared MediaStream with ${e.kind} track`)), e.kind === "audio" && (f = xe(
() => e.getRTCStatsReport(),
({ sttLatency: d, serviceLatency: S }) => {
var oe, ae, ce;
const D = de.get(!0);
let F = 0;
if (d) {
const se = ((ae = (oe = h == null ? void 0 : h.getReport()) == null ? void 0 : oe.webRTCStats) == null ? void 0 : ae.avgRtt) ?? 0;
F = se > 0 ? Math.round(se * 1e3) : 0;
}
const V = D > 0 ? D + (d ?? 0) + F : void 0, y = V !== void 0 && S !== void 0 ? V - S : void 0;
(ce = n.onFirstAudioDetected) == null || ce.call(n, { latency: V, networkLatency: y });
}
)), e.kind === "video" && ((c = n.onStreamReady) == null || c.call(n), i("CALLBACK: onSrcObjectReady"), (s = n.onSrcObjectReady) == null || s.call(n, R), _ || (_ = !0, i("CALLBACK: onConnectionStateChange(Connected)"), (u = n.onConnectionStateChange) == null || u.call(n, v.Connected, "livekit:track-subscribed")), h = je(
() => e.getRTCStatsReport(),
() => w,
Oe,
(d, S) => {
i(`Video state change: ${d}`), d === A.Start ? (E && (clearTimeout(E), E = null, i("Track subscription timeout cleared")), ye()) : d === A.Stop && Z(S);
}
), h.start());
}
function we(e, t, r) {
i(`Track unsubscribed: ${e.kind} from ${r.identity}`), e.kind === "audio" && (f == null || f.destroy(), f = null), e.kind === "video" && (Z(h == null ? void 0 : h.getReport()), h == null || h.stop(), h = null);
}
function ee(e, t) {
var o;
const r = Be[e];
r && ((o = n.onMessage) == null || o.call(n, r, { event: r, ...t }));
}
function B(e, t) {
var r, o, c, s, u;
if (e === p.ToolCallStarted) {
const d = t;
I = d.interruptible !== !1, (r = n.onInterruptibleChange) == null || r.call(n, I), b = m.ToolActive, (o = n.onAgentActivityStateChange) == null || o.call(n, m.ToolActive), (c = n.onToolEvent) == null || c.call(n, p.ToolCallStarted, d);
return;
}
if (e === p.ToolCallDone) {
(s = n.onToolEvent) == null || s.call(n, p.ToolCallDone, t);
return;
}
e === p.ToolCallError && ((u = n.onToolEvent) == null || u.call(n, p.ToolCallError, t));
}
function Ee(e, t) {
var r, o, c, s, u;
if (I = ((r = t.metadata) == null ? void 0 : r.interruptible) !== !1, (o = n.onInterruptibleChange) == null || o.call(n, I), e === p.StreamVideoCreated) {
b = m.Talking, (c = n.onAgentActivityStateChange) == null || c.call(n, m.Talking), f == null || f.arm({
sttLatency: (s = t == null ? void 0 : t.stt) == null ? void 0 : s.latency,
serviceLatency: t == null ? void 0 : t.serviceLatency
});
return;
}
I && (b = m.Idle, (u = n.onAgentActivityStateChange) == null || u.call(n, m.Idle));
}
function x(e, t) {
var s, u, d, S;
const r = ((u = (s = h == null ? void 0 : h.getReport()) == null ? void 0 : s.webRTCStats) == null ? void 0 : u.avgRtt) ?? 0, o = r > 0 ? Math.round(r / 2 * 1e3) : 0, c = { ...t, downstreamNetworkLatency: o };
g.debug && ((d = t == null ? void 0 : t.metadata) != null && d.sentiment) && (c.sentiment = {
id: t.metadata.sentiment.id,
name: t.metadata.sentiment.sentiment
}), (S = n.onMessage) == null || S.call(n, e, c), Ee(e, t);
}
function Ae(e, t) {
var r;
(r = n.onMessage) == null || r.call(n, O.Transcribe, { event: O.Transcribe, ...t }), queueMicrotask(() => {
var o;
(o = n.onAgentActivityStateChange) == null || o.call(n, m.Loading);
});
}
const Re = {
[p.ChatAnswer]: ee,
[p.ChatPartial]: ee,
[p.ToolCallStarted]: B,
[p.ToolCallDone]: B,
[p.ToolCallError]: B,
[p.StreamVideoCreated]: x,
[p.StreamVideoDone]: x,
[p.StreamVideoError]: x,
[p.StreamVideoRejected]: x,
[p.ChatAudioTranscribed]: Ae
};
function Pe(e, t, r, o) {
const c = new TextDecoder().decode(e);
try {
const s = JSON.parse(c), u = o || s.subject;
if (i("Data received:", { subject: u, data: s }), !u) return;
const d = Re[u];
d == null || d(u, s);
} catch (s) {
i("Failed to parse data channel message:", s);
}
}
function Le(e) {
var t;
i("Media devices error:", e), (t = n.onError) == null || t.call(n, new Error(U), { sessionId: C });
}
function Me(e) {
var t;
i("Encryption error:", e), (t = n.onError) == null || t.call(n, new Error(U), { sessionId: C });
}
function ke(e, t, r) {
i("Track subscription failed:", { trackSid: e, participant: t, reason: r });
}
function be(e, t, r) {
for (const [o, c] of r)
if (c.source === t && c.track) {
const s = c.track.mediaStreamTrack;
if (s === e || (s == null ? void 0 : s.id) === e.id)
return c;
}
return null;
}
async function ne(e, t, r, o, c, s) {
var D, F, V;
if (!w || !a)
throw i(`Room is not connected, cannot publish ${o} stream`), new Error("Room is not connected");
if (e.isPublishing) {
i(`${o} publish already in progress, skipping`);
return;
}
const u = r(t);
if (u.length === 0)
throw new Error(`No ${o} track found in the provided MediaStream`);
const d = u[0], S = be(d, o, c());
if (S) {
i(`${o} track is already published, skipping`, {
trackId: d.id,
publishedTrackId: (F = (D = S.track) == null ? void 0 : D.mediaStreamTrack) == null ? void 0 : F.id
}), e.publication = S;
return;
}
if ((V = e.publication) != null && V.track) {
const y = e.publication.track.mediaStreamTrack;
y !== d && (y == null ? void 0 : y.id) !== d.id && (i(`Unpublishing existing ${o} track before publishing new one`), await s());
}
i(`Publishing ${o} track from provided MediaStream`, { trackId: d.id }), e.isPublishing = !0;
try {
e.publication = await a.localParticipant.publishTrack(d, { source: o }), i(`${o} track published successfully`, { trackSid: e.publication.trackSid });
} catch (y) {
throw i(`Failed to publish ${o} track:`, y), y;
} finally {
e.isPublishing = !1;
}
}
async function te(e, t) {
if (!(!e.publication || !e.publication.track))
try {
a && (await a.localParticipant.unpublishTrack(e.publication.track, !1), i(`${t} track unpublished`));
} catch (r) {
i(`Error unpublishing ${t} track:`, r);
} finally {
e.publication = null;
}
}
async function Ie(e) {
return ne(
P,
e,
(t) => t.getAudioTracks(),
H.Source.Microphone,
() => a.localParticipant.audioTrackPublications,
Q
);
}
async function Q() {
return te(P, "Microphone");
}
async function $e(e) {
if (!w || !a)
throw i("Cannot replace microphone track: room is not connected"), new Error("Room is not connected");
if (e.kind !== "audio")
throw i("Cannot replace microphone track: not an audio track", { kind: e.kind }), new Error("Microphone track must be an audio track");
if (P.isPublishing)
throw i("Cannot replace microphone track: publish in progress"), new Error("Microphone publish in progress");
const t = P.publication;
if (!t || !t.track)
throw i("Cannot replace microphone track: no publication to replace"), new Error("No microphone publication to replace");
try {
P.isPublishing = !0, await t.track.replaceTrack(e), i("Microphone track replaced", { trackId: e.id, trackSid: t.trackSid });
} finally {
P.isPublishing = !1;
}
}
async function Ke(e) {
return ne(
J,
e,
(t) => t.getVideoTracks(),
H.Source.Camera,
() => a.localParticipant.videoTrackPublications,
q
);
}
async function q() {
return te(J, "Camera");
}
function Ve() {
R && (R.getTracks().forEach((e) => e.stop()), R = null);
}
async function z(e, t) {
var r, o;
if (!w || !a) {
i("Room is not connected for sending messages"), (r = n.onError) == null || r.call(n, new Error(U), {
sessionId: C
});
return;
}
try {
await a.localParticipant.sendText(e, { topic: t }), i("Message sent successfully:", e);
} catch (c) {
i("Failed to send message:", c), (o = n.onError) == null || o.call(n, new Error(U), { sessionId: C });
}
}
async function ie(e) {
var t;
try {
const o = JSON.parse(e).topic;
return z("", o);
} catch (r) {
i("Failed to send data channel message:", r), (t = n.onError) == null || t.call(n, new Error(U), { sessionId: C });
}
}
function _e(e) {
return z(
e,
"lk.chat"
/* Chat */
);
}
async function W(e) {
var t, r;
E && (clearTimeout(E), E = null), f == null || f.destroy(), f = null, a && ((t = n.onConnectionStateChange) == null || t.call(n, v.Disconnecting, e), await Promise.all([Q(), q()]), await a.disconnect()), Ve(), w = !1, _ = !1, (r = n.onAgentActivityStateChange) == null || r.call(n, m.Idle), b = m.Idle;
}
return {
speak(e) {
const t = typeof e == "string" ? e : JSON.stringify(e);
return z(
t,
"did.speak"
/* Speak */
);
},
disconnect: () => W("user:disconnect"),
async reconnect() {
var e, t;
if ((a == null ? void 0 : a.state) === k.Connected) {
i("Room is already connected");
return;
}
if (!a || !K || !$)
throw i("Cannot reconnect: missing room, URL or token"), new Error("Cannot reconnect: session not available");
i("Reconnecting to LiveKit room, state:", a.state), _ = !1, (e = n.onConnectionStateChange) == null || e.call(n, v.Connecting, "user:reconnect");
try {
if (await a.connect(K, $), i("Room reconnected"), w = !0, a.remoteParticipants.size === 0) {
if (i("Waiting for agent to join..."), !await new Promise((o) => {
const c = setTimeout(() => {
a == null || a.off(l.ParticipantConnected, s), o(!1);
}, 5e3), s = () => {
clearTimeout(c), a == null || a.off(l.ParticipantConnected, s), o(!0);
};
a == null || a.on(l.ParticipantConnected, s);
}))
throw i("Agent did not join within timeout"), await a.disconnect(), new Error("Agent did not rejoin the room");
i("Agent joined, reconnection successful");
}
} catch (r) {
throw i("Failed to reconnect:", r), (t = n.onConnectionStateChange) == null || t.call(n, v.Fail, "user:reconnect-failed"), r;
}
},
sendDataChannelMessage: ie,
sendTextMessage: _e,
publishMicrophoneStream: Ie,
unpublishMicrophoneStream: Q,
replaceMicrophoneTrack: $e,
publishCameraStream: Ke,
unpublishCameraStream: q,
interrupt(e) {
e !== "text" && ie(JSON.stringify({
topic: "did.interrupt"
/* Interrupt */
}));
},
registerRpcMethod(e, t) {
a == null || a.registerRpcMethod(e, t);
},
unregisterRpcMethod(e) {
a == null || a.unregisterRpcMethod(e);
},
sessionId: C,
streamId: C,
streamType: X,
interruptAvailable: Y,
isInterruptible: I
};
}
export {
We as createLiveKitStreamingManager,
pe as handleInitError
};