@kortexa-ai/react-multimodal
Version:
A set of react components and hooks to help with multimodal input
1,683 lines (1,676 loc) • 47.5 kB
JavaScript
var er = Object.defineProperty;
var rr = (n, r, c) => r in n ? er(n, r, { enumerable: !0, configurable: !0, writable: !0, value: c }) : n[r] = c;
var We = (n, r, c) => rr(n, typeof r != "symbol" ? r + "" : r, c);
import { createContext as ge, useContext as Ee, useState as V, useRef as m, useEffect as R, useCallback as f, useMemo as ee } from "react";
import { jsx as ue } from "react/jsx-runtime";
import { FilesetResolver as we, HandLandmarker as tr, GestureRecognizer as nr, PoseLandmarker as sr, FaceLandmarker as ir } from "@mediapipe/tasks-vision";
const Fe = ge(
void 0
);
Fe.displayName = "kortexa.ai:composite-media";
const Ir = () => {
const n = Ee(Fe);
if (!n)
throw new Error(
"useCompositeMedia must be used within a CompositeMediaProvider"
);
return n;
}, ne = [];
for (let n = 0; n < 256; ++n)
ne.push((n + 256).toString(16).slice(1));
function ar(n, r = 0) {
return (ne[n[r + 0]] + ne[n[r + 1]] + ne[n[r + 2]] + ne[n[r + 3]] + "-" + ne[n[r + 4]] + ne[n[r + 5]] + "-" + ne[n[r + 6]] + ne[n[r + 7]] + "-" + ne[n[r + 8]] + ne[n[r + 9]] + "-" + ne[n[r + 10]] + ne[n[r + 11]] + ne[n[r + 12]] + ne[n[r + 13]] + ne[n[r + 14]] + ne[n[r + 15]]).toLowerCase();
}
let Me;
const or = new Uint8Array(16);
function cr() {
if (!Me) {
if (typeof crypto > "u" || !crypto.getRandomValues)
throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");
Me = crypto.getRandomValues.bind(crypto);
}
return Me(or);
}
const dr = typeof crypto < "u" && crypto.randomUUID && crypto.randomUUID.bind(crypto), Ge = { randomUUID: dr };
function X(n, r, c) {
var a;
if (Ge.randomUUID && !n)
return Ge.randomUUID();
n = n || {};
const e = n.random ?? ((a = n.rng) == null ? void 0 : a.call(n)) ?? cr();
if (e.length < 16)
throw new Error("Random bytes length must be >= 16");
return e[6] = e[6] & 15 | 64, e[8] = e[8] & 63 | 128, ar(e);
}
const Se = ge(
void 0
);
Se.displayName = "kortexa.ai:microphone";
const ur = 16e3, qe = "kortexa-audio-processor", lr = `
class AudioProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.bufferSize = 512;
this.buffer = new Float32Array(this.bufferSize);
this.offset = 0;
}
process(inputs, outputs) {
const input = inputs[0]?.[0];
if (!input?.length) return true;
for (let i = 0; i < input.length; i++) {
this.buffer[this.offset++] = input[i];
if (this.offset === this.bufferSize) {
this.port.postMessage(this.buffer);
this.offset = 0;
}
}
return true;
}
}
registerProcessor('${qe}', AudioProcessor);
`;
function fr() {
const n = new Blob([lr], {
type: "application/javascript; charset=utf-8"
});
return URL.createObjectURL(n);
}
function mr({
sampleRate: n = ur,
onData: r,
onError: c
}) {
const [e, a] = V(!1), o = m(!1), t = m(void 0), T = m(void 0), y = m(void 0), D = m(void 0), A = m(r), u = m(c);
R(() => {
A.current = r, u.current = c;
}, [r, c]);
const S = f(() => {
var I, C, g;
if (!o.current) {
o.current = !0;
try {
T.current && (T.current.getTracks().forEach((O) => O.stop()), T.current = void 0), D.current && (D.current.disconnect(), D.current = void 0), y.current && (y.current.port.onmessage = null, y.current.disconnect(), y.current = void 0), ((I = t.current) == null ? void 0 : I.state) !== "closed" && ((C = t.current) == null || C.close(), t.current = void 0);
} catch (M) {
console.error("Error closing audio context:", M), (g = u.current) == null || g.call(u, "Failed to close audio context");
} finally {
a(!1), o.current = !1;
}
}
}, []), h = f(async () => {
var I;
e && S();
try {
(!t.current || t.current.state === "closed") && (t.current = new AudioContext({ sampleRate: n })), t.current.state === "suspended" && await t.current.resume(), await t.current.audioWorklet.addModule(
// `data:application/javascript;base64,${btoa(WORKLET_CODE)}`
fr()
), T.current = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: !0,
noiseSuppression: !0
}
}), D.current = t.current.createMediaStreamSource(
T.current
), y.current = new AudioWorkletNode(
t.current,
qe
), y.current.port.onmessage = (C) => {
var g;
(g = A.current) == null || g.call(A, new Float32Array(C.data));
}, D.current.connect(y.current), y.current.connect(t.current.destination), a(!0);
} catch (C) {
console.error("Error starting audio:", C), (I = u.current) == null || I.call(u, "Failed to access microphone"), S();
}
}, [n, S, e]);
return R(() => S, [S]), {
isRecording: e,
start: h,
stop: S
};
}
class Ke {
constructor() {
We(this, "listeners", /* @__PURE__ */ new Map());
}
addListener(r, c) {
this.listeners.has(r) || this.listeners.set(r, /* @__PURE__ */ new Map()), this.listeners.get(r).set(c.id, c);
}
removeListener(r, c) {
const e = this.listeners.get(r);
e && e.delete(c);
}
async dispatch(r, c) {
const e = this.listeners.get(r);
if (!e) return;
const a = [];
e.forEach((o) => {
try {
const T = o.listener(c);
T instanceof Promise && a.push(T);
} catch (t) {
console.error(`Error in ${String(r)} listener ${o.id}:`, t);
}
}), a.length > 0 && await Promise.all(a);
}
clear(r) {
r ? this.listeners.delete(r) : this.listeners.clear();
}
getListenerCount(r) {
var c;
return ((c = this.listeners.get(r)) == null ? void 0 : c.size) ?? 0;
}
}
const oe = new Ke();
function gr({ children: n, ...r }) {
const {
isRecording: c,
start: e,
stop: a
} = mr({
...r,
onData: (g) => {
var M;
(M = r.onData) == null || M.call(r, g), oe.dispatch("data", g);
},
onError: (g) => {
var M;
(M = r.onError) == null || M.call(r, g), oe.dispatch("error", g);
}
}), o = f(async () => {
try {
c || (await e(), await oe.dispatch("start"));
} catch (g) {
let M = "Failed to start microphone";
throw g instanceof Error ? M += ": " + g.message : M += ": " + String(g), oe.dispatch("error", M), g;
}
}, [c, e]), t = f(() => {
try {
c && (oe.dispatch("stop"), a());
} catch {
oe.dispatch("error", "Failed to stop microphone");
}
}, [c, a]), T = f((g) => {
const M = `kortexa-microphone-data-${X()}`;
return oe.addListener("data", { id: M, listener: g }), M;
}, []), y = f((g) => {
oe.removeListener("data", g);
}, []), D = f((g) => {
const M = `kortexa-microphone-start-${X()}`;
return oe.addListener("start", { id: M, listener: g }), M;
}, []), A = f((g) => {
oe.removeListener("start", g);
}, []), u = f((g) => {
const M = `kortexa-microphone-stop-${X()}`;
return oe.addListener("stop", { id: M, listener: g }), M;
}, []), S = f((g) => {
oe.removeListener("stop", g);
}, []), h = f((g) => {
const M = `kortexa-microphone-error-${X()}`;
return oe.addListener("error", { id: M, listener: g }), M;
}, []), I = f((g) => {
oe.removeListener("error", g);
}, []);
R(() => () => {
oe.clear();
}, []);
const C = ee(() => ({
isRecording: c,
start: o,
stop: t,
addDataListener: T,
removeDataListener: y,
addStartListener: D,
removeStartListener: A,
addStopListener: u,
removeStopListener: S,
addErrorListener: h,
removeErrorListener: I
}), [
c,
o,
t,
T,
y,
D,
A,
u,
S,
h,
I
]);
return /* @__PURE__ */ ue(Se.Provider, { value: C, children: n });
}
function vr() {
const n = Ee(Se);
if (!n) throw new Error("useMicrophone must be used within MicrophoneProvider");
return n;
}
const Ce = ge(
void 0
);
Ce.displayName = "kortexa.ai:camera";
const hr = ({
defaultFacingMode: n = "user",
defaultDeviceId: r,
requestedWidth: c,
requestedHeight: e,
requestedAspectRatio: a,
onStream: o,
onError: t
} = {}) => {
const [T, y] = V(!1), [D, A] = V(null), [u, S] = V(n), [h, I] = V(
[]
), [C, g] = V(
r
), M = f(async () => {
try {
const x = (await navigator.mediaDevices.enumerateDevices()).filter(
(L) => L.kind === "videoinput"
);
if (I(x), x.length > 0 && !C) {
const L = x.find((_) => (_.label.toLowerCase().includes("front") ? "user" : _.label.toLowerCase().includes("back") ? "environment" : null) === u);
g(
(L == null ? void 0 : L.deviceId) || x[0].deviceId
);
}
} catch (k) {
console.error("Error enumerating camera devices:", k), t == null || t("Error enumerating camera devices.");
}
}, [C, u, t]);
R(() => {
M();
}, []), R(() => {
typeof window < "u" && "ontouchstart" in window && n === "user" && !r && S("environment");
}, [n, r]);
const O = f(
async (k) => {
const x = k || C;
if (!x && h.length === 0 && (await M(), h.length === 0 || !x)) {
console.error("No camera devices available or selected."), t == null || t("No camera devices available or selected.");
return;
}
try {
const L = {
deviceId: x ? { exact: x } : void 0,
facingMode: x ? void 0 : u
// Only use facingMode if no specific deviceId
};
c && (L.width = { ideal: c }), e && (L.height = { ideal: e }), a && (L.aspectRatio = {
ideal: a
});
const _ = {
video: Object.keys(L).length > 0 ? L : !0
}, d = await navigator.mediaDevices.getUserMedia(
_
);
A(d), y(!0), o == null || o(d), x && g(x);
} catch (L) {
if (console.error("Error accessing camera:", L), t == null || t(`Error accessing camera: ${L.message}`), x && h.length > 1) {
const _ = h.find(
(d) => d.deviceId !== x
);
_ && (console.log(
`Attempting to start with device: ${_.label || _.deviceId}`
), await O(_.deviceId));
}
}
},
[
C,
u,
o,
t,
h,
M,
c,
e,
a
]
), z = f(() => {
D && (D.getTracks().forEach((k) => k.stop()), A(null), y(!1), o == null || o());
}, [D, o]), W = f(() => {
T ? z() : O();
}, [T, z, O]), q = f(async () => {
if (!T) return;
const k = u === "user" ? "environment" : "user";
S(k), z();
const x = h.find((L) => {
const _ = L.label.toLowerCase();
return k === "user" ? _.includes("front") || !_.includes("back") : k === "environment" ? _.includes("back") : !1;
});
x ? (g(x.deviceId), await O(x.deviceId)) : h.length > 0 ? (g(void 0), await O()) : t == null || t("No alternative camera found for flipping.");
}, [T, u, z, O, h, t]), b = f(
async (k) => {
k === C && T || (z(), g(k), await O(k));
},
[C, T, z, O]
);
return R(() => () => {
D && D.getTracks().forEach((k) => k.stop());
}, [D]), {
stream: D,
isRecording: T,
facingMode: u,
availableDevices: h,
currentDeviceId: C,
start: O,
stop: z,
toggle: W,
flip: q,
setDevice: b,
getDevices: M
};
}, Y = new Ke();
function pr({ children: n, ...r }) {
const {
stream: c,
isRecording: e,
facingMode: a,
availableDevices: o,
currentDeviceId: t,
start: T,
stop: y,
flip: D,
setDevice: A,
getDevices: u
} = hr({
...r,
onStream: (d) => {
var v;
(v = r.onStream) == null || v.call(r, d), Y.dispatch("stream", d);
},
onError: (d) => {
var v;
(v = r.onError) == null || v.call(r, d), Y.dispatch("error", d);
}
}), S = f(async () => {
try {
await T(), Y.dispatch("start");
} catch (d) {
let v = "Failed to start camera";
throw d instanceof Error ? v += `: ${d.message}` : v += `: ${String(d)}`, Y.dispatch("error", v), d;
}
}, [T]), h = f(() => {
try {
y(), Y.dispatch("stop");
} catch (d) {
const v = `Failed to stop camera: ${String(d)}`;
Y.dispatch("error", v);
}
}, [y]), I = f(async () => {
try {
await D();
} catch (d) {
const v = `Failed to flip camera: ${String(d)}`;
Y.dispatch("error", v);
}
}, [D]), C = f(async (d) => {
try {
await A(d);
} catch (v) {
const $ = `Failed to set device: ${String(v)}`;
Y.dispatch("error", $);
}
}, [A]);
R(() => {
Y.dispatch("facingMode", a);
}, [a]);
const g = f((d) => {
const v = `kortexa-camera-stream-${X()}`, $ = (N) => {
d(N);
};
return Y.addListener("stream", { id: v, listener: $ }), v;
}, []), M = f((d) => {
Y.removeListener("stream", d);
}, []), O = f((d) => {
const v = `kortexa-camera-start-${X()}`;
return Y.addListener("start", { id: v, listener: d }), v;
}, []), z = f((d) => {
Y.removeListener("start", d);
}, []), W = f((d) => {
const v = `kortexa-camera-stop-${X()}`;
return Y.addListener("stop", { id: v, listener: d }), v;
}, []), q = f((d) => {
Y.removeListener("stop", d);
}, []), b = f((d) => {
const v = `kortexa-camera-facingMode-${X()}`, $ = (N) => {
N !== void 0 && d(N);
};
return Y.addListener("facingMode", { id: v, listener: $ }), v;
}, []), k = f((d) => {
Y.removeListener("facingMode", d);
}, []), x = f((d) => {
const v = `kortexa-camera-error-${X()}`, $ = (N) => {
N !== void 0 && d(N);
};
return Y.addListener("error", { id: v, listener: $ }), v;
}, []), L = f((d) => {
Y.removeListener("error", d);
}, []);
R(() => () => {
Y.clear();
}, []);
const _ = ee(() => ({
isRecording: e,
stream: c,
facingMode: a,
availableDevices: o,
currentDeviceId: t,
start: S,
stop: h,
toggle: () => e ? h() : S(),
flip: I,
setDevice: C,
getDevices: u,
addStreamListener: g,
removeStreamListener: M,
addStartListener: O,
removeStartListener: z,
addStopListener: W,
removeStopListener: q,
addFacingModeListener: b,
removeFacingModeListener: k,
addErrorListener: x,
removeErrorListener: L
}), [
e,
c,
a,
o,
t,
S,
h,
I,
C,
u,
g,
M,
O,
z,
W,
q,
b,
k,
x,
L
]);
return /* @__PURE__ */ ue(Ce.Provider, { value: _, children: n });
}
const kr = () => {
const n = Ee(Ce);
if (!n) throw new Error("useCamera must be used within a CameraProvider");
return n;
}, yr = {
staticImageMode: !1,
maxNumHands: 2,
modelComplexity: 1,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip handedness for selfie view
};
function Lr({
options: n,
onInitialLoad: r,
onHandsData: c,
onError: e,
onTrackingStarted: a,
onTrackingStopped: o,
onResults: t,
enableGestures: T = !0,
gestureOptions: y,
gestureModelPath: D,
onGestureResults: A
} = {}) {
const u = m(null), S = m(null), h = m(null), I = m(null), C = m(!1), g = m(!1), [M, O] = V(!1), [z, W] = V(null), [q, b] = V(null), k = ee(
() => ({ ...yr, ...n }),
[n]
), x = m(c);
R(() => {
x.current = c;
}, [c]);
const L = m(e);
R(() => {
L.current = e;
}, [e]);
const _ = m(a);
R(() => {
_.current = a;
}, [a]);
const d = m(o);
R(() => {
d.current = o;
}, [o]);
const v = m(t);
R(() => {
v.current = t;
}, [t]);
const $ = m(r);
R(() => {
$.current = r;
}, [r]);
const N = m(A);
R(() => {
N.current = A;
}, [A]);
const P = m(/* @__PURE__ */ new Map()), K = m(
/* @__PURE__ */ new Map()
), B = m(/* @__PURE__ */ new Map()), ce = m(/* @__PURE__ */ new Map()), E = f((w, F) => {
var G, se, ie;
const H = [];
w.landmarks && w.landmarks.forEach((J, Q) => {
var Z;
const re = {
landmarks: J,
worldLandmarks: ((Z = w.worldLandmarks) == null ? void 0 : Z[Q]) || [],
handedness: w.handedness[Q][0],
// Take first handedness
gestures: (F == null ? void 0 : F.gestures[Q]) || []
// Add gesture results
};
H.push(re);
});
const U = {
detectedHands: H,
imageDimensions: {
width: ((G = h.current) == null ? void 0 : G.videoWidth) || 0,
height: ((se = h.current) == null ? void 0 : se.videoHeight) || 0
}
};
if (W(U), (ie = x.current) == null || ie.call(x, U), P.current.forEach(
(J, Q) => {
J(U);
}
), F && N.current && H.forEach((J, Q) => {
J.gestures.length > 0 && N.current(J.gestures, Q);
}), v.current)
try {
v.current(H, h.current || void 0);
} catch (J) {
const Q = J instanceof Error ? J.message : String(J);
b(Q), L.current && L.current(Q), K.current.forEach(
(re, Z) => re(Q)
);
}
!g.current && H.length > 0 && (g.current = !0, $.current && $.current());
}, [
W,
b,
x,
v,
$,
L,
N,
P,
K,
g,
h
]);
R(() => ((async () => {
try {
const F = await we.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), H = await tr.createFromOptions(F, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numHands: k.maxNumHands || 2,
minHandDetectionConfidence: k.minDetectionConfidence || 0.5,
minHandPresenceConfidence: 0.5,
minTrackingConfidence: k.minTrackingConfidence || 0.5
});
let U = null;
T && (U = await nr.createFromOptions(F, {
baseOptions: {
modelAssetPath: D || "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numHands: (y == null ? void 0 : y.numHands) || 2,
minHandDetectionConfidence: (y == null ? void 0 : y.minHandDetectionConfidence) || 0.5,
minHandPresenceConfidence: (y == null ? void 0 : y.minHandPresenceConfidence) || 0.5,
minTrackingConfidence: (y == null ? void 0 : y.minTrackingConfidence) || 0.5
})), u.current = H, S.current = U;
} catch (F) {
const H = F instanceof Error ? F.message : String(F);
b(H), L.current && L.current(H);
}
})(), () => {
u.current = null, S.current = null;
}), [k, T, y, D, L]);
const s = f(async () => {
if (!h.current || !u.current)
return;
if (h.current.paused || h.current.ended || h.current.readyState < HTMLMediaElement.HAVE_METADATA) {
C.current && (I.current = requestAnimationFrame(s));
return;
}
const w = performance.now();
try {
const F = u.current.detectForVideo(
h.current,
w
);
let H;
S.current && (H = S.current.recognizeForVideo(
h.current,
w
)), E(F, H);
} catch (F) {
const H = F instanceof Error ? F.message : String(F);
b(H), L.current && L.current(H), K.current.forEach(
(U, G) => {
U(H);
}
);
}
C.current && (I.current = requestAnimationFrame(s));
}, [
u,
S,
h,
b,
L,
K,
E
]), l = f(
async (w) => {
if (!u.current) {
b("HandLandmarker not initialized."), L.current && L.current("HandLandmarker not initialized."), K.current.forEach(
(F, H) => {
F("HandLandmarker not initialized.");
}
);
return;
}
C.current || (h.current = w, g.current = !1, C.current = !0, O(!0), I.current && cancelAnimationFrame(I.current), I.current = requestAnimationFrame(s), _.current && _.current(), B.current.forEach(
(F, H) => {
F();
}
), b(null));
},
[
s,
_,
B,
L,
K,
b
]
), j = f(() => {
!C.current && !M || (C.current = !1, O(!1), I.current && (cancelAnimationFrame(I.current), I.current = null), d.current && d.current(), ce.current.forEach(
(w, F) => {
w();
}
));
}, [M, d, ce]);
return {
isTracking: M,
handsData: z,
error: q,
startTracking: l,
stopTracking: j,
getHandLandmarker: () => u.current,
getGestureRecognizer: () => S.current,
addHandsDataListener: (w) => {
const F = `kortexa-hands-data-${X()}`;
return P.current.set(F, w), F;
},
removeHandsDataListener: (w) => {
P.current.delete(w);
},
addErrorListener: (w) => {
const F = `kortexa-hands-error-${X()}`;
return K.current.set(F, w), F;
},
removeErrorListener: (w) => {
K.current.delete(w);
},
addStartListener: (w) => {
const F = `kortexa-hands-start-${X()}`;
return B.current.set(F, w), F;
},
removeStartListener: (w) => {
B.current.delete(w);
},
addStopListener: (w) => {
const F = `kortexa-hands-stop-${X()}`;
return ce.current.set(F, w), F;
},
removeStopListener: (w) => {
ce.current.delete(w);
}
};
}
const Te = ge(
{}
);
Te.displayName = "kortexa.ai:hands-tracking";
function Er({ children: n, ...r }) {
const c = Lr(r);
return /* @__PURE__ */ ue(Te.Provider, { value: c, children: n });
}
const Mr = () => {
const n = Ee(Te);
if (n === null) throw new Error("useHands must be used within a HandsProvider");
return n;
}, wr = {
staticImageMode: !1,
modelComplexity: 1,
smoothLandmarks: !0,
enableSegmentation: !1,
smoothSegmentation: !0,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip for selfie view
};
function Xe({
options: n,
onInitialLoad: r,
onBodyData: c,
onError: e,
onTrackingStarted: a,
onTrackingStopped: o,
onResults: t,
enableSegmentation: T = !1,
outputSegmentationMasks: y = !1
} = {}) {
const D = m(null), A = m(null), u = m(null), S = m(!1), h = m(!1), [I, C] = V(!1), [g, M] = V(null), [O, z] = V(null), W = ee(
() => ({ ...wr, ...n, enableSegmentation: T }),
[n, T]
), q = m(c);
R(() => {
q.current = c;
}, [c]);
const b = m(e);
R(() => {
b.current = e;
}, [e]);
const k = m(a);
R(() => {
k.current = a;
}, [a]);
const x = m(o);
R(() => {
x.current = o;
}, [o]);
const L = m(t);
R(() => {
L.current = t;
}, [t]);
const _ = m(r);
R(() => {
_.current = r;
}, [r]);
const d = m(/* @__PURE__ */ new Map()), v = m(
/* @__PURE__ */ new Map()
), $ = m(/* @__PURE__ */ new Map()), N = m(/* @__PURE__ */ new Map()), P = f((E) => {
var j, w, F;
const s = [];
E.landmarks && E.landmarks.length > 0 && E.landmarks.forEach((H, U) => {
var se;
const G = {
landmarks: H,
worldLandmarks: ((se = E.worldLandmarks) == null ? void 0 : se[U]) || []
};
y && E.segmentationMasks && (G.segmentationMasks = []), s.push(G);
});
const l = {
detectedBodies: s,
imageDimensions: {
width: ((j = A.current) == null ? void 0 : j.videoWidth) || 0,
height: ((w = A.current) == null ? void 0 : w.videoHeight) || 0
}
};
if (M(l), (F = q.current) == null || F.call(q, l), d.current.forEach(
(H, U) => {
H(l);
}
), L.current)
try {
L.current(s, A.current || void 0);
} catch (H) {
const U = H instanceof Error ? H.message : String(H);
z(U), b.current && b.current(U), v.current.forEach(
(G, se) => {
G(U);
}
);
}
!h.current && s.length > 0 && (h.current = !0, _.current && _.current());
}, [
M,
z,
q,
L,
_,
b,
d,
v,
h,
A,
y
]);
R(() => ((async () => {
try {
const s = await we.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), l = await sr.createFromOptions(s, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/1/pose_landmarker_heavy.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numPoses: 1,
// Single pose detection
minPoseDetectionConfidence: W.minDetectionConfidence || 0.5,
minPosePresenceConfidence: 0.5,
minTrackingConfidence: W.minTrackingConfidence || 0.5,
outputSegmentationMasks: W.enableSegmentation || !1
});
D.current = l;
} catch (s) {
const l = s instanceof Error ? s.message : String(s);
z(l), b.current && b.current(l);
}
})(), () => {
D.current = null;
}), [W, b]);
const K = f(async () => {
if (!A.current || !D.current)
return;
if (A.current.paused || A.current.ended || A.current.readyState < HTMLMediaElement.HAVE_METADATA) {
S.current && (u.current = requestAnimationFrame(K));
return;
}
const E = performance.now();
try {
const s = D.current.detectForVideo(
A.current,
E
);
P(s);
} catch (s) {
const l = s instanceof Error ? s.message : String(s);
z(l), b.current && b.current(l), v.current.forEach(
(j, w) => {
j(l);
}
);
}
S.current && (u.current = requestAnimationFrame(K));
}, [
D,
A,
z,
b,
v,
P
]), B = f(
async (E) => {
if (!D.current) {
z("PoseLandmarker not initialized."), b.current && b.current("PoseLandmarker not initialized."), v.current.forEach(
(s, l) => {
s("PoseLandmarker not initialized.");
}
);
return;
}
S.current || (A.current = E, h.current = !1, S.current = !0, C(!0), u.current && cancelAnimationFrame(u.current), u.current = requestAnimationFrame(K), k.current && k.current(), $.current.forEach(
(s, l) => {
s();
}
), z(null));
},
[
K,
k,
$,
b,
v,
z
]
), ce = f(() => {
!S.current && !I || (S.current = !1, C(!1), u.current && (cancelAnimationFrame(u.current), u.current = null), x.current && x.current(), N.current.forEach(
(E, s) => {
E();
}
));
}, [I, x, N]);
return {
isTracking: I,
bodyData: g,
error: O,
startTracking: B,
stopTracking: ce,
getPoseLandmarker: () => D.current,
addBodyDataListener: (E) => {
const s = `kortexa-body-data-${X()}`;
return d.current.set(s, E), s;
},
removeBodyDataListener: (E) => {
d.current.delete(E);
},
addErrorListener: (E) => {
const s = `kortexa-body-error-${X()}`;
return v.current.set(s, E), s;
},
removeErrorListener: (E) => {
v.current.delete(E);
},
addStartListener: (E) => {
const s = `kortexa-body-start-${X()}`;
return $.current.set(s, E), s;
},
removeStartListener: (E) => {
$.current.delete(E);
},
addStopListener: (E) => {
const s = `kortexa-body-stop-${X()}`;
return N.current.set(s, E), s;
},
removeStopListener: (E) => {
N.current.delete(E);
}
};
}
const Ye = ge(
{}
);
Ye.displayName = "kortexa.ai:body-tracking";
function Fr({ children: n, ...r }) {
const c = Xe(r);
return /* @__PURE__ */ ue(Ye.Provider, { value: c, children: n });
}
const Sr = {
staticImageMode: !1,
maxNumFaces: 1,
refineLandmarks: !0,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip for selfie view
};
function Je({
options: n,
onInitialLoad: r,
onFaceData: c,
onError: e,
onTrackingStarted: a,
onTrackingStopped: o,
onResults: t,
outputFaceBlendshapes: T = !0,
outputTransformationMatrix: y = !1,
runningMode: D = "VIDEO"
} = {}) {
const A = m(null), u = m(null), S = m(null), h = m(!1), I = m(!1), [C, g] = V(!1), [M, O] = V(null), [z, W] = V(null), q = ee(
() => ({ ...Sr, ...n }),
[n]
), b = m(c);
R(() => {
b.current = c;
}, [c]);
const k = m(e);
R(() => {
k.current = e;
}, [e]);
const x = m(a);
R(() => {
x.current = a;
}, [a]);
const L = m(o);
R(() => {
L.current = o;
}, [o]);
const _ = m(t);
R(() => {
_.current = t;
}, [t]);
const d = m(r);
R(() => {
d.current = r;
}, [r]);
const v = m(/* @__PURE__ */ new Map()), $ = m(
/* @__PURE__ */ new Map()
), N = m(/* @__PURE__ */ new Map()), P = m(/* @__PURE__ */ new Map()), K = f((s) => {
var w, F, H;
const l = [];
s.faceLandmarks && s.faceLandmarks.length > 0 && s.faceLandmarks.forEach((U, G) => {
const se = {
landmarks: U
};
if (T && s.faceBlendshapes && s.faceBlendshapes[G] && (se.blendshapes = s.faceBlendshapes[G].categories), y && s.facialTransformationMatrixes && s.facialTransformationMatrixes[G]) {
const ie = s.facialTransformationMatrixes[G];
se.transformationMatrix = ie.data ? Array.from(ie.data) : [];
}
if (U.length > 0) {
let ie = 1, J = 1, Q = 0, re = 0;
U.forEach((Z) => {
ie = Math.min(ie, Z.x), J = Math.min(J, Z.y), Q = Math.max(Q, Z.x), re = Math.max(re, Z.y);
}), se.boundingBox = {
xMin: ie,
yMin: J,
width: Q - ie,
height: re - J
};
}
l.push(se);
});
const j = {
detectedFaces: l,
imageDimensions: {
width: ((w = u.current) == null ? void 0 : w.videoWidth) || 0,
height: ((F = u.current) == null ? void 0 : F.videoHeight) || 0
}
};
if (O(j), (H = b.current) == null || H.call(b, j), v.current.forEach(
(U, G) => {
U(j);
}
), _.current)
try {
_.current(l, u.current || void 0);
} catch (U) {
const G = U instanceof Error ? U.message : String(U);
W(G), k.current && k.current(G), $.current.forEach(
(se, ie) => {
se(G);
}
);
}
!I.current && l.length > 0 && (I.current = !0, d.current && d.current());
}, [
O,
W,
b,
_,
d,
k,
v,
$,
I,
u,
T,
y
]);
R(() => ((async () => {
try {
const l = await we.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), j = await ir.createFromOptions(l, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
delegate: "GPU"
},
runningMode: D,
numFaces: q.maxNumFaces || 1,
minFaceDetectionConfidence: q.minDetectionConfidence || 0.5,
minFacePresenceConfidence: 0.5,
minTrackingConfidence: q.minTrackingConfidence || 0.5,
outputFaceBlendshapes: T,
outputFacialTransformationMatrixes: y
});
A.current = j;
} catch (l) {
const j = l instanceof Error ? l.message : String(l);
W(j), k.current && k.current(j);
}
})(), () => {
A.current = null;
}), [q, D, T, y, k]);
const B = f(async () => {
if (!u.current || !A.current)
return;
if (u.current.paused || u.current.ended || u.current.readyState < HTMLMediaElement.HAVE_METADATA) {
h.current && (S.current = requestAnimationFrame(B));
return;
}
const s = performance.now();
try {
const l = A.current.detectForVideo(
u.current,
s
);
K(l);
} catch (l) {
const j = l instanceof Error ? l.message : String(l);
W(j), k.current && k.current(j), $.current.forEach(
(w, F) => {
w(j);
}
);
}
h.current && (S.current = requestAnimationFrame(B));
}, [
A,
u,
W,
k,
$,
K
]), ce = f(
async (s) => {
if (!A.current) {
W("FaceLandmarker not initialized."), k.current && k.current("FaceLandmarker not initialized."), $.current.forEach(
(l, j) => {
l("FaceLandmarker not initialized.");
}
);
return;
}
h.current || (u.current = s, I.current = !1, h.current = !0, g(!0), S.current && cancelAnimationFrame(S.current), S.current = requestAnimationFrame(B), x.current && x.current(), N.current.forEach(
(l, j) => {
l();
}
), W(null));
},
[
B,
W
]
), E = f(() => {
!h.current && !C || (h.current = !1, g(!1), S.current && (cancelAnimationFrame(S.current), S.current = null), L.current && L.current(), P.current.forEach(
(s, l) => {
s();
}
));
}, [C]);
return {
isTracking: C,
faceData: M,
error: z,
startTracking: ce,
stopTracking: E,
getFaceLandmarker: () => A.current,
getFaceDetector: () => null,
// Not implemented in this version
addFaceDataListener: (s) => {
const l = `kortexa-face-data-${X()}`;
return v.current.set(l, s), l;
},
removeFaceDataListener: (s) => {
v.current.delete(s);
},
addErrorListener: (s) => {
const l = `kortexa-face-error-${X()}`;
return $.current.set(l, s), l;
},
removeErrorListener: (s) => {
$.current.delete(s);
},
addStartListener: (s) => {
const l = `kortexa-face-start-${X()}`;
return N.current.set(l, s), l;
},
removeStartListener: (s) => {
N.current.delete(s);
},
addStopListener: (s) => {
const l = `kortexa-face-stop-${X()}`;
return P.current.set(l, s), l;
},
removeStopListener: (s) => {
P.current.delete(s);
}
};
}
const Qe = ge(
{}
);
Qe.displayName = "kortexa.ai:face-tracking";
function Cr({ children: n, ...r }) {
const c = Je(r);
return /* @__PURE__ */ ue(Qe.Provider, { value: c, children: n });
}
const Tr = "proceed";
function Dr({
children: n,
startBehavior: r
}) {
const c = vr(), e = kr(), a = Mr(), o = Xe(), t = Je(), T = m(null), y = m(null), D = m(null), [A, u] = V(!1), [S, h] = V(!1), [I, C] = V(!1), [g, M] = V(!1), [O, z] = V(!1), [W, q] = V(!1), [b, k] = V(void 0), [x, L] = V(!1), [_, d] = V(void 0), [v, $] = V(void 0), [N, P] = V(void 0), [K, B] = V(void 0), [ce, E] = V(void 0), [s, l] = V(!1), [j, w] = V(!1), [F, H] = V(!1), [U, G] = V(!1), [se, ie] = V(!1), [J, Q] = V(!1);
R(() => {
const i = c.addErrorListener((p) => {
d(p);
});
return () => {
c.removeErrorListener(i);
};
}, [c]), R(() => {
const i = e.addErrorListener((p) => {
$(p);
});
return () => {
e.removeErrorListener(i);
};
}, [e]), R(() => {
if (a) {
const i = a.addErrorListener((p) => {
P(p), l(!1);
});
return () => {
a.removeErrorListener(i);
};
}
return () => {
};
}, [a]), R(() => {
if (o) {
const i = o.addErrorListener((p) => {
B(p), w(!1);
});
return () => {
o.removeErrorListener(i);
};
}
return () => {
};
}, [o]), R(() => {
if (t) {
const i = t.addErrorListener((p) => {
E(p), H(!1);
});
return () => {
t.removeErrorListener(i);
};
}
return () => {
};
}, [t]);
const re = ee(() => c.isRecording, [c]), Z = ee(() => e.isRecording, [e]), De = ee(() => (a == null ? void 0 : a.isTracking) ?? !1, [a]), Ae = ee(() => (o == null ? void 0 : o.isTracking) ?? !1, [o]), xe = ee(() => (t == null ? void 0 : t.isTracking) ?? !1, [t]), Re = ee(() => re && Z, [re, Z]), He = ee(() => {
}, []), Ie = ee(() => e.stream ?? void 0, [e]), be = ee(() => e.facingMode, [e]), _e = ee(() => (a == null ? void 0 : a.handsData) ?? void 0, [a]), Ve = ee(() => (o == null ? void 0 : o.bodyData) ?? void 0, [o]), $e = ee(() => (t == null ? void 0 : t.faceData) ?? void 0, [t]), Pe = f((i) => {
T.current = i, i ? G(!0) : (u(!1), G(!1));
}, []), Ue = f((i) => {
y.current = i, i ? ie(!0) : (h(!1), ie(!1));
}, []), ze = f((i) => {
D.current = i, i ? Q(!0) : (C(!1), Q(!1));
}, []), le = f(async () => {
if (!a || !T.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const p = "Pre-conditions not met for starting hand tracking (hands, videoElement, camera state).";
return P(p), u(!0), Promise.reject(new Error(p));
}
if (a.isTracking)
return Promise.resolve();
const i = T.current;
u(!0), P(void 0);
try {
i.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((p, te) => {
const ae = () => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), p();
}, de = (Le) => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), te(new Error("Video element error during metadata load for hand tracking."));
};
i.addEventListener("loadedmetadata", ae), i.addEventListener("error", de);
}), await a.startTracking(i);
} catch (p) {
const te = p instanceof Error ? p.message : String(p);
throw P(te), p;
}
}, [a, e, P, u]), fe = f(async () => {
if (!o || !y.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const p = "Pre-conditions not met for starting body tracking (body, videoElement, camera state).";
return B(p), h(!0), Promise.reject(new Error(p));
}
if (o.isTracking)
return Promise.resolve();
const i = y.current;
h(!0), B(void 0);
try {
i.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((p, te) => {
const ae = () => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), p();
}, de = (Le) => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), te(new Error("Video element error during metadata load for body tracking."));
};
i.addEventListener("loadedmetadata", ae), i.addEventListener("error", de);
}), await o.startTracking(i);
} catch (p) {
const te = p instanceof Error ? p.message : String(p);
throw B(te), p;
}
}, [o, e, B, h]), me = f(async () => {
if (!t || !D.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const p = "Pre-conditions not met for starting face tracking (face, videoElement, camera state).";
return E(p), C(!0), Promise.reject(new Error(p));
}
if (t.isTracking)
return Promise.resolve();
const i = D.current;
C(!0), E(void 0);
try {
i.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((p, te) => {
const ae = () => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), p();
}, de = (Le) => {
i.removeEventListener("loadedmetadata", ae), i.removeEventListener("error", de), te(new Error("Video element error during metadata load for face tracking."));
};
i.addEventListener("loadedmetadata", ae), i.addEventListener("error", de);
}), await t.startTracking(i);
} catch (p) {
const te = p instanceof Error ? p.message : String(p);
throw E(te), p;
}
}, [t, e, E, C]), ve = f(async () => {
if (x)
return;
L(!0), k(void 0), u(!1), M(!1), d(void 0), $(void 0), P(void 0);
let i = re, p = Z;
if (c && !re)
try {
await c.start(), i = !0;
} catch {
i = !1;
}
if (e && !Z)
try {
await e.start(), p = !0;
} catch {
p = !1;
}
if (r === "halt") {
if (!i || !p) {
const ae = _ || (i ? "" : "Failed to start"), de = v || (p ? "" : "Failed to start"), Le = `Media start halted: Mic ${i ? "OK" : `FAIL (${ae})`}. Cam ${p ? "OK" : `FAIL (${de})`}.`;
k(Le), i && c && c.stop(), p && e && e.stop(), L(!1);
return;
}
} else {
const te = [];
if (i || te.push(`Mic: ${_ || "Unknown"}`), p || te.push(`Cam: ${v || "Unknown"}`), te.length > 0) {
const ae = `Failed to start: ${te.join("; ")}. Others proceeded if successful.`;
k(ae);
}
}
if (p && (e != null && e.isRecording) && e.stream) {
if (a && !a.isTracking && T.current && !A)
try {
await le();
} catch {
r === "halt" && k(`Failed to start hand tracking. Halting media start. Error: ${N || "Unknown hands error"}`);
}
if (o && !o.isTracking && y.current && !S)
try {
await fe();
} catch {
r === "halt" && k(`Failed to start body tracking. Halting media start. Error: ${K || "Unknown body error"}`);
}
if (t && !t.isTracking && D.current && !I)
try {
await me();
} catch {
r === "halt" && k(`Failed to start face tracking. Halting media start. Error: ${ce || "Unknown face error"}`);
}
}
L(!1);
}, [
c,
e,
a,
o,
t,
r,
re,
Z,
x,
T,
y,
D,
A,
S,
I,
le,
fe,
me,
_,
v,
N,
K,
ce,
d,
$,
P,
B,
E,
k,
L,
u,
h,
C,
M,
z,
q
]);
R(() => {
e != null && e.isRecording && e.stream && a && !a.isTracking && T.current && !A && !g && le().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, a, T, A, g, le]), R(() => {
e != null && e.isRecording && e.stream && o && !o.isTracking && y.current && !S && !O && fe().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, o, y, S, O, fe]), R(() => {
e != null && e.isRecording && e.stream && t && !t.isTracking && D.current && !I && !W && me().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, t, D, I, W, me]), R(() => {
e != null && e.isRecording || (u(!1), h(!1), C(!1));
}, [e == null ? void 0 : e.isRecording, u, h, C]);
const he = f(async () => {
if (a && a.isTracking)
try {
await a.stopTracking(), M(!0);
} catch (i) {
const p = i instanceof Error ? i.message : String(i);
P(p);
}
u(!1), l(!1);
}, [a, P, u]), pe = f(async () => {
if (o && o.isTracking)
try {
await o.stopTracking(), z(!0);
} catch (i) {
const p = i instanceof Error ? i.message : String(i);
B(p);
}
h(!1), w(!1);
}, [o, B, h]), ke = f(async () => {
if (t && t.isTracking)
try {
await t.stopTracking(), q(!0);
} catch (i) {
const p = i instanceof Error ? i.message : String(i);
E(p);
}
C(!1), H(!1);
}, [t, E, C]);
R(() => {
e && !e.isRecording && (a != null && a.isTracking && he(), o != null && o.isTracking && pe(), t != null && t.isTracking && ke(), u(!1), h(!1), C(!1));
}, [e, e == null ? void 0 : e.isRecording, a, o, t, he, pe, ke]);
const ye = f(() => {
c != null && c.isRecording && c.stop(), e != null && e.isRecording && e.stop(), k(void 0), d(void 0), $(void 0), P(void 0), B(void 0), E(void 0), L(!1);
}, [c, e, k, d, $, P, B, E, L]), Be = f(async () => {
re && Z ? ye() : await ve();
}, [re, Z, ve, ye]), Ne = f(async () => {
if (!(s || a != null && a.isTracking)) {
if (M(!1), l(!0), P(void 0), !a) {
P("Hand tracking service not available."), l(!1);
return;
}
if (!(e != null && e.isRecording)) {
P("Camera is not active. Cannot start hand tracking."), l(!1);
return;
}
if (!T.current) {
P("Video element not set for hand tracking."), l(!1), G(!1);
return;
}
try {
await le();
} catch {
} finally {
l(!1);
}
}
}, [a, e == null ? void 0 : e.isRecording, T, s, le, l, P]), je = f(async () => {
if (!(j || o != null && o.isTracking)) {
if (z(!1), w(!0), B(void 0), !o) {
B("Body tracking service not available."), w(!1);
return;
}
if (!(e != null && e.isRecording)) {
B("Camera is not active. Cannot start body tracking."), w(!1);
return;
}
if (!y.current) {
B("Video element not set for body tracking."), w(!1), ie(!1);
return;
}
try {
await fe();
} catch {
} finally {
w(!1);
}
}
}, [o, e == null ? void 0 : e.isRecording, y, j, fe, w, B]), Oe = f(async () => {
if (!(F || t != null && t.isTracking)) {
if (q(!1), H(!0), E(void 0), !t) {
E("Face tracking service not available."), H(!1);
return;
}
if (!(e != null && e.isRecording)) {
E("Camera is not active. Cannot start face tracking."), H(!1);
return;
}
if (!D.current) {
E("Video element not set for face tracking."), H(!1), Q(!1);
return;
}
try {
await me();
} catch {
} finally {
H(!1);
}
}
}, [t, e == null ? void 0 : e.isRecording, D, F, me, H, E]), Ze = ee(() => ({
isAudioActive: re,
isVideoActive: Z,
isHandTrackingActive: De,
isBodyTrackingActive: Ae,
isFaceTrackingActive: xe,
isMediaActive: Re,
audioStream: He,
videoStream: Ie,
videoFacingMode: be,
currentHandsData: _e,
currentBodyData: Ve,
currentFaceData: $e,
audioError: _,
videoError: v,
handsError: N,
bodyError: K,
faceError: ce,
mediaError: b,
startMedia: ve,
stopMedia: ye,
toggleMedia: Be,
cam: e,
mic: c,
hands: a,
body: o,
face: t,
setVideoElementForHands: Pe,
setVideoElementForBody: Ue,
setVideoElementForFace: ze,
startHands: Ne,
stopHands: he,
startBody: je,
stopBody: pe,
startFace: Oe,
stopFace: ke,
isStartingMedia: x,
isStartingHands: s,
isStartingBody: j,
isStartingFace: F,
isVideoElementForHandsSet: U,
isVideoElementForBodySet: se,
isVideoElementForFaceSet: J
}), [
re,
Z,
De,
Ae,
xe,
Re,
He,
Ie,
be,
_e,
Ve,
$e,
_,
v,
N,
K,
ce,
b,
ve,
ye,
Be,
e,
c,
a,
o,
t,
Pe,
Ue,
ze,
Ne,
he,
je,
pe,
Oe,
ke,
x,
s,
j,
F,
U,
se,
J
]);
return /* @__PURE__ */ ue(Fe.Provider, { value: Ze, children: n });
}
function br({
children: n,
microphoneProps: r,
cameraProps: c,
handsProps: e,
bodyProps: a,
faceProps: o,
startBehavior: t = Tr
}) {
return /* @__PURE__ */ ue(gr, { ...r, children: /* @__PURE__ */ ue(pr, { ...c, children: /* @__PURE__ */ ue(Er, { ...e, children: /* @__PURE__ */ ue(Fr, { ...a, children: /* @__PURE__ */ ue(Cr, { ...o, children: /* @__PURE__ */ ue(
Dr,
{
startBehavior: t,
children: n
}
) }) }) }) }) });
}
export {
Fr as BodyProvider,
pr as CameraProvider,
br as CompositeMediaProvider,
Cr as FaceProvider,
Er as HandsProvider,
gr as MicrophoneProvider,
Xe as useBodyControl,
kr as useCamera,
hr as useCameraDevice,
Ir as useCompositeMedia,
Je as useFaceControl,
Mr as useHands,
Lr as useHandsControl,
vr as useMicrophone,
mr as useMicrophoneDevice
};
//# sourceMappingURL=react-multimodal.js.map