@kortexa-ai/react-multimodal
Version:
A set of react components and hooks to help with multimodal input
1,626 lines (1,619 loc) • 46.3 kB
JavaScript
var Ye = Object.defineProperty;
var Je = (k, i, a) => i in k ? Ye(k, i, { enumerable: !0, configurable: !0, writable: !0, value: a }) : k[i] = a;
var Ne = (k, i, a) => Je(k, typeof i != "symbol" ? i + "" : i, a);
import { createContext as fe, useContext as Le, useState as P, useRef as f, useEffect as R, useCallback as l, useMemo as Z } from "react";
import { jsx as ce } from "react/jsx-runtime";
import { FilesetResolver as Ee, HandLandmarker as Qe, GestureRecognizer as Ze, PoseLandmarker as er, FaceLandmarker as rr } from "@mediapipe/tasks-vision";
const ye = fe(
void 0
);
ye.displayName = "kortexa.ai:composite-media";
const Fr = () => {
const k = Le(ye);
if (!k)
throw new Error(
"useCompositeMedia must be used within a CompositeMediaProvider"
);
return k;
}, Me = fe(
void 0
);
Me.displayName = "kortexa.ai:microphone";
const tr = 16e3, je = "kortexa-audio-processor", nr = `
class AudioProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.bufferSize = 512;
this.buffer = new Float32Array(this.bufferSize);
this.offset = 0;
}
process(inputs, outputs) {
const input = inputs[0]?.[0];
if (!input?.length) return true;
for (let i = 0; i < input.length; i++) {
this.buffer[this.offset++] = input[i];
if (this.offset === this.bufferSize) {
this.port.postMessage(this.buffer);
this.offset = 0;
}
}
return true;
}
}
registerProcessor('${je}', AudioProcessor);
`;
function sr() {
const k = new Blob([nr], {
type: "application/javascript; charset=utf-8"
});
return URL.createObjectURL(k);
}
function ir({
sampleRate: k = tr,
onData: i,
onError: a
}) {
const [e, o] = P(!1), s = f(!1), r = f(void 0), C = f(void 0), E = f(void 0), T = f(void 0), A = f(i), d = f(a);
R(() => {
A.current = i, d.current = a;
}, [i, a]);
const F = l(() => {
var I, D, m;
if (!s.current) {
s.current = !0;
try {
C.current && (C.current.getTracks().forEach((O) => O.stop()), C.current = void 0), T.current && (T.current.disconnect(), T.current = void 0), E.current && (E.current.port.onmessage = null, E.current.disconnect(), E.current = void 0), ((I = r.current) == null ? void 0 : I.state) !== "closed" && ((D = r.current) == null || D.close(), r.current = void 0);
} catch (M) {
console.error("Error closing audio context:", M), (m = d.current) == null || m.call(d, "Failed to close audio context");
} finally {
o(!1), s.current = !1;
}
}
}, []), v = l(async () => {
var I;
e && F();
try {
(!r.current || r.current.state === "closed") && (r.current = new AudioContext({ sampleRate: k })), r.current.state === "suspended" && await r.current.resume(), await r.current.audioWorklet.addModule(
// `data:application/javascript;base64,${btoa(WORKLET_CODE)}`
sr()
), C.current = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: !0,
noiseSuppression: !0
}
}), T.current = r.current.createMediaStreamSource(
C.current
), E.current = new AudioWorkletNode(
r.current,
je
), E.current.port.onmessage = (D) => {
var m;
(m = A.current) == null || m.call(A, new Float32Array(D.data));
}, T.current.connect(E.current), E.current.connect(r.current.destination), o(!0);
} catch (D) {
console.error("Error starting audio:", D), (I = d.current) == null || I.call(d, "Failed to access microphone"), F();
}
}, [k, F, e]);
return R(() => F, [F]), {
isRecording: e,
start: v,
stop: F
};
}
class Oe {
constructor() {
Ne(this, "listeners", /* @__PURE__ */ new Map());
}
addListener(i, a) {
this.listeners.has(i) || this.listeners.set(i, /* @__PURE__ */ new Map()), this.listeners.get(i).set(a.id, a);
}
removeListener(i, a) {
const e = this.listeners.get(i);
e && e.delete(a);
}
async dispatch(i, a) {
const e = this.listeners.get(i);
if (!e) return;
const o = [];
e.forEach((s) => {
try {
const C = s.listener(a);
C instanceof Promise && o.push(C);
} catch (r) {
console.error(`Error in ${String(i)} listener ${s.id}:`, r);
}
}), o.length > 0 && await Promise.all(o);
}
clear(i) {
i ? this.listeners.delete(i) : this.listeners.clear();
}
getListenerCount(i) {
var a;
return ((a = this.listeners.get(i)) == null ? void 0 : a.size) ?? 0;
}
}
const ie = new Oe();
function ar({ children: k, ...i }) {
const {
isRecording: a,
start: e,
stop: o
} = ir({
...i,
onData: (m) => {
var M;
(M = i.onData) == null || M.call(i, m), ie.dispatch("data", m);
},
onError: (m) => {
var M;
(M = i.onError) == null || M.call(i, m), ie.dispatch("error", m);
}
}), s = l(async () => {
try {
a || (await e(), await ie.dispatch("start"));
} catch (m) {
let M = "Failed to start microphone";
throw m instanceof Error ? M += ": " + m.message : M += ": " + String(m), ie.dispatch("error", M), m;
}
}, [a, e]), r = l(() => {
try {
a && (ie.dispatch("stop"), o());
} catch {
ie.dispatch("error", "Failed to stop microphone");
}
}, [a, o]), C = l((m) => {
const M = `kortexa-microphone-data-${Date.now()}`;
return ie.addListener("data", { id: M, listener: m }), M;
}, []), E = l((m) => {
ie.removeListener("data", m);
}, []), T = l((m) => {
const M = `kortexa-microphone-start-${Date.now()}`;
return ie.addListener("start", { id: M, listener: m }), M;
}, []), A = l((m) => {
ie.removeListener("start", m);
}, []), d = l((m) => {
const M = `kortexa-microphone-stop-${Date.now()}`;
return ie.addListener("stop", { id: M, listener: m }), M;
}, []), F = l((m) => {
ie.removeListener("stop", m);
}, []), v = l((m) => {
const M = `kortexa-microphone-error-${Date.now()}`;
return ie.addListener("error", { id: M, listener: m }), M;
}, []), I = l((m) => {
ie.removeListener("error", m);
}, []);
R(() => () => {
ie.clear();
}, []);
const D = Z(() => ({
isRecording: a,
start: s,
stop: r,
addDataListener: C,
removeDataListener: E,
addStartListener: T,
removeStartListener: A,
addStopListener: d,
removeStopListener: F,
addErrorListener: v,
removeErrorListener: I
}), [
a,
s,
r,
C,
E,
T,
A,
d,
F,
v,
I
]);
return /* @__PURE__ */ ce(Me.Provider, { value: D, children: k });
}
function or() {
const k = Le(Me);
if (!k) throw new Error("useMicrophone must be used within MicrophoneProvider");
return k;
}
const we = fe(
void 0
);
we.displayName = "kortexa.ai:camera";
const cr = ({
defaultFacingMode: k = "user",
defaultDeviceId: i,
requestedWidth: a,
requestedHeight: e,
requestedAspectRatio: o,
onStream: s,
onError: r
} = {}) => {
const [C, E] = P(!1), [T, A] = P(null), [d, F] = P(k), [v, I] = P(
[]
), [D, m] = P(
i
), M = l(async () => {
try {
const H = (await navigator.mediaDevices.enumerateDevices()).filter(
(L) => L.kind === "videoinput"
);
if (I(H), H.length > 0 && !D) {
const L = H.find((_) => (_.label.toLowerCase().includes("front") ? "user" : _.label.toLowerCase().includes("back") ? "environment" : null) === d);
m(
(L == null ? void 0 : L.deviceId) || H[0].deviceId
);
}
} catch (p) {
console.error("Error enumerating camera devices:", p), r == null || r("Error enumerating camera devices.");
}
}, [D, d, r]);
R(() => {
M();
}, []), R(() => {
typeof window < "u" && "ontouchstart" in window && k === "user" && !i && F("environment");
}, [k, i]);
const O = l(
async (p) => {
const H = p || D;
if (!H && v.length === 0 && (await M(), v.length === 0 || !H)) {
console.error("No camera devices available or selected."), r == null || r("No camera devices available or selected.");
return;
}
try {
const L = {
deviceId: H ? { exact: H } : void 0,
facingMode: H ? void 0 : d
// Only use facingMode if no specific deviceId
};
a && (L.width = { ideal: a }), e && (L.height = { ideal: e }), o && (L.aspectRatio = {
ideal: o
});
const _ = {
video: Object.keys(L).length > 0 ? L : !0
}, c = await navigator.mediaDevices.getUserMedia(
_
);
A(c), E(!0), s == null || s(c), H && m(H);
} catch (L) {
if (console.error("Error accessing camera:", L), r == null || r(`Error accessing camera: ${L.message}`), H && v.length > 1) {
const _ = v.find(
(c) => c.deviceId !== H
);
_ && (console.log(
`Attempting to start with device: ${_.label || _.deviceId}`
), await O(_.deviceId));
}
}
},
[
D,
d,
s,
r,
v,
M,
a,
e,
o
]
), B = l(() => {
T && (T.getTracks().forEach((p) => p.stop()), A(null), E(!1), s == null || s());
}, [T, s]), W = l(() => {
C ? B() : O();
}, [C, B, O]), q = l(async () => {
if (!C) return;
const p = d === "user" ? "environment" : "user";
F(p), B();
const H = v.find((L) => {
const _ = L.label.toLowerCase();
return p === "user" ? _.includes("front") || !_.includes("back") : p === "environment" ? _.includes("back") : !1;
});
H ? (m(H.deviceId), await O(H.deviceId)) : v.length > 0 ? (m(void 0), await O()) : r == null || r("No alternative camera found for flipping.");
}, [C, d, B, O, v, r]), b = l(
async (p) => {
p === D && C || (B(), m(p), await O(p));
},
[D, C, B, O]
);
return R(() => () => {
T && T.getTracks().forEach((p) => p.stop());
}, [T]), {
stream: T,
isRecording: C,
facingMode: d,
availableDevices: v,
currentDeviceId: D,
start: O,
stop: B,
toggle: W,
flip: q,
setDevice: b,
getDevices: M
};
}, X = new Oe();
function dr({ children: k, ...i }) {
const {
stream: a,
isRecording: e,
facingMode: o,
availableDevices: s,
currentDeviceId: r,
start: C,
stop: E,
flip: T,
setDevice: A,
getDevices: d
} = cr({
...i,
onStream: (c) => {
var g;
(g = i.onStream) == null || g.call(i, c), X.dispatch("streamChanged", c);
},
onError: (c) => {
var g;
(g = i.onError) == null || g.call(i, c), X.dispatch("error", c);
}
}), F = l(async () => {
try {
await C(), X.dispatch("started");
} catch (c) {
let g = "Failed to start camera";
throw c instanceof Error ? g += ": " + c.message : g += ": " + String(c), X.dispatch("error", g), c;
}
}, [C]), v = l(() => {
try {
E(), X.dispatch("stopped");
} catch (c) {
const g = `Failed to stop camera: ${c.message}`;
X.dispatch("error", g);
}
}, [E]), I = l(async () => {
try {
await T();
} catch (c) {
const g = `Failed to flip camera: ${c.message}`;
X.dispatch("error", g);
}
}, [T]), D = l(async (c) => {
try {
await A(c);
} catch (g) {
const V = `Failed to set device: ${g.message}`;
X.dispatch("error", V);
}
}, [A]);
R(() => {
X.dispatch("facingMode", o);
}, [o]);
const m = l((c) => {
const g = `kortexa-camera-stream-${Date.now()}`, V = (U) => {
c(U);
};
return X.addListener("stream", { id: g, listener: V }), g;
}, []), M = l((c) => {
X.removeListener("stream", c);
}, []), O = l((c) => {
const g = `kortexa-camera-started-${Date.now()}`;
return X.addListener("started", { id: g, listener: c }), g;
}, []), B = l((c) => {
X.removeListener("started", c);
}, []), W = l((c) => {
const g = `kortexa-camera-stopped-${Date.now()}`;
return X.addListener("stopped", { id: g, listener: c }), g;
}, []), q = l((c) => {
X.removeListener("stopped", c);
}, []), b = l((c) => {
const g = `kortexa-camera-facingMode-${Date.now()}`, V = (U) => {
U !== void 0 && c(U);
};
return X.addListener("facingMode", { id: g, listener: V }), g;
}, []), p = l((c) => {
X.removeListener("facingMode", c);
}, []), H = l((c) => {
const g = `kortexa-camera-error-${Date.now()}`, V = (U) => {
U !== void 0 && c(U);
};
return X.addListener("error", { id: g, listener: V }), g;
}, []), L = l((c) => {
X.removeListener("error", c);
}, []);
R(() => () => {
X.clear();
}, []);
const _ = Z(() => ({
isRecording: e,
stream: a,
facingMode: o,
availableDevices: s,
currentDeviceId: r,
start: F,
stop: v,
toggle: () => e ? v() : F(),
flip: I,
setDevice: D,
getDevices: d,
addStreamListener: m,
removeStreamListener: M,
addStartListener: O,
removeStartListener: B,
addStopListener: W,
removeStopListener: q,
addFacingModeListener: b,
removeFacingModeListener: p,
addErrorListener: H,
removeErrorListener: L
}), [
e,
a,
o,
s,
r,
F,
v,
I,
D,
d,
m,
M,
O,
B,
W,
q,
b,
p,
H,
L
]);
return /* @__PURE__ */ ce(we.Provider, { value: _, children: k });
}
const ur = () => {
const k = Le(we);
if (!k) throw new Error("useCamera must be used within a CameraProvider");
return k;
}, lr = {
staticImageMode: !1,
maxNumHands: 2,
modelComplexity: 1,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip handedness for selfie view
};
function fr({
options: k,
onInitialLoad: i,
onHandsData: a,
onError: e,
onTrackingStarted: o,
onTrackingStopped: s,
onResults: r,
enableGestures: C = !0,
gestureOptions: E,
gestureModelPath: T,
onGestureResults: A
} = {}) {
const d = f(null), F = f(null), v = f(null), I = f(null), D = f(!1), m = f(!1), [M, O] = P(!1), [B, W] = P(null), [q, b] = P(null), p = Z(
() => ({ ...lr, ...k }),
[k]
), H = f(a);
R(() => {
H.current = a;
}, [a]);
const L = f(e);
R(() => {
L.current = e;
}, [e]);
const _ = f(o);
R(() => {
_.current = o;
}, [o]);
const c = f(s);
R(() => {
c.current = s;
}, [s]);
const g = f(r);
R(() => {
g.current = r;
}, [r]);
const V = f(i);
R(() => {
V.current = i;
}, [i]);
const U = f(A);
R(() => {
U.current = A;
}, [A]);
const z = f(/* @__PURE__ */ new Map()), K = f(
/* @__PURE__ */ new Map()
), N = f(/* @__PURE__ */ new Map()), ae = f(/* @__PURE__ */ new Map()), y = l((w, S) => {
var G, te, ne;
const x = [];
w.landmarks && w.landmarks.forEach((Y, J) => {
var Q;
const ee = {
landmarks: Y,
worldLandmarks: ((Q = w.worldLandmarks) == null ? void 0 : Q[J]) || [],
handedness: w.handedness[J][0],
// Take first handedness
gestures: (S == null ? void 0 : S.gestures[J]) || []
// Add gesture results
};
x.push(ee);
});
const $ = {
detectedHands: x,
imageDimensions: {
width: ((G = v.current) == null ? void 0 : G.videoWidth) || 0,
height: ((te = v.current) == null ? void 0 : te.videoHeight) || 0
}
};
if (W($), (ne = H.current) == null || ne.call(H, $), z.current.forEach(
(Y, J) => Y($)
), S && U.current && x.forEach((Y, J) => {
Y.gestures.length > 0 && U.current(Y.gestures, J);
}), g.current)
try {
g.current(x, v.current || void 0);
} catch (Y) {
const J = Y instanceof Error ? Y.message : String(Y);
b(J), L.current && L.current(J), K.current.forEach(
(ee, Q) => ee(J)
);
}
!m.current && x.length > 0 && (m.current = !0, V.current && V.current());
}, [
W,
b,
H,
g,
V,
L,
U,
z,
K,
m,
v
]);
R(() => ((async () => {
try {
const S = await Ee.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), x = await Qe.createFromOptions(S, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numHands: p.maxNumHands || 2,
minHandDetectionConfidence: p.minDetectionConfidence || 0.5,
minHandPresenceConfidence: 0.5,
minTrackingConfidence: p.minTrackingConfidence || 0.5
});
let $ = null;
C && ($ = await Ze.createFromOptions(S, {
baseOptions: {
modelAssetPath: T || "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numHands: (E == null ? void 0 : E.numHands) || 2,
minHandDetectionConfidence: (E == null ? void 0 : E.minHandDetectionConfidence) || 0.5,
minHandPresenceConfidence: (E == null ? void 0 : E.minHandPresenceConfidence) || 0.5,
minTrackingConfidence: (E == null ? void 0 : E.minTrackingConfidence) || 0.5
})), d.current = x, F.current = $;
} catch (S) {
const x = S instanceof Error ? S.message : String(S);
b(x), L.current && L.current(x);
}
})(), () => {
d.current = null, F.current = null;
}), [p, C, E, T, L]);
const t = l(async () => {
if (!v.current || !d.current)
return;
if (v.current.paused || v.current.ended || v.current.readyState < HTMLMediaElement.HAVE_METADATA) {
D.current && (I.current = requestAnimationFrame(t));
return;
}
const w = performance.now();
try {
const S = d.current.detectForVideo(
v.current,
w
);
let x;
F.current && (x = F.current.recognizeForVideo(
v.current,
w
)), y(S, x);
} catch (S) {
const x = S instanceof Error ? S.message : String(S);
b(x), L.current && L.current(x), K.current.forEach(
($, G) => $(x)
);
}
D.current && (I.current = requestAnimationFrame(t));
}, [
d,
F,
v,
b,
L,
K,
y
]), u = l(
async (w) => {
if (!d.current) {
b("HandLandmarker not initialized."), L.current && L.current("HandLandmarker not initialized."), K.current.forEach(
(S, x) => S("HandLandmarker not initialized.")
);
return;
}
D.current || (v.current = w, m.current = !1, D.current = !0, O(!0), I.current && cancelAnimationFrame(I.current), I.current = requestAnimationFrame(t), _.current && _.current(), N.current.forEach(
(S, x) => S()
), b(null));
},
[
t,
_,
N,
L,
K,
b
]
), j = l(() => {
!D.current && !M || (D.current = !1, O(!1), I.current && (cancelAnimationFrame(I.current), I.current = null), c.current && c.current(), ae.current.forEach(
(w, S) => w()
));
}, [M, c, ae]);
return {
isTracking: M,
handsData: B,
error: q,
startTracking: u,
stopTracking: j,
getHandLandmarker: () => d.current,
getGestureRecognizer: () => F.current,
addHandsDataListener: (w) => {
const S = Date.now().toString() + Math.random().toString();
return z.current.set(S, w), S;
},
removeHandsDataListener: (w) => {
z.current.delete(w);
},
addErrorListener: (w) => {
const S = Date.now().toString() + Math.random().toString();
return K.current.set(S, w), S;
},
removeErrorListener: (w) => {
K.current.delete(w);
},
addStartListener: (w) => {
const S = Date.now().toString() + Math.random().toString();
return N.current.set(S, w), S;
},
removeStartListener: (w) => {
N.current.delete(w);
},
addStopListener: (w) => {
const S = Date.now().toString() + Math.random().toString();
return ae.current.set(S, w), S;
},
removeStopListener: (w) => {
ae.current.delete(w);
}
};
}
const Se = fe(
{}
);
Se.displayName = "kortexa.ai:hands-tracking";
function mr({ children: k, ...i }) {
const a = fr(i);
return /* @__PURE__ */ ce(Se.Provider, { value: a, children: k });
}
const gr = () => {
const k = Le(Se);
if (k === null) throw new Error("useHands must be used within a HandsProvider");
return k;
}, vr = {
staticImageMode: !1,
modelComplexity: 1,
smoothLandmarks: !0,
enableSegmentation: !1,
smoothSegmentation: !0,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip for selfie view
};
function We({
options: k,
onInitialLoad: i,
onBodyData: a,
onError: e,
onTrackingStarted: o,
onTrackingStopped: s,
onResults: r,
enableSegmentation: C = !1,
outputSegmentationMasks: E = !1
} = {}) {
const T = f(null), A = f(null), d = f(null), F = f(!1), v = f(!1), [I, D] = P(!1), [m, M] = P(null), [O, B] = P(null), W = Z(
() => ({ ...vr, ...k, enableSegmentation: C }),
[k, C]
), q = f(a);
R(() => {
q.current = a;
}, [a]);
const b = f(e);
R(() => {
b.current = e;
}, [e]);
const p = f(o);
R(() => {
p.current = o;
}, [o]);
const H = f(s);
R(() => {
H.current = s;
}, [s]);
const L = f(r);
R(() => {
L.current = r;
}, [r]);
const _ = f(i);
R(() => {
_.current = i;
}, [i]);
const c = f(/* @__PURE__ */ new Map()), g = f(
/* @__PURE__ */ new Map()
), V = f(/* @__PURE__ */ new Map()), U = f(/* @__PURE__ */ new Map()), z = l((y) => {
var j, w, S;
const t = [];
y.landmarks && y.landmarks.length > 0 && y.landmarks.forEach((x, $) => {
var te;
const G = {
landmarks: x,
worldLandmarks: ((te = y.worldLandmarks) == null ? void 0 : te[$]) || []
};
E && y.segmentationMasks && (G.segmentationMasks = []), t.push(G);
});
const u = {
detectedBodies: t,
imageDimensions: {
width: ((j = A.current) == null ? void 0 : j.videoWidth) || 0,
height: ((w = A.current) == null ? void 0 : w.videoHeight) || 0
}
};
if (M(u), (S = q.current) == null || S.call(q, u), c.current.forEach(
(x, $) => x(u)
), L.current)
try {
L.current(t, A.current || void 0);
} catch (x) {
const $ = x instanceof Error ? x.message : String(x);
B($), b.current && b.current($), g.current.forEach(
(G, te) => G($)
);
}
!v.current && t.length > 0 && (v.current = !0, _.current && _.current());
}, [
M,
B,
q,
L,
_,
b,
c,
g,
v,
A,
E
]);
R(() => ((async () => {
try {
const t = await Ee.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), u = await er.createFromOptions(t, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/1/pose_landmarker_heavy.task",
delegate: "GPU"
},
runningMode: "VIDEO",
numPoses: 1,
// Single pose detection
minPoseDetectionConfidence: W.minDetectionConfidence || 0.5,
minPosePresenceConfidence: 0.5,
minTrackingConfidence: W.minTrackingConfidence || 0.5,
outputSegmentationMasks: W.enableSegmentation || !1
});
T.current = u;
} catch (t) {
const u = t instanceof Error ? t.message : String(t);
B(u), b.current && b.current(u);
}
})(), () => {
T.current = null;
}), [W, b]);
const K = l(async () => {
if (!A.current || !T.current)
return;
if (A.current.paused || A.current.ended || A.current.readyState < HTMLMediaElement.HAVE_METADATA) {
F.current && (d.current = requestAnimationFrame(K));
return;
}
const y = performance.now();
try {
const t = T.current.detectForVideo(
A.current,
y
);
z(t);
} catch (t) {
const u = t instanceof Error ? t.message : String(t);
B(u), b.current && b.current(u), g.current.forEach(
(j, w) => j(u)
);
}
F.current && (d.current = requestAnimationFrame(K));
}, [
T,
A,
B,
b,
g,
z
]), N = l(
async (y) => {
if (!T.current) {
B("PoseLandmarker not initialized."), b.current && b.current("PoseLandmarker not initialized."), g.current.forEach(
(t, u) => t("PoseLandmarker not initialized.")
);
return;
}
F.current || (A.current = y, v.current = !1, F.current = !0, D(!0), d.current && cancelAnimationFrame(d.current), d.current = requestAnimationFrame(K), p.current && p.current(), V.current.forEach(
(t, u) => t()
), B(null));
},
[
K,
p,
V,
b,
g,
B
]
), ae = l(() => {
!F.current && !I || (F.current = !1, D(!1), d.current && (cancelAnimationFrame(d.current), d.current = null), H.current && H.current(), U.current.forEach(
(y, t) => y()
));
}, [I, H, U]);
return {
isTracking: I,
bodyData: m,
error: O,
startTracking: N,
stopTracking: ae,
getPoseLandmarker: () => T.current,
addBodyDataListener: (y) => {
const t = Date.now().toString() + Math.random().toString();
return c.current.set(t, y), t;
},
removeBodyDataListener: (y) => {
c.current.delete(y);
},
addErrorListener: (y) => {
const t = Date.now().toString() + Math.random().toString();
return g.current.set(t, y), t;
},
removeErrorListener: (y) => {
g.current.delete(y);
},
addStartListener: (y) => {
const t = Date.now().toString() + Math.random().toString();
return V.current.set(t, y), t;
},
removeStartListener: (y) => {
V.current.delete(y);
},
addStopListener: (y) => {
const t = Date.now().toString() + Math.random().toString();
return U.current.set(t, y), t;
},
removeStopListener: (y) => {
U.current.delete(y);
}
};
}
const Ge = fe(
{}
);
Ge.displayName = "kortexa.ai:body-tracking";
function hr({ children: k, ...i }) {
const a = We(i);
return /* @__PURE__ */ ce(Ge.Provider, { value: a, children: k });
}
const pr = {
staticImageMode: !1,
maxNumFaces: 1,
refineLandmarks: !0,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: !0
// Flip for selfie view
};
function qe({
options: k,
onInitialLoad: i,
onFaceData: a,
onError: e,
onTrackingStarted: o,
onTrackingStopped: s,
onResults: r,
outputFaceBlendshapes: C = !0,
outputTransformationMatrix: E = !1,
runningMode: T = "VIDEO"
} = {}) {
const A = f(null), d = f(null), F = f(null), v = f(!1), I = f(!1), [D, m] = P(!1), [M, O] = P(null), [B, W] = P(null), q = Z(
() => ({ ...pr, ...k }),
[k]
), b = f(a);
R(() => {
b.current = a;
}, [a]);
const p = f(e);
R(() => {
p.current = e;
}, [e]);
const H = f(o);
R(() => {
H.current = o;
}, [o]);
const L = f(s);
R(() => {
L.current = s;
}, [s]);
const _ = f(r);
R(() => {
_.current = r;
}, [r]);
const c = f(i);
R(() => {
c.current = i;
}, [i]);
const g = f(/* @__PURE__ */ new Map()), V = f(
/* @__PURE__ */ new Map()
), U = f(/* @__PURE__ */ new Map()), z = f(/* @__PURE__ */ new Map()), K = l((t) => {
var w, S, x;
const u = [];
t.faceLandmarks && t.faceLandmarks.length > 0 && t.faceLandmarks.forEach(($, G) => {
const te = {
landmarks: $
};
if (C && t.faceBlendshapes && t.faceBlendshapes[G] && (te.blendshapes = t.faceBlendshapes[G].categories), E && t.facialTransformationMatrixes && t.facialTransformationMatrixes[G]) {
const ne = t.facialTransformationMatrixes[G];
te.transformationMatrix = ne.data ? Array.from(ne.data) : [];
}
if ($.length > 0) {
let ne = 1, Y = 1, J = 0, ee = 0;
$.forEach((Q) => {
ne = Math.min(ne, Q.x), Y = Math.min(Y, Q.y), J = Math.max(J, Q.x), ee = Math.max(ee, Q.y);
}), te.boundingBox = {
xMin: ne,
yMin: Y,
width: J - ne,
height: ee - Y
};
}
u.push(te);
});
const j = {
detectedFaces: u,
imageDimensions: {
width: ((w = d.current) == null ? void 0 : w.videoWidth) || 0,
height: ((S = d.current) == null ? void 0 : S.videoHeight) || 0
}
};
if (O(j), (x = b.current) == null || x.call(b, j), g.current.forEach(
($, G) => $(j)
), _.current)
try {
_.current(u, d.current || void 0);
} catch ($) {
const G = $ instanceof Error ? $.message : String($);
W(G), p.current && p.current(G), V.current.forEach(
(te, ne) => te(G)
);
}
!I.current && u.length > 0 && (I.current = !0, c.current && c.current());
}, [
O,
W,
b,
_,
c,
p,
g,
V,
I,
d,
C,
E
]);
R(() => ((async () => {
try {
const u = await Ee.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
), j = await rr.createFromOptions(u, {
baseOptions: {
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
delegate: "GPU"
},
runningMode: T,
numFaces: q.maxNumFaces || 1,
minFaceDetectionConfidence: q.minDetectionConfidence || 0.5,
minFacePresenceConfidence: 0.5,
minTrackingConfidence: q.minTrackingConfidence || 0.5,
outputFaceBlendshapes: C,
outputFacialTransformationMatrixes: E
});
A.current = j;
} catch (u) {
const j = u instanceof Error ? u.message : String(u);
W(j), p.current && p.current(j);
}
})(), () => {
A.current = null;
}), [q, T, C, E, p]);
const N = l(async () => {
if (!d.current || !A.current)
return;
if (d.current.paused || d.current.ended || d.current.readyState < HTMLMediaElement.HAVE_METADATA) {
v.current && (F.current = requestAnimationFrame(N));
return;
}
const t = performance.now();
try {
const u = A.current.detectForVideo(
d.current,
t
);
K(u);
} catch (u) {
const j = u instanceof Error ? u.message : String(u);
W(j), p.current && p.current(j), V.current.forEach(
(w, S) => w(j)
);
}
v.current && (F.current = requestAnimationFrame(N));
}, [
A,
d,
W,
p,
V,
K
]), ae = l(
async (t) => {
if (!A.current) {
W("FaceLandmarker not initialized."), p.current && p.current("FaceLandmarker not initialized."), V.current.forEach(
(u, j) => u("FaceLandmarker not initialized.")
);
return;
}
v.current || (d.current = t, I.current = !1, v.current = !0, m(!0), F.current && cancelAnimationFrame(F.current), F.current = requestAnimationFrame(N), H.current && H.current(), U.current.forEach(
(u, j) => u()
), W(null));
},
[
N,
H,
U,
p,
V,
W
]
), y = l(() => {
!v.current && !D || (v.current = !1, m(!1), F.current && (cancelAnimationFrame(F.current), F.current = null), L.current && L.current(), z.current.forEach(
(t, u) => t()
));
}, [D, L, z]);
return {
isTracking: D,
faceData: M,
error: B,
startTracking: ae,
stopTracking: y,
getFaceLandmarker: () => A.current,
getFaceDetector: () => null,
// Not implemented in this version
addFaceDataListener: (t) => {
const u = Date.now().toString() + Math.random().toString();
return g.current.set(u, t), u;
},
removeFaceDataListener: (t) => {
g.current.delete(t);
},
addErrorListener: (t) => {
const u = Date.now().toString() + Math.random().toString();
return V.current.set(u, t), u;
},
removeErrorListener: (t) => {
V.current.delete(t);
},
addStartListener: (t) => {
const u = Date.now().toString() + Math.random().toString();
return U.current.set(u, t), u;
},
removeStartListener: (t) => {
U.current.delete(t);
},
addStopListener: (t) => {
const u = Date.now().toString() + Math.random().toString();
return z.current.set(u, t), u;
},
removeStopListener: (t) => {
z.current.delete(t);
}
};
}
const Ke = fe(
{}
);
Ke.displayName = "kortexa.ai:face-tracking";
function kr({ children: k, ...i }) {
const a = qe(i);
return /* @__PURE__ */ ce(Ke.Provider, { value: a, children: k });
}
const Lr = "proceed";
function Er({
children: k,
startBehavior: i
}) {
const a = or(), e = ur(), o = gr(), s = We(), r = qe(), C = f(null), E = f(null), T = f(null), [A, d] = P(!1), [F, v] = P(!1), [I, D] = P(!1), [m, M] = P(!1), [O, B] = P(!1), [W, q] = P(!1), [b, p] = P(void 0), [H, L] = P(!1), [_, c] = P(void 0), [g, V] = P(void 0), [U, z] = P(void 0), [K, N] = P(void 0), [ae, y] = P(void 0), [t, u] = P(!1), [j, w] = P(!1), [S, x] = P(!1), [$, G] = P(!1), [te, ne] = P(!1), [Y, J] = P(!1);
R(() => {
const n = a.addErrorListener((h) => {
c(h);
});
return () => {
a.removeErrorListener(n);
};
}, [a]), R(() => {
const n = e.addErrorListener((h) => {
V(h);
});
return () => {
e.removeErrorListener(n);
};
}, [e]), R(() => {
if (o) {
const n = o.addErrorListener((h) => {
z(h), u(!1);
});
return () => {
o.removeErrorListener(n);
};
}
return () => {
};
}, [o]), R(() => {
if (s) {
const n = s.addErrorListener((h) => {
N(h), w(!1);
});
return () => {
s.removeErrorListener(n);
};
}
return () => {
};
}, [s]), R(() => {
if (r) {
const n = r.addErrorListener((h) => {
y(h), x(!1);
});
return () => {
r.removeErrorListener(n);
};
}
return () => {
};
}, [r]);
const ee = Z(() => a.isRecording, [a]), Q = Z(() => e.isRecording, [e]), Fe = Z(() => (o == null ? void 0 : o.isTracking) ?? !1, [o]), De = Z(() => (s == null ? void 0 : s.isTracking) ?? !1, [s]), Ce = Z(() => (r == null ? void 0 : r.isTracking) ?? !1, [r]), Te = Z(() => ee && Q, [ee, Q]), Ae = Z(() => {
}, []), He = Z(() => e.stream ?? void 0, [e]), Re = Z(() => e.facingMode, [e]), xe = Z(() => (o == null ? void 0 : o.handsData) ?? void 0, [o]), Ie = Z(() => (s == null ? void 0 : s.bodyData) ?? void 0, [s]), be = Z(() => (r == null ? void 0 : r.faceData) ?? void 0, [r]), _e = l((n) => {
C.current = n, n ? G(!0) : (d(!1), G(!1));
}, []), Pe = l((n) => {
E.current = n, n ? ne(!0) : (v(!1), ne(!1));
}, []), Ve = l((n) => {
T.current = n, n ? J(!0) : (D(!1), J(!1));
}, []), de = l(async () => {
if (!o || !C.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const h = "Pre-conditions not met for starting hand tracking (hands, videoElement, camera state).";
return z(h), d(!0), Promise.reject(new Error(h));
}
if (o.isTracking)
return Promise.resolve();
const n = C.current;
d(!0), z(void 0);
try {
n.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((h, re) => {
const se = () => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), h();
}, oe = (ke) => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), re(new Error("Video element error during metadata load for hand tracking."));
};
n.addEventListener("loadedmetadata", se), n.addEventListener("error", oe);
}), await o.startTracking(n);
} catch (h) {
const re = h instanceof Error ? h.message : String(h);
throw z(re), h;
}
}, [o, e, z, d]), ue = l(async () => {
if (!s || !E.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const h = "Pre-conditions not met for starting body tracking (body, videoElement, camera state).";
return N(h), v(!0), Promise.reject(new Error(h));
}
if (s.isTracking)
return Promise.resolve();
const n = E.current;
v(!0), N(void 0);
try {
n.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((h, re) => {
const se = () => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), h();
}, oe = (ke) => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), re(new Error("Video element error during metadata load for body tracking."));
};
n.addEventListener("loadedmetadata", se), n.addEventListener("error", oe);
}), await s.startTracking(n);
} catch (h) {
const re = h instanceof Error ? h.message : String(h);
throw N(re), h;
}
}, [s, e, N, v]), le = l(async () => {
if (!r || !T.current || !(e != null && e.isRecording) || !(e != null && e.stream)) {
const h = "Pre-conditions not met for starting face tracking (face, videoElement, camera state).";
return y(h), D(!0), Promise.reject(new Error(h));
}
if (r.isTracking)
return Promise.resolve();
const n = T.current;
D(!0), y(void 0);
try {
n.readyState < HTMLMediaElement.HAVE_METADATA && await new Promise((h, re) => {
const se = () => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), h();
}, oe = (ke) => {
n.removeEventListener("loadedmetadata", se), n.removeEventListener("error", oe), re(new Error("Video element error during metadata load for face tracking."));
};
n.addEventListener("loadedmetadata", se), n.addEventListener("error", oe);
}), await r.startTracking(n);
} catch (h) {
const re = h instanceof Error ? h.message : String(h);
throw y(re), h;
}
}, [r, e, y, D]), me = l(async () => {
if (H)
return;
L(!0), p(void 0), d(!1), M(!1), c(void 0), V(void 0), z(void 0);
let n = ee, h = Q;
if (a && !ee)
try {
await a.start(), n = !0;
} catch {
n = !1;
}
if (e && !Q)
try {
await e.start(), h = !0;
} catch {
h = !1;
}
if (i === "halt") {
if (!n || !h) {
const se = _ || (n ? "" : "Failed to start"), oe = g || (h ? "" : "Failed to start"), ke = `Media start halted: Mic ${n ? "OK" : `FAIL (${se})`}. Cam ${h ? "OK" : `FAIL (${oe})`}.`;
p(ke), n && a && a.stop(), h && e && e.stop(), L(!1);
return;
}
} else {
const re = [];
if (n || re.push(`Mic: ${_ || "Unknown"}`), h || re.push(`Cam: ${g || "Unknown"}`), re.length > 0) {
const se = `Failed to start: ${re.join("; ")}. Others proceeded if successful.`;
p(se);
}
}
if (h && (e != null && e.isRecording) && e.stream) {
if (o && !o.isTracking && C.current && !A)
try {
await de();
} catch {
i === "halt" && p(`Failed to start hand tracking. Halting media start. Error: ${U || "Unknown hands error"}`);
}
if (s && !s.isTracking && E.current && !F)
try {
await ue();
} catch {
i === "halt" && p(`Failed to start body tracking. Halting media start. Error: ${K || "Unknown body error"}`);
}
if (r && !r.isTracking && T.current && !I)
try {
await le();
} catch {
i === "halt" && p(`Failed to start face tracking. Halting media start. Error: ${ae || "Unknown face error"}`);
}
}
L(!1);
}, [
a,
e,
o,
s,
r,
i,
ee,
Q,
H,
C,
E,
T,
A,
F,
I,
de,
ue,
le,
_,
g,
U,
K,
ae,
c,
V,
z,
N,
y,
p,
L,
d,
v,
D,
M,
B,
q
]);
R(() => {
e != null && e.isRecording && e.stream && o && !o.isTracking && C.current && !A && !m && de().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, o, C, A, m, de]), R(() => {
e != null && e.isRecording && e.stream && s && !s.isTracking && E.current && !F && !O && ue().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, s, E, F, O, ue]), R(() => {
e != null && e.isRecording && e.stream && r && !r.isTracking && T.current && !I && !W && le().catch(() => {
});
}, [e == null ? void 0 : e.isRecording, e == null ? void 0 : e.stream, r, T, I, W, le]), R(() => {
e != null && e.isRecording || (d(!1), v(!1), D(!1));
}, [e == null ? void 0 : e.isRecording, d, v, D]);
const ge = l(async () => {
if (o && o.isTracking)
try {
await o.stopTracking(), M(!0);
} catch (n) {
const h = n instanceof Error ? n.message : String(n);
z(h);
}
d(!1), u(!1);
}, [o, z, d]), ve = l(async () => {
if (s && s.isTracking)
try {
await s.stopTracking(), B(!0);
} catch (n) {
const h = n instanceof Error ? n.message : String(n);
N(h);
}
v(!1), w(!1);
}, [s, N, v]), he = l(async () => {
if (r && r.isTracking)
try {
await r.stopTracking(), q(!0);
} catch (n) {
const h = n instanceof Error ? n.message : String(n);
y(h);
}
D(!1), x(!1);
}, [r, y, D]);
R(() => {
e && !e.isRecording && (o != null && o.isTracking && ge(), s != null && s.isTracking && ve(), r != null && r.isTracking && he(), d(!1), v(!1), D(!1));
}, [e, e == null ? void 0 : e.isRecording, o, s, r, ge, ve, he]);
const pe = l(() => {
a != null && a.isRecording && a.stop(), e != null && e.isRecording && e.stop(), p(void 0), c(void 0), V(void 0), z(void 0), N(void 0), y(void 0), L(!1);
}, [a, e, p, c, V, z, N, y, L]), ze = l(async () => {
ee && Q ? pe() : await me();
}, [ee, Q, me, pe]), $e = l(async () => {
if (!(t || o != null && o.isTracking)) {
if (M(!1), u(!0), z(void 0), !o) {
z("Hand tracking service not available."), u(!1);
return;
}
if (!(e != null && e.isRecording)) {
z("Camera is not active. Cannot start hand tracking."), u(!1);
return;
}
if (!C.current) {
z("Video element not set for hand tracking."), u(!1), G(!1);
return;
}
try {
await de();
} catch {
} finally {
u(!1);
}
}
}, [o, e == null ? void 0 : e.isRecording, C, t, de, u, z]), Be = l(async () => {
if (!(j || s != null && s.isTracking)) {
if (B(!1), w(!0), N(void 0), !s) {
N("Body tracking service not available."), w(!1);
return;
}
if (!(e != null && e.isRecording)) {
N("Camera is not active. Cannot start body tracking."), w(!1);
return;
}
if (!E.current) {
N("Video element not set for body tracking."), w(!1), ne(!1);
return;
}
try {
await ue();
} catch {
} finally {
w(!1);
}
}
}, [s, e == null ? void 0 : e.isRecording, E, j, ue, w, N]), Ue = l(async () => {
if (!(S || r != null && r.isTracking)) {
if (q(!1), x(!0), y(void 0), !r) {
y("Face tracking service not available."), x(!1);
return;
}
if (!(e != null && e.isRecording)) {
y("Camera is not active. Cannot start face tracking."), x(!1);
return;
}
if (!T.current) {
y("Video element not set for face tracking."), x(!1), J(!1);
return;
}
try {
await le();
} catch {
} finally {
x(!1);
}
}
}, [r, e == null ? void 0 : e.isRecording, T, S, le, x, y]), Xe = Z(() => ({
isAudioActive: ee,
isVideoActive: Q,
isHandTrackingActive: Fe,
isBodyTrackingActive: De,
isFaceTrackingActive: Ce,
isMediaActive: Te,
audioStream: Ae,
videoStream: He,
videoFacingMode: Re,
currentHandsData: xe,
currentBodyData: Ie,
currentFaceData: be,
audioError: _,
videoError: g,
handsError: U,
bodyError: K,
faceError: ae,
mediaError: b,
startMedia: me,
stopMedia: pe,
toggleMedia: ze,
cam: e,
mic: a,
hands: o,
body: s,
face: r,
setVideoElementForHands: _e,
setVideoElementForBody: Pe,
setVideoElementForFace: Ve,
startHands: $e,
stopHands: ge,
startBody: Be,
stopBody: ve,
startFace: Ue,
stopFace: he,
isStartingMedia: H,
isStartingHands: t,
isStartingBody: j,
isStartingFace: S,
isVideoElementForHandsSet: $,
isVideoElementForBodySet: te,
isVideoElementForFaceSet: Y
}), [
ee,
Q,
Fe,
De,
Ce,
Te,
Ae,
He,
Re,
xe,
Ie,
be,
_,
g,
U,
K,
ae,
b,
me,
pe,
ze,
e,
a,
o,
s,
r,
_e,
Pe,
Ve,
$e,
ge,
Be,
ve,
Ue,
he,
H,
t,
j,
S,
$,
te,
Y
]);
return /* @__PURE__ */ ce(ye.Provider, { value: Xe, children: k });
}
function Dr({
children: k,
microphoneProps: i,
cameraProps: a,
handsProps: e,
bodyProps: o,
faceProps: s,
startBehavior: r = Lr
}) {
return /* @__PURE__ */ ce(ar, { ...i, children: /* @__PURE__ */ ce(dr, { ...a, children: /* @__PURE__ */ ce(mr, { ...e, children: /* @__PURE__ */ ce(hr, { ...o, children: /* @__PURE__ */ ce(kr, { ...s, children: /* @__PURE__ */ ce(
Er,
{
startBehavior: r,
children: k
}
) }) }) }) }) });
}
export {
hr as BodyProvider,
dr as CameraProvider,
Dr as CompositeMediaProvider,
kr as FaceProvider,
mr as HandsProvider,
ar as MicrophoneProvider,
We as useBodyControl,
ur as useCamera,
cr as useCameraDevice,
Fr as useCompositeMedia,
qe as useFaceControl,
gr as useHands,
fr as useHandsControl,
or as useMicrophone,
ir as useMicrophoneDevice
};
//# sourceMappingURL=react-multimodal.js.map