@nuralogix.ai/anura-web-core-sdk
Version:
Anura Web Core SDK
691 lines (683 loc) • 23.9 kB
JavaScript
var CameraPermissionError = /* @__PURE__ */ ((CameraPermissionError2) => {
CameraPermissionError2["PERMISSION_DENIED"] = "permission_denied";
CameraPermissionError2["NO_DEVICES"] = "no_devices";
CameraPermissionError2["UNKNOWN"] = "unknown";
return CameraPermissionError2;
})(CameraPermissionError || {});
const CameraControllerEvents = {
"CAMERA_STATUS": "cameraStatus",
"SELECTED_DEVICE_CHANGED": "selectedDeviceChanged",
"DEVICE_PIXEL_RATIO_CHANGED": "devicePixelRatioChanged",
"MEDIA_DEVICE_LIST_CHANGED": "mediaDeviceListChanged"
};
const roundValue = (value) => {
let result = value.toFixed(2);
if (result === "0") result = value.toPrecision(1);
return Number(result);
};
const getNewDeviceId = (deviceId, mediaDevices) => {
let newDeviceId = "";
if (mediaDevices.length > 0) {
if (!mediaDevices.find((mediaDevice) => mediaDevice.device.deviceId === deviceId) || deviceId === "") {
newDeviceId = mediaDevices[0].device.deviceId;
}
if (mediaDevices.find((mediaDevice) => mediaDevice.device.deviceId === deviceId)) {
newDeviceId = deviceId;
}
}
return newDeviceId;
};
let removeListener = null;
class CameraController extends EventTarget {
mediaDevices = [];
selectedDeviceId = "";
cameraStream = null;
cameraWidth = 0;
cameraHeight = 0;
cameraFrameRate = 0;
settingsInputIds = ["iso", "exposureTime", "focusDistance", "colorTemperature", "zoom"];
standardSettingsIds = ["contrast", "saturation", "sharpness", "brightness", "exposureCompensation"];
videoTrackCapabilities = {};
#deviceChangeInProgress = false;
constructor() {
super();
navigator.mediaDevices.ondevicechange = async () => {
if (!this.#deviceChangeInProgress) {
this.#deviceChangeInProgress = true;
await this.list();
this.#deviceChangeInProgress = false;
}
};
this.updatePixelRatio = this.updatePixelRatio.bind(this);
this.updatePixelRatio();
}
static init() {
return new this();
}
updatePixelRatio() {
if (removeListener != null) {
removeListener();
}
const mqString = `(resolution: ${window.devicePixelRatio}dppx)`;
const media = matchMedia(mqString);
media.addEventListener("change", this.updatePixelRatio);
removeListener = () => {
media.removeEventListener("change", this.updatePixelRatio);
};
setTimeout(() => {
this.#dispatch(this.#getDeviceAspectRatioChangedEvent());
}, 1e3);
this.#dispatch(this.#getDeviceAspectRatioChangedEvent());
}
/** Enumerates the list of video input devices
*
* The label field for each MediaDevice will be empty if the permission
* is not equal to `granted`
*/
async enumerate() {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const mediaDevices = devices.filter((device) => device.kind === "videoinput");
const cameras = mediaDevices.map((mediaDevice) => {
const capabilities = typeof mediaDevice.getCapabilities === "function" ? mediaDevice.getCapabilities() : {};
const deviceInfo = {
device: {
label: mediaDevice.label,
deviceId: mediaDevice.deviceId,
kind: "videoinput",
groupId: mediaDevice.groupId
},
capabilities
};
return deviceInfo;
});
return cameras;
} catch (error) {
console.error("Error enumerating devices:", error);
return [];
}
}
/**
* Requests camera permissions from the user.
* @returns Promise<RequestPermissionResult>
*/
async requestPermission() {
try {
this.cameraStream = await navigator.mediaDevices.getUserMedia({ video: true });
return { success: true };
} catch (error) {
if (error instanceof DOMException && error.name === "NotAllowedError") {
console.error("Camera permission denied.");
return { success: false, error: "permission_denied" /* PERMISSION_DENIED */, message: "Camera permission denied by user" };
} else if (error instanceof DOMException && error.name === "NotFoundError") {
console.error("No camera devices found.");
return { success: false, error: "no_devices" /* NO_DEVICES */, message: "No camera devices found on this system" };
} else {
console.error("Error requesting camera permission:", error);
return { success: false, error: "unknown" /* UNKNOWN */, message: error instanceof Error ? error.message : "Unknown error occurred" };
}
} finally {
this.stop();
}
}
/** Populates the list of `mediaDevices` and sets `selectedDeviceId`
*
* If camera permission granted it will populate `mediaDevices` with the list of available cameras and
* sets `selectedDeviceId` to that of the first available camera.
*
* If camera permission is blocked, it will set `mediaDevices` to [] and `selectedDeviceId` to an empty string
*
*/
async list() {
this.mediaDevices = await this.enumerate();
this.selectedDeviceId = getNewDeviceId(this.selectedDeviceId, this.mediaDevices);
this.#dispatch(this.#getMediaDeviceListChangedEvent());
this.#dispatch(this.#getSelectedDeviceChangedEvent());
}
/** Change selected media device Id
*
*/
setDeviceId(deviceId) {
if (this.mediaDevices.length) {
if (this.mediaDevices.find((d) => d.device.deviceId === deviceId)) {
this.selectedDeviceId = deviceId;
this.#dispatch(this.#getSelectedDeviceChangedEvent());
}
}
}
#setVideoTrackCapabilities() {
if (this.cameraStream) {
const [videoTrack] = this.cameraStream.getVideoTracks();
const videoTrackSettings = videoTrack.getSettings();
const { width, height, frameRate } = videoTrackSettings;
const capabilities = videoTrack.getCapabilities();
this.cameraWidth = width || 0;
this.cameraHeight = height || 0;
this.cameraFrameRate = frameRate || 0;
this.videoTrackCapabilities = Object.fromEntries(
[...this.settingsInputIds, ...this.standardSettingsIds].map(
(id) => {
const mediaSettingsRange = capabilities[id];
return [id, {
isSupported: mediaSettingsRange ? true : false,
min: mediaSettingsRange && mediaSettingsRange.min ? roundValue(mediaSettingsRange.min) : 0,
max: mediaSettingsRange && mediaSettingsRange.max ? roundValue(mediaSettingsRange.max) : 0,
step: mediaSettingsRange && mediaSettingsRange.step ? roundValue(mediaSettingsRange.step) : 0,
value: mediaSettingsRange && videoTrackSettings[id] ? roundValue(videoTrackSettings[id]) : 0
}];
}
)
);
}
}
async #startCameraWithFallbackResolution(facingMode) {
const fallbackConstraints = {
video: {
...facingMode ? { facingMode: { ideal: facingMode } } : { deviceId: { exact: this.selectedDeviceId } },
width: 640,
height: 360
}
};
this.cameraStream = await navigator.mediaDevices.getUserMedia(fallbackConstraints);
this.#updateSelectedDeviceIdFromStream();
this.#setVideoTrackCapabilities();
}
/** When the camera is opened by facing mode the user agent picks the device, so
* `selectedDeviceId` can be stale. Align it with the device that actually opened.
*/
#updateSelectedDeviceIdFromStream() {
if (!this.cameraStream) {
return;
}
const [track] = this.cameraStream.getVideoTracks();
const { deviceId } = track.getSettings();
if (deviceId && deviceId !== this.selectedDeviceId) {
this.selectedDeviceId = deviceId;
this.#dispatch(this.#getSelectedDeviceChangedEvent());
}
}
/**
* Starts the camera.
*
* Camera selection is performed using one of the following methods:
* - If `facingMode` is provided, the browser selects a camera matching the
* requested facing direction (`user` or `environment`). When multiple cameras
* satisfy the requested facing mode, the browser decides which camera to open.
* - If `facingMode` is not provided, the currently selected camera
* (`selectedDeviceId`) is opened.
*
* After a camera is successfully opened, `selectedDeviceId` is updated to the
* deviceId of the active camera when available. This allows a camera selected
* via `facingMode` to be reopened later using its exact `deviceId`.
*
* @param frameWidth Desired frame width. Must use a 16:9 aspect ratio.
* @param frameHeight Desired frame height. Must use a 16:9 aspect ratio.
* @param facingMode Optional camera facing direction. When provided, camera
* selection is performed by facing mode instead of `selectedDeviceId`.
* @returns `true` if the camera was opened successfully; otherwise `false`.
*/
async start(frameWidth, frameHeight, facingMode) {
this.stop();
const supports = navigator.mediaDevices.getSupportedConstraints();
if (!("facingMode" in supports)) {
throw new DOMException("Constraint 'facingMode' is not supported", "OverconstrainedError");
}
if (frameWidth / frameHeight !== 16 / 9) {
throw new Error("Resolution must have a 16:9 aspect ratio.");
}
const constraints = {
video: {
...facingMode ? { facingMode: { ideal: facingMode } } : { deviceId: { exact: this.selectedDeviceId } },
..."aspectRatio" in supports ? { aspectRatio: { exact: 16 / 9 } } : {},
frameRate: { ideal: 30, min: 15, max: 30 },
width: { ideal: frameWidth },
height: { ideal: frameHeight }
}
};
try {
this.cameraStream = await navigator.mediaDevices.getUserMedia(constraints);
this.#updateSelectedDeviceIdFromStream();
this.#setVideoTrackCapabilities();
this.#dispatch(this.#getCameraStatusEvent(true));
return true;
} catch (error) {
if (error.name != "OverconstrainedError") {
console.log("Error opening camera", error);
} else {
console.log(`This camera cannot produce the requested ${error.constraint}.`);
}
try {
console.log(`Attempt to open the camera with fallback resolution: 640 x 360 pixels`);
await this.#startCameraWithFallbackResolution(facingMode);
this.#dispatch(this.#getCameraStatusEvent(true));
return true;
} catch (fallbackError) {
console.error("Fallback resolution also failed.", fallbackError);
return false;
}
}
}
/** Return VideoTrackCapabilities
*
*/
getVideoTrackCapabilities() {
return this.videoTrackCapabilities;
}
/** Stop camera
*
*/
stop() {
if (this.cameraStream) {
this.cameraStream.getTracks().forEach((track) => track.stop());
this.cameraStream = null;
this.#dispatch(this.#getCameraStatusEvent(false));
this.cameraWidth = 0;
this.cameraHeight = 0;
this.cameraFrameRate = 0;
}
}
#getCameraStatusEvent(isOpen) {
const event = new CustomEvent(CameraControllerEvents.CAMERA_STATUS, {
detail: {
isOpen,
capabilities: this.videoTrackCapabilities
}
});
return event;
}
#getSelectedDeviceChangedEvent() {
const event = new CustomEvent(CameraControllerEvents.SELECTED_DEVICE_CHANGED, {
detail: {
deviceId: this.selectedDeviceId
}
});
return event;
}
#getMediaDeviceListChangedEvent() {
const event = new CustomEvent(CameraControllerEvents.MEDIA_DEVICE_LIST_CHANGED, {
detail: {
mediaDevices: this.mediaDevices
}
});
return event;
}
#getDeviceAspectRatioChangedEvent() {
const event = new CustomEvent(CameraControllerEvents.DEVICE_PIXEL_RATIO_CHANGED, {
detail: {
devicePixelRatio: window.devicePixelRatio
}
});
return event;
}
#dispatch(event) {
this.dispatchEvent(event);
}
}
class VideoController {
mimeCodec;
videoElement = document.createElement("video");
videoSource = document.createElement("source");
canvas = document.createElement("canvas");
mediaStream = null;
bytesDownloaded = 0;
ctx;
static init(settings) {
return new this(settings);
}
constructor(settings) {
this.mimeCodec = settings.mimeCodec;
this.videoLoadedCallback = settings.videoLoadedCallback;
this.videoEndedCallback = settings.videoEndedCallback;
this.videoFrameCallback = this.videoFrameCallback.bind(this);
this.ctx = this.canvas.getContext("2d");
}
// getBytesCallback(bytes: number): void {}
async videoLoadedCallback() {
}
videoEndedCallback() {
}
// getMediaSource() {
// // https://bitmovin.com/managed-media-source
// // New Managed Media Source in Safari 17
// if (window.ManagedMediaSource) {
// console.log('Using ManagedMediaSource');
// return new window.ManagedMediaSource();
// }
// if (window.MediaSource) {
// console.log('Using MediaSource');
// return new window.MediaSource();
// }
// return null;
// }
videoFrameCallback(now, metadata) {
if (this.ctx) this.ctx.drawImage(this.videoElement, 0, 0, this.canvas.width, this.canvas.height);
this.videoElement.requestVideoFrameCallback(this.videoFrameCallback);
}
captureFromCanvas() {
const { videoWidth, videoHeight } = this.videoElement;
this.canvas.width = videoWidth;
this.canvas.height = videoHeight;
this.videoElement.addEventListener("play", () => {
this.videoElement.requestVideoFrameCallback(this.videoFrameCallback);
});
this.videoElement.addEventListener("playing", () => {
this.mediaStream = this.canvas.captureStream();
});
this.videoElement.addEventListener("ended", () => {
if (this.mediaStream) {
this.mediaStream.getTracks().forEach((track) => track.stop());
this.videoElement.pause();
this.videoEndedCallback();
}
});
}
captureFromVideoElement() {
if ("captureStream" in HTMLVideoElement.prototype) {
this.mediaStream = this.videoElement.captureStream();
}
}
setMediaStream() {
if ("captureStream" in HTMLVideoElement.prototype) {
this.captureFromVideoElement();
this.videoElement.addEventListener("timeupdate", () => {
const { currentTime, duration } = this.videoElement;
if (duration === currentTime && this.mediaStream) {
this.mediaStream.getTracks().forEach((track) => track.stop());
this.videoElement.pause();
this.videoEndedCallback();
}
});
} else {
console.log("captureFromVideoElement not supported");
this.captureFromCanvas();
}
}
// async getBuffer(url: string) {
// this.bytesDownloaded = 0;
// let videoBuffer = new ArrayBuffer(0);
// const response = await fetch(url);
// const { body } = response;
// if (body) {
// const reader = body.getReader();
// while (true) {
// const {value, done} = await reader.read();
// if (done) break;
// this.bytesDownloaded += value.byteLength;
// this.getBytesCallback(bytesToMegaBytes(this.bytesDownloaded));
// videoBuffer = await new Blob([ videoBuffer, value ]).arrayBuffer();
// }
// return videoBuffer;
// }
// }
async init(url) {
if (typeof url === "undefined" || typeof url !== "string") return;
this.videoElement = document.createElement("video");
this.videoSource = document.createElement("source");
this.videoSource.type = this.mimeCodec;
this.videoSource.src = url;
this.videoElement.appendChild(this.videoSource);
this.videoElement.playsInline = true;
this.videoElement.onloadedmetadata = () => {
this.setMediaStream();
this.videoLoadedCallback();
};
}
}
let interval;
class ImageSequenceController extends EventTarget {
canvas = document.createElement("canvas");
ctx;
imageBitmaps = [];
canDrawImage = true;
settings = {
url: "",
imageName: {
prefix: "",
numberOfDigits: 4,
extension: ""
}
};
static init(settings, videoEndedCallback) {
return new this(settings, videoEndedCallback);
}
constructor(settings, videoEndedCallback) {
super();
this.settings = settings;
this.videoEndedCallback = videoEndedCallback;
this.ctx = this.canvas.getContext("2d");
}
videoEndedCallback() {
}
getImageName(sequence) {
const { url, imageName } = this.settings;
const { prefix, numberOfDigits, extension } = imageName;
return `${url}${prefix}${sequence.toString().padStart(numberOfDigits, "0")}.${extension}`;
}
async getImageBitmapFromUrl(url) {
const response = await fetch(url);
const fileBlob = await response.blob();
const imageBitmap = await createImageBitmap(fileBlob);
return imageBitmap;
}
async getImageBitmapFromArrayOfUrls(urls) {
const imageBitmaps = await Promise.all(urls.map(async (url) => {
const imageBitmap = await this.getImageBitmapFromUrl(url);
return imageBitmap;
}));
this.setImageBitmaps(imageBitmaps);
}
setImageBitmaps(imageBitmaps) {
this.imageBitmaps = imageBitmaps;
}
drawImageBitmapOnCanvas(imageBitmap) {
if (this.ctx) {
const { width, height } = imageBitmap;
if (this.canvas.width !== width) this.canvas.width = width;
if (this.canvas.height !== height) this.canvas.height = height;
this.ctx.drawImage(imageBitmap, 0, 0);
imageBitmap.close();
}
}
getMediaStreamReadyEvent(isReady) {
const event = new CustomEvent("imageSequenceMediaStreamReady" /* MEDIASTREAM_READY */, {
detail: {
isReady
}
});
return event;
}
drawImagesOnCanvas() {
const { length } = this.imageBitmaps;
let frameCounter = 0;
interval = setInterval(
() => {
this.drawImageBitmapOnCanvas(this.imageBitmaps[frameCounter]);
if (frameCounter === 0) {
this.dispatchEvent(this.getMediaStreamReadyEvent(true));
}
if (frameCounter === length - 1 || !this.canDrawImage) {
clearInterval(interval);
this.videoEndedCallback();
}
frameCounter += 1;
},
33
);
}
}
let brotliDecode = (bytes) => {
return new Int8Array(bytes);
};
class AssetDownloader extends EventTarget {
canDecompress = false;
static init() {
return new this();
}
/* Dispatches a custom event */
dispatch(eventType, payload) {
this.dispatchEvent(new CustomEvent(eventType, { detail: payload }));
}
// BytesDownloaded
#dispatch(event) {
this.dispatchEvent(event);
}
getBytesDownloadedEvent(bytes, uncompressedSize, url, done) {
const event = new CustomEvent("bytesDownloaded" /* BYTES_DOWNLOADED */, {
detail: {
bytes,
uncompressedSize,
url,
done
}
});
return event;
}
getBytesDownloadErrorEvent(url, error) {
const event = new CustomEvent("downloadedError" /* DOWNLOAD_ERROR */, {
detail: {
url,
error
}
});
return event;
}
/** Decompresses a Brotli compressed/based 64 encoded string and returns an ArrayBuffer */
decompressBrotli(compressedBuffer) {
const binaryString = atob(compressedBuffer);
const byteArray = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
byteArray[i] = binaryString.charCodeAt(i);
}
const int8Array = new Int8Array(byteArray.length);
for (let i = 0; i < byteArray.length; i++) {
int8Array[i] = byteArray[i] >= 128 ? byteArray[i] - 256 : byteArray[i];
}
const decompressed = brotliDecode(int8Array);
return decompressed.buffer;
}
/** Returns either an ArrayBuffer or undefined */
async fetchAsset(assetSize, path, file, decompress, compressionType) {
const url = path + file;
const uncompressedSize = assetSize.find((asset) => asset.file === file).uncompressedSize;
const dispatch = (bytes, uncompressedSize2, url2, done) => this.#dispatch(this.getBytesDownloadedEvent(bytes, uncompressedSize2, url2, done));
try {
const response = await fetch(url);
const reader = response.body?.getReader();
if (!reader) {
throw new Error("Failed to get reader from response body.");
}
let bytes = 0;
const stream = new ReadableStream({
async start(controller) {
await pump();
async function pump() {
const { done, value } = await reader.read();
if (done) {
controller.close();
dispatch(bytes, uncompressedSize, url, true);
return;
}
if (value) {
controller.enqueue(value);
bytes += value.length;
dispatch(bytes, uncompressedSize, url, false);
}
await pump();
}
}
});
const newStream = new Response(stream);
if (decompress) {
const json = await newStream.json();
const { base64EncodedValue } = json;
if (compressionType === "gzip") {
await this.#setBrotliDecode(base64EncodedValue);
return void 0;
}
const arrayBuffer = this.decompressBrotli(base64EncodedValue);
return arrayBuffer;
} else {
const arrayBuffer = await newStream.arrayBuffer();
return arrayBuffer;
}
} catch (e) {
this.#dispatch(this.getBytesDownloadErrorEvent(url, e));
}
}
async #setBrotliDecode(base64EncodedValue) {
const compressedData = Uint8Array.from(atob(base64EncodedValue), (c) => c.charCodeAt(0));
const compressedStream = new Blob([compressedData]).stream();
const decompressedStream = compressedStream.pipeThrough(new DecompressionStream("gzip"));
const decompressedArrayBuffer = await new Response(decompressedStream).arrayBuffer();
const text = new TextDecoder().decode(new Uint8Array(decompressedArrayBuffer));
const blob = new Blob([text], { type: "application/javascript" });
const blobUrl = URL.createObjectURL(blob);
const module = await import(blobUrl);
brotliDecode = module.BrotliDecode;
this.canDecompress = true;
URL.revokeObjectURL(blobUrl);
}
}
const helpers = {
/**
* Example usage:
*
* ```js
* const camera = CameraController.init();
* ```
*
*/
CameraController,
/**
* Example usage:
*
* ```js
* const video = VideoController.init({
* mimeCodec: 'video/mp4',
* getBytesCallback: () => {},
* videoLoadedCallback: async () => {}
*});
* ```
*
*/
VideoController,
/**
* Example usage:
*
* ```js
* const imageSequence = ImageSequenceController.init({
url: 'http://localhost/frames/',
imageName: {
prefix: 'img',
numberOfDigits: 4,
extension: 'jpg',
}
});
* ```
*
*/
ImageSequenceController,
/**
* Example usage:
*
* ```js
* const assetDownloader = AssetDownloader.init();
* const onBytesDownloaded = (e: CustomEvent) => {
const { bytes, uncompressedSize, url, done } = e.detail;
console.log(bytes, uncompressedSize, url, done);
};
assetDownloader.addEventListener('bytesDownloaded', onBytesDownloaded as EventListener);
const onDownloadedError = (e: CustomEvent) => {
const { error, url } = e.detail;
console.log(error, url);
};
assetDownloader.addEventListener('downloadedError', onDownloadedError as EventListener);
*
* const isSimdSupported = await assetDownloader.isSimdSupported();
* const arrayBuffer = await assetDownloader.fetchAsset(url, true, 'brotli');
* ```
*
*/
AssetDownloader
};
export { CameraControllerEvents, CameraPermissionError, helpers as default };