@nuralogix.ai/anura-web-core-sdk
Version:
Anura Web Core SDK
622 lines (604 loc) • 20.8 kB
TypeScript
/**
* The objectFit sets how the content of an element should be resized to fit its container.
*
* `contain`:
*
* The content is scaled to maintain its aspect ratio while fitting within the element's
* content box. The entire object is made to fill the box, while preserving its aspect ratio,
* so the object will be "letterboxed" or "pillarboxed" if its aspect ratio does not match the
* aspect ratio of the box.
*
* `cover`:
*
* The content is sized to maintain its aspect ratio while filling the element's entire
* content box. If the object's aspect ratio does not match the aspect ratio of its box, then
* the object will be clipped to fit.
*
* `none`:
*
* The content is not resized.
* @enum
*/
declare const objectFit: {
readonly CONTAIN: "contain";
readonly COVER: "cover";
readonly NONE: "none";
};
type ObjectFit = typeof objectFit[keyof typeof objectFit];
interface PipelineStats {
cameraFps: number;
faceTrackerFps: number;
faceCount: number;
detectionAvgMs: number;
/** Pure ONNX session.run() time inside worker (excludes message overhead) */
detectionInferenceMs: number;
detectionSkipN: number;
landmarkAvgMs: number;
/** Pure ONNX session.run() time inside worker (excludes message overhead) */
landmarkInferenceMs: number;
landmarkSkipPct: number;
costPerFrameMs: number;
}
interface DfxFace {
detected: boolean;
id: string;
poseValid: boolean;
posePoints: PosePoints,
faceRect: DfxRect;
}
interface PosePoints {
[x: string]: {
point: number[],
valid: boolean;
estimated: boolean;
quality: number;
}
}
interface DfxRect {
x: number;
y: number;
width: number;
height: number;
}
interface MeshAnnotation {
x: number;
y: number;
z: number;
}
interface Direction {
yaw: number;
pitch: number;
roll: number;
}
interface VideoElementSize {
width: number;
height: number;
offsetX: number;
offsetY: number;
}
interface Annotations {
silhouette: MeshAnnotation[],
direction: Direction;
relativeFaceSize: number;
shouldersVisible: boolean;
movementConstraintExceeded: boolean;
}
type ChunkAction = 'CHUNK::PROCESS' | 'FIRST::PROCESS' | 'LAST::PROCESS';
interface ConstraintsConfig {
backLightMaxPixels_pct: number;
backLightSearchMult: number;
backLightThresh: number;
boxCenterX_pct: number;
boxCenterY_pct: number;
boxHeight_pct: number;
boxWidth_pct: number;
cameraRot_chunkThresh: number;
cameraRot_windowThresh: number;
checkBackLight: boolean;
checkCameraMovement: boolean;
checkCentered: boolean;
checkDistance: boolean;
checkEyebrowMovement: boolean;
checkFaceDirection: boolean;
checkLighting: boolean;
checkMaxDistance: boolean;
checkMinFps: boolean;
checkMovement: boolean;
chunkMovementThresh_pct: number;
enableDebugLog: boolean;
enableFigures: boolean;
faceRotLR_thresh: number;
faceRotUD_lowerthresh: number;
faceRotUD_upperThresh: number;
hy_faceRotLR_thresh: number;
hy_maxFaceRotLR_deg: number;
hy_maxFaceRotUD_deg: number;
hy_minInterPupilDist_px: number;
hy_maxInterPupilDist_px: number;
hy_minimumFps: number;
maxEyebrowMovement_mm: number;
maxFaceRotLR_deg: number;
maxFaceRotUD_deg: number;
maxMovement_mm: number;
minInterPupilDist_px: number;
maxInterPupilDist_px: number;
minimumFps: number;
movementWindow_ms: number;
threshBright: number;
threshDark: number;
}
interface ChunkSent {
chunkNumber: number;
numberChunks: number;
startTime_s: number;
endTime_s: number;
duration_s: number;
metadata: Uint8Array;
payload: Uint8Array;
action: ChunkAction;
measurementId: string;
}
/**
* Face attribute values used to define demographic information
* @enum
*/
declare const faceAttributeValue: {
readonly SEX_NOT_PROVIDED: 1;
readonly SEX_ASSIGNED_MALE_AT_BIRTH: 2;
readonly SEX_ASSIGNED_FEMALE_AT_BIRTH: 3;
readonly DIABETES_NONE: 4;
readonly DIABETES_TYPE1: 5;
readonly DIABETES_TYPE2: 6;
readonly SMOKER_TRUE: 0;
readonly SMOKER_FALSE: 1;
readonly BLOOD_PRESSURE_MEDICATION_TRUE: 1;
readonly BLOOD_PRESSURE_MEDICATION_FALSE: 0;
};
/**
* Face tracker states
* @enum
*/
declare const faceTrackerState: {
readonly ASSETS_NOT_DOWNLOADED: "ASSETS_NOT_DOWNLOADED";
readonly NOT_LOADED: "NOT_LOADED";
readonly LOADING: "LOADING";
readonly LOADED: "LOADED";
readonly READY: "READY";
};
/**
* @useDeclaredType
* Face Tracker State
*
* `ASSETS_NOT_DOWNLOADED`: Assets required for face tracking have not been downloaded.
*
* `NOT_LOADED`: Face tracker is not loaded.
*
* `LOADING`: Face tracker is currently loading.
*
* `LOADED`: Face tracker has been loaded but is not yet ready.
*
* `READY`: Face tracker is ready to use.
*/
type FaceTrackerState = keyof typeof faceTrackerState;
/**
* Error categories enum
* @enum
*/
declare const errorCategories: {
readonly COLLECTOR: "COLLECTOR";
readonly ASSET_DOWNLOAD: "ASSET_DOWNLOAD";
readonly WEB_SOCKET: "WEB_SOCKET";
};
/**
* @useDeclaredType
* Error Categories
*
* `COLLECTOR`: Errors related to the collector, such as issues with data collection or processing.
*
* `ASSET_DOWNLOAD`: Errors related to downloading assets, such as face tracking models or other required files.
*
* `WEB_SOCKET`: WebSocket Errors.
*/
type ErrorCategories = keyof typeof errorCategories;
/**
* Constraint Feedback enum
* @enum
*/
declare const constraintFeedback: {
readonly FACE_NONE: "FaceNone";
readonly FACE_OFF_TARGET: "FaceOffTarget";
readonly FACE_DIRECTION: "FaceDirection";
readonly FACE_FAR: "FaceFar";
readonly FACE_MOVEMENT: "FaceMovement";
readonly IMAGE_BRIGHT: "ImageBright";
readonly IMAGE_DARK: "ImageDark";
readonly IMAGE_QUALITY: "ImageQuality";
readonly IMAGE_BACKLIT: "ImageBackLit";
readonly LOW_FPS: "LowFps";
};
/**
* Constraint Feedback
*
* `FaceNone`: No face detected, move face into target region.
*
* `FaceOffTarget`: Face not in target region, move face into target region.
*
* `FaceDirection`: Not looking at camera, look straight at the camera.
*
* `FaceFar`: Too far from camera, move closer to the camera.
*
* `FaceMovement`: Moving too much, hold still.
*
* `ImageBright`: Image too bright, try a darker room.
*
* `ImageDark`: Image too dark, try a brighter room.
*
* `ImageQuality`: Bad image quality, improve image quality - try alternate webcam.
*
* `ImageBackLit`: Backlit face, remove backlight behind face.
*
* `LowFps`: Framerate too low, try alternate webcam or a brighter room.
*
*/
type ConstraintFeedback = typeof constraintFeedback[keyof typeof constraintFeedback];
/**
* Constraint Status enum
* @enum
*/
declare const constraintStatus: {
readonly GOOD: "GOOD";
readonly WARN: "WARN";
readonly ERROR: "ERROR";
};
/**
* @useDeclaredType
* Constraint Feedback
*
* `Good`: indicates there is nothing presently detected in constraints
*
* `Warn`: indicates a problem that needs to be corrected
*
* `Error`: indicates a problem that has failed the measurement
*/
type ConstraintStatus = keyof typeof constraintStatus;
/**
* Real-time Result Errors enum
* @enum
*/
declare const realtimeResultErrors: {
readonly WORKER_ERROR: "WORKER_ERROR";
readonly LIVENESS_ERROR: "LIVENESS_ERROR";
readonly ANALYSIS_ERROR: "ANALYSIS_ERROR";
};
/**
* @useDeclaredType
* A measurement may have one of the following errors received from the DeepAffex® backend:
*
* `WORKER_ERROR`: It is an unrecoverable internal error at the DeepAffex® backend.
*
* `LIVENESS_ERROR`: The SNR (Signal to Noise Ratio) of the entire measurement was
* too low to meet our minimum threshold (0.5 dB). If this error is received then
* the measurement will only return one point i.e. SNR with -100 as the value.
*
* `ANALYSIS_ERROR`: indicates a problem that has failed the measurement.
* There was a failure while computing a signal. The returned JSON will have a
* `signal_id` key and a value string indicating the reason behind failure.
*/
type RealtimeResultErrors = keyof typeof realtimeResultErrors;
/**
* Notes in real-time results
* @enum
*/
declare const realtimeResultNotes: {
readonly NOTE_USED_PRED_DEMOG: "NOTE_USED_PRED_DEMOG";
readonly NOTE_SNR_BELOW_THRESHOLD: "NOTE_SNR_BELOW_THRESHOLD";
readonly NOTE_FT_LIVENSSS_FAILED: "NOTE_FT_LIVENSSS_FAILED";
readonly NOTE_MODEL_LIVENSSS_FAILED: "NOTE_MODEL_LIVENSSS_FAILED";
readonly NOTE_MISSING_MEDICAL_INFO: "NOTE_MISSING_MEDICAL_INFO";
readonly NOTE_DEGRADED_ACCURACY: "NOTE_DEGRADED_ACCURACY";
};
/**
* @useDeclaredType
* Real-time results may also contain notes for each Point. They provide additional
* information about the computation of the signal on the Cloud. If present for a
* specific DFX Point, notes will be as a value to the "Notes" key in that point.
* (e.g., for ABSI they will be at result/"Channels/"ABSI"/"Notes")."Notes" is an
* enum; multiple notes may be present against a single Point.
* They are described below:
*
* `NOTE_USED_PRED_DEMOG`: User profile data was predicted for computations since
* user-entered data was missing
*
* `NOTE_SNR_BELOW_THRESHOLD`: SNR was below 0.5 db; no additional signals were computed
*
* `NOTE_FT_LIVENSSS_FAILED`: First liveness test failed
*
* `NOTE_MODEL_LIVENSSS_FAILED`: Second liveness test failed
*
* `NOTE_MISSING_MEDICAL_INFO`: Medical history questionnaire information was not sent
*
* `NOTE_DEGRADED_ACCURACY`: Signal computation suffered from degraded accuracy
*/
type RealtimeResultNotes = keyof typeof realtimeResultNotes;
interface MeasurementOptions {
userProfileId?: string;
partnerId?: string;
}
interface LoggerSettings {
mediaPipe?: boolean;
beforeRESTCall?: boolean;
afterRESTCall?: boolean;
extractionLibWasm?: boolean;
apiClient?: boolean;
webSocket?: boolean;
extractionWorker?: boolean;
faceTrackerWorkers?: boolean;
sdk?: boolean;
}
interface Settings {
mediaElement: HTMLDivElement;
assetFolder: string;
/** If the optional `apiUrl` is not set in the SDK, it will be automatically
determined based on the token's region. This effectively ties the frontend
region to the token's region, which should be suitable for most use cases.
If the optional `apiUrl` is explicitly set in the SDK, the frontend will
communicate with that URL, regardless of the token's region. In this case,
it is the implementor’s responsibility to ensure compatibility and prevent
potential issues. Results will always be stored in the token's region.
Data processing will occur in the frontend's region, as determined by your
implementation.
If the backend only registers a license and returns a device token to
perform an anonymous measurement, the token’s region will match the
region specified in the backend's `API_URL`.
However, if the backend first registers a license to obtain a device token
and then uses that token to log in a user and return the new tokens to the
frontend, the region will be determined by the user token, not the device
token. This means that if the user was originally created in the eu-central
but the device token is in na-east, then the region would be eu-central. */
apiUrl?: string;
logger?: LoggerSettings;
metrics?: boolean;
mirrorVideo?: boolean;
displayMediaStream?: boolean;
constraintOverrides?: Partial<ConstraintsConfig>;
}
interface Demographics {
/** age: 13-120 years */
age: number;
/** height: 120-220 cm (metric). Convert from imperial if needed */
height: number;
/** weight: 30-300 kg (metric). Convert from imperial if needed */
weight: number;
/** sex: 0 (not provided), 2 (male), 3 (female) */
sex: number;
/** smoking: 1 (non-smoker), 0 (smoker) */
smoking: number;
/** bloodPressureMedication: 0 (not on medication), 1 (on medication) */
bloodPressureMedication: number;
/** diabetes: 4 (none), 5 (type 1), 6 (type 2) */
diabetes: number;
}
interface ResultsError {
Code: RealtimeResultErrors | 'OK';
Errors: {
[x: string]: {
msgs: string[];
};
};
}
type Channel = 'slow/toi4' | 'images' | 'waist' | 'metadata/bmi' | 'bloodpressure/slow' | 'physio-age/slow' | 'health' | 'face_tracker_slow/face_tracker_fast' | 'cvd-risk' | 'breathing/rois';
type Point = {
Channel: Channel;
Data: number[];
Notes?: RealtimeResultNotes[];
};
interface DFXResults {
Channels: {
[x: string]: Point;
};
Error: ResultsError;
MeasurementDataID: string;
MeasurementID: string;
MeasurementResultID: string;
Multiplier: number;
/** Status of the measurement - Only available on final results, not intermediate results */
StatusID?: 'COMPLETE' | 'PARTIAL' | 'ERROR';
}
type IsoDate = `${number}-${number}-${number}T${number}:${number}:${number}.${number}Z`;
interface SDKVersion {
webSDK: string;
extractionLib: {
version: string;
sdkId: string;
};
faceTracker: {
blazeFace: {
version: string;
backend: string;
};
faceMesh: {
version: string;
backend: string;
};
};
}
interface Drawables {
face: DfxFace;
annotations: Annotations;
starRating: number;
percentCompleted: number;
stats: PipelineStats;
}
interface IMediaElementSize {
width: number;
height: number;
x: number;
y: number;
}
interface IFrameInfo {
mediaStreamWidth: number;
mediaStreamHeight: number;
faceTrackerWidth: number;
faceTrackerHeight: number;
}
interface IMaskResize {
mediaElementSize: IMediaElementSize;
videoElementSize: VideoElementSize;
frameInfo: IFrameInfo;
isPortrait: boolean;
aspectRatio: number;
}
interface MediaElementResizeEvent extends CustomEvent {
detail: IMaskResize;
}
type WebSocketError = {
type: 'DISCONNECTED';
code: number;
reason: string;
wasClean: boolean;
} | {
type: 'ERROR';
event: Event;
};
declare class Measurement {
#private;
static VERSION: string;
readonly on: {
/**
* before REST call
* @param {IsoDate} timestamp - timestamp of the event
*/
beforeRESTCall: ((timestamp: IsoDate, actionId: number) => void) | null;
/**
* after REST call
* @param {IsoDate} timestamp - timestamp of the event
* @param {string} status - HTTP status code
* @param {unknown} error - error object
*/
afterRESTCall: ((timestamp: IsoDate, actionId: number, status: string, error: unknown) => void) | null;
/**
* bytes downloaded
* @param {number} bytes - number of bytes downloaded
* @param {number} uncompressedSize - uncompressed size of the file
* @param {string} url - download URL
* @param {boolean} done - true if download is complete
*
* Note: If server-side compression is enabled, then you're comparing the downloaded byte count
* (from the network stream) to the uncompressed file size which is not reliable.
*/
bytesDownloaded: ((bytes: number, uncompressedSize: number, url: string, done: boolean) => void) | null;
/**
* when face tracker state Changes
* @param {faceTrackerState} state - face tracker state
*/
faceTrackerStateChanged: ((state: FaceTrackerState) => void) | null;
/**
* when measurement results are received
* @param {any} results - measurement results
*/
resultsReceived: ((results: DFXResults) => void) | null;
/**
* when measurement results are received
* @param {any} results - measurement results
*/
constraintsUpdated: ((feedback: ConstraintFeedback, status: ConstraintStatus) => void) | null;
/**
* When media element size changes
* @param {event} MediaElementResizeEvent
*/
mediaElementResize: ((event: MediaElementResizeEvent) => void) | null;
/**
* When facial landmarks are updated
* @param {drawables} Drawables
*/
facialLandmarksUpdated: ((drawables: Drawables) => void) | null;
/**
* When a chunk is sent to DeepAffex
* @param {chunkSent} ChunkSent
*/
chunkSent: ((chunkSent: ChunkSent) => void) | null;
/**
* When an error occurs
* @param {ErrorCategories} category - error category
* @param {unknown} data - error data
*/
error: ((category: ErrorCategories, data: unknown | WebSocketError) => void) | null;
};
/**
* Initialize the Measurement SDK
* @param {object} settings - Initialization settings
* @returns Promise<Measurement>
*/
static init(settings: Settings): Promise<Measurement>;
private constructor();
loadMask(element: SVGSVGElement): void;
/**
* Set settings
* @param {Settings} newSettings
* @returns {boolean} true if success
*/
setSettings(newSettings: Partial<Settings>): boolean;
/**
* Returns version number
* @returns {SDKVersion} version - [Web SDK, DFX Extraction Lib, Face Tracker]
*/
getVersion(): SDKVersion;
/**
* Download assets and initialize DFX and face tracker workers
* @returns {Promise<boolean>} true if success
*/
downloadAssets(): Promise<boolean>;
/**
* Set extraction library settings
* @param {number} numberofChunks Number of chunks for extraction library collector
* @param {number} targetFPS Target FPS for extraction library collector
* @param {number} chunkDurationSeconds Chunk duration in seconds for extraction library collector
*/
setExtractionLibSettings(numberofChunks?: number, targetFPS?: number, chunkDurationSeconds?: number): void;
/**
* Set the action for the next chunk to LAST::PROCESS
*/
setNextChunkAsFinal(): Promise<void>;
startTracking(): Promise<void>;
/**
* Stop Face Tracker tracking frames and DFX extraction library collection
* If the measurement is started, it will disconnect from the Web Socket as well.
*/
stopTracking(): Promise<void>;
/**
* Disconnect from the Web Socket if the measurement is started
*/
disconnect(): Promise<void>;
/**
* Destroy Face Tracker workers and DFX extraction library worker
* @returns {boolean} true if all workers were successfully destroyed, false otherwise
*/
destroy(): Promise<boolean>;
/**
* Reset only if the face tracker assets have been downloaded
* @returns {boolean} true if the reset was successful, false otherwise
*/
reset(): Promise<boolean>;
setMediaStream(mediaStream: MediaStream): Promise<void>;
prepare(token: string, refreshToken: string, studyId: string, sdkId?: string): Promise<boolean>;
setConstraintsConfig(override: boolean): void;
/**
* Start measurement by obtaining a measurement ID, resetting the
* collection, setting number of chunks and constraints config, setting
* demographics, starting collection, and connecting to the Web Socket if
* user token is available.
*
* Returns the measurement ID.
* @param overrideConstraints
* @param measurementOptions
*/
startMeasurement(overrideConstraints?: boolean, measurementOptions?: MeasurementOptions): Promise<string>;
/**
* Set demographics
* @param demographics Demographics object
* @returns true if demographics are valid and set, false otherwise
*/
setDemographics(demographics: Demographics): boolean;
setObjectFit(fit: ObjectFit): boolean;
}
export { type Annotations, type ChunkAction, type ChunkSent, type ConstraintFeedback, type ConstraintStatus, type ConstraintsConfig, type DFXResults, type Demographics, type DfxFace, type DfxRect, type Direction, type Drawables, type ErrorCategories, type FaceTrackerState, type IsoDate, type LoggerSettings, Measurement, type MeasurementOptions, type MediaElementResizeEvent, type MeshAnnotation, type PosePoints, type RealtimeResultErrors, type RealtimeResultNotes, type ResultsError, type Settings, type WebSocketError, constraintFeedback, constraintStatus, errorCategories, faceAttributeValue, faceTrackerState, realtimeResultErrors, realtimeResultNotes };