sandai-react
Version:
React components and utilities for the Sandai 3D AI Characters.
182 lines • 6.64 kB
TypeScript
/**
* Options that are shared across different message types
*/
export interface SharedMessageOptions {
voiceName?: string;
useAudioBasedLipSync?: boolean;
emotionInferrenceType?: "sentiment" | "distilbert";
emotionIntensityMultiplier?: number;
}
/**
* Represents a message to send speech via the interaction manager.
*
* @interface SendSpeechMessage
* @property {string} type - The type of the message, which is "sendSpeech".
* @property {string} message - The message content to be spoken.
*/
export interface SendSpeechMessage {
type: "sendSpeech";
id: string;
message: string;
options?: SharedMessageOptions & {
audioFile?: string | ArrayBuffer | Blob | File;
};
}
/**
* Represents a message to send a message the model should respond to via the interaction manager.
*
* @interface SendRespondMessage
* @property {string} type - The type of the message, which is "sendRespond".
* @property {string} message - The message content the model should respond to.
*/
export interface RespondMessage {
type: "respond";
id: string;
message: string;
context?: string;
options?: SharedMessageOptions & {
modelProvider?: {
model: string;
dtype: string;
pipeline: "text" | "auto" | "mediapipe";
};
};
}
/**
* Manages interactions with the sandai iframe.
*
* This class provides methods for making the embedded system say something or stop speaking.
* It also handles sending and receiving messages between the main window and the iframe.
*
* ## Usage
* ```typescript
* const iframeElement = document.getElementById("myIframe") as HTMLIFrameElement;
* const interactionManager = new InteractionManager(iframeElement);
* interactionManager.say("Hello, world!");
* interactionManager.stop();
* ```
*
* @class InteractionManager
*
* @property {HTMLIFrameElement} _iframe - The iframe element this manager interacts with.
* @property {URL} _iframeUrl - The URL of the iframe.
* @property {Array<{ name: string; type: string; label: string }>} _saySchema - The schema for the "say" command.
* @property {Array<unknown>} _stopSchema - The schema for the "stop" command.
*
* @method say - Sends a speech message to the iframe.
* @method stop - Sends a stop speech command to the iframe.
* @method _sendMessage - Sends a message to the iframe.
* @method _listenForMessage - Listens for messages from the iframe.
* @method _getDocs - Retrieves documentation for available interaction methods.
*/
export interface StopSpeechMessage {
type: "stopSpeech";
id: string;
}
/**
* Manages interactions with the sandai iframe, including sending and stopping speech messages.
*/
export declare class InteractionManager {
private _iframe;
private _iframeUrl;
/**
* Creates an instance of `InteractionManager`.
*
* @param {HTMLIFrameElement} iframe - The sandai iframe element for interaction.
*/
constructor(iframe: HTMLIFrameElement);
private _saySchema;
/**
* Sends a message to make the character say something,
* and waits until speech has started.
*
* @param {string} message - The message content to be spoken.
* @returns {Promise<Promise<void>>} Resolves when the iframe signals that speech has started.
* The inner Promise Resolves when the iframe signals that speech has ended.
*/
say(message: string, options?: SendSpeechMessage["options"]): Promise<{
speechEndPromise: Promise<void>;
}>;
private _respondSchema;
/**
* Sends a message to make the character respond using the built-in LLM,
* and waits for a response.
*
* @param {string} message - The message content the model responds to.
* @param {string} [context] - Optional context for the model.
* @returns {Promise<string>} Resolves with the model's response.
*/
respond(message: string, context?: string, options?: RespondMessage["options"]): Promise<string>;
private _stopSchema;
/**
* Sends a message to make the character stop speaking,
* and waits until speech has actually stopped.
*
* @returns {Promise<void>} Resolves when the iframe signals that speech has stopped.
*/
stop(): Promise<void>;
/**
* Transforms a url to an array buffer
*/
_prepareUrl(url: string): Promise<ArrayBuffer>;
/**
* Transforms the options sent to the iframe, including:
* - audio files sent through [options.audioFile]
*
* The array it returns contains all the references that should have
* their ownership transferred
*/
_prepareTransferableOptions(options: SendSpeechMessage["options"]): Promise<{
options: {};
references: never[];
} | {
references: ArrayBuffer[];
options?: undefined;
}>;
/**
* Sends a message to the iframe's content window.
*
* @private
* @param {SendSpeechMessage | StopSpeechMessage | RespondMessage} data - The message data to send.
* @throws {Error} Throws an error if the iframe's `contentWindow` is not available.
*/
_sendMessage(data: SendSpeechMessage | StopSpeechMessage | RespondMessage): Promise<void>;
/**
* Stores all registered message event listeners for later removal.
*
* @private
*/
private _listeners;
/**
* Listens for messages from the iframe and invokes the provided callback when a message is received.
* Only messages from the iframe's origin are processed.
*
* The listener is stored internally so it can be removed later via `destroy()`.
*
* @private
* @param {(data: InteractionManagerServerMessage) => void} callback - The callback function to invoke with the message data.
*/
private _listenForMessage;
/**
* Cleans up all registered message event listeners previously added by `_listenForMessage`.
* Call this method when the instance is being destroyed or no longer needs to listen to messages.
*/
destroy(): void;
/**
* Generates documentation for the `InteractionManager` class, including schemas and methods.
*
* @private
* @returns {Map<string, Map<string, {schema: any[], func: Function}>>} A map containing documentation for the `say` and `stop` methods.
*/
_getDocs(): Map<string, Map<string, {
schema: any[];
func: Function;
}> | Map<string, {
schema: any[];
func: Function;
}> | Map<string, {
schema: any[];
func: Function;
}>>;
}
//# sourceMappingURL=InteractionManager.d.ts.map