sandai-react
Version:
React components and utilities for the Sandai 3D AI Characters.
456 lines • 19.9 kB
TypeScript
import { WLipSyncAudioNode, Profile } from "./wLipSync/www/index";
/**
* Represents a mouth expression with duration and a mixture of mouth
* expressions representing what different phonemes look like when
* spoken out loud.
*
* The expressions are based on VRM model expressions.
*
* The mouth expressions are represented as a number between 0 and 1,
* where 1 means the expression is fully expressed and 0 means the expression is not expressed at all.
*
* @class
* @property {number} duration - Duration of the mouth expression in milliseconds.
* @property {number} aa - Mouth expression for the phoneme "aa".
* @property {number} ee - Mouth expression for the phoneme "ee".
* @property {number} ih - Mouth expression for the phoneme "ih".
* @property {number} oh - Mouth expression for the phoneme "oh".
* @property {number} ou - Mouth expression for the phoneme "ou".
*
* @example
* ```js
* d: new MouthExpression({
* duration: characterDuration,
* aa: 0,
* ee: 0,
* ih: 0,
* oh: 0,
* ou: 0.33,
* }),
* ʈ: new MouthExpression({
* duration: characterDuration,
* aa: 0.28,
* ee: 0,
* ih: 0,
* oh: 0.24,
* ou: 0.27,
* }),
* ```
* @usage
*```js
* const mouthExpression = new MouthExpression({ duration: 40.05, aa: 0.22, ee: 0, ih: 1, oh: 0, ou: 0 });
* ```
*
*/
declare class MouthExpression {
duration: number;
aa: number;
ee: number;
ih: number;
oh: number;
ou: number;
/**
* Creates a new MouthExpression.
* @param {Object} params - The parameters for the mouth expression.
* @param {number} params.duration - The duration of the expression.
* @param {number} params.aa - The "aa" property of the expression.
* @param {number} params.ee - The "ee" property of the expression.
* @param {number} params.ih - The "ih" property of the expression.
* @param {number} params.oh - The "oh" property of the expression.
* @param {number} params.ou - The "ou" property of the expression.
*
* All values default to 0 if not provided.
*
* @returns {MouthExpression} The new MouthExpression.
*/
constructor({ duration, aa, ee, ih, oh, ou, }: {
duration?: number;
aa?: number;
ee?: number;
ih?: number;
oh?: number;
ou?: number;
});
}
/**
* Represents a face expression with duration and a mixture of emotion
* expressions a face can show.
*
* The expressions are based on VRM model expressions.
*
* The face expressions are represented as a number between 0 and 1,
* where 1 means the expression is fully expressed and 0 means the expression is not expressed at all.
*
* @class
* @property {number} duration - Duration of the face expression in milliseconds.
* @property {number} angry - Face expression for the emotion "angry".
* @property {number} happy - Face expression for the emotion "happy".
* @property {number} relaxed - Face expression for the emotion "relaxed".
* @property {number} sad - Face expression for the emotion "sad".
* @property {number} surprised - Face expression for the emotion "surprised".
*
* @example
* ```js
* new FaceExpression({
* duration: 1000,
* angry: 0.0,
* happy: 0.5,
* relaxed: 0.5,
* sad: 0.0,
* surprised: 0.0,
* }),
* ```
* @usage
*```js
* const faceExpression = new FaceExpression({ duration: 1000, angry: 0.0, happy: 0.5, relaxed: 0.5, sad: 0.0, surprised: 0.0 });
* ```
*
*/
export declare class FaceExpression {
duration: number;
angry: number;
happy: number;
relaxed: number;
sad: number;
surprised: number;
neutral: number;
emotion: string;
emotionScore: number;
/**
* Creates a new FaceExpression.
* @param {Object} params - The parameters for the mouth expression.
* @param {number} params.duration - The duration of the expression.
* @param {number} params.angry - The "angry" property of the expression.
* @param {number} params.happy - The "happy" property of the expression.
* @param {number} params.relaxed - The "relaxed" property of the expression.
* @param {number} params.sad - The "sad" property of the expression.
* @param {number} params.surprised - The "surprised" property of the expression.
* @param {number} params.neutral - The "neutral" property of the expression.
* @param {string} params.emotion - The emotion to express.
* @param {number} params.emotionScore - The score of the emotion, indicating confidence.
*
* All values default to 0 if not provided.
*
* @returns {FaceExpression} The new FaceExpression.
*/
constructor({ duration, angry, happy, relaxed, sad, surprised, neutral, emotion, emotionScore, }: {
duration?: number;
angry?: number;
happy?: number;
relaxed?: number;
sad?: number;
surprised?: number;
neutral?: number;
emotion?: string;
emotionScore?: number;
});
intensify(factor: number): void;
/**
* Creates and returns a new DistilBERT GoEmotion emotion intensity object.
*
* This static factory method initializes a complete emotion intensity mapping
* with all 28 emotion categories from the DistilBERT GoEmotion model, each
* set to zero intensity (0.0). The emotion categories cover a comprehensive
* range of human emotional states.
*
* @static
* @returns {{
* admiration: number,
* amusement: number,
* anger: number,
* annoyance: number,
* approval: number,
* caring: number,
* confusion: number,
* curiosity: number,
* desire: number,
* disappointment: number,
* disapproval: number,
* disgust: number,
* embarrassment: number,
* excitement: number,
* fear: number,
* gratitude: number,
* grief: number,
* joy: number,
* love: number,
* nervousness: number,
* optimism: number,
* pride: number,
* realization: number,
* relief: number,
* remorse: number,
* sadness: number,
* surprise: number,
* neutral: number
* }} A new emotion intensity object with all GoEmotion categories initialized to 0
*
* @example
* // Create a new emotion intensity object
* const emotions = EmotionModel.createDistilbertGoEmotion();
*
* @example
* // Set specific emotion intensities
* const emotions = EmotionModel.createDistilbertGoEmotion();
* emotions.joy = 0.8;
* emotions.sadness = 0.2;
*/
static createDistilbertGoEmotion(): {
admiration: number;
amusement: number;
anger: number;
annoyance: number;
approval: number;
caring: number;
confusion: number;
curiosity: number;
desire: number;
disappointment: number;
disapproval: number;
disgust: number;
embarrassment: number;
excitement: number;
fear: number;
gratitude: number;
grief: number;
joy: number;
love: number;
nervousness: number;
optimism: number;
pride: number;
realization: number;
relief: number;
remorse: number;
sadness: number;
surprise: number;
neutral: number;
};
private static _orderedSentimentArray;
/**
* Creates a {@link FaceExpression} from a sentiment analysis result.
*
* The sentiment's comparative score is clamped to the range [-1, 1],
* normalized to [0, 1], and mapped to an ordered list of emotions.
* The resulting emotion is then converted into a FaceExpression
* using the DistilBERT GoEmotions mapping.
*
* @param sentiment - Sentiment analysis output containing a raw score and comparative value.
* @param sentiment.score - Overall sentiment score.
* @param sentiment.comparative - Normalized sentiment score, typically in the range [-1, 1].
* @param duration - Optional duration (in milliseconds) for the expression. Defaults to -1.
* @returns A FaceExpression corresponding to the mapped sentiment.
*/
static fromSentiment(sentiment: {
score: number;
comparative: number;
}, duration?: number): FaceExpression;
/**
*
* @param {Object} sentiment
* @param {number} sentiment.score
* @param {string} sentiment.label
* @param {number} duration
* @returns {FaceExpression}
*/
static fromDistilbertGoEmotions(sentiment: {
score: number;
label: string;
}, duration?: number): FaceExpression;
}
/**
* Represents expressions that can be used to animate VRM models.
*
* @class
* @property {MouthExpression[]} mouthExpressions - The mouth expressions.
* @property {FaceExpression[]} faceExpressions - The face expressions.
*
* @example
* ```js
* const expressions = new Expressions({ipa: "hai", duration: 120}); // "hai" is the IPA string and 120 is the total duration.
* ```
*
* mouthExpressions are generated from the IPA string and are generated on object instantiation.
*
* faceExpressions require a separate call to `inferEmotionsFromAudio` or `inferEmotionsFromText`,
* since they are performance heavy and may not be needed.
*/
export declare class Expressions {
private static _sentimentAnalyzer;
static _lipSyncNodes: Record<string, Record<string, {
context: AudioContext;
node: WLipSyncAudioNode;
}>>;
static _lipSyncProfiles: Record<string, Profile>;
static _lipSyncSources: Record<string, Record<string, AudioNode | null>>;
static _lipSyncModule: WebAssembly.Module | null;
static expressionWorker: Worker | null;
mouthExpressions: MouthExpression[];
faceExpressions: FaceExpression[];
/**
* Creates a new Expressions.
*
* **important**:
* one of [ipa] or [audio] must be provided.
*
* @param {Object} params - The parameters for the expressions.
* @param {string | undefined} params.ipa - The IPA string.
* @param {Float32Array | undefined} params.audio - The audio to infer expressions from.
* @param {number | undefined} [params.duration] - The total duration of the expressions in milliseconds. If not provided, the duration is calculated from the IPA string.
* @param {number | undefined} [params.startOffset] - The start offset of the audio content in milliseconds, used to prepend an empty expression at the start.
* @param {number | undefined} [params.endOffset] - The end offset of the audio content in milliseconds, used to append an empty expression at the start.
* @returns {Expressions} The new Expressions.
*/
constructor({ ipa, duration, startOffset, endOffset, }: {
ipa: string | undefined;
audio: Float32Array | undefined;
duration?: number | undefined;
startOffset?: number | undefined;
endOffset?: number | undefined;
});
/**
* Converts an IPA string to a list of mouth expressions.
* @param {string} ipa - The IPA string to convert.
* @param {number} duration - The total duration for the expressions in milliseconds.
* @param {number} [startOffset=0] - The start offset in milliseconds to prepend as silence.
* @param {number} [endOffset=0] - The end offset in milliseconds to append as silence.
* @returns {MouthExpression[]} The list of mouth expressions.
*/
static ipa2mouth(ipa: string, duration?: number, startOffset?: number, endOffset?: number): MouthExpression[];
/**
* Creates an audio-to-mouth system that connects audio input to lip-sync analysis
* @param {string} wLipSyncProfileBinUrl - URL to the lip-sync profile binary file
* @param {string} wLipSyncAudioWorkletJsUrl - URL to the lip-sync js worklet file
* @param {string} wLipSyncModuleWasmUrl - URL to the lip-sync wasm file
* @param {string} cacheKey - Key for caching the lip-sync node instance
* @returns {Promise<{
* listen: (factory: (context: AudioContext) => AudioNode) => void,
* render: (normalizationOptions: { scale: number } | undefined) => MouthExpression,
* disconnect: () => void,
* destroy: () => void
* }>} An object with listen and render methods for audio processing
*/
audio2mouth(wLipSyncProfileBinUrl: string, wLipSyncAudioWorkletJsUrl: string, wLipSyncModuleWasmUrl: string, cacheKey?: string): Promise<{
listen: (factory: (context: AudioContext) => AudioNode) => void;
render: (normalizationOptions: {
scale: number;
} | undefined) => MouthExpression;
disconnect: () => void;
destroy: () => void;
}>;
/**
* Creates an audio-to-mouth system that connects audio input to lip-sync analysis
* @param {string} wLipSyncProfileBinUrl - URL to the lip-sync profile binary file
* @param {string} wLipSyncAudioWorkletJsUrl - URL to the lip-sync js worklet file
* @param {string} wLipSyncModuleWasmUrl - URL to the lip-sync wasm file
* @param {string} cacheKey - Key for caching the lip-sync node instance
* @returns {Promise<{
* listen: (factory: (context: AudioContext) => AudioNode) => void,
* render: (normalizationOptions: { scale: number } | undefined) => MouthExpression,
* disconnect: () => void,
* destroy: () => void
* }>} An object with listen and render methods for audio processing
*/
static audio2mouth(wLipSyncProfileBinUrl: string, wLipSyncAudioWorkletJsUrl: string, wLipSyncModuleWasmUrl: string, cacheKey?: string): Promise<{
listen: (factory: (context: AudioContext) => AudioNode) => void;
render: (normalizationOptions: {
scale: number;
} | undefined) => MouthExpression;
disconnect: () => void;
destroy: () => void;
}>;
/**
* Creates and caches an audio lip-sync node for the given profile URL.
* If a lip-sync node already exists for the profile URL, returns the cached instance.
*
* @static
* @async
* @param {string} wLipSyncProfileBinUrl - URL to the lip-sync profile binary file
* @param {string} wLipSyncAudioWorkletJsUrl - URL to the lip-sync js worklet file
* @param {string} wLipSyncModuleWasmUrl - URL to the lip-sync wasm file
* @param {string | undefined} cacheKey - The key to the binary lip-sync profile file.
* Caching will be done based on the wLipSyncProfileBinUrl if undefined.
* @returns {Promise<{node: Awaited<ReturnType<typeof createWLipSyncNode>>, context: AudioContext}>} A promise that resolves to an object containing:
* - {Object} node - The created lip-sync node instance
* - {AudioContext} context - The AudioContext associated with the lip-sync node
*
* @throws {Error} If fetching the profile fails
* @throws {Error} If parsing the binary profile fails
* @throws {Error} If creating the lip-sync node fails
*
* @example
* // Create a lip-sync node from a profile URL
* try {
* const { node, context } = await AudioLipSyncClass.createAudioLipSyncNode('/profiles/avatar-profile.bin');
* // Use the node and context for audio processing
* } catch (error) {
* console.error('Failed to create lip-sync node:', error);
* }
*
* @example
* // Subsequent calls with the same URL return cached instance
* const result1 = await AudioLipSyncClass.createAudioLipSyncNode('/profiles/same-profile.bin');
* const result2 = await AudioLipSyncClass.createAudioLipSyncNode('/profiles/same-profile.bin');
* // result1 and result2 reference the same cached instance
*/
static _createAudioLipSyncNode(wLipSyncProfileBinUrl: string, wLipSyncAudioWorkletJsUrl: string, wLipSyncModuleWasmUrl: string, cacheKey?: string | undefined): Promise<{
node: WLipSyncAudioNode;
context: AudioContext;
}>;
/**
* Infers emotions from an audio file using a Web Worker.
*
* @param {string} filePath - The path or URL to the audio file.
* @param {string} workerUrl - The URL to the Web Worker script.
* @param {function(number): void} onProgress - Callback function to handle progress updates. The progress is given as a percentage (0 to 100).
* @returns {Promise<Object>} A promise that resolves with the inferred emotions as an object.
*/
inferEmotionsFromAudio(filePath: string, workerUrl: string, onProgress: (arg0: number) => void): Promise<object>;
/**
* Infers emotions from a text string on the main thread using the sentiment package.
*
* @param {string} text - The text from which to infer emotions.
* @param {number} duration - The duration of the expressions in milliseconds.
* @returns {Promise<void>} sets this.faceExpressions to the inferred values
*/
inferEmotionsFromTextSentiment(text?: string, duration?: number): Promise<void>;
/**
* Infers emotions from a text string using a Web Worker.
*
* @param {string} text - The text from which to infer emotions.
* @param {number} duration - The duration of the expressions in milliseconds.
* @param {string} workerUrl - The URL to the Web Worker script.
* @param {function(number): void} onProgress - Callback function to handle progress updates. The progress is given as a percentage (0 to 100).
* @returns {void} sets this.faceExpressions to the inferred values
*/
inferEmotionsFromText(text: string, duration: number, workerUrl: string, onProgress: (arg0: number) => void, onnxRuntimePath?: string, modelDir?: string, modelName?: string): Promise<void>;
/**
* Static method to infer emotions from a text string using a Web Worker.
*
* @param {string} [text="Hello, beautiful World!"] - The text from which to infer emotions.
* @param {number} [duration=1000] - The duration of the expressions in milliseconds.
* @param {string} [workerUrl="expression_worker.js"] - The URL to the Web Worker script.
* @param {function(number): void} [onProgress=(progress) => {}] - Callback function to handle progress updates. The progress is given as a percentage (0 to 100).
* @returns {Promise<FaceExpression[]>} A promise that resolves with the inferred emotions as an object.
*/
static inferEmotionsFromText(text?: string, duration?: number, workerUrl?: string, onProgress?: (arg0: number) => void, onnxRuntimePath?: string, modelDir?: string, modelName?: string): Promise<FaceExpression[]>;
private static _analyzeSentiment;
private static _ensureSentimentInitialized;
/**
* Static method to infer emotions from a text string by sentiment on the main thread.
*
* @param {string} [text="Hello, beautiful World!"] - The text from which to infer emotions.
* @param {number} [duration=1000] - Duration of the audio
* @returns {Promise<FaceExpression[]>} A promise that resolves with the inferred emotions as an object.
*/
static inferEmotionsFromTextSentiment(text?: string, duration?: number): Promise<FaceExpression[]>;
/**
* Static method to infer emotions from an audio file using a Web Worker.
*
* @param {string} [filePath="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav"] - The path or URL to the audio file.
* @param {string} [workerUrl="expression_worker.js"] - The URL to the Web Worker script.
* @returns {Promise<Object>} A promise that resolves with the inferred emotions as an object.
*/
static inferEmotionsFromAudio(filePath?: string, workerUrl?: string, onProgress?: (progressState: any) => void): Promise<object>;
}
export {};
//# sourceMappingURL=expressions.d.ts.map