lm-avatar-chat-sdk/lmAvatarChat.d.ts

lm avatar chat web sdk

import { default as AliRtcEngine } from 'aliyun-rtc-sdk';
import { AliRtcEngine as AliRtcEngine_2 } from 'aliyun-rtc-sdk';
import { AliRtcPublishState } from 'aliyun-rtc-sdk';
import { default as default_2 } from 'eventemitter3';
import { LogLevelDesc } from 'loglevel';
import { nanoid } from 'nanoid';

export { AliRtcEngine }

export declare enum AliRtcOnByeType {
    /** 当前user被踢出channel */
    AliRtcOnByeBeKickedOut = 1,
    /** channel已结束，需要离开会议 */
    AliRtcOnByeChannelTerminated = 2,
    /** 相同userID在其他设备joinChannel，当前设备被下线 */
    AliRtcOnByeUserReplaced = 3
}

export declare enum ApplicationErrorCode {
    LinkChatServerFailed = 1120
}

export declare enum ApplicationErrorMsg {
    LinkChatServerFailed = "Link chat server failed"
}

/**
 * 获取音频流数据的模式
 * @enum
 */
declare enum AudioStreamMode {
    /**
     * 默认模式：通过RTC拉取音视频流
     */
    Default = 0,
    /**
     * 通过数据通道发送音频数据，接收端通过数据通道接收音频数据
     */
    DownFromDataChannel = 1
}

export declare enum AvatarAction {
    Idle = "idle",
    Listening = "listen",
    Responding = "speak",
    Thinking = "think"
}

/**
 * 云端渲染数字人
 */
export declare class CloudAvatar extends RtcAvatar {
    private _avatarVideoEle?;
    constructor(avatarConfig: IAvatarInitConfig);
    private _init;
    exit(): void;
    private _setupRTCView;
    start(dialogParams: IDialogConfig): Promise<void>;
    private _videoSubscribeStateChangedHandler;
}

/**
 * 创建数字人实例
 * @param type 数字人类型
 * @param avatarInitParams 数字人初始化参数
 * @returns 数字人实例
 */
export declare function createAvatar(type: TYAvatarType, avatarInitParams: IAvatarInitConfig, assetParams?: IAvatarAssetsConfig): CloudAvatar | LocalAvatar;

export declare interface CustomTYWebsocketConfig {
    keepAlive?: boolean | number;
    protocols?: {
        'avatar-id': string;
        'model-id': string;
        'x-fag-token': string;
    };
    url: string;
}

export declare enum EventTypes {
    'ErrorReceived' = "ErrorReceived",
    'FirstFrameReceived' = "FirstFrameReceived",
    'SpeakStarted' = "SpeakStarted",
    'SpeakEnded' = "SpeakEnded",
    'ResponseStarted' = "ResponseStarted",
    'ResponseEnded' = "ResponseEnded",
    'MessageReceived' = "MessageReceived",
    'ReadyToSpeech' = "ReadyToSpeech",
    'StateChanged' = "StateChanged",
    'StatsReceived' = "StatsReceived",
    'VolumeChanged' = "VolumeChanged"
}

export declare interface IAnimationConfig extends ISubAnimationConfig {
    other: {
        name: string;
    }[];
}

export declare type IAvatarAction = 'hello' | 'idle' | 'listen' | 'speak' | 'think';

/**
 * 3D数字人初始化额外配置
 * @interface
 */
export declare interface IAvatarAssetExt {
    /**
     * 是否开启阴影
     */
    enableShadow?: boolean;
    /**
     * 推理帧率
     */
    inferFps?: number;
    inferWasmPaths?: string;
    /**
     * 开启图片缓存,开启后，会降低渲染性能
     */
    preserveDrawingBuffer?: boolean;
    /**
     * 渲染性能模式
     */
    renderMode?: IRenderMode;
    /**
     * 垂直同步间隔
     */
    vsyncNum?: number;
}

export declare enum IAvatarAssetModelType {
    AVATAR_3D_TRADITIONAL = "AVATAR_3D_TRADITIONAL",
    AVATAR_LIVE2D = "AVATAR_LIVE2D",
    AVATAR_2D_MOBILE = "AVATAR_2D_MOBILE",
    GAUSSIAN_SPLATTING_3D = "GAUSSIAN_SPLATTING_3D"
}

export declare interface IAvatarAssetsConfig {
    url: string;
    license: string;
    secret: string;
    type?: IAvatarAssetModelType;
}

export declare enum IAvatarAudioFormat {
    f32 = "f32",
    opus = "opus",
    s16 = "s16"
}

export declare interface IAvatarInitConfig {
    appId: string;
    channel: string;
    timestamp: number;
    token: string;
    clientUserId: string;
    clientUserName?: string;
    serverUserId: string;
    avatarUserId: string;
    sessionId?: string;
    rootContainer: HTMLElement | string;
}

export declare interface IDialogConfig {
    mode: TYVoiceChatMode;
    keepAlive?: boolean;
    avatarAudioFormat?: IAvatarAudioFormat;
    outboundSampleRate?: IOutboundSampleRate;
}

/**
 trace: 详细的开发日志，用于追踪应用。
 debug: 详细的开发信息，用于调试应用。
 info: 重要事件的简要信息，如系统启动、配置等。
 warn: 系统能正常运行，但有潜在错误的情况。
 error: 由于严重的问题，某些功能无法正常运行。
 slient: 静音，不再打印日志
 */
export declare enum IEnv {
    dev = "dev",
    local = "local",
    pre = "pre",
    prod = "prod"
}

declare interface IMessage {
    action?: string;
    code?: number;
    event?: string;
    ext?: Record<string, unknown>;
    level: 'error' | 'warn' | 'info' | 'trace';
    msg?: string;
    stack?: string;
    time?: string;
}

/**
 * @interface
 * 运动数据
 */
export declare interface IMotionConfig {
    offset: Record<string, number>;
    scale: Record<string, number>;
}

export declare type IOutboundSampleRate = 48000 | 16000 | 24000;

declare interface IPanel {
    dom: HTMLCanvasElement;
    key?: string;
    name: string;
    update: (value: number, maxValue: number) => void;
}

declare type IPayload = Record<string, string | number | object>;

/**
 * 渲染性能模式
 * @enum
 */
export declare enum IRenderMode {
    /**
     * 高性能
     */
    High = "High",
    /**
     * 低性能
     */
    Low = "Low",
    /**
     * 标准性能
     */
    Standard = "Standard"
}

export declare enum IRenderType {
    cloud = "cloud",
    local = "local"
}

export declare enum IRequestToRespondType {
    prompt = "prompt",
    transcript = "transcript"
}

/**
 * rtc加入房间参数
 * @interface
 */
declare interface IRtcJoinRoomQuery {
    /**
     * rtc应用Id
     */
    appId: string;
    /**
     * rtc频道
     */
    channelId: string;
    /**
     * rtc鉴权有效的时间戳
     */
    timestamp: number;
    /**
     * rtc鉴权相关的token
     */
    token: string;
    /**
     * web端加入rtc的用户id
     */
    userId: string;
    /**
     * web端加入rtc的用户名称
     */
    userName?: string;
}

declare interface ISpeech {
    audioData: Int8Array | Uint8Array;
    endOfTask: boolean;
    taskId: string;
}

export declare interface IStatsData {
    asrAtomDelay: number;
    asrAtomStopDelay: number;
    asrCompleteTime: number;
    avatarDelay: number;
    firstAudioTime: number;
    frdFirstTime: number;
    llmDelay: number;
    llmFirstTokenTime: number;
    llmRequestTime: number;
    respondingStartTime: number;
    totalDelay: number;
    ttsAtomDelay: number;
    ttsFirstByteTime: number;
    ttsFrdDelay: number;
    ttsTotalDelay: number;
    vadDelay: number;
    vadEndTime: number;
}

export declare type ISubAnimationConfig = {
    [key in IAvatarAction]: {
        isGroup: boolean;
        size: number;
    };
};

/**
 * 端测渲染(端到端)数字人
 */
export declare class LocalAvatar extends RtcAvatar {
    private _processor;
    private _renderer?;
    private _avatarDivEle?;
    private _seq_id;
    private _renderExt;
    private _waitNextAudioClipCount;
    private _waitNextAudioClipTime?;
    private _renderModelType;
    private _assetConfig;
    static _canvas?: HTMLCanvasElement;
    constructor(avatarConfig: IAvatarInitConfig, assetConfig: IAvatarAssetsConfig);
    private _init;
    exit(): void;
    getArkitFaceFrame(): {
        arkitFace: {} | null;
    };
    getTts2FaceFrame(): {
        tts2Face: Float32Array | null;
    };
    interrupt(): void;
    sendSpeech(): void;
    private _setupRTCView;
    start(dialogParams: IDialogConfig): Promise<void>;
    private _render;
    private _bindEventTypes;
    protected _dataChannelMsgHandler(uid: string, msg: {
        data: ArrayBuffer;
    }): true | undefined;
    protected _dialogStateChangedHandler(payload: IPayload): void;
}

export declare class Logger {
    static setLevel(level: LogLevelDesc): void;
    static log(message: IMessage): void;
    static trace(message: Omit<IMessage, 'level'>): void;
    static info(message: Omit<IMessage, 'level'>): void;
    static warn(message: Omit<IMessage, 'level'>): void;
    static error(message: Omit<IMessage, 'level'>): void;
}

export { LogLevelDesc }

export { nanoid }

export declare enum PlayerEventTypes {
    'Player_EndSpeaking' = "Player_EndSpeaking",
    'Player_NoLegacy' = "Player_NoLegacy",
    'Player_StartSpeaking' = "Player_StartSpeaking",
    'Player_WaitNextAudioClip' = "Player_WaitNextAudioClip"
}

export declare enum ProcessorEventTypes {
    'Change_Status' = "Change_Status",
    'Chat_BinsizeError' = "Chat_BinsizeError"
}

/**
 * 基于WebSocket的单独输出数字人
 */
export declare class PureAvatar extends default_2<PlayerEventTypes | WsEventTypes | RecEventTypes | EventTypes> {
    private _processor;
    private _avatarDivEle?;
    private _speechHandlerQueue;
    private _ws;
    private _curState;
    private _curServiceState?;
    private _renderer?;
    private _heartBeatTimer?;
    private _renderExt;
    private _renderModelType;
    private _wsConfig;
    private _assetConfig;
    constructor(wsConfig: TYWebsocketConfig, assetConfig: IAvatarAssetsConfig);
    private _init;
    changeState(state: TYVoiceChatState): void;
    exit(): void;
    getArkitFaceFrame(): {
        arkitFace: {} | null;
    };
    getTts2FaceFrame(): {
        tts2Face: Float32Array | null;
    };
    interrupt(): void;
    onErrorReceived(func: (err: TYError) => void): void;
    onFirstFrameReceived(func: () => void): void;
    onStateChanged(func: (state: TYVoiceChatState) => void): void;
    sendSpeech(options: ISpeech): void;
    private _setupRTCView;
    start(): Promise<void>;
    stopSpeech(): void;
    private _bindEventTypes;
    private _preCheckMsgError;
    private _messageHandler;
    private _heartBeat;
    private _render;
}

export declare enum RecEventTypes {
    'REC_DATA' = "REC_DATA",
    'REC_DATAHEADER' = "REC_DATAHEADER",
    'REC_ERROR' = "REC_ERROR",
    'REC_SESSION_STARTED' = "REC_SESSION_STARTED",
    'REC_START' = "REC_START"
}

declare class Rtc {
    deviceMode: AudioStreamMode;
    private ee;
    engine?: AliRtcEngine_2;
    seiSupported: boolean;
    constructor(ee: default_2);
    init(deviceMode?: AudioStreamMode): Promise<AliRtcEngine | undefined>;
    joinRoom(query: IRtcJoinRoomQuery): Promise<void>;
    joinRoomAsLive(query: IRtcJoinRoomQuery): Promise<void>;
    leave(): void;
    muteLocalMic(mute: boolean): void;
    send(msg: string): void;
    private _audioVolumeHandler;
}

/**
 * 基于Aliyun RTC的数字人基类
 */
export declare class RtcAvatar extends default_2<EventTypes | PlayerEventTypes | ProcessorEventTypes> {
    protected _curState: TYVoiceChatState;
    protected _isReadyTimer?: NodeJS.Timeout;
    protected _isReady: boolean;
    protected _localMicMuted: boolean;
    protected _rtc: Rtc;
    private _heartBeatTimer?;
    protected _deviceId?: string;
    protected _sessionId?: string;
    protected _avatarInitConfig?: IAvatarInitConfig;
    protected _dialogConfig: IDialogConfig;
    constructor(initConfig: IAvatarInitConfig, name?: string);
    setDialogConfig(dialogConfig: IDialogConfig): void;
    exit(): void;
    interrupt(): void;
    muteLocalMic(mute: boolean): void;
    onErrorReceived(func: (err: TYError) => void): void;
    onFirstFrameReceived(func: () => void): void;
    onMessageReceived(func: (msg: TYVoiceChatMessage) => void): void;
    onReadyToSpeech(func: () => void): void;
    onStateChanged(func: (state: TYVoiceChatState) => void): void;
    onVolumeChanged(func: (data: TYVolume) => void): void;
    onPerformanceInfoTrack(func: (data: TYPerformanceInfo) => void): void;
    requestToRespond(type: IRequestToRespondType, content: string): void;
    sendSpeech(): void;
    setLogLevel(level: LogLevelDesc): void;
    stopSpeech(): void;
    protected _dataPublishStateChangedHandler(oldState: AliRtcPublishState, newState: AliRtcPublishState): void;
    protected _dataChannelMsgHandler(uid: string, msg: {
        data: ArrayBuffer;
    }): any;
    protected _respondingEnded(): void;
    private _startedHandler;
    private _heartBeat;
    private _readyHandler;
    /**
     * 发送Start信号
     * 需要根据不同渲染方式，确定发送Start信号的时间节点，所以提供函数供子类调用发送
     */
    protected sendStartSignal(renderType: IRenderType): void;
    protected startLinkDataChannel(): void;
    private _respondingStarted;
    private _respondingContent;
    private _speechStarted;
    private _speechEnded;
    private _speechContent;
    private _stopped;
    private _errorHandler;
    protected _avatarStateChangedHandler(payload: IPayload): void;
    protected _dialogStateChangedHandler(payload: IPayload): void;
    protected _timestampStatistics(payload: IPayload): void;
}

export declare enum RTCErrorCode {
    invalidAuthParams = 1000,
    authFailed = 1001,
    joinChannelFailed = 1002,
    dataChannelLostPackets = 1003,
    dataChannelBroken = 1004,
    channelDestroyed = 1005,
    rtcKickOff = 1006,
    dataChannelDelay = 1007
}

export declare enum RTCErrorMsg {
    authFailed = "rtc auth failed",
    channelDestroyed = "channel destroyed",
    dataChannelBroken = "data channel broken",
    dataChannelDelay = "data channel delay",
    dataChannelLostPackets = "data channel lost packets",
    invalidAuthParams = "invalid auth params",
    joinChannelFailed = "rtc join channel failed",
    rtcKickOff = "rtc kick off"
}

export declare enum RTCEventTypes {
    audioVolume = "audioVolume",
    bye = "bye",
    dataChannelMsg = "dataChannelMsg",
    dataPublishStateChanged = "dataPublishStateChanged",
    localDeviceException = "localDeviceException",
    mediaExtensionMsgReceived = "mediaExtensionMsgReceived",
    onOccurError = "onOccurError",
    videoSubscribeStateChanged = "videoSubscribeStateChanged",
    rtcRemoteVideoStats = "rtcRemoteVideoStats",
    rtcRemoteAudioStats = "rtcRemoteAudioStats"
}

export declare class Stats {
    static Panel(name: string, fg: string, bg: string): {
        dom: HTMLCanvasElement;
        name: string;
        update: (value: number, maxValue: number) => void;
    };
    beginTime: number;
    container: HTMLElement;
    fpsPanel?: IPanel;
    frames: number;
    memPanel?: IPanel;
    mode: number;
    msPanel?: IPanel;
    prevTime: number;
    statsArea: HTMLDivElement;
    voiceChatPanels: IPanel[];
    constructor();
    addPanel(panel: IPanel): IPanel;
    addPanels(panels: IPanel[]): IPanel[];
    begin(): void;
    end(data?: IStatsData): number;
    hide(): void;
    show(): void;
    showPanel(id: number): void;
    update(data?: IStatsData): void;
}

export declare enum SystemErrorCode {
    RTCNoSupported = 1100,
    CameraUnauthorized = 1102,
    MicUnauthorized = 1103,
    AudioPublishFailed = 1104,
    VideoPublishFailed = 1105,
    UnknownError = 1106,
    UserActivationError = 1107
}

export declare enum SystemErrorMsg {
    AudioPublishFailed = "Audio publish failed",
    CameraUnauthorized = "Camera unauthorized",
    MicUnauthorized = "Mic unauthorized",
    RTCNoSupported = "The current environment does not support WebRTC",
    UnknownError = "unknown error",
    UserActivationError = "\u5F53\u524D\u7F51\u9875\u672A\u5B58\u5728\u7528\u6237\u4EA4\u4E92\u884C\u4E3A\uFF0C\u9ED8\u8BA4\u4F1A\u88AB\u6D4F\u89C8\u5668\u62E6\u622A",
    VideoPublishFailed = "Video publish failed"
}

export declare interface TYAvatarAsset {
    avatarId?: string;
    ext?: IAvatarAssetExt;
    type?: number;
}

export declare enum TYAvatarType {
    cloudAvatar = "cloudAvatar",
    localAvatar = "localAvatar"
}

export declare interface TYError {
    code: RTCErrorCode | SystemErrorCode;
    error?: Error;
    message?: RTCErrorMsg | SystemErrorMsg;
    terminate?: boolean;
}

export declare interface TYPerformanceInfo {
    type: TYPerformanceInfoType;
    data: Record<string, string | number | Array<object>>;
}

export declare enum TYPerformanceInfoType {
    TimestampStatistics = "TimestampStatistics",// 时延统计
    RemoteRtcAudioStats = "RemoteRtcAudioStats",
    RemoteRtcVideoStats = "RemoteRtcVideoStats"
}

export declare interface TYVoiceChatMessage {
    isFinished: boolean;
    text: string;
    type: TYVoiceChatMessageType;
}

export declare enum TYVoiceChatMessageType {
    speaking = 0,
    responding = 1
}

export declare enum TYVoiceChatMode {
    duplex = "duplex",
    push2talk = "push2talk",
    tap2talk = "tap2talk"
}

export declare enum TYVoiceChatState {
    Idle = "Idle",
    Listening = "Listening",
    Responding = "Responding",
    Thinking = "Thinking"
}

export declare enum TYVoicePlatform {
    minimax = "minimax",
    nls = "nls"
}

export declare interface TYVolume {
    source: TYVolumeSourceType;
    volume: number;
}

export declare enum TYVolumeSourceType {
    mic = 0,
    player = 1
}

export declare interface TYWebsocketConfig {
    avatarId: string;
    url: string;
    rootContainer: HTMLElement | string;
    keepAlive?: boolean | number;
    modelId: string;
    xFagToken: string;
}

/**
 * WebSocket错误码
 * @enum
 */
export declare enum WSErrorCode {
    /**
     * 参数不规范
     */
    invalidParam = 400,
    /**
     * 服务内部错误
     */
    internalException = 500,
    /**
     * socket触发关闭
     */
    socketClosed = 700
}

/**
 * WebSocket错误消息
 * @enum
 */
export declare enum WSErrorMsg {
    /**
     * 服务内部错误
     */
    internalException = "aca.error.internal.exception",
    /**
     * 参数不规范
     */
    invalidParam = "aca.error.invalid.param",
    /**
     * socket触发关闭
     */
    socketClosed = "socket closed"
}

export declare enum WsEventTypes {
    'WS_CLOSE' = "WS_CLOSE",
    'WS_ERROR' = "WS_ERROR",
    'WS_MESSAGE' = "WS_MESSAGE",
    'WS_OPEN' = "WS_OPEN"
}

export { }