lm-avatar-chat-sdk/lmCloudAvatarChat.d.ts

lm avatar chat web sdk

import { default as AliRtcEngine } from 'aliyun-rtc-sdk';
import { AliRtcEngine as AliRtcEngine_2 } from 'aliyun-rtc-sdk';
import { AliRtcPublishState } from 'aliyun-rtc-sdk';
import { default as default_2 } from 'eventemitter3';
import { LogLevelDesc } from 'loglevel';

export { AliRtcEngine }

export declare enum ApplicationErrorCode {
    LinkChatServerFailed = 1120
}

export declare enum ApplicationErrorMsg {
    LinkChatServerFailed = "Link chat server failed"
}

/**
 * 获取音频流数据的模式
 * @enum
 */
declare enum AudioStreamMode {
    /**
     * 默认模式：通过RTC拉取音视频流
     */
    Default = 0,
    /**
     * 通过数据通道发送音频数据，接收端通过数据通道接收音频数据
     */
    DownFromDataChannel = 1
}

/**
 * 云端渲染数字人
 */
export declare class CloudAvatar extends RtcAvatar {
    private _avatarVideoEle?;
    private _innerVideoEle;
    private _chromaKeyProcessor;
    private _alphaMaskProcessor;
    private _stutterMonitor;
    /** ClientMotionData 中 batch_id 递增，与每段 1s 的 MotionData 对齐 */
    private _clientMotionBatchSeq;
    constructor(avatarConfig: IAvatarInitConfig);
    private _init;
    exit(): void;
    private _setupRTCView;
    /**
     * 移动端 Safari：非用户手势下对「有声」video.play() 会 NotAllowedError；抠像后音轨在输出流上，可先静音起播再恢复。
     */
    private _safePlayAvatarVideo;
    start(dialogParams: IDialogConfig): Promise<void>;
    /**
     * 将外部传入的 int16 PCM（与 .pcm 文件内容一致）按 1s 切段，封装为 JBIN MotionData，
     * 再按 format.md（同目录）约定分片后经 dataChannel 发送。
     * @param audio 一段连续 PCM（单个 {@link Int16Array}），或多段分片（{@link Int16Array[]} 再拼接）
     * @param end 是否为当前流最后一包（用于 end_of_batch）
     */
    pushAudioData(audio: Int16Array | Int16Array[], end?: boolean): void;
    /**
     * 将一整块 MotionData 二进制切条后经 RTC dataChannel 发送：
     * 先发首包（总长、片数），再按序发 Base64 片（与 format.md 一致）。
     * 切条宽度见 {@link DEFAULT_CLIENT_MOTION_SLICE_BYTES}。
     */
    private _sendClientMotionDataChunked;
    private _videoSubscribeStateChangedHandler;
    /**
     * 远端推流视频透明背景处理
     * @returns void
     */
    private _remoteTransparentBackgroundProcessor;
}

/**
 * 创建云端渲染数字人实例
 * @param avatarInitParams 云端渲染数字人初始化参数
 * @returns 云端渲染数字人实例
 */
export declare function createCloudAvatar(avatarInitParams: IAvatarInitConfig): CloudAvatar;

export declare enum EventTypes {
    'ErrorReceived' = "ErrorReceived",
    'FirstFrameReceived' = "FirstFrameReceived",
    'SpeakStarted' = "SpeakStarted",
    'SpeakEnded' = "SpeakEnded",
    'ResponseStarted' = "ResponseStarted",
    'ResponseEnded' = "ResponseEnded",
    'MessageReceived' = "MessageReceived",
    'ReadyToSpeech' = "ReadyToSpeech",
    'StateChanged' = "StateChanged",
    'StatsReceived' = "StatsReceived",
    'VolumeChanged' = "VolumeChanged"
}

export declare enum IAvatarAudioFormat {
    f32 = "f32",
    opus = "opus",
    s16 = "s16"
}

export declare interface IAvatarInitConfig {
    appId: string;
    channel: string;
    timestamp: number;
    token: string;
    clientUserId: string;
    clientUserName?: string;
    serverUserId: string;
    avatarUserId: string;
    sessionId?: string;
    rootContainer: HTMLElement | string;
    localContainer?: HTMLElement | string;
    ignoreAudioInput?: boolean;
    cameraCapturerConfig?: {
        width?: number;
        height?: number;
        bitrate?: number;
    };
}

export declare interface IDialogConfig {
    mode: TYVoiceChatMode;
    keepAlive?: boolean;
    avatarAudioFormat?: IAvatarAudioFormat;
    outboundSampleRate?: IOutboundSampleRate;
}

/**
 trace: 详细的开发日志，用于追踪应用。
 debug: 详细的开发信息，用于调试应用。
 info: 重要事件的简要信息，如系统启动、配置等。
 warn: 系统能正常运行，但有潜在错误的情况。
 error: 由于严重的问题，某些功能无法正常运行。
 slient: 静音，不再打印日志
 */
export declare enum IEnv {
    dev = "dev",
    pre = "pre",
    prod = "prod"
}

declare type IHeader = Record<string, string | number>;

declare interface IMessage {
    action?: string;
    code?: number;
    event?: string;
    ext?: Record<string, unknown>;
    level: 'error' | 'warn' | 'info' | 'trace';
    msg?: string;
    stack?: string;
    time?: string;
}

export declare type IOutboundSampleRate = 48000 | 16000 | 24000;

declare interface IPanel {
    dom: HTMLCanvasElement;
    key?: string;
    name: string;
    update: (value: number, maxValue: number) => void;
}

declare type IPayload = Record<string, string | number | object>;

declare enum IRenderType {
    cloud = "cloud",
    local = "local"
}

export declare enum IRequestToRespondType {
    prompt = "prompt",
    transcript = "transcript"
}

declare interface IRtcInitConfig {
    deviceMode?: AudioStreamMode;
    ignoreAudioInput: boolean;
    enableLocalVideo: boolean;
    cameraCapturerConfig?: IAvatarInitConfig['cameraCapturerConfig'];
}

/**
 * rtc加入房间参数
 * @interface
 */
declare interface IRtcJoinRoomQuery {
    /**
     * rtc应用Id
     */
    appId: string;
    /**
     * rtc频道
     */
    channelId: string;
    /**
     * rtc鉴权有效的时间戳
     */
    timestamp: number;
    /**
     * rtc鉴权相关的token
     */
    token: string;
    /**
     * web端加入rtc的用户id
     */
    userId: string;
    /**
     * web端加入rtc的用户名称
     */
    userName?: string;
}

declare interface IStatsData {
    asrAtomDelay: number;
    asrAtomStopDelay: number;
    asrCompleteTime: number;
    avatarDelay: number;
    firstAudioTime: number;
    frdFirstTime: number;
    llmDelay: number;
    llmFirstTokenTime: number;
    llmRequestTime: number;
    respondingStartTime: number;
    totalDelay: number;
    ttsAtomDelay: number;
    ttsFirstByteTime: number;
    ttsFrdDelay: number;
    ttsTotalDelay: number;
    vadDelay: number;
    vadEndTime: number;
}

declare enum ITransparentBackgroundType {
    realBg = "REAL_BG",
    serverReplaceBg = "SERVER_REPLACE_BG",
    chromaKey = "CHROMA_KEY",
    maskBlend = "MASK_BLEND"
}

export declare class Logger {
    static enableTracking(): void;
    static disableTracking(): void;
    static setEnv(env: IEnv): void;
    static setApp(app: string): void;
    static setLevel(level: LogLevelDesc): void;
    static setBaseExt(ext: Record<string, unknown>): void;
    static resetBaseExt(): void;
    static log(message: IMessage): void;
    static trace(message: Omit<IMessage, 'level'>): void;
    static info(message: Omit<IMessage, 'level'>): void;
    static warn(message: Omit<IMessage, 'level'>): void;
    static error(message: Omit<IMessage, 'level'>): void;
    static _enableTracking: boolean;
    static _BaseExt: {
        env: IEnv;
        app: string;
        version: any;
    };
}

export { LogLevelDesc }

declare enum PlayerEventTypes {
    'Player_EndSpeaking' = "Player_EndSpeaking",
    'Player_NoLegacy' = "Player_NoLegacy",
    'Player_StartSpeaking' = "Player_StartSpeaking",
    'Player_WaitNextAudioClip' = "Player_WaitNextAudioClip"
}

declare enum ProcessorEventTypes {
    'Change_Status' = "Change_Status",
    'Chat_BinsizeError' = "Chat_BinsizeError"
}

declare class Rtc {
    private ee;
    private initConfig;
    private silentAudioTrack;
    engine?: AliRtcEngine_2;
    constructor(ee: default_2);
    init(rtcInitParams: IRtcInitConfig): Promise<AliRtcEngine | undefined>;
    joinRoom(query: IRtcJoinRoomQuery): Promise<void>;
    joinRoomAsLive(query: IRtcJoinRoomQuery): Promise<void>;
    leave(): void;
    muteLocalMic(mute: boolean): void;
    setPlayoutVolume(volume: number): void;
    send(msg: string): void;
    private _audioVolumeHandler;
}

/**
 * 基于Aliyun RTC的数字人基类
 */
declare class RtcAvatar extends default_2<EventTypes | PlayerEventTypes | ProcessorEventTypes> {
    protected _curState: TYVoiceChatState;
    protected _isDataChannelReady: boolean;
    protected _isReadyTimer?: NodeJS.Timeout;
    protected _isReady: boolean;
    protected _localMicMuted: boolean;
    protected _rtc: Rtc;
    private _heartBeatTimer?;
    protected _deviceId?: string;
    protected _sessionId?: string;
    protected _avatarInitConfig: IAvatarInitConfig;
    protected _dialogConfig: IDialogConfig;
    protected _serviceInitData: {
        transparentBackgroundType: ITransparentBackgroundType;
    };
    constructor(initConfig: IAvatarInitConfig, name?: string);
    protected _checkUserActivation(): Promise<void>;
    protected _checkMicPermissionDenied(): Promise<boolean>;
    setDialogConfig(dialogConfig: IDialogConfig): void;
    exit(): void;
    interrupt(): void;
    muteLocalMic(mute: boolean): void;
    setVolume(volume: number): void;
    onErrorReceived(func: (err: TYError) => void): void;
    onFirstFrameReceived(func: () => void): void;
    onMessageReceived(func: (msg: TYVoiceChatMessage) => void): void;
    onReadyToSpeech(func: () => void): void;
    onStateChanged(func: (state: TYVoiceChatState) => void): void;
    onVolumeChanged(func: (data: TYVolume) => void): void;
    onPerformanceInfoTrack(func: (data: TYPerformanceInfo) => void): void;
    requestToRespond(type: IRequestToRespondType, content: string): void;
    sendSpeech(): void;
    setLogLevel(level: LogLevelDesc): void;
    stopSpeech(): void;
    protected _dataPublishStateChangedHandler(oldState: AliRtcPublishState, newState: AliRtcPublishState): void;
    protected _dataChannelMsgHandler(uid: string, msg: {
        data: ArrayBuffer;
    }): any;
    protected _respondingEnded(): void;
    private _startedHandler;
    private _heartBeat;
    private _readyHandler;
    /**
     * Start 信令 payload；云/本地默认含 mode，子类（如动作复刻）可覆盖字段组合。
     */
    protected buildStartSignalPayload(renderType: IRenderType): Record<string, unknown>;
    /**
     * 发送Start信号
     * 需要根据不同渲染方式，确定发送Start信号的时间节点，所以提供函数供子类调用发送
     */
    protected sendStartSignal(renderType: IRenderType): void;
    protected startLinkDataChannel(renderType: IRenderType): void;
    protected emitFirstFrame(): void;
    protected sendFirstFrameRendered(): void;
    private _respondingStarted;
    private _respondingContent;
    private _speechStarted;
    private _speechEnded;
    private _speechContent;
    private _stopped;
    private _errorHandler;
    protected _avatarStateChangedHandler(payload: IPayload): void;
    protected _dialogStateChangedHandler(payload: IPayload): void;
    protected _timestampStatistics(header: IHeader, payload: IPayload): void;
}

export declare enum RTCErrorCode {
    invalidAuthParams = 1000,
    authFailed = 1001,
    joinChannelFailed = 1002,
    dataChannelLostPackets = 1003,
    dataChannelBroken = 1004,
    channelDestroyed = 1005,
    rtcKickOff = 1006,
    dataChannelDelay = 1007
}

export declare enum RTCErrorMsg {
    authFailed = "rtc auth failed",
    channelDestroyed = "channel destroyed",
    dataChannelBroken = "data channel broken",
    dataChannelDelay = "data channel delay",
    dataChannelLostPackets = "data channel lost packets",
    invalidAuthParams = "invalid auth params",
    joinChannelFailed = "rtc join channel failed",
    rtcKickOff = "rtc kick off"
}

export declare enum RTCEventTypes {
    audioVolume = "audioVolume",
    bye = "bye",
    dataChannelMsg = "dataChannelMsg",
    dataPublishStateChanged = "dataPublishStateChanged",
    localDeviceException = "localDeviceException",
    mediaExtensionMsgReceived = "mediaExtensionMsgReceived",
    onOccurError = "onOccurError",
    videoSubscribeStateChanged = "videoSubscribeStateChanged",
    rtcRemoteVideoStats = "rtcRemoteVideoStats",
    rtcRemoteAudioStats = "rtcRemoteAudioStats"
}

export declare class Stats {
    static Panel(name: string, fg: string, bg: string): {
        dom: HTMLCanvasElement;
        name: string;
        update: (value: number, maxValue: number) => void;
    };
    beginTime: number;
    container: HTMLElement;
    fpsPanel?: IPanel;
    frames: number;
    memPanel?: IPanel;
    mode: number;
    msPanel?: IPanel;
    prevTime: number;
    statsArea: HTMLDivElement;
    voiceChatPanels: IPanel[];
    constructor();
    addPanel(panel: IPanel): IPanel;
    addPanels(panels: IPanel[]): IPanel[];
    begin(): void;
    end(data?: IStatsData): number;
    hide(): void;
    show(): void;
    showPanel(id: number): void;
    update(data?: IStatsData): void;
}

export declare enum SystemErrorCode {
    RTCNoSupported = 1100,
    CameraUnauthorized = 1102,
    MicUnauthorized = 1103,
    AudioPublishFailed = 1104,
    VideoPublishFailed = 1105,
    UnknownError = 1106,
    UserActivationError = 1107
}

export declare enum SystemErrorMsg {
    AudioPublishFailed = "Audio publish failed",
    CameraUnauthorized = "Camera unauthorized",
    MicUnauthorized = "Mic unauthorized",
    RTCNoSupported = "The current environment does not support WebRTC",
    UnknownError = "unknown error",
    UserActivationError = "\u5F53\u524D\u7F51\u9875\u672A\u5B58\u5728\u7528\u6237\u4EA4\u4E92\u884C\u4E3A\uFF0C\u9ED8\u8BA4\u4F1A\u88AB\u6D4F\u89C8\u5668\u62E6\u622A",
    VideoPublishFailed = "Video publish failed"
}

export declare interface TYError {
    code: RTCErrorCode | SystemErrorCode;
    error?: Error;
    message?: RTCErrorMsg | SystemErrorMsg;
    terminate?: boolean;
}

export declare interface TYPerformanceInfo {
    type: TYPerformanceInfoType;
    data: Record<string, string | number | Array<object>>;
}

export declare enum TYPerformanceInfoType {
    TimestampStatistics = "TimestampStatistics",// 时延统计
    RemoteRtcAudioStats = "RemoteRtcAudioStats",
    RemoteRtcVideoStats = "RemoteRtcVideoStats"
}

export declare interface TYVoiceChatMessage {
    isFinished: boolean;
    text: string;
    rawText: string;
    type: TYVoiceChatMessageType;
}

export declare enum TYVoiceChatMessageType {
    speaking = 0,
    responding = 1
}

export declare enum TYVoiceChatMode {
    duplex = "duplex",
    push2talk = "push2talk",
    tap2talk = "tap2talk"
}

export declare enum TYVoiceChatState {
    StandBy = "StandBy",
    Idle = "Idle",
    Listening = "Listening",
    Responding = "Responding",
    Thinking = "Thinking",
    NONE = "None"
}

export declare enum TYVoicePlatform {
    minimax = "minimax",
    nls = "nls"
}

declare interface TYVolume {
    source: TYVolumeSourceType;
    volume: number;
}

declare enum TYVolumeSourceType {
    mic = 0,
    player = 1
}

export declare const VERSION: any;

export { }