UNPKG

lm-avatar-chat-sdk

Version:

lm avatar chat web sdk

806 lines (748 loc) 24.5 kB
import { default as AliRtcEngine } from 'aliyun-rtc-sdk'; import { AliRtcEngine as AliRtcEngine_2 } from 'aliyun-rtc-sdk'; import { AliRtcPublishState } from 'aliyun-rtc-sdk'; import { default as default_2 } from 'eventemitter3'; import { default as default_3 } from 'jszip'; import { LogLevelDesc } from 'loglevel'; import * as THREE from 'three'; export { AliRtcEngine } export declare enum ApplicationErrorCode { LinkChatServerFailed = 1120 } export declare enum ApplicationErrorMsg { LinkChatServerFailed = "Link chat server failed" } /** * 获取音频流数据的模式 * @enum */ declare enum AudioStreamMode { /** * 默认模式:通过RTC拉取音视频流 */ Default = 0, /** * 通过数据通道发送音频数据,接收端通过数据通道接收音频数据 */ DownFromDataChannel = 1 } /** * 创建端侧渲染数字人实例 * @param avatarInitParams 端侧渲染数字人初始化参数 * @param assetParams 端侧渲染数字人资产参数 * @returns 端侧渲染数字人实例 */ export declare function createLocalAvatar(avatarInitParams: IAvatarInitConfig, assetParams: IAvatarAssetsConfig): LocalAvatar; export declare enum EventTypes { 'ErrorReceived' = "ErrorReceived", 'FirstFrameReceived' = "FirstFrameReceived", 'SpeakStarted' = "SpeakStarted", 'SpeakEnded' = "SpeakEnded", 'ResponseStarted' = "ResponseStarted", 'ResponseEnded' = "ResponseEnded", 'MessageReceived' = "MessageReceived", 'ReadyToSpeech' = "ReadyToSpeech", 'StateChanged' = "StateChanged", 'StatsReceived' = "StatsReceived", 'VolumeChanged' = "VolumeChanged" } declare interface IAnimationConfig extends ISubAnimationConfig { other: { name: string; }[]; } declare type IAvatarAction = 'hello' | 'idle' | 'listen' | 'speak' | 'think'; /** * 3D数字人初始化额外配置 * @interface */ declare interface IAvatarAssetExt { /** * 是否开启阴影 */ enableShadow?: boolean; /** * 推理帧率 */ inferFps?: number; /** * 开启图片缓存,开启后,会降低渲染性能 */ preserveDrawingBuffer?: boolean; /** * 渲染性能模式 */ renderMode?: IRenderMode; /** * 垂直同步间隔 */ vsyncNum?: number; } export declare enum IAvatarAssetModelType { AVATAR_3D_TRADITIONAL = "AVATAR_3D_TRADITIONAL", AVATAR_2D_MOBILE = "AVATAR_2D_MOBILE" } export declare interface IAvatarAssetsConfig { url: string; license: string; secret: string; type?: IAvatarAssetModelType; } export declare enum IAvatarAudioFormat { f32 = "f32", opus = "opus", s16 = "s16" } export declare interface IAvatarInitConfig { appId: string; channel: string; timestamp: number; token: string; clientUserId: string; clientUserName?: string; serverUserId: string; avatarUserId: string; sessionId?: string; rootContainer: HTMLElement | string; localContainer?: HTMLElement | string; ignoreAudioInput?: boolean; cameraCapturerConfig?: { width?: number; height?: number; bitrate?: number; }; } /** * @interface * 角色配置 */ declare interface ICharacterConfig { /** * 是否开启阴影 */ useLive2dBS: boolean; enableShadow: boolean; isUnlit: boolean; name: string; position: { x: number; y: number; z: number; }; rotation: { x: number; y: number; z: number; }; scale: { x: number; y: number; z: number; }; projectionType?: string; persFov?: number; orthSize?: number; camPos?: { x: number; y: number; z: number; }; camRot?: { x: number; y: number; z: number; }; physicalMaterialParams?: { roughness: number; metalness: number; emissive: [number, number, number]; }; enviromentMap?: { intensity: number; rotationX: number; rotationY: number; rotationZ: number; }; directionalLight?: { intensity: number; rotation: [number, number]; distance: number; color: number; }; shadowMap?: { bias: number; normalBias: number; far: number; near: number; left: number; right: number; top: number; bottom: number; }; } export declare interface IDialogConfig { mode: TYVoiceChatMode; keepAlive?: boolean; avatarAudioFormat?: IAvatarAudioFormat; outboundSampleRate?: IOutboundSampleRate; } /** trace: 详细的开发日志,用于追踪应用。 debug: 详细的开发信息,用于调试应用。 info: 重要事件的简要信息,如系统启动、配置等。 warn: 系统能正常运行,但有潜在错误的情况。 error: 由于严重的问题,某些功能无法正常运行。 slient: 静音,不再打印日志 */ export declare enum IEnv { dev = "dev", pre = "pre", prod = "prod" } declare type IHeader = Record<string, string | number>; declare interface IMessage { action?: string; code?: number; event?: string; ext?: Record<string, unknown>; level: 'error' | 'warn' | 'info' | 'trace'; msg?: string; stack?: string; time?: string; } /** * @interface * 运动数据 */ declare interface IMotionConfig { offset: Record<string, number>; scale: Record<string, number>; } export declare type IOutboundSampleRate = 48000 | 16000 | 24000; declare interface IPanel { dom: HTMLCanvasElement; key?: string; name: string; update: (value: number, maxValue: number) => void; } declare type IPayload = Record<string, string | number | object>; /** * 渲染性能模式 * @enum */ declare enum IRenderMode { /** * 高性能 */ High = "High", /** * 低性能 */ Low = "Low", /** * 标准性能 */ Standard = "Standard" } declare enum IRenderType { cloud = "cloud", local = "local" } export declare enum IRequestToRespondType { prompt = "prompt", transcript = "transcript" } declare interface IRtcInitConfig { deviceMode?: AudioStreamMode; ignoreAudioInput: boolean; enableLocalVideo: boolean; cameraCapturerConfig?: IAvatarInitConfig['cameraCapturerConfig']; } /** * rtc加入房间参数 * @interface */ declare interface IRtcJoinRoomQuery { /** * rtc应用Id */ appId: string; /** * rtc频道 */ channelId: string; /** * rtc鉴权有效的时间戳 */ timestamp: number; /** * rtc鉴权相关的token */ token: string; /** * web端加入rtc的用户id */ userId: string; /** * web端加入rtc的用户名称 */ userName?: string; } declare interface IStatsData { asrAtomDelay: number; asrAtomStopDelay: number; asrCompleteTime: number; avatarDelay: number; firstAudioTime: number; frdFirstTime: number; llmDelay: number; llmFirstTokenTime: number; llmRequestTime: number; respondingStartTime: number; totalDelay: number; ttsAtomDelay: number; ttsFirstByteTime: number; ttsFrdDelay: number; ttsTotalDelay: number; vadDelay: number; vadEndTime: number; } declare type ISubAnimationConfig = { [key in IAvatarAction]: { isGroup: boolean; size: number; }; }; declare enum ITransparentBackgroundType { realBg = "REAL_BG", serverReplaceBg = "SERVER_REPLACE_BG", chromaKey = "CHROMA_KEY", maskBlend = "MASK_BLEND" } export declare class Local3DRenderer { static inferenceModelZipData?: default_3; static getInstance(container: HTMLDivElement, assetConfig: { characterModelUrl: string; animationModelUrl: string; characterConfig: ICharacterConfig; animationConfig: IAnimationConfig; motionConfig: IMotionConfig; isBackgroundPng?: boolean; backgroundUrl?: string; hdrUrl?: string; lutUrl?: string; inferenceModelUrl?: string; }, ee: LocalAvatar | null, renderExt: IAvatarAssetExt): Promise<Local3DRenderer>; private clock; private vsyncNum; private vsyncCount; private motioncfg; private scene; private camera; private renderer; private background; private model; private mixer; private animManager; private chatState; private expressionData; private _ee; private _shadow; private _materials; private _directionalLight; private useLive2dBS; private _Blinking; private _BlinkingFrame; private _container; private _lutTexture; private _lutIntensity; private _lightRig; private _extInfo; private _audio2expressionProcessor; private _audio2expressionShapeKeys; private rafId; static _canvas: HTMLCanvasElement; static instance: Local3DRenderer | undefined; constructor(_container: HTMLDivElement, _param: THREE.WebGLRendererParameters, _ee: LocalAvatar | null, extInfo?: { enableShadow: boolean; isPerspective: boolean; orthSize: number; persFov: number; pixRatio: number; vsyncNum: number; directionalLightIntestion: number; directionalLightColor: number; directionalLightDistance: number; directionalLighRotation: [number, number]; shadowMapBias: number; shadowMapNormalBias: number; shadowMapFar: number; shadowMapNear: number; shadowMapLeft: number; shadowMapRight: number; shadowMapTop: number; shadowMapBottom: number; }); setupLights(): void; updateLightDirection(horizontalAngle: number, verticalAngle: number): void; setShadowMapBias(bias: number): void; setShadowMapNormalBias(normalBias: number): void; setDirectionalLightDistance(distance: number): void; dispose(): void; disposeModel(): void; getCamera(): THREE.Camera; loadBackGround(url: string): Promise<void>; private LoadGLTF; loadRGBLUT(url: string): Promise<THREE.Texture | undefined>; loadHDRI(url: string, intensity?: number, rotationX?: number, rotationY?: number, rotationZ?: number): Promise<unknown>; mapping2Live2DBS(expressionFrameData: Record<string, number>): Record<string, number>; loadModel(url_model: string, url_anim: string, avatarConfig: ICharacterConfig, animationConfig: IAnimationConfig, motionConfig: IMotionConfig): Promise<void>; /** * @param {THREE.Material} material - material that needs to be injected * @param {THREE.Texture} lutTexture - LUT texture(256x16) * @param {number} intensity - LUT intensity(0-1) */ injectLUT(material: THREE.MeshPhysicalMaterial | THREE.MeshBasicMaterial | THREE.Material | ShadowedBasicMaterial, lutTexture: THREE.Texture, intensity?: number): void; render(): void; setBackGroundPos(x?: number, y?: number, z?: number): void; setBackGroundScale(x?: number, y?: number, z?: number): void; getMaterials(): Record<string, THREE.Material>; getSelectedMaterial(name: string): THREE.Material | undefined; setMaterialEmissive(material: THREE.MeshPhysicalMaterial, color: number): void; setMaterialRoughness(material: THREE.MeshPhysicalMaterial, roughness: number): void; setMaterialMetalness(material: THREE.MeshPhysicalMaterial, metalness: number): void; setEnviromentLightMap(url: string, intensity?: number, rotation?: THREE.Vector3): Promise<unknown> | undefined; setEnviromentLightRotation(x?: number, y?: number, z?: number): void; setDirectionalLightIntensity(intensity?: number): void; setDirectionalLightColor(color: number): void; setDirectionalLightRotation(horizontal: number, vertical: number): void; setShadowMapNear(near: number): void; setShadowMapFar(far: number): void; setShadowMapLeft(left: number): void; setShadowMapRight(right: number): void; setShadowMapTop(top: number): void; setShadowMapBottom(bottom: number): void; private setExpression; setModelPos(x?: number, y?: number, z?: number): void; setModelScale(x?: number, y?: number, z?: number): void; setCameraPos(x?: number, y?: number, z?: number): void; setCameraRotation(x?: number, y?: number, z?: number): void; setFOV(fov: number): void; setCameraType(isPerspective: boolean): void; getAnimations(): Record<string, THREE.AnimationAction> | undefined; playAnimation(name: string, animationActions: Record<string, THREE.AnimationAction>): void; stopAnimation(name: string, animationActions: Record<string, THREE.AnimationAction>): void; setLUTtexture(url: string): Promise<void>; private initAudioProcessor; getAudio2expressionShapeKeys(): string[]; doAudio2expressionInference(audioData: ArrayBuffer, inputAudioSampleRate: number): Promise<number[]>; private mergeTfModelPath; } /** * 端侧渲染(端到端)数字人 */ export declare class LocalAvatar extends RtcAvatar { private _processor; private _renderer?; private _avatarDivEle?; private _seq_id; private _renderExt; private _waitNextAudioClipCount; private _waitNextAudioClipTime?; private _renderModelType; private _assetConfig; private _zipAsset?; static _canvas?: HTMLCanvasElement; constructor(avatarConfig: IAvatarInitConfig, assetConfig: IAvatarAssetsConfig); private _init; get renderModelType(): IAvatarAssetModelType; exit(): void; getArkitFaceFrame(): { arkitFace: {} | null; }; getTts2FaceFrame(): { tts2Face: Float32Array | null; }; interrupt(): void; sendSpeech(): void; doAudio2faceInference(audioData: ArrayBuffer, inputAudioSampleRate: number, targetFaceFps: number, endTag?: boolean): Promise<number[]>; resetAudio2faceStream(): void; doAudio2expressionInference(audioData: ArrayBuffer, inputAudioSampleRate: number): Promise<number[]>; getAudio2expressionShapeKeys(): string[]; private _setupRTCView; start(dialogParams: IDialogConfig): Promise<void>; private _render; private _bindEventTypes; protected _dataChannelMsgHandler(uid: string, msg: { data: ArrayBuffer; }): true | undefined; protected _dialogStateChangedHandler(payload: IPayload): void; } export declare class Logger { static enableTracking(): void; static disableTracking(): void; static setEnv(env: IEnv): void; static setApp(app: string): void; static setLevel(level: LogLevelDesc): void; static setBaseExt(ext: Record<string, unknown>): void; static resetBaseExt(): void; static log(message: IMessage): void; static trace(message: Omit<IMessage, 'level'>): void; static info(message: Omit<IMessage, 'level'>): void; static warn(message: Omit<IMessage, 'level'>): void; static error(message: Omit<IMessage, 'level'>): void; static _enableTracking: boolean; static _BaseExt: { env: IEnv; app: string; version: any; }; } export { LogLevelDesc } declare enum PlayerEventTypes { 'Player_EndSpeaking' = "Player_EndSpeaking", 'Player_NoLegacy' = "Player_NoLegacy", 'Player_StartSpeaking' = "Player_StartSpeaking", 'Player_WaitNextAudioClip' = "Player_WaitNextAudioClip" } declare enum ProcessorEventTypes { 'Change_Status' = "Change_Status", 'Chat_BinsizeError' = "Chat_BinsizeError" } declare class Rtc { private ee; private initConfig; private silentAudioTrack; engine?: AliRtcEngine_2; constructor(ee: default_2); init(rtcInitParams: IRtcInitConfig): Promise<AliRtcEngine | undefined>; joinRoom(query: IRtcJoinRoomQuery): Promise<void>; joinRoomAsLive(query: IRtcJoinRoomQuery): Promise<void>; leave(): void; muteLocalMic(mute: boolean): void; setPlayoutVolume(volume: number): void; send(msg: string): void; private _audioVolumeHandler; } /** * 基于Aliyun RTC的数字人基类 */ declare class RtcAvatar extends default_2<EventTypes | PlayerEventTypes | ProcessorEventTypes> { protected _curState: TYVoiceChatState; protected _isDataChannelReady: boolean; protected _isReadyTimer?: NodeJS.Timeout; protected _isReady: boolean; protected _localMicMuted: boolean; protected _rtc: Rtc; private _heartBeatTimer?; protected _deviceId?: string; protected _sessionId?: string; protected _avatarInitConfig: IAvatarInitConfig; protected _dialogConfig: IDialogConfig; protected _serviceInitData: { transparentBackgroundType: ITransparentBackgroundType; }; constructor(initConfig: IAvatarInitConfig, name?: string); protected _checkUserActivation(): Promise<void>; protected _checkMicPermissionDenied(): Promise<boolean>; setDialogConfig(dialogConfig: IDialogConfig): void; exit(): void; interrupt(): void; muteLocalMic(mute: boolean): void; setVolume(volume: number): void; onErrorReceived(func: (err: TYError) => void): void; onFirstFrameReceived(func: () => void): void; onMessageReceived(func: (msg: TYVoiceChatMessage) => void): void; onReadyToSpeech(func: () => void): void; onStateChanged(func: (state: TYVoiceChatState) => void): void; onVolumeChanged(func: (data: TYVolume) => void): void; onPerformanceInfoTrack(func: (data: TYPerformanceInfo) => void): void; requestToRespond(type: IRequestToRespondType, content: string): void; sendSpeech(): void; setLogLevel(level: LogLevelDesc): void; stopSpeech(): void; protected _dataPublishStateChangedHandler(oldState: AliRtcPublishState, newState: AliRtcPublishState): void; protected _dataChannelMsgHandler(uid: string, msg: { data: ArrayBuffer; }): any; protected _respondingEnded(): void; private _startedHandler; private _heartBeat; private _readyHandler; /** * Start 信令 payload;云/本地默认含 mode,子类(如动作复刻)可覆盖字段组合。 */ protected buildStartSignalPayload(renderType: IRenderType): Record<string, unknown>; /** * 发送Start信号 * 需要根据不同渲染方式,确定发送Start信号的时间节点,所以提供函数供子类调用发送 */ protected sendStartSignal(renderType: IRenderType): void; protected startLinkDataChannel(renderType: IRenderType): void; protected emitFirstFrame(): void; protected sendFirstFrameRendered(): void; private _respondingStarted; private _respondingContent; private _speechStarted; private _speechEnded; private _speechContent; private _stopped; private _errorHandler; protected _avatarStateChangedHandler(payload: IPayload): void; protected _dialogStateChangedHandler(payload: IPayload): void; protected _timestampStatistics(header: IHeader, payload: IPayload): void; } export declare enum RTCErrorCode { invalidAuthParams = 1000, authFailed = 1001, joinChannelFailed = 1002, dataChannelLostPackets = 1003, dataChannelBroken = 1004, channelDestroyed = 1005, rtcKickOff = 1006, dataChannelDelay = 1007 } export declare enum RTCErrorMsg { authFailed = "rtc auth failed", channelDestroyed = "channel destroyed", dataChannelBroken = "data channel broken", dataChannelDelay = "data channel delay", dataChannelLostPackets = "data channel lost packets", invalidAuthParams = "invalid auth params", joinChannelFailed = "rtc join channel failed", rtcKickOff = "rtc kick off" } export declare enum RTCEventTypes { audioVolume = "audioVolume", bye = "bye", dataChannelMsg = "dataChannelMsg", dataPublishStateChanged = "dataPublishStateChanged", localDeviceException = "localDeviceException", mediaExtensionMsgReceived = "mediaExtensionMsgReceived", onOccurError = "onOccurError", videoSubscribeStateChanged = "videoSubscribeStateChanged", rtcRemoteVideoStats = "rtcRemoteVideoStats", rtcRemoteAudioStats = "rtcRemoteAudioStats" } declare class ShadowedBasicMaterial extends THREE.MeshBasicMaterial { [x: string]: any; private _originalVertexShader; private _originalFragmentShader; uniforms: { shadowMap: { value: null | THREE.Texture; }; lightMatrix: { value: THREE.Matrix4; }; shadowBias: { value: number; }; shadowRadius: { value: number; }; shadowMapSize: { value: THREE.Vector2; }; }; constructor(parameters?: any); _injectVertexShader(original: string): string; _injectFragmentShader(original: string): string; updateLight(light: THREE.DirectionalLight): void; } export declare class Stats { static Panel(name: string, fg: string, bg: string): { dom: HTMLCanvasElement; name: string; update: (value: number, maxValue: number) => void; }; beginTime: number; container: HTMLElement; fpsPanel?: IPanel; frames: number; memPanel?: IPanel; mode: number; msPanel?: IPanel; prevTime: number; statsArea: HTMLDivElement; voiceChatPanels: IPanel[]; constructor(); addPanel(panel: IPanel): IPanel; addPanels(panels: IPanel[]): IPanel[]; begin(): void; end(data?: IStatsData): number; hide(): void; show(): void; showPanel(id: number): void; update(data?: IStatsData): void; } export declare enum SystemErrorCode { RTCNoSupported = 1100, CameraUnauthorized = 1102, MicUnauthorized = 1103, AudioPublishFailed = 1104, VideoPublishFailed = 1105, UnknownError = 1106, UserActivationError = 1107 } export declare enum SystemErrorMsg { AudioPublishFailed = "Audio publish failed", CameraUnauthorized = "Camera unauthorized", MicUnauthorized = "Mic unauthorized", RTCNoSupported = "The current environment does not support WebRTC", UnknownError = "unknown error", UserActivationError = "\u5F53\u524D\u7F51\u9875\u672A\u5B58\u5728\u7528\u6237\u4EA4\u4E92\u884C\u4E3A\uFF0C\u9ED8\u8BA4\u4F1A\u88AB\u6D4F\u89C8\u5668\u62E6\u622A", VideoPublishFailed = "Video publish failed" } export declare interface TYError { code: RTCErrorCode | SystemErrorCode; error?: Error; message?: RTCErrorMsg | SystemErrorMsg; terminate?: boolean; } export declare interface TYPerformanceInfo { type: TYPerformanceInfoType; data: Record<string, string | number | Array<object>>; } export declare enum TYPerformanceInfoType { TimestampStatistics = "TimestampStatistics",// 时延统计 RemoteRtcAudioStats = "RemoteRtcAudioStats", RemoteRtcVideoStats = "RemoteRtcVideoStats" } export declare interface TYVoiceChatMessage { isFinished: boolean; text: string; rawText: string; type: TYVoiceChatMessageType; } export declare enum TYVoiceChatMessageType { speaking = 0, responding = 1 } export declare enum TYVoiceChatMode { duplex = "duplex", push2talk = "push2talk", tap2talk = "tap2talk" } export declare enum TYVoiceChatState { StandBy = "StandBy", Idle = "Idle", Listening = "Listening", Responding = "Responding", Thinking = "Thinking", NONE = "None" } export declare enum TYVoicePlatform { minimax = "minimax", nls = "nls" } declare interface TYVolume { source: TYVolumeSourceType; volume: number; } declare enum TYVolumeSourceType { mic = 0, player = 1 } export declare const VERSION: any; export { }