@hahnpro/ms-speech-sdk
Version:
Microsoft Cognitive Services Speech SDK for JavaScript
1 lines • 12.2 kB
Source Map (JSON)
{"version":3,"sources":["src/sdk/AvatarSynthesizer.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,MAAM,uCAAuC,CAAC;AAC1E,OAAO,EAEH,eAAe,EACf,2BAA2B,EAC3B,mBAAmB,EACnB,oBAAoB,EACvB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EACH,YAAY,EACZ,eAAe,EACf,kBAAkB,EAGlB,YAAY,EAEZ,qBAAqB,EACrB,eAAe,EACf,WAAW,EACd,MAAM,cAAc,CAAC;AAItB;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAC9C,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC;IAC7C,OAAO,CAAC,gBAAgB,CAAe;IACvC,OAAO,CAAC,cAAc,CAAiB;IACvC;;;;;OAKG;IACI,mBAAmB,EAAE,CAAC,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,eAAe,KAAK,IAAI,CAAC;IAExF;;;;;OAKG;gBACgB,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY;IAUzE,SAAS,CAAC,yBAAyB,IAAI,IAAI;IAS3C;;;;;;;OAOG;IACU,gBAAgB,CAAC,cAAc,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAwD1F;;;;;;;OAOG;IACU,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAUnE;;;;;;;OAOG;IACU,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAUnE;;;;;;OAMG;IACU,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ/C;;;;;;;;OAQG;IACU,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAK7C;;;;;OAKG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;OAEG;IACH,IAAW,UAAU,IAAI,YAAY,EAAE,CAEtC;IAGD,SAAS,CAAC,sBAAsB,CAC5B,cAAc,EAAE,eAAe,EAC/B,iBAAiB,EAAE,2BAA2B,EAC9C,iBAAiB,EAAE,iBAAiB,GAAG,oBAAoB;IAS/D,SAAS,CAAC,0BAA0B,CAChC,eAAe,EAAE,eAAe,EAChC,kBAAkB,EAAE,iBAAiB,GAAG,oBAAoB;IAIhE,SAAS,CAAC,uBAAuB,CAAC,YAAY,EAAE,mBAAmB,GAAG,iBAAiB;cAMvE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,qBAAqB,CAAC;CAgBvF","file":"AvatarSynthesizer.d.ts","sourcesContent":["// Copyright (c) Microsoft Corporation. All rights reserved.\n// Licensed under the MIT license.\n\nimport { SpeechSynthesisConnectionFactory } from \"../common.speech/SpeechSynthesisConnectionFactory.js\";\nimport { SynthesisRestAdapter } from \"../common.speech/SynthesisRestAdapter.js\";\nimport { SynthesizerConfig } from \"../common.speech/SynthesizerConfig.js\";\nimport {\n AvatarSynthesisAdapter,\n IAuthentication,\n ISynthesisConnectionFactory,\n SpeechServiceConfig,\n SynthesisAdapterBase\n} from \"../common.speech/Exports.js\";\nimport { createNoDashGuid, Deferred, Events, EventType, PlatformEvent } from \"../common/Exports.js\";\nimport { AudioOutputFormatImpl } from \"./Audio/AudioOutputFormat.js\";\nimport {\n AvatarConfig,\n AvatarEventArgs,\n PropertyCollection,\n PropertyId,\n ResultReason,\n SpeechConfig,\n SpeechSynthesisOutputFormat,\n SpeechSynthesisResult,\n SynthesisResult,\n Synthesizer\n} from \"./Exports.js\";\nimport { Contracts } from \"./Contracts.js\";\nimport { SynthesisRequest } from \"./Synthesizer.js\";\n\n/**\n * Defines the avatar synthesizer.\n * @class AvatarSynthesizer\n * Added in version 1.33.0\n *\n * @experimental This feature is experimental and might change or have limited support.\n */\nexport class AvatarSynthesizer extends Synthesizer {\n protected privProperties: PropertyCollection;\n private privAvatarConfig: AvatarConfig;\n private privIceServers: RTCIceServer[];\n /**\n * Defines event handler for avatar events.\n * @member AvatarSynthesizer.prototype.avatarEventReceived\n * @function\n * @public\n */\n public avatarEventReceived: (sender: AvatarSynthesizer, event: AvatarEventArgs) => void;\n\n /**\n * Creates and initializes an instance of this class.\n * @constructor\n * @param {SpeechConfig} speechConfig - The speech config.\n * @param {AvatarConfig} avatarConfig - The talking avatar config.\n */\n public constructor(speechConfig: SpeechConfig, avatarConfig: AvatarConfig) {\n super(speechConfig);\n\n Contracts.throwIfNullOrUndefined(avatarConfig, \"avatarConfig\");\n\n this.privConnectionFactory = new SpeechSynthesisConnectionFactory();\n this.privAvatarConfig = avatarConfig;\n this.implCommonSynthesizeSetup();\n }\n\n protected implCommonSynthesizeSetup(): void {\n super.implCommonSynthesizeSetup();\n\n // The service checks the audio format setting while it ignores it in avatar synthesis.\n this.privAdapter.audioOutputFormat = AudioOutputFormatImpl.fromSpeechSynthesisOutputFormat(\n SpeechSynthesisOutputFormat.Riff24Khz16BitMonoPcm\n );\n }\n\n /**\n * Starts the talking avatar session and establishes the WebRTC connection.\n * @member AvatarSynthesizer.prototype.startAvatarAsync\n * @function\n * @public\n * @param {AvatarWebRTCConnectionInfo} peerConnection - The peer connection.\n * @returns {Promise<SynthesisResult>} The promise of the connection result.\n */\n public async startAvatarAsync(peerConnection: RTCPeerConnection): Promise<SynthesisResult> {\n Contracts.throwIfNullOrUndefined(peerConnection, \"peerConnection\");\n this.privIceServers = peerConnection.getConfiguration().iceServers;\n Contracts.throwIfNullOrUndefined(this.privIceServers, \"Ice servers must be set.\");\n const iceGatheringDone = new Deferred<void>();\n // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/icegatheringstatechange_event\n peerConnection.onicegatheringstatechange = (): void => {\n Events.instance.onEvent(new PlatformEvent(\"peer connection: ice gathering state: \" + peerConnection.iceGatheringState, EventType.Debug));\n if (peerConnection.iceGatheringState === \"complete\") {\n Events.instance.onEvent(new PlatformEvent(\"peer connection: ice gathering complete.\", EventType.Info));\n iceGatheringDone.resolve();\n }\n };\n peerConnection.onicecandidate = (event: RTCPeerConnectionIceEvent): void => {\n if (event.candidate) {\n Events.instance.onEvent(new PlatformEvent(\"peer connection: ice candidate: \" + event.candidate.candidate, EventType.Debug));\n } else {\n Events.instance.onEvent(new PlatformEvent(\"peer connection: ice candidate: complete\", EventType.Debug));\n iceGatheringDone.resolve();\n }\n };\n // Set a timeout for ice gathering, currently 2 seconds.\n setTimeout((): void => {\n if (peerConnection.iceGatheringState !== \"complete\") {\n Events.instance.onEvent(new PlatformEvent(\"peer connection: ice gathering timeout.\", EventType.Warning));\n iceGatheringDone.resolve();\n }\n }, 2000);\n const sdp: RTCSessionDescriptionInit = await peerConnection.createOffer();\n await peerConnection.setLocalDescription(sdp);\n await iceGatheringDone.promise;\n Events.instance.onEvent(new PlatformEvent(\"peer connection: got local SDP.\", EventType.Info));\n this.privProperties.setProperty(PropertyId.TalkingAvatarService_WebRTC_SDP, JSON.stringify(peerConnection.localDescription));\n\n const result: SpeechSynthesisResult = await this.speak(\"\", false);\n if (result.reason !== ResultReason.SynthesizingAudioCompleted) {\n return new SynthesisResult(\n result.resultId,\n result.reason,\n result.errorDetails,\n result.properties,\n );\n }\n const sdpAnswerString: string = atob(result.properties.getProperty(PropertyId.TalkingAvatarService_WebRTC_SDP));\n const sdpAnswer: RTCSessionDescription = new RTCSessionDescription(\n JSON.parse(sdpAnswerString) as RTCSessionDescriptionInit,\n );\n await peerConnection.setRemoteDescription(sdpAnswer);\n return new SynthesisResult(\n result.resultId,\n result.reason,\n undefined,\n result.properties,\n );\n }\n\n /**\n * Speaks plain text asynchronously. The rendered audio and video will be sent via the WebRTC connection.\n * @member AvatarSynthesizer.prototype.speakTextAsync\n * @function\n * @public\n * @param {string} text - The plain text to speak.\n * @returns {Promise<SynthesisResult>} The promise of the synthesis result.\n */\n public async speakTextAsync(text: string): Promise<SynthesisResult> {\n const r = await this.speak(text, false);\n return new SynthesisResult(\n r.resultId,\n r.reason,\n r.errorDetails,\n r.properties,\n );\n }\n\n /**\n * Speaks SSML asynchronously. The rendered audio and video will be sent via the WebRTC connection.\n * @member AvatarSynthesizer.prototype.speakSsmlAsync\n * @function\n * @public\n * @param {string} ssml - The SSML text to speak.\n * @returns {Promise<SynthesisResult>} The promise of the synthesis result.\n */\n public async speakSsmlAsync(ssml: string): Promise<SynthesisResult> {\n const r = await this.speak(ssml, true);\n return new SynthesisResult(\n r.resultId,\n r.reason,\n r.errorDetails,\n r.properties,\n );\n }\n\n /**\n * Speaks text asynchronously. The avatar will switch to idle state.\n * @member AvatarSynthesizer.prototype.stopSpeakingAsync\n * @function\n * @public\n * @returns {Promise<void>} The promise of the void result.\n */\n public async stopSpeakingAsync(): Promise<void> {\n while (this.synthesisRequestQueue.length() > 0) {\n const request = await this.synthesisRequestQueue.dequeue();\n request.err(\"Synthesis is canceled by user.\");\n }\n return this.privAdapter.stopSpeaking();\n }\n\n /**\n * Stops the talking avatar session and closes the WebRTC connection.\n * For now, this is the same as close().\n * You need to create a new AvatarSynthesizer instance to start a new session.\n * @member AvatarSynthesizer.prototype.stopAvatarAsync\n * @function\n * @public\n * @returns {Promise<void>} The promise of the void result.\n */\n public async stopAvatarAsync(): Promise<void> {\n Contracts.throwIfDisposed(this.privDisposed);\n return this.dispose(true);\n }\n\n /**\n * Dispose of associated resources.\n * @member AvatarSynthesizer.prototype.close\n * @function\n * @public\n */\n public async close(): Promise<void> {\n if (this.privDisposed) {\n return;\n }\n\n return this.dispose(true);\n }\n\n /**\n * Gets the ICE servers. Internal use only.\n */\n public get iceServers(): RTCIceServer[] {\n return this.privIceServers;\n }\n\n // Creates the synthesis adapter\n protected createSynthesisAdapter(\n authentication: IAuthentication,\n connectionFactory: ISynthesisConnectionFactory,\n synthesizerConfig: SynthesizerConfig): SynthesisAdapterBase {\n return new AvatarSynthesisAdapter(\n authentication,\n connectionFactory,\n synthesizerConfig,\n this,\n this.privAvatarConfig);\n }\n\n protected createRestSynthesisAdapter(\n _authentication: IAuthentication,\n _synthesizerConfig: SynthesizerConfig): SynthesisRestAdapter {\n return undefined;\n }\n\n protected createSynthesizerConfig(speechConfig: SpeechServiceConfig): SynthesizerConfig {\n const config = super.createSynthesizerConfig(speechConfig);\n config.avatarEnabled = true;\n return config;\n }\n\n protected async speak(text: string, isSSML: boolean): Promise<SpeechSynthesisResult> {\n const requestId = createNoDashGuid();\n const deferredResult = new Deferred<SpeechSynthesisResult>();\n this.synthesisRequestQueue.enqueue(new SynthesisRequest(requestId, text, isSSML,\n (e: SpeechSynthesisResult): void => {\n deferredResult.resolve(e);\n this.privSynthesizing = false;\n void this.adapterSpeak();\n },\n (e: string): void => {\n deferredResult.reject(e);\n this.privSynthesizing = false;\n }));\n void this.adapterSpeak();\n return deferredResult.promise;\n }\n}\n"]}