@voice-ping/cognitive-services-speech
Version:
VoicePing Cognitive Services Speech SDK for JavaScript forked from Microsoft
1 lines • 24 kB
Source Map (JSON)
{"version":3,"sources":["src/sdk/Transcription/ConversationTranslator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,WAAW,EAAkB,MAAM,sBAAsB,CAAC;AAEnE,OAAO,EACH,WAAW,EAMX,kBAAkB,EAGlB,gBAAgB,EAKnB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACH,+BAA+B,EAC/B,wCAAwC,EACxC,wCAAwC,EACxC,gCAAgC,EAChC,WAAW,EACd,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,oBAAY,WAAW;IACnB,QAAQ,IAAA;IAAE,UAAU,IAAA;IAAE,SAAS,IAAA;CAClC;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,uBAAuB,EAAE,WAAW;IAE/E,OAAO,CAAC,6BAA6B,CAAS;IAC9C,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,2BAA2B,CAA0B;IAC7D,OAAO,CAAC,mCAAmC,CAAa;IACxD,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,yBAAyB,CAAwB;IACzD,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,eAAe,CAAqC;IAC5D,OAAO,CAAC,UAAU,CAA2D;IAC7E,OAAO,CAAC,kBAAkB,CAA8C;IACxE,OAAO,CAAC,qBAAqB,CAAoB;gBAE9B,WAAW,CAAC,EAAE,WAAW;IAK5C,IAAW,UAAU,IAAI,kBAAkB,CAE1C;IAED,IAAW,yBAAyB,IAAI,MAAM,CAE7C;IAED,IAAW,YAAY,IAAI,WAAW,EAAE,CAEvC;IAEM,QAAQ,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,wCAAwC,KAAK,IAAI,CAAC;IACrG,sBAAsB,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,+BAA+B,KAAK,IAAI,CAAC;IAC1G,mBAAmB,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,wCAAwC,KAAK,IAAI,CAAC;IAChH,cAAc,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACnF,cAAc,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACnF,mBAAmB,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,gCAAgC,KAAK,IAAI,CAAC;IACxG,WAAW,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,gCAAgC,KAAK,IAAI,CAAC;IAChG,YAAY,EAAE,CAAC,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,gCAAgC,KAAK,IAAI,CAAC;IAExG;;;;;;;OAOG;IACI,qBAAqB,CAAC,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IACzG,qBAAqB,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IA8FzH;;;;OAIG;IACI,sBAAsB,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IA6BlE;;;;;OAKG;IACI,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IAajF;;;;OAIG;IACI,sBAAsB,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IAqDlE;;;;OAIG;IACI,qBAAqB,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI;IAyB1D,UAAU,IAAI,OAAO;IAIrB,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAerC;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA+CnC;;;;OAIG;IACH,OAAO,CAAC,0BAA0B;IAIlC,2BAA2B;IAC3B,OAAO,CAAC,iBAAiB,CAExB;IAED,OAAO,CAAC,oBAAoB,CAG3B;IAED,OAAO,CAAC,kBAAkB,CASzB;IAED,OAAO,CAAC,mBAAmB,CAE1B;IAED,OAAO,CAAC,gBAAgB,CAQvB;IAED,OAAO,CAAC,sBAAsB,CAG7B;IAED,OAAO,CAAC,sBAAsB,CAE7B;IAED;;;OAGG;IACH,OAAO,CAAC,eAAe;IAiBvB;;OAEG;IACH,OAAO,CAAC,YAAY;IAapB,OAAO,KAAK,QAAQ,GAkBnB;IAED,OAAO,CAAC,cAAc;IAatB,OAAO,CAAC,WAAW;CAWtB","file":"ConversationTranslator.d.ts","sourcesContent":["// Copyright (c) Microsoft Corporation. All rights reserved.\n// Licensed under the MIT license.\n// Multi-device Conversation is a Preview feature.\n\nimport { ConversationConnectionConfig } from \"../../common.speech/Exports\";\nimport { IDisposable, IErrorMessages } from \"../../common/Exports\";\nimport { Contracts } from \"../Contracts\";\nimport {\n AudioConfig,\n CancellationErrorCode,\n CancellationReason,\n Connection,\n ConnectionEventArgs,\n ProfanityOption,\n PropertyCollection,\n PropertyId,\n Recognizer,\n SessionEventArgs,\n SpeechTranslationConfig,\n TranslationRecognitionCanceledEventArgs,\n TranslationRecognitionEventArgs,\n TranslationRecognizer\n} from \"../Exports\";\nimport { ConversationImpl } from \"./Conversation\";\nimport {\n ConversationExpirationEventArgs,\n ConversationParticipantsChangedEventArgs,\n ConversationTranslationCanceledEventArgs,\n ConversationTranslationEventArgs,\n Participant,\n} from \"./Exports\";\nimport { Callback, IConversation } from \"./IConversation\";\nimport { IConversationTranslator } from \"./IConversationTranslator\";\n\nexport enum SpeechState {\n Inactive, Connecting, Connected\n}\n\n/***\n * Join, leave or connect to a conversation.\n */\nexport class ConversationTranslator implements IConversationTranslator, IDisposable {\n\n private privSpeechRecognitionLanguage: string;\n private privProperties: PropertyCollection;\n private privAudioConfig: AudioConfig;\n private privSpeechTranslationConfig: SpeechTranslationConfig;\n private privTranslationRecognizerConnection: Connection;\n private privIsDisposed: boolean = false;\n private privTranslationRecognizer: TranslationRecognizer;\n private privIsSpeaking: boolean = false;\n private privConversation: ConversationImpl;\n private privSpeechState: SpeechState = SpeechState.Inactive;\n private privErrors: IErrorMessages = ConversationConnectionConfig.restErrors;\n private privPlaceholderKey: string = \"abcdefghijklmnopqrstuvwxyz012345\";\n private privPlaceholderRegion: string = \"westus\";\n\n public constructor(audioConfig?: AudioConfig) {\n this.privProperties = new PropertyCollection();\n this.privAudioConfig = audioConfig;\n }\n\n public get properties(): PropertyCollection {\n return this.privProperties;\n }\n\n public get speechRecognitionLanguage(): string {\n return this.privSpeechRecognitionLanguage;\n }\n\n public get participants(): Participant[] {\n return this.privConversation?.participants;\n }\n\n public canceled: (sender: IConversationTranslator, event: ConversationTranslationCanceledEventArgs) => void;\n public conversationExpiration: (sender: IConversationTranslator, event: ConversationExpirationEventArgs) => void;\n public participantsChanged: (sender: IConversationTranslator, event: ConversationParticipantsChangedEventArgs) => void;\n public sessionStarted: (sender: IConversationTranslator, event: SessionEventArgs) => void;\n public sessionStopped: (sender: IConversationTranslator, event: SessionEventArgs) => void;\n public textMessageReceived: (sender: IConversationTranslator, event: ConversationTranslationEventArgs) => void;\n public transcribed: (sender: IConversationTranslator, event: ConversationTranslationEventArgs) => void;\n public transcribing: (sender: IConversationTranslator, event: ConversationTranslationEventArgs) => void;\n\n /**\n * Join a conversation. If this is the host, pass in the previously created Conversation object.\n * @param conversation\n * @param nickname\n * @param lang\n * @param cb\n * @param err\n */\n public joinConversationAsync(conversation: IConversation, nickname: string, cb?: Callback, err?: Callback): void;\n public joinConversationAsync(conversationId: string, nickname: string, lang: string, cb?: Callback, err?: Callback): void;\n public joinConversationAsync(conversation: any, nickname: string, param1?: string | Callback, param2?: Callback, param3?: Callback): void {\n\n try {\n\n if (typeof conversation === \"string\") {\n\n Contracts.throwIfNullOrUndefined(conversation, this.privErrors.invalidArgs.replace(\"{arg}\", \"conversation id\"));\n Contracts.throwIfNullOrWhitespace(nickname, this.privErrors.invalidArgs.replace(\"{arg}\", \"nickname\"));\n\n if (!!this.privConversation) {\n this.handleError(new Error(this.privErrors.permissionDeniedStart), param3);\n }\n\n let lang: string = param1 as string;\n if (lang === undefined || lang === null || lang === \"\") { lang = ConversationConnectionConfig.defaultLanguageCode; }\n\n // create a placecholder config\n this.privSpeechTranslationConfig = SpeechTranslationConfig.fromSubscription(\n this.privPlaceholderKey,\n this.privPlaceholderRegion);\n this.privSpeechTranslationConfig.setProfanity(ProfanityOption.Masked);\n this.privSpeechTranslationConfig.addTargetLanguage(lang);\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.SpeechServiceConnection_RecoLanguage], lang);\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.ConversationTranslator_Name], nickname);\n\n const endpoint: string = this.privProperties.getProperty(PropertyId.ConversationTranslator_Host);\n if (endpoint) {\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.ConversationTranslator_Host], endpoint);\n }\n const speechEndpointHost: string = this.privProperties.getProperty(PropertyId.SpeechServiceConnection_Host);\n if (speechEndpointHost) {\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.SpeechServiceConnection_Host], speechEndpointHost);\n }\n\n // join the conversation\n this.privConversation = new ConversationImpl(this.privSpeechTranslationConfig);\n this.privConversation.conversationTranslator = this;\n\n this.privConversation.joinConversationAsync(\n conversation,\n nickname,\n lang,\n ((result: string) => {\n\n if (!result) {\n this.handleError(new Error(this.privErrors.permissionDeniedConnect), param3);\n }\n\n this.privSpeechTranslationConfig.authorizationToken = result;\n\n // connect to the ws\n this.privConversation.startConversationAsync(\n (() => {\n this.handleCallback(param2, param3);\n }),\n ((error: any) => {\n this.handleError(error, param3);\n }));\n\n }),\n ((error: any) => {\n this.handleError(error, param3);\n }));\n\n } else if (typeof conversation === \"object\") {\n\n Contracts.throwIfNullOrUndefined(conversation, this.privErrors.invalidArgs.replace(\"{arg}\", \"conversation id\"));\n Contracts.throwIfNullOrWhitespace(nickname, this.privErrors.invalidArgs.replace(\"{arg}\", \"nickname\"));\n\n // save the nickname\n this.privProperties.setProperty(PropertyId.ConversationTranslator_Name, nickname);\n // ref the conversation object\n this.privConversation = conversation as ConversationImpl;\n // ref the conversation translator object\n this.privConversation.conversationTranslator = this;\n\n Contracts.throwIfNullOrUndefined(this.privConversation, this.privErrors.permissionDeniedConnect);\n Contracts.throwIfNullOrUndefined(this.privConversation.room.token, this.privErrors.permissionDeniedConnect);\n\n this.privSpeechTranslationConfig = conversation.config;\n\n this.handleCallback(param1 as Callback, param2);\n } else {\n this.handleError(\n new Error(this.privErrors.invalidArgs.replace(\"{arg}\", \"invalid conversation type\")),\n param2);\n }\n\n } catch (error) {\n this.handleError(error, typeof param1 === \"string\" ? param3 : param2);\n }\n }\n\n /**\n * Leave the conversation\n * @param cb\n * @param err\n */\n public leaveConversationAsync(cb?: Callback, err?: Callback): void {\n\n try {\n\n // stop the speech websocket\n this.cancelSpeech();\n\n // stop the websocket\n this.privConversation.endConversationAsync(\n (() => {\n // https delete request\n this.privConversation.deleteConversationAsync(\n (() => {\n this.handleCallback(cb, err);\n this.dispose();\n }),\n ((error: any) => {\n this.handleError(error, err);\n }));\n }),\n ((error: any) => {\n this.handleError(error, err);\n }));\n\n } catch (error) {\n this.handleError(error, err);\n }\n }\n\n /**\n * Send a text message\n * @param message\n * @param cb\n * @param err\n */\n public sendTextMessageAsync(message: string, cb?: Callback, err?: Callback): void {\n\n try {\n Contracts.throwIfNullOrUndefined(this.privConversation, this.privErrors.permissionDeniedSend);\n Contracts.throwIfNullOrWhitespace(message, this.privErrors.invalidArgs.replace(\"{arg}\", message));\n\n this.privConversation?.sendTextMessageAsync(message, cb, err);\n } catch (error) {\n\n this.handleError(error, err);\n }\n }\n\n /**\n * Start speaking\n * @param cb\n * @param err\n */\n public startTranscribingAsync(cb?: Callback, err?: Callback): void {\n\n try {\n Contracts.throwIfNullOrUndefined(this.privConversation, this.privErrors.permissionDeniedSend);\n Contracts.throwIfNullOrUndefined(this.privConversation.room.token, this.privErrors.permissionDeniedConnect);\n\n if (!this.canSpeak) {\n this.handleError(new Error(this.privErrors.permissionDeniedSend), err);\n }\n\n if (this.privTranslationRecognizer === undefined) {\n this.connectTranslatorRecognizer(\n (() => {\n this.startContinuousRecognition(\n (() => {\n this.privIsSpeaking = true;\n this.handleCallback(cb, err);\n }),\n ((error: any) => {\n\n this.privIsSpeaking = false;\n // this.fireCancelEvent(error);\n this.cancelSpeech();\n this.handleError(error, err);\n }));\n }),\n ((error: any) => {\n this.handleError(error, err);\n }));\n } else {\n this.startContinuousRecognition(\n (() => {\n this.privIsSpeaking = true;\n this.handleCallback(cb, err);\n }),\n ((error: any) => {\n this.privIsSpeaking = false;\n // this.fireCancelEvent(error);\n this.cancelSpeech();\n\n this.handleError(error, err);\n }));\n }\n\n } catch (error) {\n\n this.handleError(error, err);\n\n this.cancelSpeech();\n\n }\n }\n\n /**\n * Stop speaking\n * @param cb\n * @param err\n */\n public stopTranscribingAsync(cb?: Callback, err?: Callback): void {\n\n try {\n if (!this.privIsSpeaking) {\n // stop speech\n this.cancelSpeech();\n this.handleCallback(cb, err);\n return;\n }\n\n // stop the recognition but leave the websocket open\n this.privIsSpeaking = false;\n this.privTranslationRecognizer?.stopContinuousRecognitionAsync(() => {\n this.handleCallback(cb, err);\n }, (error: any) => {\n this.handleError(error, err);\n this.cancelSpeech();\n });\n\n } catch (error) {\n this.handleError(error, err);\n this.cancelSpeech();\n }\n }\n\n public isDisposed(): boolean {\n return this.privIsDisposed;\n }\n\n public dispose(reason?: string): void {\n if (this.isDisposed && !this.privIsSpeaking) {\n return;\n }\n this.cancelSpeech();\n this.privIsDisposed = true;\n this.privSpeechTranslationConfig?.close();\n this.privSpeechRecognitionLanguage = undefined;\n this.privProperties = undefined;\n this.privAudioConfig = undefined;\n this.privSpeechTranslationConfig = undefined;\n this.privConversation?.dispose();\n this.privConversation = undefined;\n }\n\n /**\n * Connect to the speech translation recognizer.\n * Currently there is no language validation performed before sending the SpeechLanguage code to the service.\n * If it's an invalid language the raw error will be: 'Error during WebSocket handshake: Unexpected response code: 400'\n * e.g. pass in 'fr' instead of 'fr-FR', or a text-only language 'cy'\n * @param cb\n * @param err\n */\n private connectTranslatorRecognizer(cb?: Callback, err?: Callback): void {\n\n try {\n\n if (this.privAudioConfig === undefined) {\n this.privAudioConfig = AudioConfig.fromDefaultMicrophoneInput();\n }\n\n // clear the temp subscription key if it's a participant joining\n if (this.privSpeechTranslationConfig.getProperty(PropertyId[PropertyId.SpeechServiceConnection_Key])\n === this.privPlaceholderKey) {\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.SpeechServiceConnection_Key], \"\");\n }\n\n // TODO\n const token: string = encodeURIComponent(this.privConversation.room.token);\n\n let endpointHost: string = this.privSpeechTranslationConfig.getProperty(\n PropertyId[PropertyId.SpeechServiceConnection_Host], ConversationConnectionConfig.speechHost);\n endpointHost = endpointHost.replace(\"{region}\", this.privConversation.room.cognitiveSpeechRegion);\n\n const url: string = `wss://${endpointHost}${ConversationConnectionConfig.speechPath}?${ConversationConnectionConfig.configParams.token}=${token}`;\n\n this.privSpeechTranslationConfig.setProperty(PropertyId[PropertyId.SpeechServiceConnection_Endpoint], url);\n\n this.privTranslationRecognizer = new TranslationRecognizer(this.privSpeechTranslationConfig, this.privAudioConfig);\n this.privTranslationRecognizerConnection = Connection.fromRecognizer(this.privTranslationRecognizer);\n this.privTranslationRecognizerConnection.connected = this.onSpeechConnected;\n this.privTranslationRecognizerConnection.disconnected = this.onSpeechDisconnected;\n this.privTranslationRecognizer.recognized = this.onSpeechRecognized;\n this.privTranslationRecognizer.recognizing = this.onSpeechRecognizing;\n this.privTranslationRecognizer.canceled = this.onSpeechCanceled;\n this.privTranslationRecognizer.sessionStarted = this.onSpeechSessionStarted;\n this.privTranslationRecognizer.sessionStopped = this.onSpeechSessionStopped;\n\n this.handleCallback(cb, err);\n\n } catch (error) {\n\n this.handleError(error, err);\n\n this.cancelSpeech();\n // this.fireCancelEvent(error); ?\n\n }\n }\n\n /**\n * Handle the start speaking request\n * @param cb\n * @param err\n */\n private startContinuousRecognition(cb?: Callback, err?: Callback): void {\n this.privTranslationRecognizer.startContinuousRecognitionAsync(cb, err);\n }\n\n /** Recognizer callbacks */\n private onSpeechConnected = (e: ConnectionEventArgs) => {\n this.privSpeechState = SpeechState.Connected;\n }\n\n private onSpeechDisconnected = (e: ConnectionEventArgs) => {\n this.privSpeechState = SpeechState.Inactive;\n this.cancelSpeech();\n }\n\n private onSpeechRecognized = (r: TranslationRecognizer, e: TranslationRecognitionEventArgs) => {\n // TODO: add support for getting recognitions from here if own speech\n\n // if there is an error connecting to the conversation service from the speech service the error will be returned in the ErrorDetails field.\n if (e.result?.errorDetails) {\n this.cancelSpeech();\n // TODO: format the error message contained in 'errorDetails'\n this.fireCancelEvent(e.result.errorDetails);\n }\n }\n\n private onSpeechRecognizing = (r: TranslationRecognizer, e: TranslationRecognitionEventArgs) => {\n // TODO: add support for getting recognitions from here if own speech\n }\n\n private onSpeechCanceled = (r: TranslationRecognizer, e: TranslationRecognitionCanceledEventArgs) => {\n if (this.privSpeechState !== SpeechState.Inactive) {\n try {\n this.cancelSpeech();\n } catch (error) {\n this.privSpeechState = SpeechState.Inactive;\n }\n }\n }\n\n private onSpeechSessionStarted = (r: Recognizer, e: SessionEventArgs) => {\n this.privSpeechState = SpeechState.Connected;\n\n }\n\n private onSpeechSessionStopped = (r: Recognizer, e: SessionEventArgs) => {\n this.privSpeechState = SpeechState.Inactive;\n }\n\n /**\n * Fire a cancel event\n * @param error\n */\n private fireCancelEvent(error: any): void {\n try {\n if (!!this.canceled) {\n const cancelEvent: ConversationTranslationCanceledEventArgs = new ConversationTranslationCanceledEventArgs(\n error?.reason ?? CancellationReason.Error,\n error?.errorDetails ?? error,\n error?.errorCode ?? CancellationErrorCode.RuntimeError,\n undefined,\n error?.sessionId);\n\n this.canceled(this, cancelEvent);\n }\n } catch (e) {\n //\n }\n }\n\n /**\n * Cancel the speech websocket\n */\n private cancelSpeech(): void {\n try {\n this.privIsSpeaking = false;\n this.privTranslationRecognizer?.stopContinuousRecognitionAsync();\n this.privTranslationRecognizerConnection?.closeConnection();\n this.privTranslationRecognizerConnection = undefined;\n this.privTranslationRecognizer = undefined;\n this.privSpeechState = SpeechState.Inactive;\n } catch (e) {\n // ignore the error\n }\n }\n\n private get canSpeak(): boolean {\n\n // is there a Conversation websocket available\n if (!this.privConversation.isConnected) {\n return false;\n }\n\n // is the user already speaking\n if (this.privIsSpeaking || this.privSpeechState === SpeechState.Connected || this.privSpeechState === SpeechState.Connecting) {\n return false;\n }\n\n // is the user muted\n if (this.privConversation.isMutedByHost) {\n return false;\n }\n\n return true;\n }\n\n private handleCallback(cb: Callback, err: Callback): void {\n if (!!cb) {\n try {\n cb();\n } catch (e) {\n if (!!err) {\n err(e);\n }\n }\n cb = undefined;\n }\n }\n\n private handleError(error: any, err: Callback): void {\n if (!!err) {\n if (error instanceof Error) {\n const typedError: Error = error as Error;\n err(typedError.name + \": \" + typedError.message);\n\n } else {\n err(error);\n }\n }\n }\n}\n"]}