@voice-ping/cognitive-services-speech
Version:
VoicePing Cognitive Services Speech SDK for JavaScript forked from Microsoft
472 lines (470 loc) • 32 kB
JavaScript
"use strict";
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license.
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
var Exports_1 = require("../../common/Exports");
var Promise_1 = require("../../common/Promise");
var Exports_2 = require("../../sdk/Exports");
var Exports_3 = require("../Exports");
var ConversationConnectionMessage_1 = require("./ConversationConnectionMessage");
var ConversationRequestSession_1 = require("./ConversationRequestSession");
var ConversationTranslatorEventArgs_1 = require("./ConversationTranslatorEventArgs");
var ConversationTranslatorInterfaces_1 = require("./ConversationTranslatorInterfaces");
var Exports_4 = require("./ServiceMessages/Exports");
/***
* The service adapter handles sending and receiving messages to the Conversation Translator websocket.
*/
var ConversationServiceAdapter = /** @class */ (function (_super) {
__extends(ConversationServiceAdapter, _super);
function ConversationServiceAdapter(authentication, connectionFactory, audioSource, recognizerConfig, conversationServiceConnector) {
var _this = _super.call(this, authentication, connectionFactory, audioSource, recognizerConfig, conversationServiceConnector) || this;
_this.privLastPartialUtteranceId = "";
_this.sendMessageAsync = function (message) {
var sink = new Promise_1.Sink();
_this.fetchConversationConnection().continueWith(function (antecedent) {
try {
if (antecedent.isError) {
sink.reject(antecedent.error);
}
else {
antecedent.result.send(new ConversationConnectionMessage_1.ConversationConnectionMessage(Exports_1.MessageType.Text, message))
.continueWith(function (innerAntecedent) {
try {
if (innerAntecedent.isError) {
sink.reject(innerAntecedent.error);
}
else {
sink.resolve(innerAntecedent.result);
}
}
catch (e) {
sink.reject("Unhandled inner error: " + e);
}
});
}
}
catch (e) {
sink.reject("Unhandled error: " + e);
}
});
return new Exports_1.Promise(sink);
};
_this.noOp = function () {
// operation not supported
};
/**
* Process incoming websocket messages
*/
_this.receiveConversationMessageOverride = function (successCallback, errorCallBack) {
// we won't rely on the cascading promises of the connection since we want to continually be available to receive messages
var communicationCustodian = new Exports_1.Deferred();
_this.fetchConversationConnection().on(function (connection) {
return connection.read().onSuccessContinueWithPromise(function (message) {
var isDisposed = _this.isDisposed();
var terminateMessageLoop = (!_this.isDisposed() && _this.terminateMessageLoop);
var sessionId = _this.privConversationRequestSession.sessionId;
var sendFinal = false;
if (isDisposed || terminateMessageLoop) {
// We're done.
communicationCustodian.resolve(undefined);
return Exports_1.PromiseHelper.fromResult(undefined);
}
if (!message) {
return _this.receiveConversationMessageOverride();
}
try {
switch (message.conversationMessageType.toLowerCase()) {
case "info":
case "participant_command":
case "command":
var commandPayload = Exports_4.CommandResponsePayload.fromJSON(message.textBody);
switch (commandPayload.command.toLowerCase()) {
/**
* 'ParticpantList' is the first message sent to the user after the websocket connection has opened.
* The consuming client must wait for this message to arrive
* before starting to send their own data.
*/
case "participantlist":
var participantsPayload = Exports_4.ParticipantsListPayloadResponse.fromJSON(message.textBody);
var participantsResult = participantsPayload.participants.map(function (p) {
var participant = {
avatar: p.avatar,
displayName: p.nickname,
id: p.participantId,
isHost: p.ishost,
isMuted: p.ismuted,
isUsingTts: p.usetts,
preferredLanguage: p.locale
};
return participant;
});
if (!!_this.privConversationServiceConnector.participantsListReceived) {
_this.privConversationServiceConnector.participantsListReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantsListEventArgs(participantsPayload.roomid, participantsPayload.token, participantsPayload.translateTo, participantsPayload.profanityFilter, participantsPayload.roomProfanityFilter, participantsPayload.roomLocked, participantsPayload.muteAll, participantsResult, sessionId));
}
break;
/**
* 'SetTranslateToLanguages' represents the list of languages being used in the Conversation by all users(?).
* This is sent at the start of the Conversation
*/
case "settranslatetolanguages":
if (!!_this.privConversationServiceConnector.participantUpdateCommandReceived) {
_this.privConversationServiceConnector.participantUpdateCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantAttributeEventArgs(commandPayload.participantId, ConversationTranslatorInterfaces_1.ConversationTranslatorCommandTypes.setTranslateToLanguages, commandPayload.value, sessionId));
}
break;
/**
* 'SetProfanityFiltering' lets the client set the level of profanity filtering.
* If sent by the participant the setting will effect only their own profanity level.
* If sent by the host, the setting will effect all participants including the host.
* Note: the profanity filters differ from Speech Service (?): 'marked', 'raw', 'removed', 'tagged'
*/
case "setprofanityfiltering":
if (!!_this.privConversationServiceConnector.participantUpdateCommandReceived) {
_this.privConversationServiceConnector.participantUpdateCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantAttributeEventArgs(commandPayload.participantId, ConversationTranslatorInterfaces_1.ConversationTranslatorCommandTypes.setProfanityFiltering, commandPayload.value, sessionId));
}
break;
/**
* 'SetMute' is sent if the participant has been muted by the host.
* Check the 'participantId' to determine if the current user has been muted.
*/
case "setmute":
if (!!_this.privConversationServiceConnector.participantUpdateCommandReceived) {
_this.privConversationServiceConnector.participantUpdateCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantAttributeEventArgs(commandPayload.participantId, ConversationTranslatorInterfaces_1.ConversationTranslatorCommandTypes.setMute, commandPayload.value, sessionId));
}
break;
/**
* 'SetMuteAll' is sent if the Conversation has been muted by the host.
*/
case "setmuteall":
if (!!_this.privConversationServiceConnector.muteAllCommandReceived) {
_this.privConversationServiceConnector.muteAllCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.MuteAllEventArgs(commandPayload.value, sessionId));
}
break;
/**
* 'RoomExpirationWarning' is sent towards the end of the Conversation session to give a timeout warning.
*/
case "roomexpirationwarning":
if (!!_this.privConversationServiceConnector.conversationExpiration) {
_this.privConversationServiceConnector.conversationExpiration(_this.privConversationServiceConnector, new Exports_2.ConversationExpirationEventArgs(commandPayload.value, _this.privConversationRequestSession.sessionId));
}
break;
/**
* 'SetUseTts' is sent as a confirmation if the user requests TTS to be turned on or off.
*/
case "setusetts":
if (!!_this.privConversationServiceConnector.participantUpdateCommandReceived) {
_this.privConversationServiceConnector.participantUpdateCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantAttributeEventArgs(commandPayload.participantId, ConversationTranslatorInterfaces_1.ConversationTranslatorCommandTypes.setUseTTS, commandPayload.value, sessionId));
}
break;
/**
* 'SetLockState' is set if the host has locked or unlocked the Conversation.
*/
case "setlockstate":
if (!!_this.privConversationServiceConnector.lockRoomCommandReceived) {
_this.privConversationServiceConnector.lockRoomCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.LockRoomEventArgs(commandPayload.value, sessionId));
}
break;
/**
* 'ChangeNickname' is received if a user changes their display name.
* Any cached particpiants list should be updated to reflect the display name.
*/
case "changenickname":
if (!!_this.privConversationServiceConnector.participantUpdateCommandReceived) {
_this.privConversationServiceConnector.participantUpdateCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantAttributeEventArgs(commandPayload.participantId, ConversationTranslatorInterfaces_1.ConversationTranslatorCommandTypes.changeNickname, commandPayload.nickname, sessionId));
}
break;
/**
* 'JoinSession' is sent when a user joins the Conversation.
*/
case "joinsession":
var joinParticipantPayload = Exports_4.ParticipantPayloadResponse.fromJSON(message.textBody);
var joiningParticipant = {
avatar: joinParticipantPayload.avatar,
displayName: joinParticipantPayload.nickname,
id: joinParticipantPayload.participantId,
isHost: joinParticipantPayload.ishost,
isMuted: joinParticipantPayload.ismuted,
isUsingTts: joinParticipantPayload.usetts,
preferredLanguage: joinParticipantPayload.locale,
};
if (!!_this.privConversationServiceConnector.participantJoinCommandReceived) {
_this.privConversationServiceConnector.participantJoinCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantEventArgs(joiningParticipant, sessionId));
}
break;
/**
* 'LeaveSession' is sent when a user leaves the Conversation'.
*/
case "leavesession":
var leavingParticipant = {
id: commandPayload.participantId
};
if (!!_this.privConversationServiceConnector.participantLeaveCommandReceived) {
_this.privConversationServiceConnector.participantLeaveCommandReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ParticipantEventArgs(leavingParticipant, sessionId));
}
break;
/**
* 'DisconnectSession' is sent when a user is disconnected from the session (e.g. network problem).
* Check the 'ParticipantId' to check whether the message is for the current user.
*/
case "disconnectsession":
var disconnectParticipant = {
id: commandPayload.participantId
};
break;
/**
* Message not recognized.
*/
default:
break;
}
break;
/**
* 'partial' (or 'hypothesis') represents a unfinalized speech message.
*/
case "partial":
/**
* 'final' (or 'phrase') represents a finalized speech message.
*/
case "final":
var speechPayload = Exports_4.SpeechResponsePayload.fromJSON(message.textBody);
var speechResult = new Exports_2.ConversationTranslationResult(speechPayload.participantId, _this.getTranslations(speechPayload.translations), speechPayload.language, undefined, undefined, speechPayload.recognition, undefined, undefined, message.textBody, undefined);
if (speechPayload.isFinal) {
// check the length, sometimes empty finals are returned
if (speechResult.text !== undefined && speechResult.text.length > 0) {
sendFinal = true;
}
else if (speechPayload.id === _this.privLastPartialUtteranceId) {
// send final as normal. We had a non-empty partial for this same utterance
// so sending the empty final is important
sendFinal = true;
}
else {
// suppress unneeded final
}
if (sendFinal) {
if (!!_this.privConversationServiceConnector.translationReceived) {
_this.privConversationServiceConnector.translationReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ConversationReceivedTranslationEventArgs(ConversationTranslatorInterfaces_1.ConversationTranslatorMessageTypes.final, speechResult, sessionId));
}
}
}
else if (speechResult.text !== undefined) {
_this.privLastPartialUtteranceId = speechPayload.id;
if (!!_this.privConversationServiceConnector.translationReceived) {
_this.privConversationServiceConnector.translationReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ConversationReceivedTranslationEventArgs(ConversationTranslatorInterfaces_1.ConversationTranslatorMessageTypes.partial, speechResult, sessionId));
}
}
break;
/**
* "translated_message" is a text message or instant message (IM).
*/
case "translated_message":
var textPayload = Exports_4.TextResponsePayload.fromJSON(message.textBody);
var textResult = new Exports_2.ConversationTranslationResult(textPayload.participantId, _this.getTranslations(textPayload.translations), textPayload.language, undefined, undefined, textPayload.originalText, undefined, undefined, undefined, message.textBody, undefined);
if (!!_this.privConversationServiceConnector.translationReceived) {
_this.privConversationServiceConnector.translationReceived(_this.privConversationServiceConnector, new ConversationTranslatorEventArgs_1.ConversationReceivedTranslationEventArgs(ConversationTranslatorInterfaces_1.ConversationTranslatorMessageTypes.instantMessage, textResult, sessionId));
}
break;
default:
// ignore any unsupported message types
break;
}
}
catch (e) {
// continue
}
return _this.receiveConversationMessageOverride();
});
}, function (error) {
_this.terminateMessageLoop = true;
});
return communicationCustodian.promise();
};
_this.fetchConversationConnection = function () {
return _this.configConnection();
};
_this.privConversationServiceConnector = conversationServiceConnector;
_this.privConversationAuthentication = authentication;
_this.receiveMessageOverride = _this.receiveConversationMessageOverride;
_this.recognizeOverride = _this.noOp;
_this.connectImplOverride = _this.conversationConnectImpl;
_this.configConnectionOverride = _this.configConnection;
_this.fetchConnectionOverride = _this.fetchConversationConnection;
_this.disconnectOverride = _this.privDisconnect;
_this.privConversationRequestSession = new ConversationRequestSession_1.ConversationRequestSession(Exports_1.createNoDashGuid());
_this.privConversationConnectionFactory = connectionFactory;
_this.privConversationIsDisposed = false;
return _this;
}
ConversationServiceAdapter.prototype.isDisposed = function () {
return this.privConversationIsDisposed;
};
ConversationServiceAdapter.prototype.dispose = function (reason) {
this.privConversationIsDisposed = true;
if (this.privConnectionConfigPromise) {
this.privConnectionConfigPromise.onSuccessContinueWith(function (connection) {
connection.dispose(reason);
});
}
};
ConversationServiceAdapter.prototype.sendMessage = function (message) {
this.fetchConversationConnection().onSuccessContinueWith(function (connection) {
connection.send(new ConversationConnectionMessage_1.ConversationConnectionMessage(Exports_1.MessageType.Text, message));
});
};
ConversationServiceAdapter.prototype.privDisconnect = function () {
if (this.terminateMessageLoop) {
return;
}
this.cancelRecognition(this.privConversationRequestSession.sessionId, this.privConversationRequestSession.requestId, Exports_2.CancellationReason.Error, Exports_2.CancellationErrorCode.NoError, "Disconnecting");
this.terminateMessageLoop = true;
if (this.privConversationConnectionPromise.result().isCompleted) {
if (!this.privConversationConnectionPromise.result().isError) {
this.privConversationConnectionPromise.result().result.dispose();
this.privConversationConnectionPromise = null;
}
}
else {
this.privConversationConnectionPromise.onSuccessContinueWith(function (connection) {
connection.dispose();
});
}
};
ConversationServiceAdapter.prototype.processTypeSpecificMessages = function (connectionMessage, successCallback, errorCallBack) {
return true;
};
// Cancels recognition.
ConversationServiceAdapter.prototype.cancelRecognition = function (sessionId, requestId, cancellationReason, errorCode, error) {
this.terminateMessageLoop = true;
var cancelEvent = new Exports_2.ConversationTranslationCanceledEventArgs(cancellationReason, error, errorCode, undefined, sessionId);
try {
if (!!this.privConversationServiceConnector.canceled) {
this.privConversationServiceConnector.canceled(this.privConversationServiceConnector, cancelEvent);
}
}
catch (_a) {
// continue on error
}
};
/**
* Establishes a websocket connection to the end point.
* @param isUnAuthorized
*/
ConversationServiceAdapter.prototype.conversationConnectImpl = function (isUnAuthorized) {
var _this = this;
if (isUnAuthorized === void 0) { isUnAuthorized = false; }
if (this.privConversationConnectionPromise) {
if (this.privConversationConnectionPromise.result().isCompleted &&
(this.privConversationConnectionPromise.result().isError
|| this.privConversationConnectionPromise.result().result.state() === Exports_1.ConnectionState.Disconnected)) {
this.privConnectionId = null;
this.privConversationConnectionPromise = null;
this.terminateMessageLoop = true;
return this.conversationConnectImpl();
}
else {
return this.privConversationConnectionPromise;
}
}
this.privConversationAuthFetchEventId = Exports_1.createNoDashGuid();
// keep the connectionId for reconnect events
if (this.privConnectionId === undefined) {
this.privConnectionId = Exports_1.createNoDashGuid();
}
this.privConversationRequestSession.onPreConnectionStart(this.privConversationAuthFetchEventId, this.privConnectionId);
var authPromise = isUnAuthorized ? this.privConversationAuthentication.fetchOnExpiry(this.privConversationAuthFetchEventId) : this.privConversationAuthentication.fetch(this.privConversationAuthFetchEventId);
this.privConversationConnectionPromise = authPromise
.continueWithPromise(function (result) {
if (result.isError) {
_this.privConversationRequestSession.onAuthCompleted(true, result.error);
throw new Error(result.error);
}
else {
_this.privConversationRequestSession.onAuthCompleted(false);
}
var connection = _this.privConversationConnectionFactory.create(_this.privRecognizerConfig, result.result, _this.privConnectionId);
// Attach to the underlying event. No need to hold onto the detach pointers as in the event the connection goes away,
// it'll stop sending events.
connection.events.attach(function (event) {
_this.connectionEvents.onEvent(event);
});
return connection.open().onSuccessContinueWithPromise(function (response) {
if (response.statusCode === 200) {
_this.privConversationRequestSession.onPreConnectionStart(_this.privConversationAuthFetchEventId, _this.privConnectionId);
_this.privConversationRequestSession.onConnectionEstablishCompleted(response.statusCode);
var sessionStartEventArgs = new Exports_2.SessionEventArgs(_this.privConversationRequestSession.sessionId);
if (!!_this.privConversationServiceConnector.connectionOpened) {
_this.privConversationServiceConnector.connectionOpened(_this.privConversationServiceConnector, sessionStartEventArgs);
}
return Exports_1.PromiseHelper.fromResult(connection);
}
else if (response.statusCode === 403 && !isUnAuthorized) {
return _this.conversationConnectImpl(true);
}
else {
_this.privConversationRequestSession.onConnectionEstablishCompleted(response.statusCode, response.reason);
return Exports_1.PromiseHelper.fromError("Unable to contact server. StatusCode: " + response.statusCode + ", " + _this.privRecognizerConfig.parameters.getProperty(Exports_2.PropertyId.SpeechServiceConnection_Endpoint) + " Reason: " + response.reason);
}
});
});
this.privConnectionLoop = this.startMessageLoop();
return this.privConversationConnectionPromise;
};
ConversationServiceAdapter.prototype.startMessageLoop = function () {
var _this = this;
this.terminateMessageLoop = false;
var messageRetrievalPromise = this.receiveConversationMessageOverride();
return messageRetrievalPromise.on(function (r) {
return true;
}, function (error) {
_this.cancelRecognition(_this.privRequestSession ? _this.privRequestSession.sessionId : "", _this.privRequestSession ? _this.privRequestSession.requestId : "", Exports_2.CancellationReason.Error, Exports_2.CancellationErrorCode.RuntimeError, error);
});
};
// Takes an established websocket connection to the endpoint
ConversationServiceAdapter.prototype.configConnection = function () {
if (this.privConnectionConfigPromise) {
if (this.privConnectionConfigPromise.result().isCompleted &&
(this.privConnectionConfigPromise.result().isError
|| this.privConnectionConfigPromise.result().result.state() === Exports_1.ConnectionState.Disconnected)) {
this.privConnectionConfigPromise = null;
return this.configConnection();
}
else {
return this.privConnectionConfigPromise;
}
}
if (this.terminateMessageLoop) {
return Exports_1.PromiseHelper.fromResult(undefined);
}
this.privConnectionConfigPromise = this.conversationConnectImpl()
.onSuccessContinueWith(function (connection) {
return connection;
});
return this.privConnectionConfigPromise;
};
ConversationServiceAdapter.prototype.getTranslations = function (serviceResultTranslations) {
var translations;
if (undefined !== serviceResultTranslations) {
translations = new Exports_2.Translations();
for (var _i = 0, serviceResultTranslations_1 = serviceResultTranslations; _i < serviceResultTranslations_1.length; _i++) {
var translation = serviceResultTranslations_1[_i];
translations.set(translation.lang, translation.translation);
}
}
return translations;
};
return ConversationServiceAdapter;
}(Exports_3.ServiceRecognizerBase));
exports.ConversationServiceAdapter = ConversationServiceAdapter;
//# sourceMappingURL=ConversationServiceAdapter.js.map