UNPKG

@inworld/web-core

Version:
269 lines (268 loc) 11.5 kB
import { ControlEventAction, CustomEventType, DataChunkDataType, } from '../../../proto/ai/inworld/packets/packets.pb.js'; import { InworlControlAction, InworldPacketType, InworldTextPacketType, MicrophoneMode, } from '../../common/data_structures/index.js'; import { Character } from '../character.entity.js'; import { ItemOperation } from '../entities/item_operation.js'; import { AudioEvent } from './audio.entity.js'; import { CancelResponsesEvent } from './cancel_responses.entity.js'; import { ControlEvent } from './control.entity.js'; import { EmotionEvent } from './emotion/emotion.entity.js'; import { LatencyReportEvent } from './latency/latency_report.entity.js'; import { LogsEvent } from './log.entity.js'; import { NarratedAction } from './narrated_action.entity.js'; import { OperationStatusEvent } from './operation_status.entity.js'; import { PacketId } from './packet_id.entity.js'; import { Routing } from './routing.entity.js'; import { SilenceEvent } from './silence.entity.js'; import { TaskEvent } from './task.entity.js'; import { TextEvent } from './text.entity.js'; import { TriggerEvent } from './trigger.entity.js'; export class InworldPacket { constructor(props) { this.type = InworldPacketType.UNKNOWN; this.packetId = props.packetId; this.routing = props.routing; this.date = props.date; this.type = props.type; if (this.isText()) { this.text = props.text; } if (this.isAudio()) { this.audio = props.audio; } if (this.isControl()) { this.control = props.control; } if (this.isEmotion()) { this.emotions = props.emotions; } if (this.isLog()) { this.log = props.log; } if (this.isTask()) { this.task = props.task; } if (this.isTrigger()) { this.trigger = props.trigger; } if (this.isSilence()) { this.silence = props.silence; } if (this.isCancelResponse()) { this.cancelResponses = props.cancelResponses; } if (this.isNarratedAction()) { this.narratedAction = props.narratedAction; } if (this.isSceneMutationResponse() || this.isSceneMutationRequest()) { this.sceneMutation = props.sceneMutation; } if (this.isEntitiesItemOperation()) { this.entitiesItemsOperation = props.entitiesItemsOperation; } if (this.isOperationStatus()) { this.operationStatus = props.operationStatus; } if (this.isLatencyReport()) { this.latencyReport = props.latencyReport; } } isText() { return this.type === InworldPacketType.TEXT; } isAudio() { return this.type === InworldPacketType.AUDIO; } isControl() { return this.type === InworldPacketType.CONTROL; } isTask() { return this.type === InworldPacketType.TASK; } isTrigger() { return this.type === InworldPacketType.TRIGGER; } isEmotion() { return this.type === InworldPacketType.EMOTION; } isLog() { return this.type === InworldPacketType.LOG; } isInteractionEnd() { return (this.isControl() && this.control.action === InworlControlAction.INTERACTION_END); } isTTSPlaybackMute() { return (this.isControl() && this.control.action === InworlControlAction.TTS_PLAYBACK_MUTE); } isTTSPlaybackUnmute() { return (this.isControl() && this.control.action === InworlControlAction.TTS_PLAYBACK_UNMUTE); } isWarning() { return (this.isControl() && this.control.action === InworlControlAction.WARNING); } isSilence() { return this.type === InworldPacketType.SILENCE; } isCancelResponse() { return this.type === InworldPacketType.CANCEL_RESPONSE; } isNarratedAction() { return this.type === InworldPacketType.NARRATED_ACTION; } isSceneMutationRequest() { return this.type === InworldPacketType.SCENE_MUTATION_REQUEST; } isSceneMutationResponse() { return this.type === InworldPacketType.SCENE_MUTATION_RESPONSE; } isEntitiesItemOperation() { return this.type === InworldPacketType.ENTITIES_ITEM_OPERATION; } isOperationStatus() { return this.type === InworldPacketType.OPERATION_STATUS; } isLatencyReport() { return this.type === InworldPacketType.LATENCY_REPORT; } isPingPongReport() { return this.isLatencyReport() && !!this.latencyReport.pingPong; } isPerceivedLatencyReport() { return this.isLatencyReport() && !!this.latencyReport.perceivedLatency; } isSpeechRecognitionResult() { return (this.isText() && this.routing.source.isPlayer && this.text.final && this.text.type === InworldTextPacketType.SPEECH_TO_TEXT); } isPlayerTypeInText() { return (this.isText() && this.routing.source.isPlayer && this.text.type === InworldTextPacketType.TYPED_IN); } isNonSpeechPacket() { return this.isTrigger() || this.isNarratedAction(); } isPushToTalkAudioSessionStart() { return (this.isControl() && this.control.action === InworlControlAction.AUDIO_SESSION_START && this.control.audioSessionStart.mode === MicrophoneMode.EXPECT_AUDIO_END); } isAudioSessionEnd() { return (this.isControl() && this.control.action === InworlControlAction.AUDIO_SESSION_END); } shouldHaveConversationId() { return (this.isAudio() || this.isText() || this.isTrigger() || this.isTask() || this.isNarratedAction() || this.isSilence()); } static fromProto(proto) { var _a, _b, _c, _d; const type = this.getType(proto); return new InworldPacket(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ type, date: proto.timestamp, packetId: PacketId.fromProto(proto.packetId), routing: Routing.fromProto(proto.routing) }, (type === InworldPacketType.TRIGGER && { trigger: TriggerEvent.fromProto(proto.custom), })), (type === InworldPacketType.TASK && { task: TaskEvent.fromProto(proto.custom), })), (type === InworldPacketType.TEXT && { text: TextEvent.fromProto(proto.text), })), (type === InworldPacketType.AUDIO && { audio: AudioEvent.fromProto(proto.dataChunk), })), (type === InworldPacketType.CONTROL && { control: ControlEvent.fromProto(proto.control), })), (type === InworldPacketType.SILENCE && { silence: SilenceEvent.fromProto(proto.dataChunk), })), (type === InworldPacketType.EMOTION && { emotions: EmotionEvent.fromProto(proto.emotion), })), (type === InworldPacketType.LATENCY_REPORT && { latencyReport: LatencyReportEvent.fromProto(proto.latencyReport), })), (type === InworldPacketType.LOG && { log: LogsEvent.fromProto(proto.log), })), (type === InworldPacketType.CANCEL_RESPONSE && { cancelResponses: CancelResponsesEvent.fromProto(proto.mutation), })), (type === InworldPacketType.NARRATED_ACTION && { narratedAction: NarratedAction.fromProto(proto.action), })), (type === InworldPacketType.ENTITIES_ITEM_OPERATION && { entitiesItemsOperation: ItemOperation.fromProto(proto.entitiesItemsOperation), })), (type === InworldPacketType.OPERATION_STATUS && { operationStatus: OperationStatusEvent.fromProto(proto.operationStatus), })), ([ InworldPacketType.SCENE_MUTATION_REQUEST, InworldPacketType.SCENE_MUTATION_RESPONSE, ].includes(type) && { sceneMutation: Object.assign(Object.assign(Object.assign(Object.assign({}, (((_a = proto.mutation) === null || _a === void 0 ? void 0 : _a.loadScene) && { name: proto.mutation.loadScene.name, })), (((_b = proto.mutation) === null || _b === void 0 ? void 0 : _b.loadCharacters) && { addedCharacterNames: proto.mutation.loadCharacters.name.map((c) => c.name), })), (((_c = proto.mutation) === null || _c === void 0 ? void 0 : _c.unloadCharacters) && { removedCharacterIds: proto.mutation.unloadCharacters.agents.map((c) => c.agentId), })), (((_d = proto.control) === null || _d === void 0 ? void 0 : _d.currentSceneStatus) && { name: proto.control.currentSceneStatus.sceneName, description: proto.control.currentSceneStatus.sceneDescription, displayName: proto.control.currentSceneStatus.sceneDisplayName, loadedCharacters: proto.control.currentSceneStatus.agents.map((agent) => Character.fromProto(agent)), })), }))); } static getType(packet) { var _a, _b, _c, _d, _e, _f; if (((_a = packet.mutation) === null || _a === void 0 ? void 0 : _a.loadScene) || ((_b = packet.mutation) === null || _b === void 0 ? void 0 : _b.loadCharacters) || ((_c = packet.mutation) === null || _c === void 0 ? void 0 : _c.unloadCharacters)) { return InworldPacketType.SCENE_MUTATION_REQUEST; } else if (((_d = packet.control) === null || _d === void 0 ? void 0 : _d.action) === ControlEventAction.CURRENT_SCENE_STATUS) { return InworldPacketType.SCENE_MUTATION_RESPONSE; } else if (packet.text) { return InworldPacketType.TEXT; } else if (packet.dataChunk && packet.dataChunk.type === DataChunkDataType.AUDIO) { return InworldPacketType.AUDIO; } else if (packet.dataChunk && packet.dataChunk.type === DataChunkDataType.SILENCE) { return InworldPacketType.SILENCE; } else if (packet.latencyReport) { return InworldPacketType.LATENCY_REPORT; } else if (packet.custom && packet.custom.type === CustomEventType.TASK) { return InworldPacketType.TASK; } else if (packet.custom) { return InworldPacketType.TRIGGER; } else if (packet.control) { return InworldPacketType.CONTROL; } else if (packet.emotion) { return InworldPacketType.EMOTION; } else if (packet.log) { return InworldPacketType.LOG; } else if ((_e = packet.mutation) === null || _e === void 0 ? void 0 : _e.cancelResponses) { return InworldPacketType.CANCEL_RESPONSE; } else if ((_f = packet.action) === null || _f === void 0 ? void 0 : _f.narratedAction) { return InworldPacketType.NARRATED_ACTION; } else if (packet.entitiesItemsOperation) { return InworldPacketType.ENTITIES_ITEM_OPERATION; } else if (packet.operationStatus) { return InworldPacketType.OPERATION_STATUS; } else { return InworldPacketType.UNKNOWN; } } }