UNPKG

homebridge-nest-accfactory

Version:

Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras

1,459 lines (1,215 loc) 50.5 kB
// NexusTalk // Part of homebridge-nest-accfactory // // Protocol-specific StreamTransport for Nest legacy NexusTalk streams. // // Manages bidirectional media transport over secure TLS connections using // Nest's proprietary protobuf-based NexusTalk protocol. // // NexusTalk owns: // - TLS connection/session lifecycle // - Nexus authentication and authorisation // - Protobuf message framing and parsing // - Playback session control // - H264 video access-unit assembly // - Direct AAC audio frame emission // - Speex talkback/audio send handling // - Redirect, reconnect, stall, host-change, and backend-closure recovery // - Queued outbound control messages while socket/auth state is unavailable // - Transport-level runtime updates for auth, host, and device availability // // NexusTalk receives normalised transport options from the parent camera device // and emits complete media frames plus transport state transitions to the // configured media consumer. It does not own device raw-data processing, // buffering, output pacing, HomeKit live/record sessions, fallback frames, // or ffmpeg process handling. // // Features: // - TLS-encrypted Nexus backend connection management // - Shared protobuf schema/type caching via protobuf.js helper module // - Protobuf message serialization for proprietary NexusTalk protocol // - Multiplexed media and control messages over a single connection // - Buffered outbound control-message queue using RingBuffer // - H264 video access-unit assembly before media emission // - Direct AAC audio frame emission with Nexus-derived media timing // - Two-way audio/talkback support via Speex // - Buffered packet parsing with bounded memory protection // // Notes: // - Video is delivered as H264 NAL units and assembled into complete Annex-B access units before emission // - Audio is delivered as AAC frames and emitted directly // - NexusTalk media is delivered over ordered TLS, so playback packets are not RTP-jitter buffered // - Emitted timestamps are source media timeline values; Streamer owns output playout timing // - Protobuf schemas and message types are shared/cached globally to avoid repeated protobufjs parsing across multiple camera instances // // Note: Based on foundational work from https://github.com/Brandawg93/homebridge-nest-cam // // Code version 2026.05.18 // Mark Hulskamp 'use strict'; // Define nodejs module requirements import { Buffer } from 'node:buffer'; import { setInterval, clearInterval } from 'node:timers'; import path from 'node:path'; import tls from 'tls'; import crypto from 'crypto'; // Define our modules import Streamer from './streamer.js'; import StreamTransport from './streamtransport.js'; import RingBuffer from './ringbuffer.js'; import H264 from './h264.js'; import { getProtoType } from './protobuf.js'; // Define constants import { USER_AGENT, __dirname } from './consts.js'; const PING_INTERVAL = 15000; // Ping interval to Nexus server while stream active const STALLED_TIMEOUT = 10000; // Time with no playback packets before stream is considered stalled const PENDING_MESSAGE_QUEUE_CAPACITY = 64; // Initial slot count for pending outbound control messages const MAX_PENDING_MESSAGES = 256; // Hard cap for queued outbound control messages while unauthorised/disconnected const INITIAL_PACKET_BUFFER_SIZE = 256 * 1024; const MAX_PACKET_BUFFER_SIZE = 10 * 1024 * 1024; const MAX_PACKET_PAYLOAD_SIZE = 5 * 1024 * 1024; const MEDIA_TYPE = { PING: 1, HELLO: 100, PING_CAMERA: 101, AUDIO_PAYLOAD: 102, START_PLAYBACK: 103, STOP_PLAYBACK: 104, CLOCK_SYNC_ECHO: 105, LATENCY_MEASURE: 106, TALKBACK_LATENCY: 107, METADATA_REQUEST: 108, OK: 200, ERROR: 201, PLAYBACK_BEGIN: 202, PLAYBACK_END: 203, PLAYBACK_PACKET: 204, LONG_PLAYBACK_PACKET: 205, CLOCK_SYNC: 206, REDIRECT: 207, TALKBACK_BEGIN: 208, TALKBACK_END: 209, METADATA: 210, METADATA_ERROR: 211, AUTHORIZE_REQUEST: 212, }; // Blank audio in AAC format, mono channel @48000 const AAC_MONO_48000_BLANK = Buffer.from([ 0xff, 0xf1, 0x4c, 0x40, 0x03, 0x9f, 0xfc, 0xde, 0x02, 0x00, 0x4c, 0x61, 0x76, 0x63, 0x35, 0x39, 0x2e, 0x31, 0x38, 0x2e, 0x31, 0x30, 0x30, 0x00, 0x02, 0x30, 0x40, 0x0e, ]); const MAX_PENDING_VIDEO_PARTS = 200; const MAX_PENDING_VIDEO_BYTES = 4 * 1024 * 1024; // NexusTalk object export default class NexusTalk extends StreamTransport { nexustalk_host = undefined; // Main NexusTalk streaming host token = undefined; useGoogleAuth = false; // Nest vs Google auth // Internal protobuf message types for this transport #protobufTypes = { AudioPayload: undefined, StartPlayback: undefined, StopPlayback: undefined, AuthoriseRequest: undefined, Hello: undefined, Redirect: undefined, PlaybackBegin: undefined, PlaybackPacket: undefined, PlaybackEnd: undefined, Error: undefined, }; #socket = undefined; // TCP socket object #packetBuffer = undefined; // Incoming packet buffer #packetOffset = undefined; // Current write offset in packet buffer #packetReadIndex = 0; // Current read offset for packet parsing loop // Control messages may be queued before the TLS session is authorised. #messages = new RingBuffer(0, PENDING_MESSAGE_QUEUE_CAPACITY, PENDING_MESSAGE_QUEUE_CAPACITY * 4); #authorised = false; // Have we been authorised by Nexus #sessionId = undefined; // Active Nexus playback session ID #host = undefined; // Current host connected to #pingTimer = undefined; // Timer object for ping interval #stalledTimer = undefined; // Interval object for no received data checks #lastPacketAt = undefined; // Last playback packet receipt time in ms #reconnectPending = false; // Reconnect requested once socket closes #reconnectHost = undefined; // Host to reconnect to #sessionStartTime = undefined; // Shared session time anchor in ms for all playback channels #channels = { video: { id: undefined, startOffset: 0, mediaTime: undefined, pendingTimestamp: undefined, pendingKeyFrame: false, pendingParts: [], pendingBytes: 0, lastSPS: undefined, lastPPS: undefined, }, audio: { id: undefined, startOffset: 0, mediaTime: undefined, }, }; constructor(options = {}) { super(options); let protoPath = path.join(__dirname, 'protobuf/nest/nexustalk.proto'); // Cache protobuf message types up front so we avoid repeated lookup() // calls during connect, control messaging, and packet handling paths. // Types are shared globally via protobuf.js helper caching. this.#protobufTypes.AudioPayload = getProtoType(protoPath, 'nest.nexustalk.v1.AudioPayload', this.log); this.#protobufTypes.StartPlayback = getProtoType(protoPath, 'nest.nexustalk.v1.StartPlayback', this.log); this.#protobufTypes.StopPlayback = getProtoType(protoPath, 'nest.nexustalk.v1.StopPlayback', this.log); this.#protobufTypes.AuthoriseRequest = getProtoType(protoPath, 'nest.nexustalk.v1.AuthoriseRequest', this.log); this.#protobufTypes.Hello = getProtoType(protoPath, 'nest.nexustalk.v1.Hello', this.log); this.#protobufTypes.Redirect = getProtoType(protoPath, 'nest.nexustalk.v1.Redirect', this.log); this.#protobufTypes.PlaybackBegin = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackBegin', this.log); this.#protobufTypes.PlaybackPacket = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackPacket', this.log); this.#protobufTypes.PlaybackEnd = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackEnd', this.log); this.#protobufTypes.Error = getProtoType(protoPath, 'nest.nexustalk.v1.Error', this.log); // Setup Nexus-specific codec defaults; StreamTransport owns the shared media shape. this.video.codec = StreamTransport.CODEC_TYPE.H264; this.audio.codec = StreamTransport.CODEC_TYPE.AAC; this.audio.channels = 1; this.audio.blank = AAC_MONO_48000_BLANK; this.talkback.codec = StreamTransport.CODEC_TYPE.SPEEX; this.talkback.sampleRate = 16000; this.talkback.channels = 1; this.update(options); } // Class functions async doOpen(options = {}) { let connectHost = typeof options?.host === 'string' && options.host !== '' ? options.host : this.nexustalk_host; if (typeof connectHost !== 'string' || connectHost === '') { this.setState(StreamTransport.STATE.CLOSED, { reason: 'host-missing' }); return; } if (this.#socket !== undefined && this.#socket.destroyed === false) { // Existing socket is still opening/open, avoid duplicate concurrent connects. return; } clearInterval(this.#pingTimer); this.#stopStalledMonitor(); this.#pingTimer = undefined; this.#sessionId = undefined; // No session ID yet this.#authorised = false; this.#resetPacketState(true); this.#host = connectHost; // Update internal host name since we're about to connect this.setState(StreamTransport.STATE.CONNECTING, { host: connectHost }); // Wrap tls.connect() in a Promise so we can await the TLS handshake. try { await new Promise((resolve, reject) => { let socket = tls.connect({ host: connectHost, port: 1443 }, () => { if (this.#socket !== socket) { resolve(); return; } // Opened connection to Nexus server, so now need to authenticate ourselves. this.setState(StreamTransport.STATE.CONNECTED, { host: connectHost }); socket.setKeepAlive(true); // Keep socket connection alive this.#authenticate(false); // Send authentication request resolve(); // Allow await connect() to continue }); this.#socket = socket; socket.on('error', (error) => { if (this.#socket !== socket) { return; } // TLS error (could be refused, timeout, etc.). this?.log?.warn?.('TLS error on connect to "%s": %s', connectHost, String(error)); this.#authorised = false; // Since we had an error, we cannot be authorised reject(error); }); socket.on('end', () => { // Do nothing. The socket close handler owns terminal state/reconnect handling. }); socket.on('data', (data) => { if (this.#socket !== socket) { return; } this.#handleNexusData(data); }); socket.on('close', () => { let reconnectHost = undefined; if (this.#socket !== socket) { return; } clearInterval(this.#pingTimer); this.#stopStalledMonitor(); this.#pingTimer = undefined; this.#authorised = false; this.#socket = undefined; // Clear socket object this.#sessionId = undefined; // Not an active session anymore if (this.#reconnectPending !== true) { this.#resetChannelDetails(); } if (this.hasConsumers() === true && this.#reconnectPending !== true && this.closing !== true) { this.#requestReconnect(connectHost, 'service-close', { host: connectHost }); } if (this.#reconnectPending === true) { reconnectHost = this.#reconnectHost; this.#reconnectPending = false; this.#reconnectHost = undefined; this.#host = undefined; if (this.hasConsumers() === true && typeof reconnectHost === 'string' && reconnectHost !== '') { this.open({ host: reconnectHost }); return; } } this.#host = undefined; this.setState(StreamTransport.STATE.CLOSED, { reason: 'socket-close', host: connectHost, }); }); }); } catch (error) { this?.log?.error?.('Failed to connect to "%s": %s', connectHost, String(error)); if (this.#socket?.destroyed === true) { this.#socket = undefined; } if (this.hasConsumers() === true) { this.#requestReconnect(connectHost, 'connect-failed'); this.open({ host: this.#reconnectHost }); return; } this.setState(StreamTransport.STATE.CLOSED, { reason: 'connect-failed', host: connectHost, }); } } async doClose(stopStreamFirst = true) { let reconnecting = this.#reconnectPending === true; let hadSocket = this.#socket !== undefined; let closeHost = this.#host; // Close an authenticated socket stream gracefully. // Clear any running timers before closing socket to prevent race conditions. clearInterval(this.#pingTimer); this.#stopStalledMonitor(); this.#pingTimer = undefined; if (reconnecting !== true) { this.setState(StreamTransport.STATE.CLOSING, { host: closeHost }); // Flush the final pending video frame before resetting channel state. // This path is skipped during reconnect, otherwise the final frame from // the old session can repopulate Streamer timing state just before the // new session starts. this.#flushPendingVideo(this.#channels.video); } if (this.#socket !== undefined) { if (stopStreamFirst === true) { this.#stopNexusData(); } try { this.#socket.destroy(); } catch { // Empty } } if (reconnecting !== true) { // Do not emit CLOSED here when a socket exists. // The terminal closed state is owned by the socket 'close' handler. // For reconnect/redirect paths we keep the state as RECONNECTING // until the next connect attempt begins. this.#clearMessageQueue(0); if (hadSocket !== true) { this.setState(StreamTransport.STATE.CLOSED, { reason: 'closed', host: closeHost, }); } } // Always reset channel/session state. this.#resetChannelDetails(); this.#sessionId = undefined; // Not an active session anymore this.#resetPacketState(true); } async doUpdate(options = {}) { let newToken = undefined; let newHost = undefined; let newUuid = undefined; let newGoogleAuth = undefined; let hadToken = typeof this.token === 'string' && this.token !== ''; let hadHost = typeof this.nexustalk_host === 'string' && this.nexustalk_host !== ''; // Validate options object. if (typeof options !== 'object' || options === null) { return; } // Normalise updated values from transport options. // Undefined means "leave existing value unchanged". // // Streamer passes a shared transport update payload: // { // uuid, // host, // apiAccess // } // // NexusTalk derives: // - token from apiAccess.token // - Google auth mode from apiAccess.oauth2 presence newUuid = typeof options?.uuid === 'string' && options.uuid !== '' ? options.uuid : undefined; newHost = typeof options?.host === 'string' && options.host !== '' ? options.host : undefined; newToken = typeof options?.apiAccess?.token === 'string' && options.apiAccess.token !== '' ? options.apiAccess.token : undefined; newGoogleAuth = typeof options?.apiAccess?.oauth2 === 'string' && options.apiAccess.oauth2 !== ''; // Update device UUID if supplied. Used for logging, reconnects and transport identity. if (typeof newUuid === 'string') { this.uuid = newUuid; } // Update access token if changed. // Active sessions continue until reconnect; new requests use the updated token. if (typeof newToken === 'string' && newToken !== this.token) { if (hadToken === true && this.hasConsumers() === true) { this?.log?.debug?.('Access token has changed for uuid "%s" while NexusTalk session is active. Updating stored token.', this.uuid); } this.token = newToken; } // Update authentication mode. // Google OAuth availability determines whether Google auth is used. if (this.useGoogleAuth !== newGoogleAuth) { this.useGoogleAuth = newGoogleAuth; } // Host changes require reconnecting the active NexusTalk session. // Existing sockets cannot migrate between hosts safely. if (typeof newHost === 'string' && newHost !== this.nexustalk_host) { if (hadHost === true) { this?.log?.debug?.( 'NexusTalk host has changed for uuid "%s" from "%s" to "%s"%s', this.uuid, this.nexustalk_host, newHost, this.hasConsumers() === true ? '. Reconnecting active session.' : '', ); } this.nexustalk_host = newHost; // Force reconnect when consumers are active, avoiding duplicate reconnect requests. if (hadHost === true && this.hasConsumers() === true && this.#reconnectPending !== true) { this.#requestReconnect(newHost, 'host-update'); await this.close(); } } } doSendAudio(talkingBuffer) { let AudioPayload = undefined; let encodedData = undefined; if (Buffer.isBuffer(talkingBuffer) !== true || this.#sessionId === undefined) { return; } AudioPayload = this.#protobufTypes.AudioPayload; if (AudioPayload === undefined || AudioPayload === null) { return; } try { encodedData = AudioPayload.encode( AudioPayload.create({ payload: talkingBuffer, sessionId: this.#sessionId, codec: 0, // SPEEX CODEC ID in NexusTalk is 0 sampleRate: this.talkback.sampleRate, }), ).finish(); } catch (error) { this?.log?.debug?.('AudioPayload encode failed for uuid "%s": %s', this.uuid, String(error)); return; } this.#sendMessage(MEDIA_TYPE.AUDIO_PAYLOAD, encodedData); } #startNexusData() { // Setup streaming profiles. // Use the highest profile as the main profile, with lower profiles as fallback. let otherProfiles = ['VIDEO_H264_530KBIT_L31', 'VIDEO_H264_100KBIT_L30']; // Include AAC profile for audio. otherProfiles.push('AUDIO_AAC'); let StartPlayback = this.#protobufTypes.StartPlayback; if (StartPlayback === undefined || StartPlayback === null) { return; } let encodedData = null; try { encodedData = StartPlayback.encode( StartPlayback.fromObject({ sessionId: Math.floor(Math.random() * 1000000), // larger range to reduce collisions profile: 'VIDEO_H264_2MBIT_L40', otherProfiles: otherProfiles, profileNotFoundAction: 'REDIRECT', }), ).finish(); } catch (error) { this?.log?.debug?.('StartPlayback encode failed for uuid "%s": %s', this.uuid, String(error)); return; } this.#sendMessage(MEDIA_TYPE.START_PLAYBACK, encodedData); } #stopNexusData() { if (this.#sessionId === undefined) { return; } let StopPlayback = this.#protobufTypes.StopPlayback; if (StopPlayback === undefined || StopPlayback === null) { return; } let encodedData = null; try { encodedData = StopPlayback.encode( StopPlayback.fromObject({ sessionId: this.#sessionId, }), ).finish(); } catch (error) { this?.log?.debug?.('StopPlayback encode failed for uuid "%s": %s', this.uuid, String(error)); return; } this.#sendMessage(MEDIA_TYPE.STOP_PLAYBACK, encodedData); } #sendNow(type, data) { let header = undefined; // Raw socket send path for already-connected / already-authorised messages. // This bypasses queue logic and is used by both direct sends and queued flushes. if (Buffer.isBuffer(data) !== true || this.#canWrite() !== true) { return false; } if (type === MEDIA_TYPE.LONG_PLAYBACK_PACKET) { header = Buffer.alloc(5); header.writeUInt8(type, 0); header.writeUInt32BE(data.length, 1); } else { header = Buffer.alloc(3); header.writeUInt8(type, 0); header.writeUInt16BE(data.length, 1); } // Write composed message to NexusTalk without concatenating buffers. this.#socket.cork(); try { this.#socket.write(header); this.#socket.write(data); } finally { this.#socket.uncork(); } return true; } #queueMessage(type, data) { // Queue outbound control messages until socket/auth state is ready. // Oldest messages are dropped once the hard cap is reached so memory stays bounded. if (Buffer.isBuffer(data) !== true) { return; } if (this.#messages instanceof RingBuffer !== true) { this.#messages = new RingBuffer(0, PENDING_MESSAGE_QUEUE_CAPACITY); } if (this.#messages.size >= MAX_PENDING_MESSAGES) { this.#messages.shift(1); this?.log?.warn?.('Dropped oldest pending NexusTalk message for uuid "%s" as queue is full', this.uuid); } this.#messages.push({ type: type, data: data }); } #flushQueuedMessages() { let queuedMessage = undefined; // Drain queued outbound messages using the raw send path. // Do not call #sendMessage() from here or messages could be re-queued. if (this.#messages instanceof RingBuffer !== true) { return; } while (this.#messages.size > 0) { if (this.#canWrite(true) !== true) { break; } queuedMessage = this.#messages.getByOffset(0); if (typeof queuedMessage !== 'object' || queuedMessage === null) { this.#messages.shift(1); continue; } if (this.#sendNow(queuedMessage.type, queuedMessage.data) !== true) { break; } this.#messages.shift(1); } } #sendMessage(type, data) { if (Buffer.isBuffer(data) !== true) { return; } // Bootstrap/auth messages must be allowed before authorisation exists if (type === MEDIA_TYPE.HELLO || type === MEDIA_TYPE.AUTHORIZE_REQUEST) { if (this.#canWrite() === true) { this.#sendNow(type, data); } return; } // Normal messages require an authorised writable socket if (this.#canWrite(true) !== true) { this.#queueMessage(type, data); return; } this.#sendNow(type, data); } #authenticate(reauthorise) { this.#authorised = false; // We're no longer authorised let authoriseRequest = null; let AuthoriseRequest = this.#protobufTypes.AuthoriseRequest; if (AuthoriseRequest === undefined || AuthoriseRequest === null) { return; } try { authoriseRequest = AuthoriseRequest.encode( AuthoriseRequest.fromObject(this.useGoogleAuth === true ? { oliveToken: this.token } : { sessionToken: this.token }), ).finish(); } catch (error) { this?.log?.debug?.('AuthoriseRequest encode failed for uuid "%s": %s', this.uuid, String(error)); return; } if (reauthorise === true) { if (this.#canWrite() !== true) { return; } this?.log?.debug?.('Re-authentication requested to "%s"', this.#host); this.#sendMessage(MEDIA_TYPE.AUTHORIZE_REQUEST, authoriseRequest); return; } // This isn't a re-authorise request, so perform 'Hello' packet let Hello = this.#protobufTypes.Hello; if (Hello === undefined || Hello === null) { return; } let encodedData = null; try { encodedData = Hello.encode( Hello.fromObject({ protocolVersion: 'VERSION_3', uuid: typeof this.uuid === 'string' && this.uuid.includes('.') === true ? this.uuid.split(/[.]+/)[1] : this.uuid, requireConnectedCamera: false, userAgent: USER_AGENT, deviceId: crypto.randomUUID(), clientType: 'IOS', authoriseRequest: authoriseRequest, }), ).finish(); } catch (error) { this?.log?.debug?.('Hello encode failed for uuid "%s": %s', this.uuid, String(error)); return; } this.#sendMessage(MEDIA_TYPE.HELLO, encodedData); } #handleRedirect(payload) { let redirectToHost = undefined; if (Buffer.isBuffer(payload) === true && this.#protobufTypes.Redirect !== undefined && this.#protobufTypes.Redirect !== null) { let decodedMessage = undefined; try { decodedMessage = this.#protobufTypes.Redirect.decode(payload); } catch (error) { this?.log?.debug?.('Redirect packet decode failed for uuid "%s": %s', this.uuid, String(error)); return; } redirectToHost = decodedMessage?.newHost; } if (typeof redirectToHost !== 'string' || redirectToHost === '') { return; } this.#requestReconnect(redirectToHost, 'redirect', { fromHost: this.#host, toHost: redirectToHost }); this.close(); } #handlePlaybackBegin(payload) { let decodedMessage = undefined; let now = Date.now(); let videoStream = undefined; let audioStream = undefined; let sessionStart = undefined; let startDelta = 0; if ( Buffer.isBuffer(payload) !== true || this.#protobufTypes.PlaybackBegin === undefined || this.#protobufTypes.PlaybackBegin === null ) { return; } if (this.closing === true || this.closed === true) { // We received a PlaybackBegin message but we're already closing/closed // so ignore this message since we're not going to be able to do anything with it. return; } try { decodedMessage = this.#protobufTypes.PlaybackBegin.decode(payload).toJSON(); } catch (error) { this?.log?.debug?.('PlaybackBegin decode failed for uuid "%s": %s', this.uuid, String(error)); return; } // Reset current playback channel timing/state before applying new channel details. this.#resetChannelDetails(); if (Array.isArray(decodedMessage?.channels) === true) { videoStream = decodedMessage.channels.find((stream) => stream?.codec === this.video.codec.toUpperCase()); audioStream = decodedMessage.channels.find((stream) => stream?.codec === this.audio.codec.toUpperCase()); } if (videoStream === undefined && audioStream === undefined) { this?.log?.warn?.('PlaybackBegin contained no usable NexusTalk audio/video channels for uuid "%s"', this.uuid); this.close(); return; } // Use the earliest available stream start time as the shared session anchor. if (typeof videoStream?.startTime === 'number' && typeof audioStream?.startTime === 'number') { sessionStart = Math.min(videoStream.startTime, audioStream.startTime) * 1000; } if (typeof sessionStart !== 'number' && typeof videoStream?.startTime === 'number') { sessionStart = videoStream.startTime * 1000; } if (typeof sessionStart !== 'number' && typeof audioStream?.startTime === 'number') { sessionStart = audioStream.startTime * 1000; } if (typeof sessionStart !== 'number') { sessionStart = now; } this.#sessionStartTime = sessionStart; if (typeof videoStream === 'object' && videoStream !== null) { this.#channels.video.id = videoStream.channelId; // Store video stream details reported by NexusTalk. // FPS and resolution are learned dynamically during playback from // observed frame timing and SPS data respectively. this.video.profile = typeof videoStream.profile === 'string' ? videoStream.profile : undefined; this.video.clockRate = Number.isFinite(videoStream.sampleRate) === true && videoStream.sampleRate > 0 ? videoStream.sampleRate : undefined; this.video.bitrate = videoStream?.profile?.includes?.('2MBIT') === true ? 2000000 : videoStream?.profile?.includes?.('530KBIT') === true ? 530000 : videoStream?.profile?.includes?.('100KBIT') === true ? 100000 : undefined; // Audio/video channels may start slightly offset from each other. // Clamp offset so reconnect jitter or backend timing anomalies do not // produce unrealistic media timestamps. startDelta = typeof videoStream.startTime === 'number' ? videoStream.startTime * 1000 - sessionStart : 0; this.#channels.video.startOffset = Math.max(-250, Math.min(250, Math.round(startDelta))); this.#channels.video.mediaTime = sessionStart + this.#channels.video.startOffset; } if (typeof audioStream === 'object' && audioStream !== null) { this.#channels.audio.id = audioStream.channelId; // Store audio stream details reported by NexusTalk. this.audio.profile = typeof audioStream.profile === 'string' ? audioStream.profile : undefined; this.audio.sampleRate = Number.isFinite(audioStream.sampleRate) === true && audioStream.sampleRate > 0 ? audioStream.sampleRate : 48000; this.audio.frameDuration = Number.isFinite(this.audio.sampleRate) === true && this.audio.sampleRate > 0 ? (1024 / this.audio.sampleRate) * 1000 : undefined; this.audio.channels = Number.isFinite(audioStream.channels) === true && audioStream.channels > 0 ? audioStream.channels : 1; this.audio.bitrate = Number.isFinite(audioStream.bitrate) === true && audioStream.bitrate > 0 ? audioStream.bitrate : undefined; // Audio/video channels may start slightly offset from each other. // Clamp offset so reconnect jitter or backend timing anomalies do not // produce unrealistic media timestamps. startDelta = typeof audioStream.startTime === 'number' ? audioStream.startTime * 1000 - sessionStart : 0; this.#channels.audio.startOffset = Math.max(-250, Math.min(250, Math.round(startDelta))); this.#channels.audio.mediaTime = sessionStart + this.#channels.audio.startOffset; } // Store active playback session id for stop-playback and talkback messages. this.#sessionId = decodedMessage?.sessionId; this.#lastPacketAt = Date.now(); this.#startStalledMonitor(); } #handlePlaybackPacket(payload) { let decodedMessage = undefined; let Type = this.#protobufTypes.PlaybackPacket; let video = undefined; let audio = undefined; let timestamp = 0; let data = undefined; let keyFrame = false; let nalUnits = undefined; let now = Date.now(); if (this.closing === true || this.closed === true) { // We received a PlaybackPacket message but we're already closing/closed, // so ignore it. return; } if ( Buffer.isBuffer(payload) !== true || this.#protobufTypes.PlaybackPacket === undefined || this.#protobufTypes.PlaybackPacket === null ) { return; } try { decodedMessage = Type.decode(payload); } catch (error) { this?.log?.debug?.('Playback packet decode failed for uuid "%s": %s', this.uuid, String(error)); return; } // Update the last packet receipt time used by the stalled monitor. this.#lastPacketAt = now; if (decodedMessage?.channelId === undefined) { return; } // Handle video packet. if (decodedMessage.channelId === this.#channels.video?.id) { video = this.#channels.video; timestamp = this.#calculateTimestamp(decodedMessage.timestampDelta, video, this.video.clockRate); data = this.#getPayloadBuffer(decodedMessage.payload); if (typeof timestamp !== 'number' || Buffer.isBuffer(data) !== true || data.length === 0) { return; } // Normalise incoming NexusTalk video payloads to Annex-B once, here. // This gives the consumer a stable "complete access unit in Annex-B format" // contract and avoids rebuilding multi-NAL video later. if (data.indexOf(H264.NALUS.START_CODE) !== 0) { data = H264.wrapAnnexB(data); } nalUnits = H264.getNALUnits(data); for (let nalu of nalUnits) { if (nalu.type === H264.NALUS.TYPES.SPS) { video.lastSPS = Buffer.from(nalu.data); } if (nalu.type === H264.NALUS.TYPES.PPS) { video.lastPPS = Buffer.from(nalu.data); } if (nalu.type === H264.NALUS.TYPES.IDR) { keyFrame = true; } } // A new timestamp means the prior access unit/frame is complete. if (typeof video.pendingTimestamp === 'number' && timestamp !== video.pendingTimestamp) { this.#flushPendingVideo(video); } // Initialise pending frame state for this timestamp. if (typeof video.pendingTimestamp !== 'number') { video.pendingTimestamp = timestamp; video.pendingKeyFrame = false; video.pendingParts = []; video.pendingBytes = 0; } if (Array.isArray(video.pendingParts) !== true) { video.pendingParts = []; } if (typeof video.pendingBytes !== 'number') { video.pendingBytes = 0; } // Append Annex-B NAL payload to the current pending frame. video.pendingParts.push(data); video.pendingBytes += data.length; // Mark pending frame as keyframe if any IDR was observed. if (keyFrame === true) { video.pendingKeyFrame = true; } // Guard against pathological growth if a frame never flushes cleanly. if (video.pendingParts.length > MAX_PENDING_VIDEO_PARTS || video.pendingBytes > MAX_PENDING_VIDEO_BYTES) { this.recordVideoDrop('oversized-pending-frame'); this?.log?.warn?.( 'Resetting oversized pending NexusTalk video frame for uuid "%s" (%s parts, %s bytes)', this.uuid, video.pendingParts.length, video.pendingBytes, ); this.#resetPendingVideo(video); } return; } // Handle audio packet. if (decodedMessage.channelId === this.#channels.audio?.id) { audio = this.#channels.audio; timestamp = this.#calculateTimestamp(decodedMessage.timestampDelta, audio, this.audio.sampleRate); data = this.#getPayloadBuffer(decodedMessage.payload); if (typeof timestamp !== 'number' || Buffer.isBuffer(data) !== true || data.length === 0) { return; } this.emitMedia({ type: Streamer.MEDIA_TYPE.AUDIO, codec: this.audio.codec, profile: this.audio.profile, sampleRate: this.audio.sampleRate, channels: this.audio.channels, bitrate: this.audio.bitrate, frameDuration: this.audio.frameDuration, timestamp: timestamp, keyFrame: false, data: data, }); return; } } #handlePlaybackEnd(payload) { let decodedMessage = undefined; if (Buffer.isBuffer(payload) !== true || this.#protobufTypes.PlaybackEnd === undefined || this.#protobufTypes.PlaybackEnd === null) { return; } try { decodedMessage = this.#protobufTypes.PlaybackEnd.decode(payload).toJSON(); } catch (error) { this?.log?.debug?.('PlaybackEnd decode failed for uuid "%s": %s', this.uuid, String(error)); return; } // Flush any final buffered NexusTalk video frame before ending playback. // NexusTalk frames are emitted when the timestamp changes, so the final frame // would otherwise be lost if the stream ends without a newer timestamp arriving. this.#flushPendingVideo(this.#channels.video); if (this.#sessionId !== undefined && decodedMessage.reason === 'USER_ENDED_SESSION') { // Normal playback ended ie: when we stopped playback. this?.log?.debug?.('Playback ended on "%s"', this.#host); this.close(); return; } if (decodedMessage.reason === 'ERROR_LEAF_NODE_CANNOT_REACH_CAMERA') { // Camera is currently unreachable from the Nest backend. // Do not reconnect immediately, otherwise we hammer the Dropcam service // until the device comes back online. Streamer/device update handling will // reconnect when the camera reports online again. this?.log?.debug?.( 'Playback ended on "%s" because camera uuid "%s" is currently unreachable. Waiting for device online update.', this.#host, this.uuid, ); this.setState(StreamTransport.STATE.CLOSING, { reason: 'camera-unreachable', host: this.#host, }); this.close(); return; } // Error during playback, so we'll attempt to restart by reconnection to host. this?.log?.debug?.('Playback ended on "%s" with error "%s". Attempting reconnection', this.#host, decodedMessage.reason); this.#requestReconnect(this.#host, 'playback-end'); this.close(); } #handleNexusError(payload) { // Decode error packet if (Buffer.isBuffer(payload) === true && this.#protobufTypes.Error !== undefined && this.#protobufTypes.Error !== null) { let decodedMessage = undefined; try { decodedMessage = this.#protobufTypes.Error.decode(payload).toJSON(); } catch (error) { this?.log?.debug?.('Error packet decode failed for uuid "%s": %s', this.uuid, String(error)); return; } if (decodedMessage.code === 'ERROR_AUTHORIZATION_FAILED') { this?.log?.debug?.('Authorisation failed on "%s" for uuid "%s". Reconnecting NexusTalk session.', this.#host, this.uuid); this.#requestReconnect(this.#host, 'reauthorise'); this.close(); } else { // NexusStreamer Error, packet.message contains the message this?.log?.debug?.('NexusTalk error from "%s": %s', this.#host, decodedMessage.message); } } } #handleNexusData(data) { if (Buffer.isBuffer(data) !== true || data.length === 0) { return; } // Inbound NexusTalk packets arrive as a length-prefixed byte stream over TLS. // We accumulate bytes into a growable buffer and parse packets using a read index // so we do not memmove the buffer on every decoded packet. if (Buffer.isBuffer(this.#packetBuffer) !== true) { this.#packetBuffer = Buffer.allocUnsafe(INITIAL_PACKET_BUFFER_SIZE); } if (typeof this.#packetOffset !== 'number') { this.#packetOffset = 0; this.#packetReadIndex = 0; } // Check if we need to compact the buffer to make room for incoming data. // Only compact if: (1) not enough space for incoming data, AND (2) significant prefix waste let unreadBytes = this.#packetOffset - this.#packetReadIndex; let availableSpace = this.#packetBuffer.length - this.#packetOffset; if (availableSpace < data.length && this.#packetReadIndex > 0) { // Smart compaction: only if we have enough unread data worth preserving // or if we're critically low on space if (unreadBytes > 0 || availableSpace + this.#packetReadIndex < data.length) { // Compact unread bytes to the front of the buffer. if (unreadBytes > 0) { this.#packetBuffer.copy(this.#packetBuffer, 0, this.#packetReadIndex, this.#packetOffset); } this.#packetOffset = unreadBytes; this.#packetReadIndex = 0; } } // If still not enough space or buffer uninitialized, grow it if (this.#packetOffset + data.length > this.#packetBuffer.length) { if (this.#packetBuffer.length >= MAX_PACKET_BUFFER_SIZE) { // 10MB max buffer this?.log?.warn?.('Packet buffer exceeded maximum size, resetting for uuid "%s"', this.uuid); this.#resetPacketState(false); return; } let requiredSize = this.#packetOffset + data.length; let newSize = this.#packetBuffer.length; while (newSize < requiredSize && newSize < MAX_PACKET_BUFFER_SIZE) { newSize *= 2; } newSize = Math.min(newSize, MAX_PACKET_BUFFER_SIZE); if (requiredSize > newSize) { this?.log?.warn?.('Packet buffer required size exceeded maximum, resetting for uuid "%s"', this.uuid); this.#resetPacketState(false); return; } let newBuffer = Buffer.allocUnsafe(newSize); if (this.#packetOffset > this.#packetReadIndex) { this.#packetBuffer.copy(newBuffer, 0, this.#packetReadIndex, this.#packetOffset); } this.#packetOffset -= this.#packetReadIndex; this.#packetReadIndex = 0; this.#packetBuffer = newBuffer; } data.copy(this.#packetBuffer, this.#packetOffset); this.#packetOffset += data.length; // Parse as many complete packets as are currently available in the unread window. while (this.#packetOffset - this.#packetReadIndex >= 3) { let packetType = this.#packetBuffer.readUInt8(this.#packetReadIndex); let headerSize = 3; let packetSize = 0; if (packetType === MEDIA_TYPE.LONG_PLAYBACK_PACKET) { if (this.#packetOffset - this.#packetReadIndex < 5) { break; } headerSize = 5; packetSize = this.#packetBuffer.readUInt32BE(this.#packetReadIndex + 1); } else { packetSize = this.#packetBuffer.readUInt16BE(this.#packetReadIndex + 1); } if (packetSize > MAX_PACKET_PAYLOAD_SIZE) { // invalid size this?.log?.warn?.('Invalid packet size %d, resetting buffer for uuid "%s"', packetSize, this.uuid); this.#resetPacketState(true); break; } if (this.#packetOffset - this.#packetReadIndex < headerSize + packetSize) { break; } let protoBufPayload = this.#packetBuffer.subarray( this.#packetReadIndex + headerSize, this.#packetReadIndex + headerSize + packetSize, ); this.#packetReadIndex += headerSize + packetSize; switch (packetType) { case MEDIA_TYPE.PING: { break; } case MEDIA_TYPE.OK: { // Process any pending messages we have stored. this.#authorised = true; // OK message, means we're connected and authorised to Nexus this.#flushQueuedMessages(); // Periodically send PING message to keep stream alive clearInterval(this.#pingTimer); this.#pingTimer = setInterval(() => { this.#sendMessage(MEDIA_TYPE.PING, Buffer.alloc(0)); }, PING_INTERVAL); // Start processing data this.#startNexusData(); break; } case MEDIA_TYPE.ERROR: { this.#handleNexusError(protoBufPayload); break; } case MEDIA_TYPE.PLAYBACK_BEGIN: { this.#handlePlaybackBegin(protoBufPayload); break; } case MEDIA_TYPE.PLAYBACK_END: { this.#handlePlaybackEnd(protoBufPayload); break; } case MEDIA_TYPE.PLAYBACK_PACKET: case MEDIA_TYPE.LONG_PLAYBACK_PACKET: { this.#handlePlaybackPacket(protoBufPayload); break; } case MEDIA_TYPE.REDIRECT: { this.#handleRedirect(protoBufPayload); break; } case MEDIA_TYPE.TALKBACK_BEGIN: { this?.log?.debug?.('Talking started on uuid "%s"', this.uuid); break; } case MEDIA_TYPE.TALKBACK_END: { this?.log?.debug?.('Talking ended on uuid "%s"', this.uuid); break; } default: { this?.log?.debug?.('Unknown packet type "%d" received from "%s"', packetType, this.#host); break; } } } // Normalise fully consumed parser state to keep offsets small and stable. if (this.#packetReadIndex === this.#packetOffset) { this.#packetReadIndex = 0; this.#packetOffset = 0; } } #requestReconnect(host, reason, context = undefined) { // Request a reconnect once the current socket is closed. // This does NOT perform the reconnect immediately. // The actual reconnect is handled centrally in the socket 'close' handler. let details = typeof context === 'object' && context !== null ? context : {}; let hasRoute = typeof details.fromHost === 'string' || typeof details.toHost === 'string'; if (typeof host === 'string' && host !== '') { this.#reconnectHost = host; } if ((this.#reconnectHost ?? '') === '') { this.#reconnectHost = this.#host ?? this.nexustalk_host; } if (this.#reconnectPending === true) { return; } this.#reconnectPending = true; this.setState(StreamTransport.STATE.RECONNECTING, { ...details, reason: reason, host: hasRoute === true ? details.host : (details.host ?? this.#reconnectHost), }); } #canWrite(requiresAuthorisation = false) { return ( this.#socket?.readyState === 'open' && this.#socket?.writable === true && (requiresAuthorisation !== true || this.#authorised === true) ); } #clearMessageQueue(resetStartIndex = 0) { if (this.#messages instanceof RingBuffer === true) { this.#messages.clear(resetStartIndex); return; } this.#messages = new RingBuffer(resetStartIndex, PENDING_MESSAGE_QUEUE_CAPACITY); } #resetPacketState(reuseBuffer = true) { if (reuseBuffer === true && Buffer.isBuffer(this.#packetBuffer) === true) { this.#packetOffset = 0; this.#packetReadIndex = 0; return; } this.#packetBuffer = undefined; this.#packetOffset = undefined; this.#packetReadIndex = 0; } #startStalledMonitor() { if (this.#stalledTimer !== undefined) { return; } this.#stalledTimer = setInterval( () => { if (typeof this.#lastPacketAt !== 'number') { return; } if (Date.now() - this.#lastPacketAt <= STALLED_TIMEOUT) { return; } this?.log?.debug?.( 'No NexusTalk playback packets received for uuid "%s" in the past %s seconds. Closing connection', this.uuid, Math.round(STALLED_TIMEOUT / 1000), ); this.#requestReconnect(this.#host, 'stalled'); this.close(); }, Math.max(1000, Math.round(STALLED_TIMEOUT / 2)), ); } #stopStalledMonitor() { clearInterval(this.#stalledTimer); this.#stalledTimer = undefined; this.#lastPacketAt = undefined; } #calculateTimestamp(delta, stream, sampleRate) { let deltaMs = 0; // Convert NexusTalk timestamp deltas into monotonic source media time // anchored to the shared playback session start time. Streamer later maps // this source media time onto each output's playout schedule. if (typeof stream?.mediaTime !== 'number') { stream.mediaTime = typeof this.#sessionStartTime === 'number' ? this.#sessionStartTime + (stream?.startOffset ?? 0) : Date.now(); } if (Number.isFinite(sampleRate) !== true || sampleRate <= 0) { return stream.mediaTime; } if (typeof delta === 'number') { deltaMs = (delta / sampleRate) * 1000; if (Number.isFinite(deltaMs) !== true || deltaMs < 0) { deltaMs = 0; } } if (deltaMs > 0) { stream.mediaTime += deltaMs; } return stream.mediaTime; } #getPayloadBuffer(payload) { // Copy payload bytes out of protobuf-decoded objects before retaining them. // Streamer treats media buffers as immutable, but protobufjs byte fields may // alias the reusable NexusTalk packet buffer, which is compacted/overwritten // as more TLS data arrives. if (Buffer.isBuffer(payload) === true) { return Buffer.from(payload); } if (payload instanceof Uint8Array) { return Buffer.from(payload); } // Fallback: protobufjs may expose bytes as a base64-encoded string. if (typeof payload === 'string') { return Buffer.from(payload, 'base64'); } return undefined; } #resetPendingVideo(video) { if (typeof video !== 'object' || video === null) { return; } video.pendingTimestamp = undefined; video.pendingKeyFrame = false; if (Array.isArray(video.pendingParts) === true) { video.pendingParts.length = 0; } else { video.pendingParts = []; } video.pendingBytes = 0; } #flushPendingVideo(video) { let pendingTimestamp = undefined; let pendingData = undefined; let accessUnit = undefined; if (typeof video !== 'object' || video === null) { return; } // Nothing buffered, so just clear any stale pending state if (Array.isArray(video.pendingParts) !== true || video.pendingParts.length === 0 || typeof video.pendingTimestamp !== 'number') { this.#resetPendingVideo(video); return; } pendingTimestamp = video.pendingTimestamp; accessUnit = H264.buildAccessUnit(video.pendingParts); pendingData = accessUnit?.data; if (Buffer.isBuffer(pendingData) !== true || pendingData.length === 0) { this.#resetPendingVideo(video); return; } if (this.canAcceptMediaFrame('video', { keyFrame: video.pendingKeyFrame }) !== true) { this.recordVideoDrop(video.pendingKeyFrame === true ? 'suppressed-keyframe-recovery' : 'suppressed-delta-recovery'); this.#resetPendingVideo(video); return; } this.markMediaFrame('video', { keyFrame: video.pendingKeyFrame }); try { this.emitMedia({ type: Streamer.MEDIA_TYPE.VIDEO, codec: this.video.codec, profile: this.video.profile, bitrate: this.video.bitrate, timestamp: pendingTimestamp, keyFrame: video.pendingKeyFrame, codecConfig: { sps: video.lastSPS, pps: video.lastPPS, hasSPS: accessUnit?.hasSPS === true, hasPPS: accessUnit?.hasPPS === true, }, data: pendingData, }); if (this.connected === true && this.ready !== true) { // Transition to READY only after a complete video frame has actually // been emitted to Streamer. this.setState(StreamTransport.STATE.READY, { host: this.#host, sessionId: this.#sessionId }); } } finally { this.#resetPendingVideo(video); } } #resetChannelDetails() { this.#sessionStartTime = undefined; // Reset video channel details. this.#channels.video.id = undefined; this.#channels.video.startOffset = 0; this.#channels.video.mediaTime = undefined; this.#channels.video.pendingTimestamp = undefined; this.#channels.video.pendingKeyFrame = f