homebridge-nest-accfactory
Version:
Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras
1,456 lines (1,214 loc) • 51.1 kB
JavaScript
// NexusTalk
// Part of homebridge-nest-accfactory
//
// Handles streaming connections with Nest legacy 'Nexus' backend systems.
// Manages bidirectional media streams (audio/video) over secure TLS connections
// using Nest's proprietary protobuf-based protocol.
//
// Extends Streamer base class to provide Nest-specific streaming transport,
// feeding H264 video and AAC audio directly into the shared Streamer pipeline.
//
// Responsibilities:
// - Establish and manage TLS connection to Nexus backend hosts
// - Perform session authentication and authorisation
// - Exchange protobuf messages for session control and media transport
// - Receive and forward H264 video and AAC audio into Streamer
// - Handle two-way talkback audio using Speex encoding
// - Queue outbound control messages while socket/auth state is unavailable
// - Reconnect cleanly on redirect, stall, host change, or backend closure
//
// Features:
// - TLS-encrypted connection management with Nexus backend
// - Shared protobuf schema/type caching via protobuf.js helper module
// - Protobuf message serialization for proprietary NexusTalk protocol
// - Multiplexed media and control messages over a single connection
// - Buffered outbound control-message queue using RingBuffer
// - Direct media injection into Streamer with minimal processing overhead
// - Two-way audio (talkback) support via Speex
// - Buffered packet parsing with bounded memory protection
//
// Notes:
// - Video is delivered as H264 NAL units and assembled into complete access units
// before being injected into Streamer
// - Audio is delivered as AAC frames and passed directly into Streamer
// - Protobuf schemas and message types are shared/cached globally to avoid
// repeated protobufjs parsing across multiple camera instances
//
// Architecture:
// - Extends Streamer and injects media using addMedia()
// - Uses protobuf-based message framing over TLS
// - Maintains per-stream playback timing/channel state
// - Supports reconnect-driven lifecycle handling for redirects and stalls
//
// Note: Based on foundational work from https://github.com/Brandawg93/homebridge-nest-cam
//
// Code version 2026.05.07
// Mark Hulskamp
'use strict';
// Define nodejs module requirements
import { Buffer } from 'node:buffer';
import { setInterval, clearInterval } from 'node:timers';
import path from 'node:path';
import tls from 'tls';
import crypto from 'crypto';
// Define our modules
import Streamer, { RingBuffer } from './streamer.js';
import { getProtoType } from './protobuf.js';
// Define constants
import { USER_AGENT, __dirname } from './consts.js';
const PING_INTERVAL = 15000; // Ping interval to nexus server while stream active
const STALLED_TIMEOUT = 10000; // Time with no playback packets received before we consider stream stalled and attempt restart
const PENDING_MESSAGE_QUEUE_CAPACITY = 64; // Initial slot count for pending outbound control messages
const MAX_PENDING_MESSAGES = 256; // Hard cap for queued outbound control messages while unauthorised/disconnected
const INITIAL_PACKET_BUFFER_SIZE = 256 * 1024;
const MAX_PACKET_BUFFER_SIZE = 10 * 1024 * 1024;
const MAX_PACKET_PAYLOAD_SIZE = 5 * 1024 * 1024;
const MEDIA_TYPE = {
PING: 1,
HELLO: 100,
PING_CAMERA: 101,
AUDIO_PAYLOAD: 102,
START_PLAYBACK: 103,
STOP_PLAYBACK: 104,
CLOCK_SYNC_ECHO: 105,
LATENCY_MEASURE: 106,
TALKBACK_LATENCY: 107,
METADATA_REQUEST: 108,
OK: 200,
ERROR: 201,
PLAYBACK_BEGIN: 202,
PLAYBACK_END: 203,
PLAYBACK_PACKET: 204,
LONG_PLAYBACK_PACKET: 205,
CLOCK_SYNC: 206,
REDIRECT: 207,
TALKBACK_BEGIN: 208,
TALKBACK_END: 209,
METADATA: 210,
METADATA_ERROR: 211,
AUTHORIZE_REQUEST: 212,
};
// Blank audio in AAC format, mono channel @48000
const AAC_MONO_48000_BLANK = Buffer.from([
0xff, 0xf1, 0x4c, 0x40, 0x03, 0x9f, 0xfc, 0xde, 0x02, 0x00, 0x4c, 0x61, 0x76, 0x63, 0x35, 0x39, 0x2e, 0x31, 0x38, 0x2e, 0x31, 0x30, 0x30,
0x00, 0x02, 0x30, 0x40, 0x0e,
]);
const MAX_PENDING_VIDEO_PARTS = 200;
const MAX_PENDING_VIDEO_BYTES = 4 * 1024 * 1024;
// nexusTalk object
export default class NexusTalk extends Streamer {
nexustalk_host = undefined; // Main nexustalk streaming host
token = undefined;
useGoogleAuth = false; // Nest vs google auth
blankAudio = AAC_MONO_48000_BLANK;
// Internal data only for this class
#protobufTypes = {
AudioPayload: undefined,
StartPlayback: undefined,
StopPlayback: undefined,
AuthoriseRequest: undefined,
Hello: undefined,
Redirect: undefined,
PlaybackBegin: undefined,
PlaybackPacket: undefined,
PlaybackEnd: undefined,
Error: undefined,
};
#socket = undefined; // TCP socket object
#packetBuffer = undefined; // Incoming packet buffer
#packetOffset = undefined; // Current offset in packet buffer
#packetReadIndex = 0; // Current read offset for packet parsing loop
// Pending outbound control messages while socket is unavailable/unauthorised
#messages = new RingBuffer(0, PENDING_MESSAGE_QUEUE_CAPACITY);
#authorised = false; // Have we been authorised
#sessionId = undefined; // Session ID
#host = undefined; // Current host connected to
#pingTimer = undefined; // Timer object for ping interval
#stalledTimer = undefined; // Interval object for no received data checks
#lastPacketAt = undefined; // Last playback packet receipt time in ms
#reconnectPending = false; // Reconnect requested once socket closes
#reconnectHost = undefined; // Host to reconnect to
#reconnectReason = undefined; // Reason for reconnect
#sessionStartTime = undefined; // Shared session time anchor in ms for all playback channels
#channels = {
video: {
id: undefined,
codec: Streamer.CODEC_TYPE.H264,
profile: undefined,
startOffset: 0,
mediaTime: undefined,
sampleRate: undefined,
lastEmittedTimestamp: undefined,
pendingTimestamp: undefined,
pendingKeyFrame: false,
pendingParts: [],
pendingBytes: 0,
},
audio: {
id: undefined,
codec: Streamer.CODEC_TYPE.AAC,
profile: undefined,
startOffset: 0,
mediaTime: undefined,
sampleRate: undefined,
},
};
#talkback = {
active: false,
codec: Streamer.CODEC_TYPE.SPEEX,
sampleRate: 16000,
channels: 1,
lastPacketTime: undefined,
};
// Codecs being used for video, audio and talking
get codecs() {
return {
video: this.#channels.video.codec,
audio: this.#channels.audio.codec,
talkback: this.#talkback.codec,
};
}
// Capabilities supported by this streamer
get capabilities() {
return {
live: true,
record: true,
talkback: true,
buffering: true,
};
}
constructor(uuid, deviceData, options) {
super(uuid, deviceData, options);
let protoPath = path.join(__dirname, 'protobuf/nest/nexustalk.proto');
// Cache protobuf message types up front so we avoid repeated lookup()
// calls during connect, control messaging, and packet handling paths.
// Types are shared globally via protobuf.js helper caching.
this.#protobufTypes.AudioPayload = getProtoType(protoPath, 'nest.nexustalk.v1.AudioPayload', this.log);
this.#protobufTypes.StartPlayback = getProtoType(protoPath, 'nest.nexustalk.v1.StartPlayback', this.log);
this.#protobufTypes.StopPlayback = getProtoType(protoPath, 'nest.nexustalk.v1.StopPlayback', this.log);
this.#protobufTypes.AuthoriseRequest = getProtoType(protoPath, 'nest.nexustalk.v1.AuthoriseRequest', this.log);
this.#protobufTypes.Hello = getProtoType(protoPath, 'nest.nexustalk.v1.Hello', this.log);
this.#protobufTypes.Redirect = getProtoType(protoPath, 'nest.nexustalk.v1.Redirect', this.log);
this.#protobufTypes.PlaybackBegin = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackBegin', this.log);
this.#protobufTypes.PlaybackPacket = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackPacket', this.log);
this.#protobufTypes.PlaybackEnd = getProtoType(protoPath, 'nest.nexustalk.v1.PlaybackEnd', this.log);
this.#protobufTypes.Error = getProtoType(protoPath, 'nest.nexustalk.v1.Error', this.log);
// Store data we need from the device data passed in
this.token = deviceData?.apiAccess?.token;
this.nexustalk_host = deviceData?.nexustalk_host; // Host we'll connect to
this.useGoogleAuth = typeof deviceData?.apiAccess?.oauth2 === 'string' && deviceData?.apiAccess?.oauth2 !== '';
}
// Class functions
async connect(options = {}) {
let connectHost = typeof options?.host === 'string' && options.host !== '' ? options.host : this.nexustalk_host;
if (this.online !== true || this.videoEnabled !== true) {
return;
}
if (typeof connectHost !== 'string' || connectHost === '') {
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSED, 'host-missing');
return;
}
if (this.#socket !== undefined && this.#socket.destroyed === false) {
// Existing socket is still opening/open, avoid duplicate concurrent connects.
return;
}
clearInterval(this.#pingTimer);
this.#stopStalledMonitor();
this.#pingTimer = undefined;
this.#sessionId = undefined; // No session ID yet
this.#authorised = false;
this.#resetPacketState(true);
this?.log?.debug?.('Connection started to "%s"', connectHost);
this.#host = connectHost; // Update internal host name since we’re about to connect
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CONNECTING);
// Wrap tls.connect() in a Promise so we can await the TLS handshake
try {
await new Promise((resolve, reject) => {
let socket = tls.connect({ host: connectHost, port: 1443 }, () => {
if (this.#socket !== socket) {
resolve();
return;
}
// Opened connection to Nexus server, so now need to authenticate ourselves
this?.log?.debug?.('Connection established to "%s"', connectHost);
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CONNECTED);
socket.setKeepAlive(true); // Keep socket connection alive
this.#Authenticate(false); // Send authentication request
resolve(); // Allow await connect() to continue
});
this.#socket = socket;
socket.on('error', (error) => {
if (this.#socket !== socket) {
return;
}
// TLS error (could be refused, timeout, etc.)
this?.log?.warn?.('TLS error on connect to "%s": %s', connectHost, String(error));
this.#authorised = false; // Since we had an error, we can't be authorised
reject(error);
});
socket.on('end', () => {
// Do nothing
});
socket.on('data', (data) => {
if (this.#socket !== socket) {
return;
}
this.#handleNexusData(data);
});
socket.on('close', () => {
if (this.#socket !== socket) {
return;
}
clearInterval(this.#pingTimer);
this.#stopStalledMonitor();
this.#pingTimer = undefined;
this.#authorised = false; // Since connection closed, we can't be authorised anymore
this.#socket = undefined; // Clear socket object
this.#sessionId = undefined; // Not an active session anymore
this.#host = undefined;
if (this.hasActiveStreams() === true && this.#reconnectPending !== true) {
this.#requestReconnect(connectHost, 'service-close');
}
if (this.#reconnectPending === true && this.hasActiveStreams() === true) {
let reconnectHost = this.#reconnectHost;
let reconnectReason = this.#reconnectReason;
this.#reconnectPending = false;
this.#reconnectHost = undefined;
this.#reconnectReason = undefined;
if (typeof reconnectHost === 'string' && reconnectHost !== '') {
this?.log?.debug?.(
'Connection closed to "%s", %s to "%s"',
connectHost,
reconnectReason === 'redirect' ? 'redirecting' : 'attempting reconnection',
reconnectHost,
);
this.requestSourceConnect({ host: reconnectHost, forceReconnect: true }).catch((error) => {
this?.log?.debug?.('Error reconnecting NexusTalk for uuid "%s": %s', this.nest_google_device_uuid, String(error));
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSED, 'reconnect-failed');
});
return;
}
}
this?.log?.debug?.('Connection closed to "%s"', connectHost);
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSED, 'socket-close');
});
});
} catch (error) {
this?.log?.error?.('Failed to connect to "%s": %s', connectHost, String(error));
if (this.#socket === undefined && this.hasActiveStreams() === true) {
this.#requestReconnect(connectHost, 'connect-failed');
this.requestSourceConnect({ host: this.#reconnectHost }).catch((connectError) => {
this?.log?.debug?.('Error reconnecting NexusTalk for uuid "%s": %s', this.nest_google_device_uuid, String(connectError));
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSED, 'reconnect-failed');
});
return;
}
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSED, 'connect-failed');
}
}
async close(stopStreamFirst = true) {
let reconnecting = this.#reconnectPending === true;
// Close an authenticated socket stream gracefully
// Clear any running timers before closing socket to prevent race conditions
clearInterval(this.#pingTimer);
this.#stopStalledMonitor();
this.#pingTimer = undefined;
if (this.#socket !== undefined) {
let socket = this.#socket;
if (stopStreamFirst === true) {
await this.#stopNexusData();
}
try {
socket.destroy();
} catch {
// Empty
}
}
if (reconnecting !== true) {
// Do not emit SOURCE_CLOSED here.
// The terminal closed state is owned by the socket 'close' handler.
// For reconnect/redirect paths we keep the state as SOURCE_RECONNECTING
// until the next connect attempt begins.
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_CLOSING);
// Flush any final pending NexusTalk video frame before resetting channel state.
// Do not do this during reconnect, otherwise the final frame from the old session
// can repopulate Streamer timing state just before the new session starts.
this.#flushPendingVideo(this.#channels.video);
this.#clearMessageQueue(0);
}
// Always reset channel/session state
this.#resetChannelDetails();
this.#sessionId = undefined; // Not an active session anymore
this.#resetPacketState(true);
}
async onUpdate(deviceData) {
let newToken = undefined;
let newHost = undefined;
if (typeof deviceData !== 'object' || deviceData === null) {
return;
}
newToken =
typeof deviceData?.apiAccess?.token === 'string' && deviceData.apiAccess.token !== '' ? deviceData.apiAccess.token : undefined;
newHost = typeof deviceData?.nexustalk_host === 'string' && deviceData.nexustalk_host !== '' ? deviceData.nexustalk_host : undefined;
if (typeof newToken === 'string' && newToken !== this.token) {
if (this.hasActiveStreams() === true) {
this?.log?.debug?.(
'Access token has changed for uuid "%s" while NexusTalk session is active. Updating stored token.',
this.nest_google_device_uuid,
);
}
this.token = newToken;
}
if (typeof newHost === 'string' && newHost !== this.nexustalk_host) {
this?.log?.debug?.(
'NexusTalk host has changed for uuid "%s" from "%s" to "%s"%s',
this.nest_google_device_uuid,
this.nexustalk_host,
newHost,
this.hasActiveStreams() === true ? '. Reconnecting active session.' : '',
);
this.nexustalk_host = newHost;
if (this.hasActiveStreams() === true && this.#reconnectPending !== true) {
this.#requestReconnect(newHost, 'host-update');
await this.requestSourceClose();
}
}
}
async onShutdown() {
await this.requestSourceClose(); // Gracefully stop stream and close socket
}
sendTalkback(talkingBuffer) {
let AudioPayload = undefined;
let encodedData = undefined;
if (Buffer.isBuffer(talkingBuffer) !== true || this.#sessionId === undefined) {
return;
}
AudioPayload = this.#protobufTypes.AudioPayload;
if (AudioPayload === undefined || AudioPayload === null) {
return;
}
try {
encodedData = AudioPayload.encode(
AudioPayload.fromObject({
payload: talkingBuffer,
sessionId: this.#sessionId,
codec: this.#talkback.codec,
sampleRate: this.#talkback.sampleRate,
}),
).finish();
} catch (error) {
this?.log?.debug?.('AudioPayload encode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
this.#talkback.lastPacketTime = Date.now();
this.#sendMessage(MEDIA_TYPE.AUDIO_PAYLOAD, encodedData);
}
#startNexusData() {
if (this.videoEnabled === false || this.online === false) {
return;
}
// Setup streaming profiles
// We'll use the highest profile as the main, with others for fallback
let otherProfiles = ['VIDEO_H264_530KBIT_L31', 'VIDEO_H264_100KBIT_L30'];
if (this.audioEnabled === true) {
// Include AAC profile if audio is enabled on camera
otherProfiles.push('AUDIO_AAC');
}
let StartPlayback = this.#protobufTypes.StartPlayback;
if (StartPlayback === undefined || StartPlayback === null) {
return;
}
let encodedData = null;
try {
encodedData = StartPlayback.encode(
StartPlayback.fromObject({
sessionId: Math.floor(Math.random() * 1000000), // larger range to reduce collisions
profile: 'VIDEO_H264_2MBIT_L40',
otherProfiles: otherProfiles,
profileNotFoundAction: 'REDIRECT',
}),
).finish();
} catch (error) {
this?.log?.debug?.('StartPlayback encode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
this.#sendMessage(MEDIA_TYPE.START_PLAYBACK, encodedData);
}
#stopNexusData() {
if (this.#sessionId === undefined) {
return;
}
let StopPlayback = this.#protobufTypes.StopPlayback;
if (StopPlayback === undefined || StopPlayback === null) {
return;
}
let encodedData = null;
try {
encodedData = StopPlayback.encode(
StopPlayback.fromObject({
sessionId: this.#sessionId,
}),
).finish();
} catch (error) {
this?.log?.debug?.('StopPlayback encode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
this.#sendMessage(MEDIA_TYPE.STOP_PLAYBACK, encodedData);
}
#sendNow(type, data) {
let header = undefined;
// Raw socket send path for already-connected / already-authorised messages.
// This bypasses queue logic and is used by both direct sends and queued flushes.
if (Buffer.isBuffer(data) !== true || this.#canWrite() !== true) {
return false;
}
if (type === MEDIA_TYPE.LONG_PLAYBACK_PACKET) {
header = Buffer.alloc(5);
header.writeUInt8(type, 0);
header.writeUInt32BE(data.length, 1);
} else {
header = Buffer.alloc(3);
header.writeUInt8(type, 0);
header.writeUInt16BE(data.length, 1);
}
// Write composed message to NexusTalk without concatenating buffers.
this.#socket.cork();
try {
this.#socket.write(header);
this.#socket.write(data);
} finally {
this.#socket.uncork();
}
return true;
}
#queueMessage(type, data) {
// Queue outbound control messages until socket/auth state is ready.
// Oldest messages are dropped once the hard cap is reached so memory stays bounded.
if (Buffer.isBuffer(data) !== true) {
return;
}
if (this.#messages instanceof RingBuffer !== true) {
this.#messages = new RingBuffer(0, PENDING_MESSAGE_QUEUE_CAPACITY);
}
if (this.#messages.size >= MAX_PENDING_MESSAGES) {
this.#messages.shift(1);
this?.log?.warn?.('Dropped oldest pending NexusTalk message for uuid "%s" as queue is full', this.nest_google_device_uuid);
}
this.#messages.push({ type: type, data: data });
}
#flushQueuedMessages() {
let queuedMessage = undefined;
// Drain queued outbound messages using the raw send path.
// Do not call #sendMessage() from here or messages could be re-queued.
if (this.#messages instanceof RingBuffer !== true) {
return;
}
while (this.#messages.size > 0) {
if (this.#canWrite(true) !== true) {
break;
}
queuedMessage = this.#messages.getByOffset(0);
if (typeof queuedMessage !== 'object' || queuedMessage === null) {
this.#messages.shift(1);
continue;
}
if (this.#sendNow(queuedMessage.type, queuedMessage.data) !== true) {
break;
}
this.#messages.shift(1);
}
}
#sendMessage(type, data) {
if (Buffer.isBuffer(data) !== true) {
return;
}
// Bootstrap/auth messages must be allowed before authorisation exists
if (type === MEDIA_TYPE.HELLO || type === MEDIA_TYPE.AUTHORIZE_REQUEST) {
if (this.#canWrite() === true) {
this.#sendNow(type, data);
}
return;
}
// Normal messages require an authorised writable socket
if (this.#canWrite(true) !== true) {
this.#queueMessage(type, data);
return;
}
this.#sendNow(type, data);
}
#Authenticate(reauthorise) {
this.#authorised = false; // We're no longer authorised
let authoriseRequest = null;
let AuthoriseRequest = this.#protobufTypes.AuthoriseRequest;
if (AuthoriseRequest === undefined || AuthoriseRequest === null) {
return;
}
try {
authoriseRequest = AuthoriseRequest.encode(
AuthoriseRequest.fromObject(this.useGoogleAuth === true ? { oliveToken: this.token } : { sessionToken: this.token }),
).finish();
} catch (error) {
this?.log?.debug?.('AuthoriseRequest encode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
if (reauthorise === true) {
if (this.#canWrite() !== true) {
return;
}
this?.log?.debug?.('Re-authentication requested to "%s"', this.#host);
this.#sendMessage(MEDIA_TYPE.AUTHORIZE_REQUEST, authoriseRequest);
return;
}
// This isn't a re-authorise request, so perform 'Hello' packet
let Hello = this.#protobufTypes.Hello;
if (Hello === undefined || Hello === null) {
return;
}
let encodedData = null;
try {
encodedData = Hello.encode(
Hello.fromObject({
protocolVersion: 'VERSION_3',
uuid: this.nest_google_device_uuid.split(/[._]+/)[1],
requireConnectedCamera: false,
userAgent: USER_AGENT,
deviceId: crypto.randomUUID(),
clientType: 'IOS',
authoriseRequest: authoriseRequest,
}),
).finish();
} catch (error) {
this?.log?.debug?.('Hello encode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
this?.log?.debug?.('Performing authentication to "%s"', this.#host);
this.#sendMessage(MEDIA_TYPE.HELLO, encodedData);
}
#handleRedirect(payload) {
let redirectToHost = undefined;
if (Buffer.isBuffer(payload) === true && this.#protobufTypes.Redirect !== undefined && this.#protobufTypes.Redirect !== null) {
let decodedMessage = undefined;
try {
decodedMessage = this.#protobufTypes.Redirect.decode(payload);
} catch (error) {
this?.log?.debug?.('Redirect packet decode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
redirectToHost = decodedMessage?.newHost;
} else if (typeof payload === 'string' && payload !== '') {
// Payload parameter is a string, we'll assume this is a direct hostname
redirectToHost = payload;
}
if (typeof redirectToHost !== 'string' || redirectToHost === '') {
return;
}
this?.log?.debug?.('Redirect requested from "%s" to "%s"', this.#host, redirectToHost);
this.#requestReconnect(redirectToHost, 'redirect');
this.requestSourceClose();
}
#handlePlaybackBegin(payload) {
let decodedMessage = undefined;
let now = Date.now();
let videoStream = undefined;
let audioStream = undefined;
let sessionStart = undefined;
let startDelta = 0;
if (
Buffer.isBuffer(payload) !== true ||
this.#protobufTypes.PlaybackBegin === undefined ||
this.#protobufTypes.PlaybackBegin === null
) {
return;
}
if (this.sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSING || this.sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSED) {
// We received a PlaybackBegin message but we're already closing/closed
// so ignore this message since we're not going to be able to do anything with it
return;
}
try {
decodedMessage = this.#protobufTypes.PlaybackBegin.decode(payload).toJSON();
} catch (error) {
this?.log?.debug?.('PlaybackBegin decode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
// Reset current playback channel timing/state before applying new channel details
this.#resetChannelDetails();
if (Array.isArray(decodedMessage?.channels) === true) {
videoStream = decodedMessage.channels.find((stream) => stream?.codec === this.codecs.video.toUpperCase());
audioStream = decodedMessage.channels.find((stream) => stream?.codec === this.codecs.audio.toUpperCase());
}
// Use the earliest available stream start time as the shared session anchor
if (typeof videoStream?.startTime === 'number' && typeof audioStream?.startTime === 'number') {
sessionStart = Math.min(videoStream.startTime, audioStream.startTime) * 1000;
}
if (typeof sessionStart !== 'number' && typeof videoStream?.startTime === 'number') {
sessionStart = videoStream.startTime * 1000;
}
if (typeof sessionStart !== 'number' && typeof audioStream?.startTime === 'number') {
sessionStart = audioStream.startTime * 1000;
}
if (typeof sessionStart !== 'number') {
sessionStart = now;
}
this.#sessionStartTime = sessionStart;
if (typeof videoStream === 'object' && videoStream !== null) {
this.#channels.video.id = videoStream.channelId;
this.#channels.video.profile = videoStream.profile;
this.#channels.video.sampleRate = videoStream.sampleRate;
startDelta = typeof videoStream.startTime === 'number' ? videoStream.startTime * 1000 - sessionStart : 0;
this.#channels.video.startOffset = Math.max(-250, Math.min(250, Math.round(startDelta)));
this.#channels.video.mediaTime = sessionStart + this.#channels.video.startOffset;
}
if (typeof audioStream === 'object' && audioStream !== null) {
this.#channels.audio.id = audioStream.channelId;
this.#channels.audio.profile = audioStream.profile;
this.#channels.audio.sampleRate = audioStream.sampleRate;
startDelta = typeof audioStream.startTime === 'number' ? audioStream.startTime * 1000 - sessionStart : 0;
this.#channels.audio.startOffset = Math.max(-250, Math.min(250, Math.round(startDelta)));
this.#channels.audio.mediaTime = sessionStart + this.#channels.audio.startOffset;
}
// Store active playback session id for stop-playback and talkback messages.
this.#sessionId = decodedMessage?.sessionId;
this.#lastPacketAt = Date.now();
this.#startStalledMonitor();
this?.log?.debug?.('Playback started from "%s" with session ID "%s"', this.#host, this.#sessionId);
}
#handlePlaybackPacket(payload) {
if (this.sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSING || this.sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSED) {
// We received a PlaybackPacket message but we're already closing/closed
// so ignore this message since we're not going to be able to do anything with it
return;
}
if (
Buffer.isBuffer(payload) !== true ||
this.#protobufTypes.PlaybackPacket === undefined ||
this.#protobufTypes.PlaybackPacket === null
) {
return;
}
// Decode playback packet
let decodedMessage = undefined;
let Type = this.#protobufTypes.PlaybackPacket;
let video = undefined;
let audio = undefined;
let timestamp = 0;
let data = undefined;
let keyFrame = false;
try {
decodedMessage = Type.decode(payload);
} catch (error) {
this?.log?.debug?.('Playback packet decode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
// Update the last packet receipt time used by the stalled monitor.
this.#lastPacketAt = Date.now();
if (decodedMessage?.channelId === undefined) {
return;
}
// Handle video packet
if (decodedMessage.channelId === this.#channels.video?.id) {
video = this.#channels.video;
timestamp = this.#calculateTimestamp(decodedMessage.timestampDelta, video, 80);
data = this.#getPayloadBuffer(decodedMessage.payload);
if (typeof timestamp !== 'number' || Buffer.isBuffer(data) !== true || data.length === 0) {
return;
}
keyFrame = this.#hasH264NAL(data, Streamer.H264NALUS.TYPES.IDR);
if (this.sourceState === Streamer.MESSAGE_TYPE.SOURCE_CONNECTED) {
// Transition to READY now that we have real video packets
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_READY);
}
// New timestamp means the previous buffered NALs belong to the prior frame/access unit
if (typeof video.pendingTimestamp === 'number' && timestamp !== video.pendingTimestamp) {
this.#flushPendingVideo(video);
}
// Initialise pending frame state for this timestamp
if (typeof video.pendingTimestamp !== 'number') {
video.pendingTimestamp = timestamp;
video.pendingKeyFrame = false;
if (Array.isArray(video.pendingParts) === true) {
video.pendingParts.length = 0;
} else {
video.pendingParts = [];
}
video.pendingBytes = 0;
}
if (Array.isArray(video.pendingParts) !== true) {
video.pendingParts = [];
}
if (typeof video.pendingBytes !== 'number') {
video.pendingBytes = 0;
}
// Normalise incoming NexusTalk video payloads to Annex-B once, here.
// This gives Streamer a stable "complete access unit in Annex-B format" contract
// and avoids it having to rebuild multi-NAL video again later.
if (data.indexOf(Streamer.H264NALUS.START_CODE) !== 0) {
let buffer = Buffer.allocUnsafe(Streamer.H264NALUS.START_CODE.length + data.length);
Streamer.H264NALUS.START_CODE.copy(buffer, 0);
data.copy(buffer, Streamer.H264NALUS.START_CODE.length);
data = buffer;
}
video.pendingParts.push(data);
video.pendingBytes += data.length;
if (keyFrame === true) {
video.pendingKeyFrame = true;
}
// Guard against pathological growth if a frame never flushes cleanly
if (video.pendingParts.length > MAX_PENDING_VIDEO_PARTS || video.pendingBytes > MAX_PENDING_VIDEO_BYTES) {
this?.log?.warn?.(
'Resetting oversized pending NexusTalk video frame for uuid "%s" (%s parts, %s bytes)',
this.nest_google_device_uuid,
video.pendingParts.length,
video.pendingBytes,
);
this.#resetPendingVideo(video);
return;
}
return;
}
// Handle audio packet
if (decodedMessage.channelId === this.#channels.audio?.id) {
audio = this.#channels.audio;
timestamp = this.#calculateTimestamp(decodedMessage.timestampDelta, audio, 120);
data = this.#getPayloadBuffer(decodedMessage.payload);
if (typeof timestamp !== 'number' || Buffer.isBuffer(data) !== true || data.length === 0) {
return;
}
this.addMedia({
type: Streamer.MEDIA_TYPE.AUDIO,
codec: this.codecs.audio,
profile: typeof audio?.profile === 'string' ? audio.profile : undefined,
sampleRate: Number.isFinite(audio?.sampleRate) === true && audio.sampleRate > 0 ? audio.sampleRate : undefined,
channels: Number.isFinite(audio?.channels) === true && audio.channels > 0 ? audio.channels : 1, // NexusTalk is typically mono
bitrate: Number.isFinite(audio?.bitrate) === true && audio.bitrate > 0 ? audio.bitrate : undefined,
timestamp: timestamp,
keyFrame: false,
data: data,
});
}
}
#handlePlaybackEnd(payload) {
// Decode playback ended packet
if (Buffer.isBuffer(payload) === true && this.#protobufTypes.PlaybackEnd !== undefined && this.#protobufTypes.PlaybackEnd !== null) {
let decodedMessage = undefined;
try {
decodedMessage = this.#protobufTypes.PlaybackEnd.decode(payload).toJSON();
} catch (error) {
this?.log?.debug?.('PlaybackEnd decode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
// Flush any final buffered NexusTalk video frame before ending playback.
// NexusTalk frames are emitted when the timestamp changes, so the final frame
// would otherwise be lost if the stream ends without a newer timestamp arriving.
this.#flushPendingVideo(this.#channels.video);
if (this.#sessionId !== undefined && decodedMessage.reason === 'USER_ENDED_SESSION') {
// Normal playback ended ie: when we stopped playback
this?.log?.debug?.('Playback ended on "%s"', this.#host);
this.requestSourceClose();
return;
}
// Error during playback, so we'll attempt to restart by reconnection to host
this?.log?.debug?.('Playback ended on "%s" with error "%s". Attempting reconnection', this.#host, decodedMessage.reason);
this.#requestReconnect(this.#host, 'playback-end');
this.requestSourceClose(); // Close existing socket
}
}
#handleNexusError(payload) {
// Decode error packet
if (Buffer.isBuffer(payload) === true && this.#protobufTypes.Error !== undefined && this.#protobufTypes.Error !== null) {
let decodedMessage = undefined;
try {
decodedMessage = this.#protobufTypes.Error.decode(payload).toJSON();
} catch (error) {
this?.log?.debug?.('Error packet decode failed for uuid "%s": %s', this.nest_google_device_uuid, String(error));
return;
}
if (decodedMessage.code === 'ERROR_AUTHORIZATION_FAILED') {
this?.log?.debug?.(
'Authorisation failed on "%s" for uuid "%s". Reconnecting NexusTalk session.',
this.#host,
this.nest_google_device_uuid,
);
this.#requestReconnect(this.#host, 'reauthorise');
this.requestSourceClose();
} else {
// NexusStreamer Error, packet.message contains the message
this?.log?.debug?.('NexusTalk error from "%s": %s', this.#host, decodedMessage.message);
}
}
}
#handleTalkbackBegin(payload) {
// No payload fields currently required here
if (Buffer.isBuffer(payload) === true) {
this.#talkback.active = true;
this.#talkback.lastPacketTime = undefined; // reset timing for new session
this?.log?.debug?.('Talking started on uuid "%s"', this.nest_google_device_uuid);
}
}
#handleTalkbackEnd(payload) {
// No payload fields currently required here
if (Buffer.isBuffer(payload) === true) {
this.#talkback.active = false;
this.#talkback.lastPacketTime = undefined;
this?.log?.debug?.('Talking ended on uuid "%s"', this.nest_google_device_uuid);
}
}
#handleNexusData(data) {
if (Buffer.isBuffer(data) !== true || data.length === 0) {
return;
}
// Inbound NexusTalk packets arrive as a length-prefixed byte stream over TLS.
// We accumulate bytes into a growable buffer and parse packets using a read index
// so we do not memmove the buffer on every decoded packet.
if (Buffer.isBuffer(this.#packetBuffer) !== true) {
this.#packetBuffer = Buffer.allocUnsafe(INITIAL_PACKET_BUFFER_SIZE);
}
if (typeof this.#packetOffset !== 'number') {
this.#packetOffset = 0;
this.#packetReadIndex = 0;
}
// Check if we need to compact the buffer to make room for incoming data.
// Only compact if: (1) not enough space for incoming data, AND (2) significant prefix waste
let unreadBytes = this.#packetOffset - this.#packetReadIndex;
let availableSpace = this.#packetBuffer.length - this.#packetOffset;
if (availableSpace < data.length && this.#packetReadIndex > 0) {
// Smart compaction: only if we have enough unread data worth preserving
// or if we're critically low on space
if (unreadBytes > 0 || availableSpace + this.#packetReadIndex < data.length) {
// Compact unread bytes to the front of the buffer.
if (unreadBytes > 0) {
this.#packetBuffer.copy(this.#packetBuffer, 0, this.#packetReadIndex, this.#packetOffset);
}
this.#packetOffset = unreadBytes;
this.#packetReadIndex = 0;
}
}
// If still not enough space or buffer uninitialized, grow it
if (this.#packetOffset + data.length > this.#packetBuffer.length) {
if (this.#packetBuffer.length >= MAX_PACKET_BUFFER_SIZE) {
// 10MB max buffer
this?.log?.warn?.('Packet buffer exceeded maximum size, resetting for uuid "%s"', this.nest_google_device_uuid);
this.#resetPacketState(false);
return;
}
let requiredSize = this.#packetOffset + data.length;
let newSize = this.#packetBuffer.length;
while (newSize < requiredSize && newSize < MAX_PACKET_BUFFER_SIZE) {
newSize *= 2;
}
newSize = Math.min(newSize, MAX_PACKET_BUFFER_SIZE);
if (requiredSize > newSize) {
this?.log?.warn?.('Packet buffer required size exceeded maximum, resetting for uuid "%s"', this.nest_google_device_uuid);
this.#resetPacketState(false);
return;
}
let newBuffer = Buffer.allocUnsafe(newSize);
if (this.#packetOffset > this.#packetReadIndex) {
this.#packetBuffer.copy(newBuffer, 0, this.#packetReadIndex, this.#packetOffset);
}
this.#packetOffset -= this.#packetReadIndex;
this.#packetReadIndex = 0;
this.#packetBuffer = newBuffer;
}
data.copy(this.#packetBuffer, this.#packetOffset);
this.#packetOffset += data.length;
// Parse as many complete packets as are currently available in the unread window.
while (this.#packetOffset - this.#packetReadIndex >= 3) {
let packetType = this.#packetBuffer.readUInt8(this.#packetReadIndex);
let headerSize = 3;
let packetSize = 0;
if (packetType === MEDIA_TYPE.LONG_PLAYBACK_PACKET) {
if (this.#packetOffset - this.#packetReadIndex < 5) {
break;
}
headerSize = 5;
packetSize = this.#packetBuffer.readUInt32BE(this.#packetReadIndex + 1);
} else {
packetSize = this.#packetBuffer.readUInt16BE(this.#packetReadIndex + 1);
}
if (packetSize > MAX_PACKET_PAYLOAD_SIZE) {
// invalid size
this?.log?.warn?.('Invalid packet size %d, resetting buffer for uuid "%s"', packetSize, this.nest_google_device_uuid);
this.#resetPacketState(true);
break;
}
if (this.#packetOffset - this.#packetReadIndex < headerSize + packetSize) {
break;
}
let protoBufPayload = this.#packetBuffer.subarray(
this.#packetReadIndex + headerSize,
this.#packetReadIndex + headerSize + packetSize,
);
this.#packetReadIndex += headerSize + packetSize;
switch (packetType) {
case MEDIA_TYPE.PING: {
break;
}
case MEDIA_TYPE.OK: {
// Process any pending messages we have stored.
this.#authorised = true; // OK message, means we're connected and authorised to Nexus
this.#flushQueuedMessages();
// Periodically send PING message to keep stream alive
clearInterval(this.#pingTimer);
this.#pingTimer = setInterval(() => {
this.#sendMessage(MEDIA_TYPE.PING, Buffer.alloc(0));
}, PING_INTERVAL);
// Start processing data
this.#startNexusData();
break;
}
case MEDIA_TYPE.ERROR: {
this.#handleNexusError(protoBufPayload);
break;
}
case MEDIA_TYPE.PLAYBACK_BEGIN: {
this.#handlePlaybackBegin(protoBufPayload);
break;
}
case MEDIA_TYPE.PLAYBACK_END: {
this.#handlePlaybackEnd(protoBufPayload);
break;
}
case MEDIA_TYPE.PLAYBACK_PACKET:
case MEDIA_TYPE.LONG_PLAYBACK_PACKET: {
this.#handlePlaybackPacket(protoBufPayload);
break;
}
case MEDIA_TYPE.REDIRECT: {
this.#handleRedirect(protoBufPayload);
break;
}
case MEDIA_TYPE.TALKBACK_BEGIN: {
this.#handleTalkbackBegin(protoBufPayload);
break;
}
case MEDIA_TYPE.TALKBACK_END: {
this.#handleTalkbackEnd(protoBufPayload);
break;
}
default: {
this?.log?.debug?.('Unknown packet type "%d" received from "%s"', packetType, this.#host);
break;
}
}
}
// Normalise fully consumed parser state to keep offsets small and stable.
if (this.#packetReadIndex === this.#packetOffset) {
this.#packetReadIndex = 0;
this.#packetOffset = 0;
}
}
#requestReconnect(host, reason) {
// Request a reconnect once the current socket is closed.
// This does NOT perform the reconnect immediately.
// The actual reconnect is handled centrally in the socket 'close' handler.
if (typeof host === 'string' && host !== '') {
this.#reconnectHost = host;
}
if ((this.#reconnectHost ?? '') === '') {
this.#reconnectHost = this.#host ?? this.nexustalk_host;
}
this.#reconnectReason = reason;
if (this.#reconnectPending === true) {
return;
}
this.#reconnectPending = true;
this.setSourceState(Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING, reason);
}
#canWrite(requiresAuthorisation = false) {
return (
this.#socket?.readyState === 'open' &&
this.#socket?.writable === true &&
(requiresAuthorisation !== true || this.#authorised === true)
);
}
#clearMessageQueue(resetStartIndex = 0) {
if (this.#messages instanceof RingBuffer === true) {
this.#messages.clear(resetStartIndex);
return;
}
this.#messages = new RingBuffer(resetStartIndex, PENDING_MESSAGE_QUEUE_CAPACITY);
}
#resetPacketState(reuseBuffer = true) {
if (reuseBuffer === true && Buffer.isBuffer(this.#packetBuffer) === true) {
this.#packetOffset = 0;
this.#packetReadIndex = 0;
return;
}
this.#packetBuffer = undefined;
this.#packetOffset = undefined;
this.#packetReadIndex = 0;
}
#startStalledMonitor() {
if (this.#stalledTimer !== undefined) {
return;
}
this.#stalledTimer = setInterval(
() => {
if (typeof this.#lastPacketAt !== 'number') {
return;
}
if (Date.now() - this.#lastPacketAt <= STALLED_TIMEOUT) {
return;
}
this?.log?.debug?.(
'No NexusTalk playback packets received for uuid "%s" in the past %s seconds. Closing connection',
this.nest_google_device_uuid,
Math.round(STALLED_TIMEOUT / 1000),
);
this.#requestReconnect(this.#host, 'stalled');
this.requestSourceClose();
},
Math.max(1000, Math.round(STALLED_TIMEOUT / 2)),
);
}
#stopStalledMonitor() {
clearInterval(this.#stalledTimer);
this.#stalledTimer = undefined;
this.#lastPacketAt = undefined;
}
#hasH264NAL(data, nalType) {
let offset = 0;
// Check whether the supplied H264 payload contains a given NAL type.
// Supports both raw single-NAL payloads and Annex-B formatted buffers.
if (Buffer.isBuffer(data) !== true || data.length === 0) {
return false;
}
// Raw NAL unit without Annex B start code
if ((data[0] & 0x1f) === nalType) {
return true;
}
while (offset < data.length) {
// 4-byte Annex B start code
if (
offset + 3 < data.length &&
data[offset] === 0x00 &&
data[offset + 1] === 0x00 &&
data[offset + 2] === 0x00 &&
data[offset + 3] === 0x01
) {
offset += 4;
if (offset < data.length && (data[offset] & 0x1f) === nalType) {
return true;
}
continue;
}
// 3-byte Annex B start code
if (offset + 2 < data.length && data[offset] === 0x00 && data[offset + 1] === 0x00 && data[offset + 2] === 0x01) {
offset += 3;
if (offset < data.length && (data[offset] & 0x1f) === nalType) {
return true;
}
continue;
}
offset++;
}
return false;
}
#calculateTimestamp(delta, stream, maxStepMs = undefined) {
let deltaMs = 0;
// Convert NexusTalk timestamp deltas into a monotonic media timestamp
// anchored to the shared playback session start time.
if (typeof stream?.mediaTime !== 'number') {
stream.mediaTime = typeof this.#sessionStartTime === 'number' ? this.#sessionStartTime + (stream?.startOffset ?? 0) : Date.now();
}
if (typeof stream?.sampleRate !== 'number' || Number.isFinite(stream.sampleRate) !== true || stream.sampleRate <= 0) {
return stream.mediaTime;
}
if (typeof delta === 'number') {
deltaMs = (delta / stream.sampleRate) * 1000;
if (Number.isFinite(deltaMs) !== true || deltaMs < 0) {
deltaMs = 0;
}
if (typeof maxStepMs === 'number' && maxStepMs > 0 && deltaMs > maxStepMs) {
deltaMs = maxStepMs;
}
}
if (deltaMs > 0) {
stream.mediaTime += deltaMs;
}
return stream.mediaTime;
}
#getPayloadBuffer(payload) {
// Copy payload bytes out of protobuf-decoded objects so downstream media
// handling never depends on protobufjs backing-buffer lifetime/aliasing.
// This does add one allocation per packet, so revisit if profiling shows
// meaningful GC pressure in NexusTalk playback.
if (Buffer.isBuffer(payload) === true) {
return Buffer.from(payload);
}
if (payload instanceof Uint8Array) {
return Buffer.from(payload);
}
// Fallback: assume it's a base64-encoded string
if (typeof payload === 'string') {
return Buffer.from(payload, 'base64');
}
return undefined;
}
#resetPendingVideo(video) {
if (typeof video !== 'object' || video === null) {
return;
}
video.pendingTimestamp = undefined;
video.pendingKeyFrame = false;
if (Array.isArray(video.pendingParts) === true) {
video.pendingParts.length = 0;
} else {
video.pendingParts = [];
}
video.pendingBytes = 0;
}
#flushPendingVideo(video) {
let pendingTimestamp = undefined;
let minimumStep = 1;
let pendingData = undefined;
if (typeof video !== 'object' || video === null) {
return;
}
// Nothing buffered, so just clear any stale pending state
if (Array.isArray(video.pendingParts) !== true || video.pendingParts.length === 0 || typeof video.pendingTimestamp !== 'number') {
this.#resetPendingVideo(video);
return;
}
pendingTimestamp = video.pendingTimestamp;
// If we know FPS, use it to keep flushed frame timestamps moving forward
// by at least one frame interval when duplicate/regressing timestamps occur
if (typeof this.video?.fps === 'number' && Number.isFinite(this.video.fps) === true && this.video.fps > 0) {
minimumStep = Math.max(1, Math.round(1000 / this.video.fps));
}
if (typeof video.lastEmittedTimestamp === 'number' && pendingTimestamp <= video.lastEmittedTimestamp) {
pendingTimestamp = video.lastEmittedTimestamp + minimumStep;
}
video.lastEmittedTimestamp = pendingTimestamp;
// Avoid Buffer.concat() for the common/small case where a NexusTalk frame
// only has a single pending Annex-B NAL/access unit part.
if (video.pendingParts.length === 1) {
pendingData = video.pendingParts[0];
}
if (video.pendingParts.length > 1) {
pendingData = Buffer.concat(video.pendingParts, video.pendingBytes);
}
if (Buffer.isBuffer(pendingData) !== true || pendingData.length === 0) {
this.#resetPendingVideo(video);
return;
}
this.addMedia({
type: Streamer.MEDIA_TYPE.VIDEO,
codec: this.codecs.video,
profile: typeof video?.profile === 'string' ? video.profile : undefined,
bitrate:
video?.profile?.includes?.('2MBIT') === true
? 2000000
: video?.profile?.includes?.('530KBIT') === true
? 530000
: video?.profile?.includes?.('100KBIT') === true
? 100000
: undefined,
timestamp: pendingTimestamp,
keyFrame: video.pendingKeyFrame,
data: pendingData,
});
this.#resetPendingVideo(video);
}
#resetChannelD