homebridge-nest-accfactory
Version:
Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras
1,529 lines (1,289 loc) • 91 kB
JavaScript
// streamer
// Part of homebridge-nest-accfactory
//
// Base class for all Camera/Doorbell streaming.
//
// Maintains a shared rotating media buffer using RingBuffer and allows multiple
// HomeKit clients to consume the same upstream source for live viewing, buffering,
// and/or recording. Each live or recording output maintains its own cursor into
// the shared buffer.
//
// Note:
// - RingBuffer is also exposed for reuse by transport implementations
// (e.g. NexusTalk, WebRTC) for efficient queueing and buffering.
//
// IMPORTANT:
// Extending classes are responsible for managing the upstream stream source
// (e.g. WebRTC, NexusTalk) and MUST notify Streamer of source lifecycle changes
// using setSourceState(..). This ensures correct buffer handling, connection
// lifecycle behaviour, fallback frame behaviour, and output startup logic.
//
// Typical source lifecycle states are:
// - SOURCE_CONNECTING -> when initiating connection to upstream source
// - SOURCE_CONNECTED -> when transport is established but media may not yet be flowing
// - SOURCE_READY -> when media items are flowing and valid
// - SOURCE_RECONNECTING -> when source is retrying after interruption
// - SOURCE_CLOSING -> when source teardown has started
// - SOURCE_CLOSED -> when the source has stopped or disconnected
//
// Failure to signal correct source state may result in:
// - stale or frozen video on startup
// - incorrect buffer positioning
// - missing recordings or live stream failures
//
// Media Model:
// - Streamer expects complete media frames (not partial frames such as raw NALU fragments)
// - Video should be provided as complete H264 NAL units or access units
// - Audio should be provided as complete frames (AAC or PCM)
// - Streamer handles pacing, buffering, and fan-out to outputs
//
// Buffering Model:
// - A shared RingBuffer stores recent media frames/items
// - Appends and trimming are O(1); no front-splice copying occurs
// - Buffer capacity grows only when required; re-packing only occurs during growth
// - Live and recording sessions read from the buffer using independent cursors
// - Items are referenced using logical indexes; physical storage may wrap
//
// Live Streaming Behaviour:
// - Live outputs attach at the current live edge
// - Video is suppressed until the next keyframe arrives when decoder-safe startup is required
// - This avoids replaying stale buffered media and keeps latency low
//
// Recording Behaviour:
// - Recording outputs start from a requested timestamp when provided
// - The closest retained media item to the requested time is selected
// - Cursor selection does NOT enforce keyframe alignment
//
// - Decoder safety is handled during playout:
// - If requireKeyFrameStart is true:
// - Video is suppressed until the next keyframe (IDR) is seen
// - If requireKeyFrameStart is false:
// - Playback begins immediately from the selected cursor
//
// - This separation allows:
// - Accurate time-based recording start
// - Flexible handling of decoder requirements per output type
//
// - If the requested time falls outside the retained buffer:
// - Playback falls back to the nearest valid retained position
//
// H264 Handling:
// - Streamer manages Annex-B start codes for H264 output
// - Extending classes should provide clean NAL units without start codes where possible
// - SPS/PPS are tracked and injected as needed for decoder startup
//
// Synchronisation:
// - Audio output may be suppressed until a video keyframe (IDR) is seen when required by the output policy
// - This ensures correct A/V alignment for HomeKit consumers
//
// Extending classes are expected to implement or override, as needed:
//
// streamer.connect(options)
// streamer.close()
// streamer.onUpdate(deviceData)
// streamer.onShutdown()
//
// The following should be implemented when supported by the transport:
//
// streamer.sendTalkback(talkingBuffer)
//
// The following getters should be overridden:
//
// streamer.codecs <- return object with codecs being used (video, audio, talkback)
// streamer.capabilities <- return object with streaming capabilities (live, record, talkback, buffering)
//
// The following properties should be overridden when required:
//
// blankAudio - Buffer containing a blank audio segment for the type of audio being used
//
// Code version 2026.05.06
// Mark Hulskamp
'use strict';
// Define nodejs module requirements
import { Buffer } from 'node:buffer';
import { setInterval, clearInterval, setTimeout, clearTimeout } from 'node:timers';
import fs from 'fs';
import path from 'node:path';
import { PassThrough } from 'stream';
// Define our modules
import HomeKitDevice from './HomeKitDevice.js';
// Define constants
import { TIMERS, RESOURCE_FRAMES, RESOURCE_PATH, LOG_LEVELS, __dirname } from './consts.js';
const MAX_BUFFERED_ITEMS_PER_OUTPUT_PER_TICK = 20; // Prevent one output starving others
const STREAM_FRAME_INTERVAL = 1000 / 30; // 30fps approx
const OUTPUT_LOOP_INTERVAL = 10; // Shared output scheduler interval
const OUTPUT_BUDGET_LOG_INTERVAL = 30000; // Throttle per-streamer over-budget debug logs
const OUTPUT_PLAYOUT_POLICY = {
live: {
requireKeyFrameStart: false,
playoutDelayMs: 250,
maxLagBehindLiveMs: 1000,
dueTolerance: 12,
dueSlack: 18,
catchupExitThresholdMs: 300,
catchupAudioBurstLimit: 3,
normalVideoBurstLimit: 2,
normalAudioBurstLimit: 4,
catchupVideoBurstLimit: 6,
},
record: {
requireKeyFrameStart: true,
playoutDelayMs: 200,
maxLagBehindLiveMs: 1200,
dueTolerance: 12,
dueSlack: 20,
catchupExitThresholdMs: 350,
catchupAudioBurstLimit: 6,
normalVideoBurstLimit: 3,
normalAudioBurstLimit: 6,
catchupVideoBurstLimit: 10,
},
};
// Default capacity for generic RingBuffer usage.
// This should remain a sensible general-purpose default and not be tied to
// any specific streaming or media retention behaviour.
const RINGBUFFER_DEFAULT_CAPACITY = 1024;
// Maximum number of items the RingBuffer is allowed to allocate.
//
// RingBuffer capacity is measured in item slots, not bytes.
// This prevents unbounded memory growth if producers outpace consumers
// or if trimming/output flow stalls unexpectedly.
//
// The buffer grows dynamically up to this limit using doubling growth.
// Once reached, additional growth attempts are rejected.
const RINGBUFFER_MAX_CAPACITY = 8192;
// Initial capacity used by Streamer for its shared media buffer.
// This may diverge from the generic RingBuffer default as buffering strategy,
// retention window, or media characteristics evolve.
const STREAMER_INITIAL_BUFFER_CAPACITY = 1024;
// RingBuffer
//
// Dynamically-sized circular buffer with logical indexing.
// - O(1) push and shift (no array reallocation or front-splice)
// - Grows by doubling when capacity is exceeded, up to RINGBUFFER_MAX_CAPACITY
// - Maintains a monotonically increasing logical startIndex
// so external consumers can track absolute positions even as data is trimmed
//
// Terminology:
// - head: physical index of the first logical item
// - size: number of valid items currently stored
// - startIndex: logical index of the first item (external reference point)
//
// Access is done via logical offsets (0..size-1), which are translated
// to physical positions via modulo arithmetic.
export class RingBuffer {
items = [];
capacity = 0;
head = 0;
size = 0;
startIndex = 0;
constructor(startIndex = 0, capacity = RINGBUFFER_DEFAULT_CAPACITY) {
// capacity = physical storage size (number of slots in the circular array)
// Clamp to RINGBUFFER_MAX_CAPACITY so callers cannot bypass the growth limit
this.capacity =
Number.isInteger(capacity) === true && capacity > 0 ? Math.min(capacity, RINGBUFFER_MAX_CAPACITY) : RINGBUFFER_DEFAULT_CAPACITY;
// Preallocate backing array to avoid resizing on every push
this.items = new Array(this.capacity);
// startIndex = logical index of the first item in the buffer
// (used by external consumers to map absolute positions)
this.startIndex = Number.isInteger(startIndex) === true && startIndex >= 0 ? startIndex : 0;
}
physicalOffset(offset) {
// Convert a logical offset (0..size-1) into a physical array index.
// head = physical position of logical offset 0.
// We wrap using modulo to stay within the circular buffer.
if (Number.isInteger(offset) !== true || offset < 0) {
return -1;
}
return (this.head + offset) % this.capacity;
}
getByOffset(offset) {
let physicalOffset = 0;
// Bounds check against current logical size
if (Number.isInteger(offset) !== true || offset < 0 || offset >= this.size) {
return undefined;
}
// Translate logical offset -> physical slot
physicalOffset = this.physicalOffset(offset);
// Return stored item (or undefined if something went wrong)
return physicalOffset >= 0 ? this.items[physicalOffset] : undefined;
}
grow() {
// Prevent unbounded buffer growth
// If the maximum capacity has already been reached, reject expansion
if (this.capacity >= RINGBUFFER_MAX_CAPACITY) {
return false;
}
// Double capacity to amortize growth cost (O(n), but infrequent)
// Clamp growth to the configured maximum capacity
let newCapacity = Math.min(this.capacity * 2, RINGBUFFER_MAX_CAPACITY);
let newItems = new Array(newCapacity);
let index = 0;
// Re-pack items linearly starting at index 0
// This removes wrap-around and resets head to 0
while (index < this.size) {
newItems[index] = this.getByOffset(index);
index++;
}
this.items = newItems;
this.capacity = newCapacity;
// After re-pack, logical offset 0 is now at physical index 0
this.head = 0;
return true;
}
push(item) {
let tailOffset = 0;
// Grow if buffer is full (no overwriting policy here)
if (this.size >= this.capacity) {
// Reject push if the buffer can no longer expand
if (this.grow() !== true) {
return false;
}
}
// Tail = logical position "size"
tailOffset = this.physicalOffset(this.size);
if (tailOffset < 0) {
return false;
}
// Insert at tail and increase size
this.items[tailOffset] = item;
this.size++;
return true;
}
shift(count, resetStartIndex = undefined) {
let removeCount = 0;
let index = 0;
let physicalOffset = 0;
// Remove N items from the head (logical front of buffer)
if (Number.isInteger(count) !== true || count <= 0) {
return false;
}
removeCount = Math.min(count, this.size);
// Clear references so GC can reclaim memory
while (index < removeCount) {
physicalOffset = this.physicalOffset(index);
if (physicalOffset >= 0) {
this.items[physicalOffset] = undefined;
}
index++;
}
this.head = (this.head + removeCount) % this.capacity;
this.size -= removeCount;
this.startIndex += removeCount;
if (this.size === 0) {
this.head = 0;
if (Number.isInteger(resetStartIndex) === true && resetStartIndex >= 0) {
this.startIndex = resetStartIndex;
}
}
return true;
}
clear(resetStartIndex = 0) {
let index = 0;
let physicalOffset = 0;
while (index < this.size) {
physicalOffset = this.physicalOffset(index);
if (physicalOffset >= 0) {
this.items[physicalOffset] = undefined;
}
index++;
}
this.head = 0;
this.size = 0;
if (Number.isInteger(resetStartIndex) === true && resetStartIndex >= 0) {
this.startIndex = resetStartIndex;
}
return true;
}
}
// Streamer object
export default class Streamer {
static H264NALUS = {
START_CODE: Buffer.from([0x00, 0x00, 0x00, 0x01]),
TYPES: {
SLICE_NON_IDR: 1,
SLICE_PART_A: 2,
SLICE_PART_B: 3,
SLICE_PART_C: 4,
IDR: 5, // Instantaneous Decoder Refresh
SEI: 6,
SPS: 7,
PPS: 8,
AUD: 9,
END_SEQUENCE: 10,
END_STREAM: 11,
STAP_A: 24,
FU_A: 28,
},
};
static STREAM_TYPE = {
LIVE: 'live',
RECORD: 'record',
BUFFER: 'buffer',
};
static MEDIA_TYPE = {
VIDEO: 'video',
AUDIO: 'audio',
TALK: 'talk',
METADATA: 'meta',
};
static CODEC_TYPE = {
H264: 'h264',
AAC: 'aac',
OPUS: 'opus',
PCM: 'pcm',
SPEEX: 'speex',
META: 'meta',
UNKNOWN: 'undefined',
};
static MESSAGE = 'Streamer.onMessage'; // Message type for HomeKitDevice to listen for
static MESSAGE_TYPE = {
// Action type messages
START_LIVE: 'start-live',
STOP_LIVE: 'stop-live',
START_RECORD: 'start-record',
STOP_RECORD: 'stop-record',
START_BUFFER: 'start-buffer',
STOP_BUFFER: 'stop-buffer',
// Status type messages
SOURCE_CONNECTED: 'source-connected',
SOURCE_CONNECTING: 'source-connecting',
SOURCE_READY: 'source-ready',
SOURCE_RECONNECTING: 'source-reconnecting',
SOURCE_CLOSING: 'source-closing',
SOURCE_CLOSED: 'source-closed',
};
// Shared global scheduler for all active Streamer instances
// This avoids having one timer per camera and reduces event loop overhead
static #streamers = new Map(); // uuid => Streamer instance
static #timer = undefined; // Shared timer for all active streamers
supportDump = false; // Enable support for dumping stats on demand for this streamer instance
log = undefined; // Logging function object
videoEnabled = undefined; // Video stream on camera enabled or not
audioEnabled = undefined; // Audio from camera enabled or not
online = undefined; // Camera online or not
migrating = undefined; // Device is transferring/migrating between APIs
nest_google_device_uuid = undefined; // Nest/Google UUID of the device connecting
blankAudio = undefined; // Blank audio 'frame' for the type of audio being used, to be defined in subclass if audio is supported
video = {
codec: undefined,
profile: undefined,
width: undefined,
height: undefined,
fps: undefined,
bitrate: undefined,
};
audio = {
codec: undefined,
profile: undefined,
sampleRate: undefined,
channels: undefined,
bitrate: undefined,
frameDuration: undefined,
};
// Internal data only for this class
#HomeKitDeviceUUID = undefined; // HomeKitDevice uuid for this streamer
#bufferDuration = 0; // Duration of media to keep in the shared buffer based on media timestamps
#bufferEnabled = false; // Retained buffering policy flag owned by Streamer
#buffer = undefined; // Shared rotating ring buffer used by buffering, live and recording outputs
#outputs = new Map(); // Live and recording outputs keyed by session id
#cameraFrames = {}; // H264 resource frames for offline, video off, transferring
#sequenceCounters = {}; // Sequence counters for item types
#itemIndex = 0; // Monotonic item index for shared buffer cursor tracking
#videoState = {}; // Video state tracking
#audioState = {}; // Audio state tracking
#lastFallbackFrameTime = 0; // Timer for pacing fallback frames
#lastBudgetLogTime = 0; // Last time budget processing was sampled/logged
#outputErrors = 0; // Consecutive output loop failures for this instance
#lastMediaTime = {}; // Track last buffered media time per type for fallback ordering guards
#sourceState = Streamer.MESSAGE_TYPE.SOURCE_CLOSED; // Track stream source state from messages for internal logic and logging
#connectOptions = {}; // Store options from connect to use on reconnects
#lifecycleQueue = Promise.resolve(); // Serializes source connect/close operations to avoid lifecycle races
#stats = {
source: {
connectingAt: undefined,
connectedAt: undefined,
readyAt: undefined,
lastItemAt: undefined,
lastVideoItemAt: undefined,
lastAudioItemAt: undefined,
lastKeyframeAt: undefined,
firstVideoItemAt: undefined,
firstAudioItemAt: undefined,
firstKeyframeAt: undefined,
reconnects: 0,
},
items: {
video: 0,
audio: 0,
talk: 0,
metadata: 0,
keyframes: 0,
},
};
// Codecs being used for video, audio and talking
get codecs() {
return {
video: undefined,
audio: undefined,
talkback: undefined,
};
}
// Capabilities supported by this streamer
get capabilities() {
return {
live: false,
record: false,
talkback: false,
buffering: false,
};
}
// Get current stream source state
get sourceState() {
return this.#sourceState;
}
constructor(uuid, deviceData, options) {
if (Object.values(LOG_LEVELS).every((fn) => typeof options?.log?.[fn] === 'function')) {
this.log = options.log;
}
this.#HomeKitDeviceUUID = uuid;
HomeKitDevice.message(uuid, Streamer.MESSAGE, this);
HomeKitDevice.message(uuid, HomeKitDevice.UPDATE, this);
HomeKitDevice.message(uuid, HomeKitDevice.TIMER, this);
HomeKitDevice.message(uuid, HomeKitDevice.SHUTDOWN, this);
this.migrating = deviceData?.migrating === true;
this.online = deviceData?.online === true;
this.videoEnabled = deviceData?.streaming_enabled === true;
this.audioEnabled = deviceData?.audio_enabled === true;
this.nest_google_device_uuid = deviceData?.nest_google_device_uuid;
const loadFrameResource = (filename, label) => {
let buffer = undefined;
let file = path.resolve(__dirname, RESOURCE_PATH, filename);
if (fs.existsSync(file) === true) {
buffer = fs.readFileSync(file);
if (buffer.indexOf(Streamer.H264NALUS.START_CODE) === 0) {
buffer = buffer.subarray(Streamer.H264NALUS.START_CODE.length);
}
} else {
this.log?.warn?.('Failed to load %s video resource for "%s"', label, deviceData.description);
}
return buffer;
};
this.#cameraFrames = {
offline: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFFLINE, 'offline'),
off: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFF, 'video off'),
transfer: loadFrameResource(RESOURCE_FRAMES.CAMERA_TRANSFER, 'transferring'),
};
this.#lastFallbackFrameTime = Date.now();
this.supportDump = options?.supportDump === true;
// Setup buffer duration if passed in as an option, otherwise default to 5s
// Clamp to sane bounds to avoid invalid or excessive buffer sizes
this.#bufferDuration =
Number.isInteger(options?.bufferDuration) === true && options.bufferDuration > 0
? Math.min(Math.max(options.bufferDuration, 2000), 15000)
: 5000;
}
async onUpdate(deviceData) {
if (typeof deviceData !== 'object') {
return;
}
if (deviceData?.migrating !== undefined && this.migrating !== deviceData?.migrating) {
this.migrating = deviceData.migrating;
}
if (deviceData?.nest_google_device_uuid !== undefined && this.nest_google_device_uuid !== deviceData?.nest_google_device_uuid) {
this.nest_google_device_uuid = deviceData?.nest_google_device_uuid;
if (this.hasActiveStreams() === true) {
await this.#doClose();
await this.#doConnect();
}
}
if (
(deviceData?.online !== undefined && this.online !== deviceData.online) ||
(deviceData?.streaming_enabled !== undefined && this.videoEnabled !== deviceData.streaming_enabled) ||
(deviceData?.audio_enabled !== undefined && this.audioEnabled !== deviceData.audio_enabled)
) {
if (deviceData?.online !== undefined) {
this.online = deviceData.online === true;
}
if (deviceData?.streaming_enabled !== undefined) {
this.videoEnabled = deviceData.streaming_enabled === true;
}
if (deviceData?.audio_enabled !== undefined) {
this.audioEnabled = deviceData.audio_enabled === true;
}
if (this.hasActiveStreams() === true) {
if (this.online === false || this.videoEnabled === false || this.audioEnabled === false) {
await this.#doClose();
return;
}
await this.#doConnect();
}
}
}
async onMessage(type, message) {
if (typeof type !== 'string' || type === '') {
return;
}
let sessionID = message?.sessionID !== undefined ? String(message.sessionID) : undefined;
let options = typeof message?.options === 'object' && message.options !== null ? message.options : undefined;
if (type === Streamer.MESSAGE_TYPE.START_BUFFER) {
// Enable retained buffer and ensure source is connected.
// This does not create an output stream, only prepares buffering for future use.
if (this.capabilities.buffering !== true) {
this?.log?.debug?.('Buffering is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
await this.#startBuffering(options);
return;
}
if (type === Streamer.MESSAGE_TYPE.STOP_BUFFER) {
// Disable retained buffer.
// If no outputs are active, this may also allow the source to close.
if (this.capabilities.buffering !== true) {
this?.log?.debug?.('Buffering is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
await this.#stopBuffering();
return;
}
if (type === Streamer.MESSAGE_TYPE.START_LIVE) {
// Start a live streaming output for HomeKit.
// This creates PassThrough streams and begins feeding real-time data.
if (this.capabilities.live !== true) {
this?.log?.debug?.('Live streaming is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.LIVE, options);
}
if (type === Streamer.MESSAGE_TYPE.STOP_LIVE) {
// Stop a live streaming output.
// Cleans up streams and may close source if no other outputs remain.
if (this.capabilities.live !== true) {
this?.log?.debug?.('Live streaming is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.LIVE);
return;
}
if (type === Streamer.MESSAGE_TYPE.START_RECORD) {
// Start a recording output (HKSV).
// Uses retained buffer and selects a start position based on requested time.
// Decoder safety (keyframe alignment) is handled during playout.
if (this.capabilities.record !== true) {
this?.log?.debug?.('Recording is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.RECORD, options);
}
if (type === Streamer.MESSAGE_TYPE.STOP_RECORD) {
// Stop a recording output.
// If no sessionID is provided, stops the first active recording stream.
if (this.capabilities.record !== true) {
this?.log?.debug?.('Recording is unsupported for "%s"', this.nest_google_device_uuid);
return;
}
await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.RECORD);
return;
}
if (
type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING ||
type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTED ||
type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING ||
type === Streamer.MESSAGE_TYPE.SOURCE_CLOSING ||
type === Streamer.MESSAGE_TYPE.SOURCE_CLOSED ||
type === Streamer.MESSAGE_TYPE.SOURCE_READY
) {
// Handle lifecycle events from the underlying stream source (WebRTC, NexusTalk, etc).
// These events reflect connection state and readiness for media delivery.
// Ignore duplicate state transitions to avoid log spam and unnecessary stat updates
if (this.#sourceState === type) {
return;
}
// Prevent reconnecting -> connecting downgrade, which can occur during retry loops
if (this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING && type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING) {
return;
}
this.#sourceState = type;
// Reset transient state when connection is starting, restarting, or fully closed
if (
type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING ||
type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING ||
type === Streamer.MESSAGE_TYPE.SOURCE_CLOSED
) {
this.#resetSourceState();
this.#resetSourceStats();
}
// Track timing and reconnect metrics for diagnostics/support dump
if (typeof this.#stats === 'object' && this.#stats !== null) {
let now = Date.now();
// Initial connection attempt
if (type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING) {
this.#stats.source.connectingAt = now;
}
// Transport connected (but not yet ready for frames)
if (type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTED) {
this.#stats.source.connectedAt = now;
}
// Source ready and delivering media
if (type === Streamer.MESSAGE_TYPE.SOURCE_READY) {
this.#stats.source.readyAt = now;
}
// Count reconnect attempts separately
if (type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING) {
this.#stats.source.reconnects = (this.#stats.source.reconnects ?? 0) + 1;
}
}
// Log unified source state for visibility across different transport implementations
this?.log?.debug?.(
'Stream source is "%s" for uuid "%s"%s',
type,
this.nest_google_device_uuid,
typeof message?.reason === 'string' && message.reason !== '' ? ' (' + message.reason + ')' : '',
);
return;
}
}
async onShutdown() {
Streamer.#removeStreamer(this);
await this.stopEverything();
}
async stopEverything() {
if (this.hasActiveStreams() === true) {
this?.log?.debug?.('Stopped buffering, live and recording from device uuid "%s"', this.nest_google_device_uuid);
for (let output of this.#outputs.values()) {
this.#cleanupOutput(output);
}
this.#outputs.clear();
this.#resetRetainedState();
this.#syncSchedulerState();
await this.#doClose();
}
}
addMedia(media) {
let mediaType = undefined;
let codec = undefined;
let now = Date.now();
let data = undefined;
let audioFrameDuration = 0;
let sequence = 0;
let sourceTimestamp = 0;
let mediaTime = 0;
let keyFrame = false;
let h264Result = undefined;
// Validate incoming media object
if (typeof media !== 'object' || media === null) {
return;
}
mediaType = typeof media.type === 'string' ? media.type.toLowerCase() : undefined;
keyFrame = media?.keyFrame === true;
// Validate media type and payload
if (
typeof mediaType !== 'string' ||
mediaType.trim() === '' ||
(mediaType !== Streamer.MEDIA_TYPE.VIDEO &&
mediaType !== Streamer.MEDIA_TYPE.AUDIO &&
mediaType !== Streamer.MEDIA_TYPE.TALK &&
mediaType !== Streamer.MEDIA_TYPE.METADATA) ||
Buffer.isBuffer(media.data) !== true ||
media.data.length === 0
) {
return;
}
// Do not process if no active outputs
if (this.hasActiveStreams() !== true) {
return;
}
data = media.data;
// Ensure shared buffer exists before proceeding
this.#ensureSharedBuffer();
if (this.#buffer === undefined) {
return;
}
// Resolve codec (explicit media override first, otherwise fallback to configured codecs)
codec =
typeof media?.codec === 'string'
? media.codec.toLowerCase()
: mediaType === Streamer.MEDIA_TYPE.VIDEO
? this.codecs?.video
: mediaType === Streamer.MEDIA_TYPE.AUDIO
? this.codecs?.audio
: mediaType === Streamer.MEDIA_TYPE.TALK
? this.codecs?.talkback
: mediaType === Streamer.MEDIA_TYPE.METADATA
? Streamer.CODEC_TYPE.META
: undefined;
if (typeof codec !== 'string' || codec.trim() === '') {
return;
}
// Update advertised video stream properties
if (mediaType === Streamer.MEDIA_TYPE.VIDEO) {
this.video.codec = codec;
if (media?.profile?.trim?.() !== '') {
this.video.profile = media.profile;
}
if (Number.isFinite(media?.bitrate) === true && media.bitrate > 0) {
this.video.bitrate = media.bitrate;
}
}
// Update advertised audio stream properties
if (mediaType === Streamer.MEDIA_TYPE.AUDIO) {
this.audio.codec = codec;
if (media?.profile?.trim?.() !== '') {
this.audio.profile = media.profile;
}
if (Number.isFinite(media?.sampleRate) === true && media.sampleRate > 0) {
this.audio.sampleRate = media.sampleRate;
}
if (Number.isFinite(media?.channels) === true && media.channels > 0) {
this.audio.channels = media.channels;
}
if (Number.isFinite(media?.bitrate) === true && media.bitrate > 0) {
this.audio.bitrate = media.bitrate;
}
}
// Initialise sequence counter if required
if (typeof this.#sequenceCounters?.[mediaType] !== 'number') {
this.#sequenceCounters[mediaType] = 0;
}
// Use provided sequence/timestamp or fallback to generated values
sequence = Number.isFinite(media?.sequence) === true ? media.sequence : this.#sequenceCounters[mediaType]++;
sourceTimestamp = Number.isFinite(media?.timestamp) === true ? Math.round(media.timestamp) : now;
// Ensure monotonic media time (never goes backwards)
if (typeof this.#lastMediaTime?.[mediaType] !== 'number') {
this.#lastMediaTime[mediaType] = 0;
}
mediaTime = sourceTimestamp < this.#lastMediaTime[mediaType] ? this.#lastMediaTime[mediaType] : sourceTimestamp;
this.#lastMediaTime[mediaType] = mediaTime;
// Codec-specific H264 processing
if (mediaType === Streamer.MEDIA_TYPE.VIDEO && codec === Streamer.CODEC_TYPE.H264) {
h264Result = this.#processH264VideoMedia(data, sourceTimestamp, keyFrame, now);
if (typeof h264Result !== 'object' || h264Result === null) {
return;
}
data = h264Result.data;
keyFrame = h264Result.keyFrame === true;
}
// Audio frame timing smoothing
if (mediaType === Streamer.MEDIA_TYPE.AUDIO) {
if (typeof this.#audioState.lastSourceFrameTime === 'number') {
audioFrameDuration = sourceTimestamp - this.#audioState.lastSourceFrameTime;
if (audioFrameDuration > 0) {
let isOutlier = false;
if (typeof this.audio.frameDuration === 'number' && this.audio.frameDuration > 0) {
let deviance = Math.abs(audioFrameDuration - this.audio.frameDuration) / this.audio.frameDuration;
if (deviance > 0.5) {
isOutlier = true;
}
}
if (isOutlier !== true) {
if (typeof this.audio.frameDuration === 'number') {
this.audio.frameDuration = this.audio.frameDuration * 0.8 + audioFrameDuration * 0.2;
} else {
this.audio.frameDuration = audioFrameDuration;
}
}
}
}
this.#audioState.lastSourceFrameTime = sourceTimestamp;
}
// Update source/item stats for support dump and diagnostics
if (this.supportDump === true && typeof this.#stats?.source === 'object' && this.#stats.source !== null) {
if (keyFrame === true && typeof this.#stats.source.firstKeyframeAt !== 'number') {
this.#stats.source.firstKeyframeAt = now;
}
this.#stats.source.lastItemAt = now;
if (mediaType === Streamer.MEDIA_TYPE.VIDEO) {
this.#stats.source.lastVideoItemAt = now;
if (typeof this.#stats.source.firstVideoItemAt !== 'number') {
this.#stats.source.firstVideoItemAt = now;
}
}
if (mediaType === Streamer.MEDIA_TYPE.AUDIO) {
this.#stats.source.lastAudioItemAt = now;
if (typeof this.#stats.source.firstAudioItemAt !== 'number') {
this.#stats.source.firstAudioItemAt = now;
}
}
if (keyFrame === true) {
this.#stats.source.lastKeyframeAt = now;
}
}
if (this.supportDump === true && typeof this.#stats?.items === 'object' && this.#stats.items !== null) {
if (mediaType === Streamer.MEDIA_TYPE.VIDEO) {
this.#stats.items.video++;
}
if (mediaType === Streamer.MEDIA_TYPE.AUDIO) {
this.#stats.items.audio++;
}
if (mediaType === Streamer.MEDIA_TYPE.TALK) {
this.#stats.items.talk++;
}
if (mediaType === Streamer.MEDIA_TYPE.METADATA) {
this.#stats.items.metadata++;
}
if (keyFrame === true) {
this.#stats.items.keyframes++;
}
}
// Push final packet into shared buffer
let pushed = this.#buffer.push({
index: this.#itemIndex,
type: mediaType,
codec: codec,
time: mediaTime,
sourceTimestamp: sourceTimestamp,
sequence: sequence,
keyFrame: keyFrame === true,
data: data,
});
if (pushed === true) {
// Increment buffer index if push was successful
this.#itemIndex++;
}
}
#processH264VideoMedia(data, sourceTimestamp, keyFrame, now) {
let nalUnits = undefined;
let containsFrame = false;
let resolution = undefined;
let isAnnexB = false;
let previousFPS = undefined;
let frameDelta = 0;
let fps = 0;
// Split incoming payload into NAL units
nalUnits = this.#getH264NALUnits(data);
if (nalUnits.length === 0) {
return undefined;
}
// Derive Annex-B from parser behaviour:
// - Non-Annex-B returns original buffer reference
// - Annex-B returns subarray views
isAnnexB = nalUnits[0].data !== data;
for (let nalu of nalUnits) {
// SPS -> update resolution and cache
if (nalu.type === Streamer.H264NALUS.TYPES.SPS) {
this.#videoState.lastSPS = Buffer.from(nalu.data);
resolution = this.#decodeH264SPS(nalu.data);
if (typeof resolution === 'object' && resolution !== null) {
if (Number.isInteger(resolution.width) === true && resolution.width > 0) {
this.video.width = resolution.width;
}
if (Number.isInteger(resolution.height) === true && resolution.height > 0) {
this.video.height = resolution.height;
}
}
}
// PPS -> cache for future keyframes
if (nalu.type === Streamer.H264NALUS.TYPES.PPS) {
this.#videoState.lastPPS = Buffer.from(nalu.data);
}
// IDR (keyframe)
if (nalu.type === Streamer.H264NALUS.TYPES.IDR) {
this.#videoState.lastIDR = Buffer.from(nalu.data);
containsFrame = true;
keyFrame = true;
}
// Non-IDR slice still represents a frame
if (nalu.type === Streamer.H264NALUS.TYPES.SLICE_NON_IDR) {
containsFrame = true;
}
}
// Convert non-Annex-B to Annex-B format for downstream compatibility
if (isAnnexB !== true) {
if (nalUnits.length !== 1) {
return undefined;
}
data = Buffer.concat([Streamer.H264NALUS.START_CODE, nalUnits[0].data]);
}
// FPS estimation based on source timestamps
if (containsFrame === true) {
if (typeof this.#videoState.lastSourceFrameTime === 'number' && sourceTimestamp > this.#videoState.lastSourceFrameTime) {
frameDelta = sourceTimestamp - this.#videoState.lastSourceFrameTime;
previousFPS = this.video.fps;
if (
typeof previousFPS !== 'number' ||
previousFPS <= 0 ||
Math.abs(frameDelta - 1000 / previousFPS) / (1000 / previousFPS) <= 0.5
) {
fps = 1000 / frameDelta;
if (typeof previousFPS === 'number') {
this.video.fps = previousFPS * 0.8 + fps * 0.2;
} else {
this.video.fps = fps;
}
if (
this.#videoState.streamInfoLogged !== true &&
typeof this.video.width === 'number' &&
typeof this.video.height === 'number' &&
typeof this.video.fps === 'number'
) {
this.#videoState.streamInfoLogged = true;
this?.log?.debug?.(
'Receiving incoming stream from device "%s": %s %sx%s @ %sfps',
this.nest_google_device_uuid,
this.video.codec,
this.video.width,
this.video.height,
Math.round(this.video.fps),
);
}
if (typeof previousFPS === 'number' && typeof this.video.fps === 'number') {
if (
Math.abs(Math.round(this.video.fps) - Math.round(previousFPS)) >= 3 &&
(typeof this.#videoState.lastFPSLogTime !== 'number' || now - this.#videoState.lastFPSLogTime >= 30000)
) {
this.#videoState.lastFPSLogTime = now;
this?.log?.debug?.('FPS from device "%s" has changed to %sfps', this.nest_google_device_uuid, Math.round(this.video.fps));
}
}
}
}
this.#videoState.lastSourceFrameTime = sourceTimestamp;
}
// Track last keyframe index for buffer trimming / recording alignment
if (keyFrame === true) {
this.#videoState.lastIDRIndex = this.#itemIndex;
}
return {
data: data,
keyFrame: keyFrame,
};
}
async #startBuffering(options = {}) {
this.#ensureSharedBuffer();
if (this.#bufferEnabled === true) {
return;
}
this.#bufferEnabled = true;
this.log?.debug?.('Started buffering from device uuid "%s"', this.nest_google_device_uuid);
this.#syncSchedulerState();
await this.#doConnect(options);
}
async #stopBuffering() {
if (this.#bufferEnabled !== true) {
return;
}
this.#bufferEnabled = false;
this.log?.debug?.('Stopped buffering from device uuid "%s"', this.nest_google_device_uuid);
if (this.isStreaming() === false) {
this.#resetRetainedState();
this.#syncSchedulerState();
await this.#doClose();
return;
}
this.#syncSchedulerState();
}
async #createOutput(sessionID, type, options) {
let existing = undefined;
let output = undefined;
let video = undefined;
let audio = null;
let talkback = null;
let includeAudio = options?.includeAudio === true;
let waitForReady = Number.isInteger(options?.waitForReady) === true ? options.waitForReady : 0;
let startCursor = this.#itemIndex;
let buffer = undefined;
let itemsLength = 0;
let bufferStart = 0;
let item = undefined;
let recordTime = options?.recordTime;
let closestOffset = -1;
let closestDelta = Number.POSITIVE_INFINITY;
let index = 0;
let itemTime = 0;
let startTime = Date.now();
// Validate session id
if (typeof sessionID !== 'string' || sessionID === '') {
return;
}
// Check for existing output with this session id
existing = this.#outputs.get(sessionID);
// Only allow a single record output, regardless of session id
if (type === Streamer.STREAM_TYPE.RECORD && existing === undefined) {
for (output of this.#outputs.values()) {
if (output?.type === Streamer.STREAM_TYPE.RECORD) {
existing = output;
break;
}
}
}
// Reuse existing output when possible, otherwise reject type conflict
if (existing !== undefined) {
if (existing.type !== type) {
this?.log?.warn?.(
'Cannot start output for device uuid "%s" and session id "%s" as it is already in use for "%s"',
this.nest_google_device_uuid,
sessionID,
existing.type,
);
return;
}
return {
video: existing.video,
audio: existing.audio,
talkback: existing.talkback,
};
}
// Ensure retained buffer exists and start/connect source if needed
this.#ensureSharedBuffer();
await this.#doConnect(options);
buffer = this.#buffer;
if (buffer instanceof RingBuffer !== true) {
return;
}
// Create streams for this output
video = new PassThrough();
audio = includeAudio === true ? new PassThrough() : null;
talkback = type === Streamer.STREAM_TYPE.LIVE && includeAudio === true ? new PassThrough({ highWaterMark: 1024 * 16 }) : null;
// Prevent unhandled stream errors from bubbling
video?.on?.('error', () => {});
audio?.on?.('error', () => {});
talkback?.on?.('error', () => {});
// Determine initial cursor for recording
if (type === Streamer.STREAM_TYPE.RECORD) {
itemsLength = buffer.size;
bufferStart = buffer.startIndex;
closestOffset = -1;
closestDelta = Number.POSITIVE_INFINITY;
index = 0;
// Default to buffer start if valid
if (typeof bufferStart === 'number') {
startCursor = bufferStart;
}
// Recording should start from the retained position closest to the requested
// record time only. Decoder/keyframe safety is handled later during playout
// inside #processBufferedOutput(), not here.
if (itemsLength !== 0 && typeof recordTime === 'number' && Number.isFinite(recordTime) === true) {
while (index < itemsLength) {
item = buffer.getByOffset(index);
if (typeof item?.time === 'number') {
itemTime = Math.abs(item.time - recordTime);
if (itemTime < closestDelta) {
closestDelta = itemTime;
closestOffset = index;
}
}
index++;
}
if (closestOffset !== -1) {
startCursor = buffer.getByOffset(closestOffset)?.index;
}
}
// Never allow cursor to point before current retained window
if (typeof bufferStart === 'number' && startCursor < bufferStart) {
startCursor = bufferStart;
}
}
// Live streaming should attach at the live edge.
// We do not backtrack live outputs into retained media here.
// Any decoder startup/keyframe handling remains the responsibility of
// #processBufferedOutput().
if (type === Streamer.STREAM_TYPE.LIVE) {
startCursor = this.#itemIndex;
}
// Create output state
// Each output (LIVE / RECORD / BUFFER) consumes from the shared ring buffer
// using its own cursor and playout timing model.
//
// Key concepts:
// - cursor: current read position in the shared buffer (absolute index)
// - catchingUp: used when starting behind live edge (mainly RECORD) to fast-drain
// - sourceBaseTime / wallclockBaseTime:
// map source timestamps -> real time for paced playback
// - policy: defines how this output consumes media (latency vs continuity tradeoff)
//
// IMPORTANT:
// - All smoothing / pacing happens in Streamer (not in NexusTalk/WebRTC)
// - Each output has its own independent timing model
// - Policies MUST be tuned per output type (do not unify blindly)
output = {
sessionID: sessionID,
type: type,
// Writable streams (ffmpeg pipes etc)
video: video,
audio: audio,
talkback: talkback,
talkbackTimeout: undefined,
// Whether audio should be written for this output
includeAudio: includeAudio,
// Read cursor into shared ring buffer
// RECORD starts from the retained position closest to requested time
// LIVE starts at the current live edge
cursor: startCursor,
// Catch-up mode:
// - RECORD starts in catch-up to drain historical buffer
// - LIVE starts at the live edge so no catch-up is required initially
catchingUp: type === Streamer.STREAM_TYPE.RECORD,
catchupTicks: 0,
catchupStableFrames: 0,
// Codec / decoder state tracking
sentCodecConfig: false, // SPS/PPS sent
seenKeyFrame: false, // first keyframe seen
// Last time a video frame was written (used for fallback timing)
lastVideoWriteTime: 0,
// Time mapping for paced playback:
// wallclockTime = wallclockBaseTime + (item.time - sourceBaseTime)
sourceBaseTime: undefined,
wallclockBaseTime: undefined,
// Playout policy:
// Defines how aggressively we stay near live edge vs preserve continuity
//
// LIVE:
// - small delay (~100ms)
// - prefers low latency
//
// RECORD:
// - slightly delayed paced playback
// - preserves continuity while draining retained media
policy: { ...(OUTPUT_PLAYOUT_POLICY[type] ?? OUTPUT_PLAYOUT_POLICY.live) },
// Debug / instrumentation stats (used for tuning pacing behaviour)
stats: {
startedAt: Date.now(),
firstWriteAt: undefined,
firstVideoWriteAt: undefined,
firstAudioWriteAt: undefined,
writes: { total: 0, video: 0, audio: 0 },
drops: { videoBeforeKeyframe: 0, audioBeforeKeyframe: 0, bufferTrimmed: 0 },
},
};
// Attach talkback handling for live streams
if (talkback !== null) {
talkback.on('data', (data) => {
if (typeof this?.sendTalkback === 'function') {
this.sendTalkback(data);
clearTimeout(output.talkbackTimeout);
output.talkbackTimeout = setTimeout(() => {
this.sendTalkback(Buffer.alloc(0));
}, TIMERS.TALKBACK_AUDIO.interval);
}
});
talkback.on('close', () => {
clearTimeout(output?.talkbackTimeout);
if (typeof this?.sendTalkback === 'function') {
this.sendTalkback(Buffer.alloc(0));
}
});
}
// Register output before any optional readiness wait
this.#outputs.set(sessionID, output);
this.#syncSchedulerState();
// Optionally wait for source readiness before returning stream handles
if (waitForReady > 0) {
while (Date.now() - startTime < waitForReady) {
if (
this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_READY ||
this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSED ||
this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING
) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 25));
}
}
this?.log?.debug?.('Started %s stream from device uuid "%s" and session id "%s"', type, this.nest_google_device_uuid, sessionID);
return {
video: video,
audio: audio,
talkback: talkback,
};
}
async #stopOutput(sessionID, type) {
let output = undefined;
let hasOtherLiveOutputs = false;
let id = undefined;
let activeOutput = undefined;
// Resolve output by session id if provided
if (typeof sessionID === 'string' && sessionID !== '') {
output = this.#outputs.get(sessionID);
}
// For recording, allow stopping the first active record stream if no session id was provided
if (output === undefined && type === Streamer.STREAM_TYPE.RECORD) {
for (let candidate of this.#outputs.values()) {
if (candidate?.type === Streamer.STREAM_TYPE.RECORD) {
output = candidate;
break;
}
}
}
// Nothing matched to stop
if (output === undefined) {
return;
}
// Ensure we are not stopping a mismatched type (e.g. live vs record)
if (output.type !== type) {
this?.log?.warn?.(
'Cannot stop stream for device uuid "%s" and session id "%s" as it is type "%s" not "%s"',
this.nest_google_device_uuid,
output.sessionID,
output.type,
type,
);
return;
}
// If this is the last live output and support dump is enabled, log per-output stats before cleanup
if (output.type === Streamer.STREAM_TYPE.LIVE && this.supportDump === true) {
for ([id, activeOutput] of this.#outputs) {
if (id !== output.sessionID && activeOutput?.type === Streamer.STREAM_TYPE.LIVE) {
hasOtherLiveOutputs = true;
break;
}
}
if (hasOtherLiveOutputs !== true) {
this.#outputStats(output, Date.now());
}
}
this?.log?.debug?.(
'Stopping %s stream from device uuid "%s" and session id "%s"',
type,
this.nest_google_device_uuid,
output.sessionID,
);
// Cleanup streams, timers, and any talkback state
this.#cleanupOutput(output);
// Remove from active outputs
this.#outputs.delete(output.sessionID);
// Clear retained state when last output stops and buffering is disabled.
// This prevents stale buffered media being reused by the next session.
if (this.#outputs.size === 0 && this.#bufferEnabled !== true) {
this.#resetRetainedState();
}
// Update scheduler based on remaining activity
this.#syncSchedulerState();
// If nothing remains active, fully close underlying source
if (this.isStreaming() === false && this.isBuffering() === false) {
await this.#doClose();
}
}
isBuffering() {
return this.#bufferEnabled === true;
}
isStreaming() {
return this.#outputs.size !== 0;
}
isRecording() {
for (let output of this.#outputs.values()) {
if (output?.type === Streamer.STREAM_TYPE.RECORD) {
return true;
}
}
return false;
}
isLiveStreaming() {
for (let output of this.#outputs.values()) {
if (output?.type === Streamer.STREAM_TYPE.LIVE) {
return true;
}
}
return false;
}
hasActiveStreams() {
return this.#bufferEnabled === true || this.#outputs.size !== 0;
}
isSourceReady() {
return this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_READY;
}
setSourceState(type, reason) {
if (typeof type !== 'string' || type === '') {
return;
}
HomeKitDevice.message(this.#HomeKitDeviceUUID, Streamer.MESSAGE, type, { reason: reason });
}