UNPKG

homebridge-nest-accfactory

Version:

Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras

1,529 lines (1,289 loc) 91 kB
// streamer // Part of homebridge-nest-accfactory // // Base class for all Camera/Doorbell streaming. // // Maintains a shared rotating media buffer using RingBuffer and allows multiple // HomeKit clients to consume the same upstream source for live viewing, buffering, // and/or recording. Each live or recording output maintains its own cursor into // the shared buffer. // // Note: // - RingBuffer is also exposed for reuse by transport implementations // (e.g. NexusTalk, WebRTC) for efficient queueing and buffering. // // IMPORTANT: // Extending classes are responsible for managing the upstream stream source // (e.g. WebRTC, NexusTalk) and MUST notify Streamer of source lifecycle changes // using setSourceState(..). This ensures correct buffer handling, connection // lifecycle behaviour, fallback frame behaviour, and output startup logic. // // Typical source lifecycle states are: // - SOURCE_CONNECTING -> when initiating connection to upstream source // - SOURCE_CONNECTED -> when transport is established but media may not yet be flowing // - SOURCE_READY -> when media items are flowing and valid // - SOURCE_RECONNECTING -> when source is retrying after interruption // - SOURCE_CLOSING -> when source teardown has started // - SOURCE_CLOSED -> when the source has stopped or disconnected // // Failure to signal correct source state may result in: // - stale or frozen video on startup // - incorrect buffer positioning // - missing recordings or live stream failures // // Media Model: // - Streamer expects complete media frames (not partial frames such as raw NALU fragments) // - Video should be provided as complete H264 NAL units or access units // - Audio should be provided as complete frames (AAC or PCM) // - Streamer handles pacing, buffering, and fan-out to outputs // // Buffering Model: // - A shared RingBuffer stores recent media frames/items // - Appends and trimming are O(1); no front-splice copying occurs // - Buffer capacity grows only when required; re-packing only occurs during growth // - Live and recording sessions read from the buffer using independent cursors // - Items are referenced using logical indexes; physical storage may wrap // // Live Streaming Behaviour: // - Live outputs attach at the current live edge // - Video is suppressed until the next keyframe arrives when decoder-safe startup is required // - This avoids replaying stale buffered media and keeps latency low // // Recording Behaviour: // - Recording outputs start from a requested timestamp when provided // - The closest retained media item to the requested time is selected // - Cursor selection does NOT enforce keyframe alignment // // - Decoder safety is handled during playout: // - If requireKeyFrameStart is true: // - Video is suppressed until the next keyframe (IDR) is seen // - If requireKeyFrameStart is false: // - Playback begins immediately from the selected cursor // // - This separation allows: // - Accurate time-based recording start // - Flexible handling of decoder requirements per output type // // - If the requested time falls outside the retained buffer: // - Playback falls back to the nearest valid retained position // // H264 Handling: // - Streamer manages Annex-B start codes for H264 output // - Extending classes should provide clean NAL units without start codes where possible // - SPS/PPS are tracked and injected as needed for decoder startup // // Synchronisation: // - Audio output may be suppressed until a video keyframe (IDR) is seen when required by the output policy // - This ensures correct A/V alignment for HomeKit consumers // // Extending classes are expected to implement or override, as needed: // // streamer.connect(options) // streamer.close() // streamer.onUpdate(deviceData) // streamer.onShutdown() // // The following should be implemented when supported by the transport: // // streamer.sendTalkback(talkingBuffer) // // The following getters should be overridden: // // streamer.codecs <- return object with codecs being used (video, audio, talkback) // streamer.capabilities <- return object with streaming capabilities (live, record, talkback, buffering) // // The following properties should be overridden when required: // // blankAudio - Buffer containing a blank audio segment for the type of audio being used // // Code version 2026.05.06 // Mark Hulskamp 'use strict'; // Define nodejs module requirements import { Buffer } from 'node:buffer'; import { setInterval, clearInterval, setTimeout, clearTimeout } from 'node:timers'; import fs from 'fs'; import path from 'node:path'; import { PassThrough } from 'stream'; // Define our modules import HomeKitDevice from './HomeKitDevice.js'; // Define constants import { TIMERS, RESOURCE_FRAMES, RESOURCE_PATH, LOG_LEVELS, __dirname } from './consts.js'; const MAX_BUFFERED_ITEMS_PER_OUTPUT_PER_TICK = 20; // Prevent one output starving others const STREAM_FRAME_INTERVAL = 1000 / 30; // 30fps approx const OUTPUT_LOOP_INTERVAL = 10; // Shared output scheduler interval const OUTPUT_BUDGET_LOG_INTERVAL = 30000; // Throttle per-streamer over-budget debug logs const OUTPUT_PLAYOUT_POLICY = { live: { requireKeyFrameStart: false, playoutDelayMs: 250, maxLagBehindLiveMs: 1000, dueTolerance: 12, dueSlack: 18, catchupExitThresholdMs: 300, catchupAudioBurstLimit: 3, normalVideoBurstLimit: 2, normalAudioBurstLimit: 4, catchupVideoBurstLimit: 6, }, record: { requireKeyFrameStart: true, playoutDelayMs: 200, maxLagBehindLiveMs: 1200, dueTolerance: 12, dueSlack: 20, catchupExitThresholdMs: 350, catchupAudioBurstLimit: 6, normalVideoBurstLimit: 3, normalAudioBurstLimit: 6, catchupVideoBurstLimit: 10, }, }; // Default capacity for generic RingBuffer usage. // This should remain a sensible general-purpose default and not be tied to // any specific streaming or media retention behaviour. const RINGBUFFER_DEFAULT_CAPACITY = 1024; // Maximum number of items the RingBuffer is allowed to allocate. // // RingBuffer capacity is measured in item slots, not bytes. // This prevents unbounded memory growth if producers outpace consumers // or if trimming/output flow stalls unexpectedly. // // The buffer grows dynamically up to this limit using doubling growth. // Once reached, additional growth attempts are rejected. const RINGBUFFER_MAX_CAPACITY = 8192; // Initial capacity used by Streamer for its shared media buffer. // This may diverge from the generic RingBuffer default as buffering strategy, // retention window, or media characteristics evolve. const STREAMER_INITIAL_BUFFER_CAPACITY = 1024; // RingBuffer // // Dynamically-sized circular buffer with logical indexing. // - O(1) push and shift (no array reallocation or front-splice) // - Grows by doubling when capacity is exceeded, up to RINGBUFFER_MAX_CAPACITY // - Maintains a monotonically increasing logical startIndex // so external consumers can track absolute positions even as data is trimmed // // Terminology: // - head: physical index of the first logical item // - size: number of valid items currently stored // - startIndex: logical index of the first item (external reference point) // // Access is done via logical offsets (0..size-1), which are translated // to physical positions via modulo arithmetic. export class RingBuffer { items = []; capacity = 0; head = 0; size = 0; startIndex = 0; constructor(startIndex = 0, capacity = RINGBUFFER_DEFAULT_CAPACITY) { // capacity = physical storage size (number of slots in the circular array) // Clamp to RINGBUFFER_MAX_CAPACITY so callers cannot bypass the growth limit this.capacity = Number.isInteger(capacity) === true && capacity > 0 ? Math.min(capacity, RINGBUFFER_MAX_CAPACITY) : RINGBUFFER_DEFAULT_CAPACITY; // Preallocate backing array to avoid resizing on every push this.items = new Array(this.capacity); // startIndex = logical index of the first item in the buffer // (used by external consumers to map absolute positions) this.startIndex = Number.isInteger(startIndex) === true && startIndex >= 0 ? startIndex : 0; } physicalOffset(offset) { // Convert a logical offset (0..size-1) into a physical array index. // head = physical position of logical offset 0. // We wrap using modulo to stay within the circular buffer. if (Number.isInteger(offset) !== true || offset < 0) { return -1; } return (this.head + offset) % this.capacity; } getByOffset(offset) { let physicalOffset = 0; // Bounds check against current logical size if (Number.isInteger(offset) !== true || offset < 0 || offset >= this.size) { return undefined; } // Translate logical offset -> physical slot physicalOffset = this.physicalOffset(offset); // Return stored item (or undefined if something went wrong) return physicalOffset >= 0 ? this.items[physicalOffset] : undefined; } grow() { // Prevent unbounded buffer growth // If the maximum capacity has already been reached, reject expansion if (this.capacity >= RINGBUFFER_MAX_CAPACITY) { return false; } // Double capacity to amortize growth cost (O(n), but infrequent) // Clamp growth to the configured maximum capacity let newCapacity = Math.min(this.capacity * 2, RINGBUFFER_MAX_CAPACITY); let newItems = new Array(newCapacity); let index = 0; // Re-pack items linearly starting at index 0 // This removes wrap-around and resets head to 0 while (index < this.size) { newItems[index] = this.getByOffset(index); index++; } this.items = newItems; this.capacity = newCapacity; // After re-pack, logical offset 0 is now at physical index 0 this.head = 0; return true; } push(item) { let tailOffset = 0; // Grow if buffer is full (no overwriting policy here) if (this.size >= this.capacity) { // Reject push if the buffer can no longer expand if (this.grow() !== true) { return false; } } // Tail = logical position "size" tailOffset = this.physicalOffset(this.size); if (tailOffset < 0) { return false; } // Insert at tail and increase size this.items[tailOffset] = item; this.size++; return true; } shift(count, resetStartIndex = undefined) { let removeCount = 0; let index = 0; let physicalOffset = 0; // Remove N items from the head (logical front of buffer) if (Number.isInteger(count) !== true || count <= 0) { return false; } removeCount = Math.min(count, this.size); // Clear references so GC can reclaim memory while (index < removeCount) { physicalOffset = this.physicalOffset(index); if (physicalOffset >= 0) { this.items[physicalOffset] = undefined; } index++; } this.head = (this.head + removeCount) % this.capacity; this.size -= removeCount; this.startIndex += removeCount; if (this.size === 0) { this.head = 0; if (Number.isInteger(resetStartIndex) === true && resetStartIndex >= 0) { this.startIndex = resetStartIndex; } } return true; } clear(resetStartIndex = 0) { let index = 0; let physicalOffset = 0; while (index < this.size) { physicalOffset = this.physicalOffset(index); if (physicalOffset >= 0) { this.items[physicalOffset] = undefined; } index++; } this.head = 0; this.size = 0; if (Number.isInteger(resetStartIndex) === true && resetStartIndex >= 0) { this.startIndex = resetStartIndex; } return true; } } // Streamer object export default class Streamer { static H264NALUS = { START_CODE: Buffer.from([0x00, 0x00, 0x00, 0x01]), TYPES: { SLICE_NON_IDR: 1, SLICE_PART_A: 2, SLICE_PART_B: 3, SLICE_PART_C: 4, IDR: 5, // Instantaneous Decoder Refresh SEI: 6, SPS: 7, PPS: 8, AUD: 9, END_SEQUENCE: 10, END_STREAM: 11, STAP_A: 24, FU_A: 28, }, }; static STREAM_TYPE = { LIVE: 'live', RECORD: 'record', BUFFER: 'buffer', }; static MEDIA_TYPE = { VIDEO: 'video', AUDIO: 'audio', TALK: 'talk', METADATA: 'meta', }; static CODEC_TYPE = { H264: 'h264', AAC: 'aac', OPUS: 'opus', PCM: 'pcm', SPEEX: 'speex', META: 'meta', UNKNOWN: 'undefined', }; static MESSAGE = 'Streamer.onMessage'; // Message type for HomeKitDevice to listen for static MESSAGE_TYPE = { // Action type messages START_LIVE: 'start-live', STOP_LIVE: 'stop-live', START_RECORD: 'start-record', STOP_RECORD: 'stop-record', START_BUFFER: 'start-buffer', STOP_BUFFER: 'stop-buffer', // Status type messages SOURCE_CONNECTED: 'source-connected', SOURCE_CONNECTING: 'source-connecting', SOURCE_READY: 'source-ready', SOURCE_RECONNECTING: 'source-reconnecting', SOURCE_CLOSING: 'source-closing', SOURCE_CLOSED: 'source-closed', }; // Shared global scheduler for all active Streamer instances // This avoids having one timer per camera and reduces event loop overhead static #streamers = new Map(); // uuid => Streamer instance static #timer = undefined; // Shared timer for all active streamers supportDump = false; // Enable support for dumping stats on demand for this streamer instance log = undefined; // Logging function object videoEnabled = undefined; // Video stream on camera enabled or not audioEnabled = undefined; // Audio from camera enabled or not online = undefined; // Camera online or not migrating = undefined; // Device is transferring/migrating between APIs nest_google_device_uuid = undefined; // Nest/Google UUID of the device connecting blankAudio = undefined; // Blank audio 'frame' for the type of audio being used, to be defined in subclass if audio is supported video = { codec: undefined, profile: undefined, width: undefined, height: undefined, fps: undefined, bitrate: undefined, }; audio = { codec: undefined, profile: undefined, sampleRate: undefined, channels: undefined, bitrate: undefined, frameDuration: undefined, }; // Internal data only for this class #HomeKitDeviceUUID = undefined; // HomeKitDevice uuid for this streamer #bufferDuration = 0; // Duration of media to keep in the shared buffer based on media timestamps #bufferEnabled = false; // Retained buffering policy flag owned by Streamer #buffer = undefined; // Shared rotating ring buffer used by buffering, live and recording outputs #outputs = new Map(); // Live and recording outputs keyed by session id #cameraFrames = {}; // H264 resource frames for offline, video off, transferring #sequenceCounters = {}; // Sequence counters for item types #itemIndex = 0; // Monotonic item index for shared buffer cursor tracking #videoState = {}; // Video state tracking #audioState = {}; // Audio state tracking #lastFallbackFrameTime = 0; // Timer for pacing fallback frames #lastBudgetLogTime = 0; // Last time budget processing was sampled/logged #outputErrors = 0; // Consecutive output loop failures for this instance #lastMediaTime = {}; // Track last buffered media time per type for fallback ordering guards #sourceState = Streamer.MESSAGE_TYPE.SOURCE_CLOSED; // Track stream source state from messages for internal logic and logging #connectOptions = {}; // Store options from connect to use on reconnects #lifecycleQueue = Promise.resolve(); // Serializes source connect/close operations to avoid lifecycle races #stats = { source: { connectingAt: undefined, connectedAt: undefined, readyAt: undefined, lastItemAt: undefined, lastVideoItemAt: undefined, lastAudioItemAt: undefined, lastKeyframeAt: undefined, firstVideoItemAt: undefined, firstAudioItemAt: undefined, firstKeyframeAt: undefined, reconnects: 0, }, items: { video: 0, audio: 0, talk: 0, metadata: 0, keyframes: 0, }, }; // Codecs being used for video, audio and talking get codecs() { return { video: undefined, audio: undefined, talkback: undefined, }; } // Capabilities supported by this streamer get capabilities() { return { live: false, record: false, talkback: false, buffering: false, }; } // Get current stream source state get sourceState() { return this.#sourceState; } constructor(uuid, deviceData, options) { if (Object.values(LOG_LEVELS).every((fn) => typeof options?.log?.[fn] === 'function')) { this.log = options.log; } this.#HomeKitDeviceUUID = uuid; HomeKitDevice.message(uuid, Streamer.MESSAGE, this); HomeKitDevice.message(uuid, HomeKitDevice.UPDATE, this); HomeKitDevice.message(uuid, HomeKitDevice.TIMER, this); HomeKitDevice.message(uuid, HomeKitDevice.SHUTDOWN, this); this.migrating = deviceData?.migrating === true; this.online = deviceData?.online === true; this.videoEnabled = deviceData?.streaming_enabled === true; this.audioEnabled = deviceData?.audio_enabled === true; this.nest_google_device_uuid = deviceData?.nest_google_device_uuid; const loadFrameResource = (filename, label) => { let buffer = undefined; let file = path.resolve(__dirname, RESOURCE_PATH, filename); if (fs.existsSync(file) === true) { buffer = fs.readFileSync(file); if (buffer.indexOf(Streamer.H264NALUS.START_CODE) === 0) { buffer = buffer.subarray(Streamer.H264NALUS.START_CODE.length); } } else { this.log?.warn?.('Failed to load %s video resource for "%s"', label, deviceData.description); } return buffer; }; this.#cameraFrames = { offline: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFFLINE, 'offline'), off: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFF, 'video off'), transfer: loadFrameResource(RESOURCE_FRAMES.CAMERA_TRANSFER, 'transferring'), }; this.#lastFallbackFrameTime = Date.now(); this.supportDump = options?.supportDump === true; // Setup buffer duration if passed in as an option, otherwise default to 5s // Clamp to sane bounds to avoid invalid or excessive buffer sizes this.#bufferDuration = Number.isInteger(options?.bufferDuration) === true && options.bufferDuration > 0 ? Math.min(Math.max(options.bufferDuration, 2000), 15000) : 5000; } async onUpdate(deviceData) { if (typeof deviceData !== 'object') { return; } if (deviceData?.migrating !== undefined && this.migrating !== deviceData?.migrating) { this.migrating = deviceData.migrating; } if (deviceData?.nest_google_device_uuid !== undefined && this.nest_google_device_uuid !== deviceData?.nest_google_device_uuid) { this.nest_google_device_uuid = deviceData?.nest_google_device_uuid; if (this.hasActiveStreams() === true) { await this.#doClose(); await this.#doConnect(); } } if ( (deviceData?.online !== undefined && this.online !== deviceData.online) || (deviceData?.streaming_enabled !== undefined && this.videoEnabled !== deviceData.streaming_enabled) || (deviceData?.audio_enabled !== undefined && this.audioEnabled !== deviceData.audio_enabled) ) { if (deviceData?.online !== undefined) { this.online = deviceData.online === true; } if (deviceData?.streaming_enabled !== undefined) { this.videoEnabled = deviceData.streaming_enabled === true; } if (deviceData?.audio_enabled !== undefined) { this.audioEnabled = deviceData.audio_enabled === true; } if (this.hasActiveStreams() === true) { if (this.online === false || this.videoEnabled === false || this.audioEnabled === false) { await this.#doClose(); return; } await this.#doConnect(); } } } async onMessage(type, message) { if (typeof type !== 'string' || type === '') { return; } let sessionID = message?.sessionID !== undefined ? String(message.sessionID) : undefined; let options = typeof message?.options === 'object' && message.options !== null ? message.options : undefined; if (type === Streamer.MESSAGE_TYPE.START_BUFFER) { // Enable retained buffer and ensure source is connected. // This does not create an output stream, only prepares buffering for future use. if (this.capabilities.buffering !== true) { this?.log?.debug?.('Buffering is unsupported for "%s"', this.nest_google_device_uuid); return; } await this.#startBuffering(options); return; } if (type === Streamer.MESSAGE_TYPE.STOP_BUFFER) { // Disable retained buffer. // If no outputs are active, this may also allow the source to close. if (this.capabilities.buffering !== true) { this?.log?.debug?.('Buffering is unsupported for "%s"', this.nest_google_device_uuid); return; } await this.#stopBuffering(); return; } if (type === Streamer.MESSAGE_TYPE.START_LIVE) { // Start a live streaming output for HomeKit. // This creates PassThrough streams and begins feeding real-time data. if (this.capabilities.live !== true) { this?.log?.debug?.('Live streaming is unsupported for "%s"', this.nest_google_device_uuid); return; } return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.LIVE, options); } if (type === Streamer.MESSAGE_TYPE.STOP_LIVE) { // Stop a live streaming output. // Cleans up streams and may close source if no other outputs remain. if (this.capabilities.live !== true) { this?.log?.debug?.('Live streaming is unsupported for "%s"', this.nest_google_device_uuid); return; } await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.LIVE); return; } if (type === Streamer.MESSAGE_TYPE.START_RECORD) { // Start a recording output (HKSV). // Uses retained buffer and selects a start position based on requested time. // Decoder safety (keyframe alignment) is handled during playout. if (this.capabilities.record !== true) { this?.log?.debug?.('Recording is unsupported for "%s"', this.nest_google_device_uuid); return; } return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.RECORD, options); } if (type === Streamer.MESSAGE_TYPE.STOP_RECORD) { // Stop a recording output. // If no sessionID is provided, stops the first active recording stream. if (this.capabilities.record !== true) { this?.log?.debug?.('Recording is unsupported for "%s"', this.nest_google_device_uuid); return; } await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.RECORD); return; } if ( type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING || type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTED || type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING || type === Streamer.MESSAGE_TYPE.SOURCE_CLOSING || type === Streamer.MESSAGE_TYPE.SOURCE_CLOSED || type === Streamer.MESSAGE_TYPE.SOURCE_READY ) { // Handle lifecycle events from the underlying stream source (WebRTC, NexusTalk, etc). // These events reflect connection state and readiness for media delivery. // Ignore duplicate state transitions to avoid log spam and unnecessary stat updates if (this.#sourceState === type) { return; } // Prevent reconnecting -> connecting downgrade, which can occur during retry loops if (this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING && type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING) { return; } this.#sourceState = type; // Reset transient state when connection is starting, restarting, or fully closed if ( type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING || type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING || type === Streamer.MESSAGE_TYPE.SOURCE_CLOSED ) { this.#resetSourceState(); this.#resetSourceStats(); } // Track timing and reconnect metrics for diagnostics/support dump if (typeof this.#stats === 'object' && this.#stats !== null) { let now = Date.now(); // Initial connection attempt if (type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTING) { this.#stats.source.connectingAt = now; } // Transport connected (but not yet ready for frames) if (type === Streamer.MESSAGE_TYPE.SOURCE_CONNECTED) { this.#stats.source.connectedAt = now; } // Source ready and delivering media if (type === Streamer.MESSAGE_TYPE.SOURCE_READY) { this.#stats.source.readyAt = now; } // Count reconnect attempts separately if (type === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING) { this.#stats.source.reconnects = (this.#stats.source.reconnects ?? 0) + 1; } } // Log unified source state for visibility across different transport implementations this?.log?.debug?.( 'Stream source is "%s" for uuid "%s"%s', type, this.nest_google_device_uuid, typeof message?.reason === 'string' && message.reason !== '' ? ' (' + message.reason + ')' : '', ); return; } } async onShutdown() { Streamer.#removeStreamer(this); await this.stopEverything(); } async stopEverything() { if (this.hasActiveStreams() === true) { this?.log?.debug?.('Stopped buffering, live and recording from device uuid "%s"', this.nest_google_device_uuid); for (let output of this.#outputs.values()) { this.#cleanupOutput(output); } this.#outputs.clear(); this.#resetRetainedState(); this.#syncSchedulerState(); await this.#doClose(); } } addMedia(media) { let mediaType = undefined; let codec = undefined; let now = Date.now(); let data = undefined; let audioFrameDuration = 0; let sequence = 0; let sourceTimestamp = 0; let mediaTime = 0; let keyFrame = false; let h264Result = undefined; // Validate incoming media object if (typeof media !== 'object' || media === null) { return; } mediaType = typeof media.type === 'string' ? media.type.toLowerCase() : undefined; keyFrame = media?.keyFrame === true; // Validate media type and payload if ( typeof mediaType !== 'string' || mediaType.trim() === '' || (mediaType !== Streamer.MEDIA_TYPE.VIDEO && mediaType !== Streamer.MEDIA_TYPE.AUDIO && mediaType !== Streamer.MEDIA_TYPE.TALK && mediaType !== Streamer.MEDIA_TYPE.METADATA) || Buffer.isBuffer(media.data) !== true || media.data.length === 0 ) { return; } // Do not process if no active outputs if (this.hasActiveStreams() !== true) { return; } data = media.data; // Ensure shared buffer exists before proceeding this.#ensureSharedBuffer(); if (this.#buffer === undefined) { return; } // Resolve codec (explicit media override first, otherwise fallback to configured codecs) codec = typeof media?.codec === 'string' ? media.codec.toLowerCase() : mediaType === Streamer.MEDIA_TYPE.VIDEO ? this.codecs?.video : mediaType === Streamer.MEDIA_TYPE.AUDIO ? this.codecs?.audio : mediaType === Streamer.MEDIA_TYPE.TALK ? this.codecs?.talkback : mediaType === Streamer.MEDIA_TYPE.METADATA ? Streamer.CODEC_TYPE.META : undefined; if (typeof codec !== 'string' || codec.trim() === '') { return; } // Update advertised video stream properties if (mediaType === Streamer.MEDIA_TYPE.VIDEO) { this.video.codec = codec; if (media?.profile?.trim?.() !== '') { this.video.profile = media.profile; } if (Number.isFinite(media?.bitrate) === true && media.bitrate > 0) { this.video.bitrate = media.bitrate; } } // Update advertised audio stream properties if (mediaType === Streamer.MEDIA_TYPE.AUDIO) { this.audio.codec = codec; if (media?.profile?.trim?.() !== '') { this.audio.profile = media.profile; } if (Number.isFinite(media?.sampleRate) === true && media.sampleRate > 0) { this.audio.sampleRate = media.sampleRate; } if (Number.isFinite(media?.channels) === true && media.channels > 0) { this.audio.channels = media.channels; } if (Number.isFinite(media?.bitrate) === true && media.bitrate > 0) { this.audio.bitrate = media.bitrate; } } // Initialise sequence counter if required if (typeof this.#sequenceCounters?.[mediaType] !== 'number') { this.#sequenceCounters[mediaType] = 0; } // Use provided sequence/timestamp or fallback to generated values sequence = Number.isFinite(media?.sequence) === true ? media.sequence : this.#sequenceCounters[mediaType]++; sourceTimestamp = Number.isFinite(media?.timestamp) === true ? Math.round(media.timestamp) : now; // Ensure monotonic media time (never goes backwards) if (typeof this.#lastMediaTime?.[mediaType] !== 'number') { this.#lastMediaTime[mediaType] = 0; } mediaTime = sourceTimestamp < this.#lastMediaTime[mediaType] ? this.#lastMediaTime[mediaType] : sourceTimestamp; this.#lastMediaTime[mediaType] = mediaTime; // Codec-specific H264 processing if (mediaType === Streamer.MEDIA_TYPE.VIDEO && codec === Streamer.CODEC_TYPE.H264) { h264Result = this.#processH264VideoMedia(data, sourceTimestamp, keyFrame, now); if (typeof h264Result !== 'object' || h264Result === null) { return; } data = h264Result.data; keyFrame = h264Result.keyFrame === true; } // Audio frame timing smoothing if (mediaType === Streamer.MEDIA_TYPE.AUDIO) { if (typeof this.#audioState.lastSourceFrameTime === 'number') { audioFrameDuration = sourceTimestamp - this.#audioState.lastSourceFrameTime; if (audioFrameDuration > 0) { let isOutlier = false; if (typeof this.audio.frameDuration === 'number' && this.audio.frameDuration > 0) { let deviance = Math.abs(audioFrameDuration - this.audio.frameDuration) / this.audio.frameDuration; if (deviance > 0.5) { isOutlier = true; } } if (isOutlier !== true) { if (typeof this.audio.frameDuration === 'number') { this.audio.frameDuration = this.audio.frameDuration * 0.8 + audioFrameDuration * 0.2; } else { this.audio.frameDuration = audioFrameDuration; } } } } this.#audioState.lastSourceFrameTime = sourceTimestamp; } // Update source/item stats for support dump and diagnostics if (this.supportDump === true && typeof this.#stats?.source === 'object' && this.#stats.source !== null) { if (keyFrame === true && typeof this.#stats.source.firstKeyframeAt !== 'number') { this.#stats.source.firstKeyframeAt = now; } this.#stats.source.lastItemAt = now; if (mediaType === Streamer.MEDIA_TYPE.VIDEO) { this.#stats.source.lastVideoItemAt = now; if (typeof this.#stats.source.firstVideoItemAt !== 'number') { this.#stats.source.firstVideoItemAt = now; } } if (mediaType === Streamer.MEDIA_TYPE.AUDIO) { this.#stats.source.lastAudioItemAt = now; if (typeof this.#stats.source.firstAudioItemAt !== 'number') { this.#stats.source.firstAudioItemAt = now; } } if (keyFrame === true) { this.#stats.source.lastKeyframeAt = now; } } if (this.supportDump === true && typeof this.#stats?.items === 'object' && this.#stats.items !== null) { if (mediaType === Streamer.MEDIA_TYPE.VIDEO) { this.#stats.items.video++; } if (mediaType === Streamer.MEDIA_TYPE.AUDIO) { this.#stats.items.audio++; } if (mediaType === Streamer.MEDIA_TYPE.TALK) { this.#stats.items.talk++; } if (mediaType === Streamer.MEDIA_TYPE.METADATA) { this.#stats.items.metadata++; } if (keyFrame === true) { this.#stats.items.keyframes++; } } // Push final packet into shared buffer let pushed = this.#buffer.push({ index: this.#itemIndex, type: mediaType, codec: codec, time: mediaTime, sourceTimestamp: sourceTimestamp, sequence: sequence, keyFrame: keyFrame === true, data: data, }); if (pushed === true) { // Increment buffer index if push was successful this.#itemIndex++; } } #processH264VideoMedia(data, sourceTimestamp, keyFrame, now) { let nalUnits = undefined; let containsFrame = false; let resolution = undefined; let isAnnexB = false; let previousFPS = undefined; let frameDelta = 0; let fps = 0; // Split incoming payload into NAL units nalUnits = this.#getH264NALUnits(data); if (nalUnits.length === 0) { return undefined; } // Derive Annex-B from parser behaviour: // - Non-Annex-B returns original buffer reference // - Annex-B returns subarray views isAnnexB = nalUnits[0].data !== data; for (let nalu of nalUnits) { // SPS -> update resolution and cache if (nalu.type === Streamer.H264NALUS.TYPES.SPS) { this.#videoState.lastSPS = Buffer.from(nalu.data); resolution = this.#decodeH264SPS(nalu.data); if (typeof resolution === 'object' && resolution !== null) { if (Number.isInteger(resolution.width) === true && resolution.width > 0) { this.video.width = resolution.width; } if (Number.isInteger(resolution.height) === true && resolution.height > 0) { this.video.height = resolution.height; } } } // PPS -> cache for future keyframes if (nalu.type === Streamer.H264NALUS.TYPES.PPS) { this.#videoState.lastPPS = Buffer.from(nalu.data); } // IDR (keyframe) if (nalu.type === Streamer.H264NALUS.TYPES.IDR) { this.#videoState.lastIDR = Buffer.from(nalu.data); containsFrame = true; keyFrame = true; } // Non-IDR slice still represents a frame if (nalu.type === Streamer.H264NALUS.TYPES.SLICE_NON_IDR) { containsFrame = true; } } // Convert non-Annex-B to Annex-B format for downstream compatibility if (isAnnexB !== true) { if (nalUnits.length !== 1) { return undefined; } data = Buffer.concat([Streamer.H264NALUS.START_CODE, nalUnits[0].data]); } // FPS estimation based on source timestamps if (containsFrame === true) { if (typeof this.#videoState.lastSourceFrameTime === 'number' && sourceTimestamp > this.#videoState.lastSourceFrameTime) { frameDelta = sourceTimestamp - this.#videoState.lastSourceFrameTime; previousFPS = this.video.fps; if ( typeof previousFPS !== 'number' || previousFPS <= 0 || Math.abs(frameDelta - 1000 / previousFPS) / (1000 / previousFPS) <= 0.5 ) { fps = 1000 / frameDelta; if (typeof previousFPS === 'number') { this.video.fps = previousFPS * 0.8 + fps * 0.2; } else { this.video.fps = fps; } if ( this.#videoState.streamInfoLogged !== true && typeof this.video.width === 'number' && typeof this.video.height === 'number' && typeof this.video.fps === 'number' ) { this.#videoState.streamInfoLogged = true; this?.log?.debug?.( 'Receiving incoming stream from device "%s": %s %sx%s @ %sfps', this.nest_google_device_uuid, this.video.codec, this.video.width, this.video.height, Math.round(this.video.fps), ); } if (typeof previousFPS === 'number' && typeof this.video.fps === 'number') { if ( Math.abs(Math.round(this.video.fps) - Math.round(previousFPS)) >= 3 && (typeof this.#videoState.lastFPSLogTime !== 'number' || now - this.#videoState.lastFPSLogTime >= 30000) ) { this.#videoState.lastFPSLogTime = now; this?.log?.debug?.('FPS from device "%s" has changed to %sfps', this.nest_google_device_uuid, Math.round(this.video.fps)); } } } } this.#videoState.lastSourceFrameTime = sourceTimestamp; } // Track last keyframe index for buffer trimming / recording alignment if (keyFrame === true) { this.#videoState.lastIDRIndex = this.#itemIndex; } return { data: data, keyFrame: keyFrame, }; } async #startBuffering(options = {}) { this.#ensureSharedBuffer(); if (this.#bufferEnabled === true) { return; } this.#bufferEnabled = true; this.log?.debug?.('Started buffering from device uuid "%s"', this.nest_google_device_uuid); this.#syncSchedulerState(); await this.#doConnect(options); } async #stopBuffering() { if (this.#bufferEnabled !== true) { return; } this.#bufferEnabled = false; this.log?.debug?.('Stopped buffering from device uuid "%s"', this.nest_google_device_uuid); if (this.isStreaming() === false) { this.#resetRetainedState(); this.#syncSchedulerState(); await this.#doClose(); return; } this.#syncSchedulerState(); } async #createOutput(sessionID, type, options) { let existing = undefined; let output = undefined; let video = undefined; let audio = null; let talkback = null; let includeAudio = options?.includeAudio === true; let waitForReady = Number.isInteger(options?.waitForReady) === true ? options.waitForReady : 0; let startCursor = this.#itemIndex; let buffer = undefined; let itemsLength = 0; let bufferStart = 0; let item = undefined; let recordTime = options?.recordTime; let closestOffset = -1; let closestDelta = Number.POSITIVE_INFINITY; let index = 0; let itemTime = 0; let startTime = Date.now(); // Validate session id if (typeof sessionID !== 'string' || sessionID === '') { return; } // Check for existing output with this session id existing = this.#outputs.get(sessionID); // Only allow a single record output, regardless of session id if (type === Streamer.STREAM_TYPE.RECORD && existing === undefined) { for (output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.RECORD) { existing = output; break; } } } // Reuse existing output when possible, otherwise reject type conflict if (existing !== undefined) { if (existing.type !== type) { this?.log?.warn?.( 'Cannot start output for device uuid "%s" and session id "%s" as it is already in use for "%s"', this.nest_google_device_uuid, sessionID, existing.type, ); return; } return { video: existing.video, audio: existing.audio, talkback: existing.talkback, }; } // Ensure retained buffer exists and start/connect source if needed this.#ensureSharedBuffer(); await this.#doConnect(options); buffer = this.#buffer; if (buffer instanceof RingBuffer !== true) { return; } // Create streams for this output video = new PassThrough(); audio = includeAudio === true ? new PassThrough() : null; talkback = type === Streamer.STREAM_TYPE.LIVE && includeAudio === true ? new PassThrough({ highWaterMark: 1024 * 16 }) : null; // Prevent unhandled stream errors from bubbling video?.on?.('error', () => {}); audio?.on?.('error', () => {}); talkback?.on?.('error', () => {}); // Determine initial cursor for recording if (type === Streamer.STREAM_TYPE.RECORD) { itemsLength = buffer.size; bufferStart = buffer.startIndex; closestOffset = -1; closestDelta = Number.POSITIVE_INFINITY; index = 0; // Default to buffer start if valid if (typeof bufferStart === 'number') { startCursor = bufferStart; } // Recording should start from the retained position closest to the requested // record time only. Decoder/keyframe safety is handled later during playout // inside #processBufferedOutput(), not here. if (itemsLength !== 0 && typeof recordTime === 'number' && Number.isFinite(recordTime) === true) { while (index < itemsLength) { item = buffer.getByOffset(index); if (typeof item?.time === 'number') { itemTime = Math.abs(item.time - recordTime); if (itemTime < closestDelta) { closestDelta = itemTime; closestOffset = index; } } index++; } if (closestOffset !== -1) { startCursor = buffer.getByOffset(closestOffset)?.index; } } // Never allow cursor to point before current retained window if (typeof bufferStart === 'number' && startCursor < bufferStart) { startCursor = bufferStart; } } // Live streaming should attach at the live edge. // We do not backtrack live outputs into retained media here. // Any decoder startup/keyframe handling remains the responsibility of // #processBufferedOutput(). if (type === Streamer.STREAM_TYPE.LIVE) { startCursor = this.#itemIndex; } // Create output state // Each output (LIVE / RECORD / BUFFER) consumes from the shared ring buffer // using its own cursor and playout timing model. // // Key concepts: // - cursor: current read position in the shared buffer (absolute index) // - catchingUp: used when starting behind live edge (mainly RECORD) to fast-drain // - sourceBaseTime / wallclockBaseTime: // map source timestamps -> real time for paced playback // - policy: defines how this output consumes media (latency vs continuity tradeoff) // // IMPORTANT: // - All smoothing / pacing happens in Streamer (not in NexusTalk/WebRTC) // - Each output has its own independent timing model // - Policies MUST be tuned per output type (do not unify blindly) output = { sessionID: sessionID, type: type, // Writable streams (ffmpeg pipes etc) video: video, audio: audio, talkback: talkback, talkbackTimeout: undefined, // Whether audio should be written for this output includeAudio: includeAudio, // Read cursor into shared ring buffer // RECORD starts from the retained position closest to requested time // LIVE starts at the current live edge cursor: startCursor, // Catch-up mode: // - RECORD starts in catch-up to drain historical buffer // - LIVE starts at the live edge so no catch-up is required initially catchingUp: type === Streamer.STREAM_TYPE.RECORD, catchupTicks: 0, catchupStableFrames: 0, // Codec / decoder state tracking sentCodecConfig: false, // SPS/PPS sent seenKeyFrame: false, // first keyframe seen // Last time a video frame was written (used for fallback timing) lastVideoWriteTime: 0, // Time mapping for paced playback: // wallclockTime = wallclockBaseTime + (item.time - sourceBaseTime) sourceBaseTime: undefined, wallclockBaseTime: undefined, // Playout policy: // Defines how aggressively we stay near live edge vs preserve continuity // // LIVE: // - small delay (~100ms) // - prefers low latency // // RECORD: // - slightly delayed paced playback // - preserves continuity while draining retained media policy: { ...(OUTPUT_PLAYOUT_POLICY[type] ?? OUTPUT_PLAYOUT_POLICY.live) }, // Debug / instrumentation stats (used for tuning pacing behaviour) stats: { startedAt: Date.now(), firstWriteAt: undefined, firstVideoWriteAt: undefined, firstAudioWriteAt: undefined, writes: { total: 0, video: 0, audio: 0 }, drops: { videoBeforeKeyframe: 0, audioBeforeKeyframe: 0, bufferTrimmed: 0 }, }, }; // Attach talkback handling for live streams if (talkback !== null) { talkback.on('data', (data) => { if (typeof this?.sendTalkback === 'function') { this.sendTalkback(data); clearTimeout(output.talkbackTimeout); output.talkbackTimeout = setTimeout(() => { this.sendTalkback(Buffer.alloc(0)); }, TIMERS.TALKBACK_AUDIO.interval); } }); talkback.on('close', () => { clearTimeout(output?.talkbackTimeout); if (typeof this?.sendTalkback === 'function') { this.sendTalkback(Buffer.alloc(0)); } }); } // Register output before any optional readiness wait this.#outputs.set(sessionID, output); this.#syncSchedulerState(); // Optionally wait for source readiness before returning stream handles if (waitForReady > 0) { while (Date.now() - startTime < waitForReady) { if ( this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_READY || this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_CLOSED || this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_RECONNECTING ) { break; } await new Promise((resolve) => setTimeout(resolve, 25)); } } this?.log?.debug?.('Started %s stream from device uuid "%s" and session id "%s"', type, this.nest_google_device_uuid, sessionID); return { video: video, audio: audio, talkback: talkback, }; } async #stopOutput(sessionID, type) { let output = undefined; let hasOtherLiveOutputs = false; let id = undefined; let activeOutput = undefined; // Resolve output by session id if provided if (typeof sessionID === 'string' && sessionID !== '') { output = this.#outputs.get(sessionID); } // For recording, allow stopping the first active record stream if no session id was provided if (output === undefined && type === Streamer.STREAM_TYPE.RECORD) { for (let candidate of this.#outputs.values()) { if (candidate?.type === Streamer.STREAM_TYPE.RECORD) { output = candidate; break; } } } // Nothing matched to stop if (output === undefined) { return; } // Ensure we are not stopping a mismatched type (e.g. live vs record) if (output.type !== type) { this?.log?.warn?.( 'Cannot stop stream for device uuid "%s" and session id "%s" as it is type "%s" not "%s"', this.nest_google_device_uuid, output.sessionID, output.type, type, ); return; } // If this is the last live output and support dump is enabled, log per-output stats before cleanup if (output.type === Streamer.STREAM_TYPE.LIVE && this.supportDump === true) { for ([id, activeOutput] of this.#outputs) { if (id !== output.sessionID && activeOutput?.type === Streamer.STREAM_TYPE.LIVE) { hasOtherLiveOutputs = true; break; } } if (hasOtherLiveOutputs !== true) { this.#outputStats(output, Date.now()); } } this?.log?.debug?.( 'Stopping %s stream from device uuid "%s" and session id "%s"', type, this.nest_google_device_uuid, output.sessionID, ); // Cleanup streams, timers, and any talkback state this.#cleanupOutput(output); // Remove from active outputs this.#outputs.delete(output.sessionID); // Clear retained state when last output stops and buffering is disabled. // This prevents stale buffered media being reused by the next session. if (this.#outputs.size === 0 && this.#bufferEnabled !== true) { this.#resetRetainedState(); } // Update scheduler based on remaining activity this.#syncSchedulerState(); // If nothing remains active, fully close underlying source if (this.isStreaming() === false && this.isBuffering() === false) { await this.#doClose(); } } isBuffering() { return this.#bufferEnabled === true; } isStreaming() { return this.#outputs.size !== 0; } isRecording() { for (let output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.RECORD) { return true; } } return false; } isLiveStreaming() { for (let output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.LIVE) { return true; } } return false; } hasActiveStreams() { return this.#bufferEnabled === true || this.#outputs.size !== 0; } isSourceReady() { return this.#sourceState === Streamer.MESSAGE_TYPE.SOURCE_READY; } setSourceState(type, reason) { if (typeof type !== 'string' || type === '') { return; } HomeKitDevice.message(this.#HomeKitDeviceUUID, Streamer.MESSAGE, type, { reason: reason }); }