UNPKG

homebridge-nest-accfactory

Version:

Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras

1,443 lines (1,225 loc) 86.4 kB
// Streamer // Part of homebridge-nest-accfactory // // Base class for HomeKit Camera/Doorbell media output. // // Streamer owns the HomeKit-facing media pipeline: // - shared media retention via MediaTimeline // - live and recording output fan-out // - adaptive per-output pacing, live jitter smoothing, and catch-up behaviour // - decoder-safe startup handling // - H264 SPS/PPS bootstrap for outputs // - fallback video frame injection when the camera is offline, video is disabled, or migrating // - optional talkback stream wiring to the active StreamTransport // // Streamer does not implement protocol-specific streaming. // // Protocol-specific connection handling is owned by StreamTransport subclasses // such as WebRTC and NexusTalk. A transport is injected into Streamer through // constructor options and is responsible for: // - authentication/signalling/session setup // - reconnect/recovery behaviour // - packet receive and protocol parsing // - media frame assembly // - codec metadata and source media stats // - lifecycle state reporting // // Transport boundary: // - Transport emits complete media frames using consumer.media(media) // - Transport media timestamps describe source media time // - Transport reports lifecycle state using consumer.state(state, reason) // - Streamer converts source media time into per-output playout timing // - Streamer consumes callbacks and writes retained/timed output streams // // Media model: // - Streamer expects complete media frames, not partial RTP/NAL fragments // - Video should be complete H264 NAL units or access units // - Audio should be complete AAC/PCM/Opus/etc frames as produced by the transport // - Streamer preserves media ordering using monotonic source timeline timestamps // - Transport-specific jitter repair and frame assembly stay below the StreamTransport boundary // // Buffering model: // - A shared MediaTimeline stores recent media items // - MediaTimeline uses one ordered RingBuffer plus video/audio/keyframe indexes // - Live and recording sessions read independently from the same retained timeline // - Each output maintains media-specific cursors plus a protected retention cursor // // Live streaming behaviour: // - Live outputs attach at the current live edge // - Output policy controls startup keyframe requirements, burst limits, and playout delay // - Live playout delay adapts upward during transient jitter and relaxes back toward low latency // - Catch-up mode drains retained media faster when an output falls too far behind live // - This keeps latency low while still allowing decoder-safe startup and smoother recovery // // Recording behaviour: // - Recording outputs can start from a requested timestamp // - The closest retained media item to the requested time is selected // - Decoder/keyframe safety is handled during playout, not during cursor selection // // H264 handling: // - Transport assembles H264 media // - Streamer caches SPS/PPS for output bootstrap // - Streamer expects H264 transport media to already be Annex-B access units // - Streamer injects SPS/PPS before first output keyframe when required // // Statistics: // - StreamTransport owns lifecycle/source media stats // - MediaTimeline owns retained buffer/index stats // - Streamer owns per-output write/drop/playout stats // // Code version 2026.05.18 // Mark Hulskamp 'use strict'; // Define nodejs module requirements import { Buffer } from 'node:buffer'; import { setInterval, clearInterval, setTimeout, clearTimeout } from 'node:timers'; import fs from 'fs'; import path from 'node:path'; import { PassThrough } from 'stream'; // Define our modules import HomeKitDevice from './HomeKitDevice.js'; import MediaTimeline from './mediatimeline.js'; import StreamTransport from './streamtransport.js'; import H264 from './h264.js'; // Define constants import { TIMERS, RESOURCE_FRAMES, RESOURCE_PATH, LOG_LEVELS, __dirname } from './consts.js'; const MAX_BUFFERED_ITEMS_PER_OUTPUT_PER_TICK = 20; // Prevent one output starving others const STREAM_FRAME_INTERVAL = 1000 / 30; // 30fps approx const OUTPUT_LOOP_INTERVAL = 10; // Shared output scheduler interval const OUTPUT_BUDGET_LOG_INTERVAL = 30000; // Throttle per-streamer over-budget debug logs const OUTPUT_STABLE_PLAYOUT_TARGET = 20; // Ticks before live playout delay relaxes back down const OUTPUT_KEYFRAME_AUDIO_PRIORITY_MS = 120; // Let due audio go first around large H264 keyframes const OUTPUT_AUDIO_OVERDUE_PRIORITY_MS = 40; // Do not let expensive video writes extend an audible audio gap const OUTPUT_EXPENSIVE_VIDEO_BYTES = 64 * 1024; // Treat large frames as cooperative-scheduler boundaries const OUTPUT_VIDEO_HIGH_WATER_MARK = 512 * 1024; // Enough room for a few large H264 access units const OUTPUT_AUDIO_HIGH_WATER_MARK = 128 * 1024; // Enough room for normal PCM read cadence without false stalls const OUTPUT_PLAYOUT_POLICY = { live: { requireKeyFrameStart: false, allowAudioBeforeKeyFrame: true, playoutDelayMs: 380, minPlayoutDelayMs: 340, maxPlayoutDelayMs: 620, playoutAdjustStepMs: 20, maxLagBehindLiveMs: 1000, dueTolerance: 12, dueSlack: 18, catchupExitThresholdMs: 300, catchupAudioBurstLimit: 6, normalVideoBurstLimit: 2, normalAudioBurstLimit: 8, catchupVideoBurstLimit: 4, }, record: { requireKeyFrameStart: true, allowAudioBeforeKeyFrame: false, playoutDelayMs: 200, maxLagBehindLiveMs: 1200, dueTolerance: 12, dueSlack: 20, catchupExitThresholdMs: 350, catchupAudioBurstLimit: 6, normalVideoBurstLimit: 3, normalAudioBurstLimit: 6, catchupVideoBurstLimit: 10, }, }; // Initial capacity used by Streamer for its shared media buffer. // This may diverge from the generic RingBuffer default as buffering strategy, // retention window, or media characteristics evolve. // Define constants const STREAMER_INITIAL_BUFFER_CAPACITY = 1024; const STREAMER_MAX_BUFFER_CAPACITY = 8192; const STREAMER_AUDIO_GAP_LOG_MS = 120; // Support-dump audio gap threshold for clearly audible source/output pauses const STREAMER_AUDIO_GAP_LOG_INTERVAL_MS = 30000; // Throttle targeted audio gap diagnostics const STREAMER_AV_CORRELATION_WINDOW_MS = 750; // Correlate audio gaps with nearby accepted keyframes // Streamer object export default class Streamer { static STREAM_TYPE = { LIVE: 'live', RECORD: 'record', BUFFER: 'buffer', }; static MEDIA_TYPE = { VIDEO: 'video', AUDIO: 'audio', TALK: 'talk', METADATA: 'meta', }; static CODEC_TYPE = StreamTransport.CODEC_TYPE; static MESSAGE = 'Streamer.onMessage'; // Message type for HomeKitDevice to listen for static MESSAGE_TYPE = { // Action type messages START_LIVE: 'start-live', STOP_LIVE: 'stop-live', START_RECORD: 'start-record', STOP_RECORD: 'stop-record', START_BUFFER: 'start-buffer', STOP_BUFFER: 'stop-buffer', }; // Shared global scheduler for all active Streamer instances // This avoids having one timer per camera and reduces event loop overhead static #streamers = new Map(); // uuid => Streamer instance static #timer = undefined; // Shared timer for all active streamers supportDump = false; // Enable support for dumping stats on demand for this streamer instance log = undefined; // Logging function object videoEnabled = undefined; // Video stream on camera enabled or not audioEnabled = undefined; // Audio from camera enabled or not online = undefined; // Camera online or not migrating = undefined; // Device is transferring/migrating between APIs nest_google_device_uuid = undefined; // Nest/Google UUID of the device connecting // Internal data only for this class #transport = undefined; // Active protocol transport #HomeKitDeviceUUID = undefined; // HomeKitDevice uuid for this streamer #bufferDuration = 0; // Duration of media to keep in the shared buffer based on media timestamps #bufferEnabled = false; // Retained buffering policy flag owned by Streamer #timeline = undefined; // Shared media timeline used by buffering, live and recording outputs #outputs = new Map(); // Live and recording outputs keyed by session id #cameraFrames = {}; // H264 resource frames for offline, video off, transferring #sequenceCounters = {}; // Sequence counters for item types #itemIndex = 0; // Monotonic item index for shared media timeline cursor tracking #videoState = {}; // Video state tracking #lastFallbackFrameTime = 0; // Timer for pacing fallback frames #lastBudgetLogTime = 0; // Last time budget processing was sampled/logged #outputErrors = 0; // Consecutive output loop failures for this instance #lastMediaTime = {}; // Track last buffered media time per type for fallback ordering guards #lastSourceAudioAt = undefined; // Last time Streamer received transport audio #lastSourceAudioGapLogTime = undefined; // Throttle source-audio gap diagnostics #lastKeyframeAt = undefined; // Last accepted keyframe arrival at Streamer #lastKeyframeBytes = undefined; // Size of last accepted keyframe #lastTimelineDropLogTime = undefined; // Throttle retained timeline capacity diagnostics #connectOptions = {}; // Store options from connect to use on reconnects #lifecycleQueue = Promise.resolve(); // Serializes source connect/close operations to avoid lifecycle races // Codecs currently being used by the active transport. get codecs() { return ( this.#transport?.codecs ?? { video: undefined, audio: undefined, talkback: undefined, } ); } constructor(uuid, deviceData, options = {}) { if (Object.values(LOG_LEVELS).every((fn) => typeof options?.log?.[fn] === 'function')) { this.log = options.log; } this.#HomeKitDeviceUUID = uuid; HomeKitDevice.message(uuid, Streamer.MESSAGE, this); HomeKitDevice.message(uuid, HomeKitDevice.UPDATE, this); HomeKitDevice.message(uuid, HomeKitDevice.TIMER, this); this.migrating = deviceData?.migrating === true; this.online = deviceData?.online === true; this.videoEnabled = deviceData?.streaming_enabled === true; this.audioEnabled = deviceData?.audio_enabled === true; this.nest_google_device_uuid = deviceData?.nest_google_device_uuid; const loadFrameResource = (filename, label) => { let buffer = undefined; let file = path.resolve(__dirname, RESOURCE_PATH, filename); if (fs.existsSync(file) === true) { buffer = fs.readFileSync(file); // Strip Annex-B start code from bundled fallback frames. // Streamer adds start codes during output when required. if (buffer.indexOf(H264.NALUS.START_CODE) === 0) { buffer = buffer.subarray(H264.NALUS.START_CODE.length); } } else { this.log?.warn?.('Failed to load %s video resource for "%s"', label, deviceData.description); } return buffer; }; // Load bundled fallback frames used when: // - camera is offline // - video is disabled // - Nest device is migrating between APIs this.#cameraFrames = { offline: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFFLINE, 'offline'), off: loadFrameResource(RESOURCE_FRAMES.CAMERA_OFF, 'video off'), transfer: loadFrameResource(RESOURCE_FRAMES.CAMERA_TRANSFER, 'transferring'), }; this.#lastFallbackFrameTime = Date.now(); this.supportDump = options?.supportDump === true; // Setup retained media buffer duration. // Clamp to sane bounds to avoid invalid or excessive retention sizes. this.#bufferDuration = Number.isInteger(options?.bufferDuration) === true && options.bufferDuration > 0 ? Math.min(Math.max(options.bufferDuration, 2000), 15000) : 5000; // Attach transport backend (WebRTC / NexusTalk). // Transport owns protocol connection, media parsing and state. // Streamer owns buffering, pacing, fan-out and HomeKit stream handling. if (options?.transport instanceof StreamTransport) { this.#transport = options.transport; // Wire transport callbacks into Streamer. // Media and lifecycle events flow through this interface. this.#transport.consumer = { // Incoming media from transport. media: (media) => { this.addMedia(media); }, // Transport state changes (connecting, ready, reconnecting, etc). state: (state, reason) => { this.onMessage(state, { reason: reason }); }, // Whether there are active outputs consuming media. active: () => { return this.hasActiveStreams(); }, }; } } async onUpdate(deviceData) { let reconnect = false; let wasOnline = this.online === true; let becameOnline = false; let becameAvailable = false; let transportOptions = undefined; if (typeof deviceData !== 'object' || deviceData === null) { return; } // Streamer-owned display/fallback state. // These drive offline/video-disabled fallback frame behaviour. if (deviceData?.migrating !== undefined) { this.migrating = deviceData.migrating === true; } if (deviceData?.online !== undefined) { this.online = deviceData.online === true; } if (deviceData?.streaming_enabled !== undefined) { this.videoEnabled = deviceData.streaming_enabled === true; } if (deviceData?.audio_enabled !== undefined) { this.audioEnabled = deviceData.audio_enabled === true; } // Detect when a camera becomes available again after being offline. // We only want to reconnect if we transitioned from unavailable -> available, // not on every normal update tick. becameOnline = wasOnline !== true && this.online === true; becameAvailable = becameOnline === true && this.videoEnabled === true; // Transport-owned connection/auth/source options. // Transport subclasses decide which fields matter. transportOptions = { uuid: deviceData?.nest_google_device_uuid, host: deviceData?.nexustalk_host, apiAccess: deviceData?.apiAccess, fieldTest: deviceData?.apiAccess?.fieldTest === true, }; // Source identity changed. // Existing transport session cannot be trusted, so force reconnect. if (deviceData?.nest_google_device_uuid !== undefined && this.nest_google_device_uuid !== deviceData.nest_google_device_uuid) { this.nest_google_device_uuid = deviceData.nest_google_device_uuid; reconnect = true; } // Allow transport to refresh runtime configuration such as: // - access tokens // - Google/Nest auth mode // - NexusTalk host // - field test settings if (typeof this.#transport?.update === 'function') { await this.#transport.update(transportOptions); } // No active buffering/live/record consumers. // Nothing upstream needs reconnecting yet. if (this.hasActiveStreams() !== true) { return; } // Camera unavailable or video disabled. // Close transport but keep outputs alive so fallback frames continue. if (this.online === false || this.videoEnabled === false) { await this.#doClose(); return; } // Reconnect transport when: // - source identity changed // - camera has come back online // - transport was previously closed while outputs remained active // // This covers offline -> online recovery while buffering/live view // continues to exist. if (reconnect === true || becameAvailable === true || this.#transport?.closed === true) { await this.#doConnect({ forceReconnect: true }); } } async onMessage(type, message) { if (typeof type !== 'string' || type === '') { return; } let sessionID = message?.sessionID !== undefined ? String(message.sessionID) : undefined; let options = typeof message?.options === 'object' && message.options !== null ? message.options : undefined; if (type === Streamer.MESSAGE_TYPE.START_BUFFER) { // Enable retained buffer and ensure source is connected. // This does not create an output stream, only prepares buffering for future use. await this.#startBuffering(options); return; } if (type === Streamer.MESSAGE_TYPE.STOP_BUFFER) { // Disable retained buffer. // If no outputs are active, this may also allow the source to close. await this.#stopBuffering(); return; } if (type === Streamer.MESSAGE_TYPE.START_LIVE) { // Start a live streaming output for HomeKit. // This creates PassThrough streams and begins feeding real-time data. return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.LIVE, options); } if (type === Streamer.MESSAGE_TYPE.STOP_LIVE) { // Stop a live streaming output. // Cleans up streams and may close source if no other outputs remain. await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.LIVE); return; } if (type === Streamer.MESSAGE_TYPE.START_RECORD) { // Start a recording output (HKSV). // Uses retained buffer and selects a start position based on requested time. // Decoder safety (keyframe alignment) is handled during playout. return await this.#createOutput(sessionID, Streamer.STREAM_TYPE.RECORD, options); } if (type === Streamer.MESSAGE_TYPE.STOP_RECORD) { // Stop a recording output. // If no sessionID is provided, stops the first active recording stream. await this.#stopOutput(sessionID, Streamer.STREAM_TYPE.RECORD); return; } if (Object.values(StreamTransport.STATE).includes(type) === true) { // Reset transient media/output source tracking when a new transport lifecycle // starts, restarts, or fully closes. if ( type === StreamTransport.STATE.CONNECTING || type === StreamTransport.STATE.RECONNECTING || type === StreamTransport.STATE.CLOSED ) { this.#resetSourceState(); } return; } } async stopEverything() { let stopped = []; // Capture active work before cleanup so the shutdown log reflects what // was actually running, rather than always claiming every mode was active. if (this.isBuffering() === true) { stopped.push('buffering'); } if (this.isLiveStreaming() === true) { stopped.push('live'); } if (this.isRecording() === true) { stopped.push('recording'); } if (stopped.length > 0) { this?.log?.debug?.('Stopped %s from device uuid "%s"', stopped.join(', '), this.nest_google_device_uuid); } for (let output of this.#outputs.values()) { this.#cleanupOutput(output); } this.#outputs.clear(); this.#resetRetainedState(); // Full teardown owns scheduler cleanup directly. Normal start/stop paths // still use #syncSchedulerState() as outputs/buffering change incrementally. Streamer.#removeStreamer(this); // Always close underlying transport during shutdown/cleanup. await this.#doClose(); } addMedia(media) { let addedIndex = undefined; let mediaType = undefined; let codec = undefined; let now = Date.now(); let data = undefined; let sequence = 0; let sourceTimestamp = 0; let mediaTime = 0; let minimumMediaStep = 1; let keyFrame = false; let codecConfig = undefined; let hasSPS = false; let hasPPS = false; let sourceAudioGapMs = 0; let keyframeAgeMs = undefined; // Validate incoming media object. if (typeof media !== 'object' || media === null) { return; } mediaType = typeof media.type === 'string' ? media.type.toLowerCase() : undefined; keyFrame = media?.keyFrame === true; // Validate media type and payload. if ( typeof mediaType !== 'string' || mediaType.trim() === '' || (mediaType !== Streamer.MEDIA_TYPE.VIDEO && mediaType !== Streamer.MEDIA_TYPE.AUDIO && mediaType !== Streamer.MEDIA_TYPE.TALK && mediaType !== Streamer.MEDIA_TYPE.METADATA) || Buffer.isBuffer(media.data) !== true || media.data.length === 0 ) { return; } // Do not process if no active outputs or retained buffer. if (this.hasActiveStreams() !== true) { return; } data = media.data; if (mediaType === Streamer.MEDIA_TYPE.AUDIO) { if (typeof this.#lastSourceAudioAt === 'number') { sourceAudioGapMs = now - this.#lastSourceAudioAt; if ( this.supportDump === true && sourceAudioGapMs >= STREAMER_AUDIO_GAP_LOG_MS && (typeof this.#lastSourceAudioGapLogTime !== 'number' || now - this.#lastSourceAudioGapLogTime >= STREAMER_AUDIO_GAP_LOG_INTERVAL_MS) ) { keyframeAgeMs = typeof this.#lastKeyframeAt === 'number' ? now - this.#lastKeyframeAt : undefined; this.#lastSourceAudioGapLogTime = now; this?.log?.debug?.( 'Streamer source audio gap for device uuid "%s": gapMs=%s keyframeAgeMs=%s keyframeBytes=%s', this.nest_google_device_uuid, Math.round(sourceAudioGapMs), Number.isFinite(keyframeAgeMs) === true && keyframeAgeMs <= STREAMER_AV_CORRELATION_WINDOW_MS ? Math.round(keyframeAgeMs) : '-', Number.isFinite(this.#lastKeyframeBytes) === true ? this.#lastKeyframeBytes : '-', ); } } this.#lastSourceAudioAt = now; } // Ensure shared media timeline exists before proceeding. this.#ensureMediaTimeline(); if (this.#timeline === undefined) { return; } // Resolve codec. // Transport owns codec metadata, but explicit media codec wins if provided. codec = typeof media?.codec === 'string' ? media.codec.toLowerCase() : mediaType === Streamer.MEDIA_TYPE.VIDEO ? this.codecs?.video : mediaType === Streamer.MEDIA_TYPE.AUDIO ? this.codecs?.audio : mediaType === Streamer.MEDIA_TYPE.TALK ? this.codecs?.talkback : mediaType === Streamer.MEDIA_TYPE.METADATA ? StreamTransport.CODEC_TYPE.META : undefined; if (typeof codec !== 'string' || codec.trim() === '') { return; } // H264 transport contract: // - transports emit complete Annex-B access units // - Streamer stores access units unchanged // - output writes item.data directly // - Streamer only caches clean SPS/PPS for optional decoder bootstrap if (mediaType === Streamer.MEDIA_TYPE.VIDEO && codec === StreamTransport.CODEC_TYPE.H264) { codecConfig = typeof media?.codecConfig === 'object' && media.codecConfig !== null ? media.codecConfig : undefined; if (codecConfig !== undefined) { hasSPS = codecConfig.hasSPS === true; hasPPS = codecConfig.hasPPS === true; if (Buffer.isBuffer(codecConfig.sps) === true && codecConfig.sps.length > 0) { this.#videoState.lastSPS = codecConfig.sps; } if (Buffer.isBuffer(codecConfig.pps) === true && codecConfig.pps.length > 0) { this.#videoState.lastPPS = codecConfig.pps; } } if (keyFrame === true) { this.#lastKeyframeAt = now; this.#lastKeyframeBytes = data.length; } } // Initialise sequence counter if required. if (typeof this.#sequenceCounters?.[mediaType] !== 'number') { this.#sequenceCounters[mediaType] = 0; } // Use provided transport sequence/source timestamp or fallback to generated values. sequence = Number.isFinite(media?.sequence) === true ? media.sequence : this.#sequenceCounters[mediaType]++; sourceTimestamp = Number.isFinite(media?.timestamp) === true ? Math.round(media.timestamp) : now; // Ensure monotonic media time. // StreamTransport owns source media timing metadata, but Streamer still // protects MediaTimeline ordering if timestamps repeat or go backwards. if (typeof this.#lastMediaTime?.[mediaType] !== 'number') { this.#lastMediaTime[mediaType] = 0; } if (sourceTimestamp <= this.#lastMediaTime[mediaType]) { if ( mediaType === Streamer.MEDIA_TYPE.VIDEO && Number.isFinite(this.#transport?.video?.fps) === true && this.#transport.video.fps > 0 ) { minimumMediaStep = Math.max(1, Math.round(1000 / this.#transport.video.fps)); } if ( mediaType === Streamer.MEDIA_TYPE.AUDIO && Number.isFinite(this.#transport?.audio?.frameDuration) === true && this.#transport.audio.frameDuration > 0 ) { minimumMediaStep = Math.max(1, Math.round(this.#transport.audio.frameDuration)); } if (mediaType === Streamer.MEDIA_TYPE.VIDEO && minimumMediaStep === 1) { minimumMediaStep = Math.round(STREAM_FRAME_INTERVAL); } mediaTime = this.#lastMediaTime[mediaType] + minimumMediaStep; } else { mediaTime = sourceTimestamp; } this.#lastMediaTime[mediaType] = mediaTime; // Push final packet into shared media timeline. addedIndex = this.#timeline.add({ type: mediaType, codec: codec, time: mediaTime, sourceTimestamp: sourceTimestamp, sequence: sequence, keyFrame: keyFrame === true, hasSPS: hasSPS, hasPPS: hasPPS, data: data, }); if (typeof addedIndex === 'number') { // Keep Streamer item index aligned with MediaTimeline. // MediaTimeline owns index assignment for retained media items. this.#itemIndex = this.#timeline.nextIndex; } else if ( typeof this.#lastTimelineDropLogTime !== 'number' || now - this.#lastTimelineDropLogTime >= STREAMER_AUDIO_GAP_LOG_INTERVAL_MS ) { this.#lastTimelineDropLogTime = now; this?.log?.warn?.( 'Dropped %s media for device uuid "%s" because retained timeline is full: size=%s capacity=%s maxCapacity=%s bytes=%s', mediaType, this.nest_google_device_uuid, this.#timeline.stats?.size ?? '-', this.#timeline.stats?.capacity ?? '-', this.#timeline.stats?.maxCapacity ?? '-', data.length, ); } } async #startBuffering(options = {}) { this.#ensureMediaTimeline(); if (this.#timeline === undefined) { return; } this.#bufferEnabled = true; this.log?.debug?.('Started buffering from device uuid "%s"', this.nest_google_device_uuid); this.#syncSchedulerState(); await this.#doConnect(options); } async #stopBuffering() { if (this.#bufferEnabled !== true) { return; } this.#bufferEnabled = false; this.log?.debug?.('Stopped buffering from device uuid "%s"', this.nest_google_device_uuid); if (this.isStreaming() === false) { this.#resetRetainedState(); this.#syncSchedulerState(); await this.#doClose(); return; } this.#syncSchedulerState(); } async #createOutput(sessionID, type, options) { let existing = undefined; let output = undefined; let video = undefined; let audio = null; let talkback = null; let includeAudio = options?.includeAudio === true && this.audioEnabled === true; let waitForReady = Number.isInteger(options?.waitForReady) === true ? options.waitForReady : 0; let startCursor = this.#itemIndex; let timeline = undefined; let timelineStart = 0; let item = undefined; let recordTime = options?.recordTime; let startTime = Date.now(); // Validate session id. if (typeof sessionID !== 'string' || sessionID === '') { return; } // Check for existing output with this session id. existing = this.#outputs.get(sessionID); // Only allow a single record output, regardless of session id. if (type === Streamer.STREAM_TYPE.RECORD && existing === undefined) { for (output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.RECORD) { existing = output; break; } } } // Reuse existing output when possible, otherwise reject type conflict. if (existing !== undefined) { if (existing.type !== type) { this?.log?.warn?.( 'Cannot start output for device uuid "%s" and session id "%s" as it is already in use for "%s"', this.nest_google_device_uuid, sessionID, existing.type, ); return; } return { video: existing.video, audio: existing.audio, talkback: existing.talkback, }; } // Ensure retained timeline exists and start/connect source if needed. this.#ensureMediaTimeline(); await this.#doConnect(options); timeline = this.#timeline; if (timeline instanceof MediaTimeline !== true) { return; } // Create streams for this output. video = new PassThrough({ highWaterMark: OUTPUT_VIDEO_HIGH_WATER_MARK }); audio = includeAudio === true ? new PassThrough({ highWaterMark: OUTPUT_AUDIO_HIGH_WATER_MARK }) : null; talkback = type === Streamer.STREAM_TYPE.LIVE && includeAudio === true ? new PassThrough({ highWaterMark: 1024 * 16 }) : null; // Prevent unhandled stream errors from bubbling. video?.on?.('error', () => {}); audio?.on?.('error', () => {}); talkback?.on?.('error', () => {}); // Determine initial cursor for recording. if (type === Streamer.STREAM_TYPE.RECORD) { timelineStart = timeline.startIndex; // Default to the retained timeline start. if (typeof timelineStart === 'number') { startCursor = timelineStart; } // Recording should start from the retained position closest to the requested // record time only. Decoder/keyframe safety is handled later during playout // inside #processBufferedOutput(), not here. if (timeline.empty !== true && typeof recordTime === 'number' && Number.isFinite(recordTime) === true) { item = timeline.closestToTime(recordTime); if (typeof item?.index === 'number') { startCursor = item.index; } } // Never allow cursor to point before current retained window. if (typeof timelineStart === 'number' && startCursor < timelineStart) { startCursor = timelineStart; } } // Live streaming should attach at the live edge. // We do not backtrack live outputs into retained media here. // Any decoder startup/keyframe handling remains the responsibility of // #processBufferedOutput(). if (type === Streamer.STREAM_TYPE.LIVE) { startCursor = this.#itemIndex; } // Create output state. // Each output consumes from the shared media timeline using independent video // and audio cursors with one shared playout timing model. // // Key concepts: // - videoCursor/audioCursor: next media-specific read positions (absolute indexes) // - cursor: earliest media cursor, retained for shared timeline trimming // - catchingUp: used when starting behind live edge (mainly RECORD) to fast-drain // - sourceBaseTime / wallclockBaseTime: // map source timestamps -> real time for paced playback // - policy: defines how this output consumes media (latency vs continuity tradeoff) // // IMPORTANT: // - All smoothing / pacing happens in Streamer (not in NexusTalk/WebRTC) // - Each output has its own independent timing model // - Policies MUST be tuned per output type (do not unify blindly) output = { sessionID: sessionID, type: type, // Writable streams (ffmpeg pipes etc). video: video, audio: audio, talkback: talkback, talkbackTimeout: undefined, // Whether audio should be written for this output. includeAudio: includeAudio, // Protected read cursor into shared media timeline. // The media cursors below do the actual draining; this remains the minimum // retained position used by timeline trimming. cursor: startCursor, videoCursor: startCursor, audioCursor: startCursor, // Catch-up mode: // - RECORD starts in catch-up to drain historical timeline // - LIVE starts at the live edge so no catch-up is required initially catchingUp: type === Streamer.STREAM_TYPE.RECORD, catchupTicks: 0, catchupStableFrames: 0, // Codec / decoder state tracking. sentCodecConfig: false, // SPS/PPS sent seenKeyFrame: false, // first keyframe seen // Last time a video frame was written (used for fallback timing). lastVideoWriteTime: 0, // Time mapping for paced playback: // wallclockTime = wallclockBaseTime + (item.time - sourceBaseTime) sourceBaseTime: undefined, wallclockBaseTime: undefined, playoutDelayMs: undefined, stablePlayoutTicks: 0, // Playout policy: // Defines how aggressively we stay near live edge vs preserve continuity. // // LIVE: // - attaches at the live edge with a modest playout delay // - prefers stable low-latency delivery over dumping bursts into ffmpeg // // RECORD: // - slightly delayed paced playback // - preserves continuity while draining retained media policy: { ...(OUTPUT_PLAYOUT_POLICY[type] ?? OUTPUT_PLAYOUT_POLICY.live) }, // Debug / instrumentation stats (used for tuning pacing behaviour). stats: { startedAt: Date.now(), firstWriteAt: undefined, firstVideoWriteAt: undefined, firstAudioWriteAt: undefined, writes: { total: 0, video: 0, audio: 0 }, drops: { videoBeforeKeyframe: 0, audioBeforeKeyframe: 0, bufferTrimmed: 0 }, diagnostics: { lastVideoWriteAt: undefined, lastAudioWriteAt: undefined, maxVideoWriteGapMs: 0, maxAudioWriteGapMs: 0, audioWriteGapsOver100Ms: 0, audioWriteGapsOver250Ms: 0, audioWriteGapsOver500Ms: 0, audioBlockedBehindVideo: 0, videoBackpressureStops: 0, maxAudioQueuedBytes: 0, maxBlockedAudioLagMs: 0, lastAudioDiagnosticWriteAt: undefined, lastAudioDiagnosticLogTime: undefined, }, }, }; // Attach talkback handling for live streams. if (talkback !== null) { talkback.on('data', (data) => { this.#transport?.sendAudio?.(data); clearTimeout(output.talkbackTimeout); output.talkbackTimeout = setTimeout(() => { this.#transport?.sendAudio?.(Buffer.alloc(0)); }, TIMERS.TALKBACK_AUDIO.interval); }); talkback.on('close', () => { clearTimeout(output?.talkbackTimeout); this.#transport?.sendAudio?.(Buffer.alloc(0)); }); } // Register output before any optional readiness wait. this.#outputs.set(sessionID, output); this.#syncSchedulerState(); // Optionally wait for source readiness before returning stream handles. if (waitForReady > 0) { while (Date.now() - startTime < waitForReady) { if (this.#transport?.ready === true || this.#transport?.closed === true || this.#transport?.reconnecting === true) { break; } await new Promise((resolve) => setTimeout(resolve, 25)); } } this?.log?.debug?.('Started %s stream from device uuid "%s" and session id "%s"', type, this.nest_google_device_uuid, sessionID); return { video: video, audio: audio, talkback: talkback, }; } async #stopOutput(sessionID, type) { let output = undefined; let hasOtherLiveOutputs = false; let id = undefined; let activeOutput = undefined; // Resolve output by session id if provided if (typeof sessionID === 'string' && sessionID !== '') { output = this.#outputs.get(sessionID); } // For recording, allow stopping the first active record stream if no session id was provided if (output === undefined && type === Streamer.STREAM_TYPE.RECORD) { for (let candidate of this.#outputs.values()) { if (candidate?.type === Streamer.STREAM_TYPE.RECORD) { output = candidate; break; } } } // Nothing matched to stop if (output === undefined) { return; } // Ensure we are not stopping a mismatched type (e.g. live vs record) if (output.type !== type) { this?.log?.warn?.( 'Cannot stop stream for device uuid "%s" and session id "%s" as it is type "%s" not "%s"', this.nest_google_device_uuid, output.sessionID, output.type, type, ); return; } // If this is the last live output and support dump is enabled, log per-output stats before cleanup if (output.type === Streamer.STREAM_TYPE.LIVE && this.supportDump === true) { for ([id, activeOutput] of this.#outputs) { if (id !== output.sessionID && activeOutput?.type === Streamer.STREAM_TYPE.LIVE) { hasOtherLiveOutputs = true; break; } } if (hasOtherLiveOutputs !== true) { await this.#transport?.refreshDiagnostics?.(); this.#outputStats(output, Date.now()); } } this?.log?.debug?.( 'Stopping %s stream from device uuid "%s" and session id "%s"', type, this.nest_google_device_uuid, output.sessionID, ); // Cleanup streams, timers, and any talkback state this.#cleanupOutput(output); // Remove from active outputs this.#outputs.delete(output.sessionID); // Clear retained state when last output stops and buffering is disabled. // This prevents stale buffered media being reused by the next session. if (this.#outputs.size === 0 && this.#bufferEnabled !== true) { this.#resetRetainedState(); } // Update scheduler based on remaining activity this.#syncSchedulerState(); // If nothing remains active, fully close underlying source if (this.isStreaming() === false && this.isBuffering() === false) { await this.#doClose(); } } isBuffering() { return this.#bufferEnabled === true; } isStreaming() { return this.#outputs.size !== 0; } isRecording() { for (let output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.RECORD) { return true; } } return false; } isLiveStreaming() { for (let output of this.#outputs.values()) { if (output?.type === Streamer.STREAM_TYPE.LIVE) { return true; } } return false; } hasActiveStreams() { return this.#bufferEnabled === true || this.#outputs.size !== 0; } async requestSourceConnect(options = undefined) { return await this.#doConnect(options); } async requestSourceClose() { return await this.#doClose(); } #queueLifecycle(task) { // Chain lifecycle operations sequentially so connect/close/reconnect // actions cannot overlap or race each other. let run = this.#lifecycleQueue.then(async () => { return await task(); }); // Keep the internal queue alive even if a lifecycle task fails. // This prevents a rejected promise from permanently breaking the queue // and suppresses unhandled rejection warnings for internal sequencing. this.#lifecycleQueue = run.catch((error) => { this?.log?.debug?.( 'Streamer lifecycle queue task failed for device uuid "%s": %s', this.nest_google_device_uuid, error?.message || String(error), ); }); // Return the original task promise so callers still receive failures. return run; } async #doConnect(options = undefined) { return await this.#queueLifecycle(async () => { let forceReconnect = options?.forceReconnect === true; if (this.online !== true || this.videoEnabled !== true) { return; } if (typeof options === 'object' && options !== null) { this.#connectOptions = { ...(typeof this.#connectOptions === 'object' && this.#connectOptions !== null ? this.#connectOptions : {}), ...options, }; if (Object.prototype.hasOwnProperty.call(options, 'host') !== true) { delete this.#connectOptions.host; } } if (forceReconnect !== true && this.#transport?.closed !== true && this.#transport?.reconnecting !== true) { return; } if (typeof this.#transport?.open === 'function') { await this.#transport.open(this.#connectOptions); } }); } async #doClose() { return await this.#queueLifecycle(async () => { this.#resetSourceState(); if (typeof this.#transport?.close === 'function') { await this.#transport.close(); } }); } #cleanupOutput(output) { if (typeof output !== 'object' || output === null) { return; } clearTimeout(output?.talkbackTimeout); output?.video?.removeAllListeners?.(); output?.audio?.removeAllListeners?.(); output?.talkback?.removeAllListeners?.(); output?.video?.end?.(); output?.audio?.end?.(); output?.talkback?.end?.(); } #resetRetainedState() { this.#bufferEnabled = false; this.#timeline = undefined; this.#sequenceCounters = {}; this.#itemIndex = 0; this.#videoState = {}; this.#lastMediaTime = {}; } #ensureMediaTimeline() { if (this.#timeline === undefined) { this.#timeline = new MediaTimeline(this.#itemIndex, STREAMER_INITIAL_BUFFER_CAPACITY, STREAMER_MAX_BUFFER_CAPACITY); } } #resetSourceState() { this.#videoState = {}; this.#lastMediaTime = {}; // A transport reconnect means a new encoded media session. // Existing HomeKit outputs may still be open, but their decoder state must // be reset so the next real H264 keyframe can be bootstrapped again. for (let output of this.#outputs.values()) { output.seenKeyFrame = false; output.sentCodecConfig = false; output.sourceBaseTime = undefined; output.wallclockBaseTime = undefined; output.playoutDelayMs = undefined; output.stablePlayoutTicks = 0; output.catchingUp = false; output.catchupTicks = 0; output.catchupStableFrames = 0; // Move active outputs to the current live edge. // This avoids replaying stale pre-offline media after fallback frames. output.cursor = this.#itemIndex; output.videoCursor = this.#itemIndex; output.audioCursor = this.#itemIndex; } } #ensureOutputDrops(outputStats) { if (typeof outputStats !== 'object' || outputStats === null) { return undefined; } if (typeof outputStats.drops !== 'object' || outputStats.drops === null) { outputStats.drops = { videoBeforeKeyframe: 0, audioBeforeKeyframe: 0, bufferTrimmed: 0 }; } return outputStats.drops; } #statsWrite(output, type, dateNow) { let outputStats = output?.stats; let outputWrites = outputStats?.writes; let diagnostics = outputStats?.diagnostics; let gapMs = 0; let keyframeAgeMs = undefined; if (type === Streamer.MEDIA_TYPE.AUDIO && typeof diagnostics === 'object' && diagnostics !== null) { if (typeof diagnostics.lastAudioDiagnosticWriteAt === 'number') { gapMs = dateNow - diagnostics.lastAudioDiagnosticWriteAt; if ( this.supportDump === true && gapMs >= STREAMER_AUDIO_GAP_LOG_MS && (typeof diagnostics.lastAudioDiagnosticLogTime !== 'number' || dateNow - diagnostics.lastAudioDiagnosticLogTime >= STREAMER_AUDIO_GAP_LOG_INTERVAL_MS) ) { keyframeAgeMs = typeof this.#lastKeyframeAt === 'number' ? dateNow - this.#lastKeyframeAt : undefined; diagnostics.lastAudioDiagnosticLogTime = dateNow; this?.log?.debug?.( 'Streamer output audio write gap for device uuid "%s": session="%s" type="%s" gapMs=%s keyframeAgeMs=%s ' + 'keyframeBytes=%s catchingUp=%s playoutDelayMs=%s', this.nest_google_device_uuid, output?.sessionID ?? '-', output?.type ?? '-', Math.round(gapMs), Number.isFinite(keyframeAgeMs) === true && keyframeAgeMs <= STREAMER_AV_CORRELATION_WINDOW_MS ? Math.round(keyframeAgeMs) : '-', Number.isFinite(this.#lastKeyframeBytes) === true ? this.#lastKeyframeBytes : '-', output?.catchingUp === true ? 'true' : 'false', Number.isFinite(output?.playoutDelayMs) === true ? Math.round(output.playoutDelayMs) : '-', ); } } diagnostics.lastAudioDiagnosticWriteAt = dateNow; } if (this.supportDump !== true) { return; } if (typeof outputStats !== 'object' || outputStats === null) { return; } if (typeof outputStats.firstWriteAt !== 'number') { outputStats.firstWriteAt = dateNow; } if (type === Streamer.MEDIA_TYPE.VIDEO && typeof outputStats.firstVideoWriteAt !== 'number') { outputStats.firstVideoWriteAt = dateNow; } if (type === Streamer.MEDIA_TYPE.AUDIO && typeof outputStats.firstAudioWriteAt !== 'number') { outputStats.firstAudioWriteAt = dateNow; } if (typeof outputWrites !== 'object' || outputWrites === null) { return; } if (typeof diagnostics !== 'object' || diagnostics === null) { outputStats.diagnostics = { lastVideoWriteAt: undefined, lastAudioWriteAt: undefined, maxVideoWriteGapMs: 0, maxAudioWriteGapMs: 0, audioWriteGapsOver100Ms: 0, audioWriteGapsOver250Ms: 0, audioWriteGapsOver500Ms: 0, audioBlockedBehindVideo: 0, videoBackpressureStops: 0, maxAudioQueuedBytes: 0, maxBlockedAudioLagMs: 0, lastAudioDiagnosticWriteAt: undefined, lastAudioDiagnosticLogTime: undefined, }; diagnostics = outputStats.diagnostics; } if (typeof outputWrites.total !== 'number') { outputWrites.total = 0; outputWrites.video = 0; outputWrites.audio = 0; } outputWrites.total++; if (type === Streamer.MEDIA_TYPE.VIDEO) { outputWrites.video++; if (typeof diagnostics.lastVideoWriteAt === 'number') { gapMs = dateNow - diagnostics.lastVideoWriteAt; if (gapMs > diagnostics.maxVideoWriteGapMs) { diagnostics.maxVideoWriteGapMs = gapMs; } } diagnostics.lastVideoWriteAt = dateNow; } if (type === Streamer.MEDIA_TYPE.AUDIO) { outputWrites.audio++; if (typeof diagnostics.lastAudioWriteAt === 'number') { gapMs = dateNow - diagnostics.lastAudioWriteAt; if (gapMs > diagnostics.maxAudioWriteGapMs) { diagnostics.maxAudioWriteGapMs = gapMs; } if (gapMs > 100) { diagnostics.audioWriteGapsOver100Ms++; } if (gapMs > 250) { diagnostics.audioWriteGapsOver250Ms++; } if (gapMs > 500) { diagnostics.audioWriteGapsOver500Ms++; } } diagnostics.lastAudioWriteAt = dateNow; } } #statsDrop(output, type) { let outputStats = output?.stats; if (this.supportDump !== true) { return; } if (typeof outputStats !== 'object' || outputStats === null) { return; } let outputDrops = this.#ensureOutputDrops(outputStats); if (type === Streamer.MEDIA_TYPE.VIDEO) { outputDrops.videoBeforeKeyframe++; } if (type === Streamer.MEDIA_TYPE.AUDIO) { outputDrops.audioBeforeKeyframe++; } } #writeFallback(output, fallbackFrame, dateNow = undefined) { let outputVideo = undefined; let outputAudio = undefined; let isH264 = false; if (Buffer.isBuffer(fallbackFrame) !== true || typeof output !== 'object' || output === null) { return; } outputVideo = output.video; outputAudio = output.audio; isH264 = this.codecs?.video === StreamTransport.CODEC_TYPE.H264; if (typeof dateNow !== 'number') { dateNow = Date.now(); } this.#statsWrite(output, Streamer.MEDIA_TYPE.VIDEO, dateNow); if (isH264 === true) { outputVideo.write(H264.NALUS.START_CODE); } outputVideo.write(fallbackFrame); if (output.includeAudio === true && Buffer.isBuffer(this.#transport?.audio?.blank) === true) { this.#statsWrite(output, Streamer.MEDIA_TYPE.AUDIO, dateNow); outputAudio.write(this.#transport.audio.blank); } } #processBufferedOutput(output, dateNow, budgetMs) { let timeline = this.#timeline; let startIndex = undefined; let timelineEnd = 0; let processed = 0; let item = undefined; let videoItem = undefined; let audioItem = undefined; let latestItem = undefined; let latestItemTime = undefined; let anchorTime = undefined; let nextCursor = 0; let selectedType = undefined; let stopReason = ''; let diagnostics = undefined; let outputVideo = undefined; let outputAudio = undefined; let includeAudio = false; let isH264Output = false; let isLiveOutput = false; let isRecordOutput = false; let lastSPS = undefined; let lastPPS = undefined; let hasSPS = false; let hasPPS = false; let keyFrameHasSPS = false; let keyFrameHasPPS = false; let policy = undefined; let requireKeyFrameStart = false; let allowAudioBeforeKeyFrame = false; let playoutDelayMs = 0; let minPlayoutDelayMs = 0; let maxPlayoutDelayMs = 0; let playoutAdjustStepMs = 0; let maxLagBehindLiveMs = 0; let dueTolerance = 0; let dueSlack = 0; let catchupExitThresholdMs = 0; let catchupAudioBurstLimit = 0; let catchupVideoBurstLimit = 0; let normalAudioBurstLimit = 0; let normalVideoBurstLimit = 0; let state = undefined; let shouldCatchUp = false; let catchupExitedThisTick = false; let budgetDeadline = 0; let itemLag = 0;