homebridge-nest-accfactory
Version:
Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras
1,261 lines (1,080 loc) • 84.2 kB
JavaScript
// WebRTC
// Part of homebridge-nest-accfactory
//
// Implements WebRTC-based streaming for Google Nest cameras using Google Home
// Foyer/gRPC signaling and control.
// Handles peer connection setup, RTP media processing, talkback audio,
// and integration with the Streamer pipeline for HomeKit live streaming
// and recording.
//
// Responsibilities:
// - Establish and manage RTCPeerConnection using the werift library
// - Use Google Home Foyer gRPC transport for signaling and stream control
// - Handle ICE negotiation and connection state lifecycle
// - Receive and process RTP media streams (H264 video, Opus audio, RTX)
// - Track RTP timing, sequence continuity, and stream health
// - Reorder bounded RTP audio/video jitter queues without owning output pacing
// - Parse H264 RTP payloads including:
// - Single NAL units
// - STAP-A aggregation packets
// - FU-A fragmented NAL reassembly
// - RTX retransmission recovery
// - Assemble complete H264 access units and emit Annex-B frames
// - Inject SPS/PPS before IDR frames when required
// - Decode Opus audio to PCM for downstream Streamer output
// - Inject media into Streamer for live streaming and HKSV recording
// - Support two-way audio (talkback) via outbound RTP/Opus pipeline
// - Detect stream stalls and perform automatic reconnect handling
//
// Features:
// - Secure media transport over DTLS-SRTP
// - Google Home Foyer gRPC signaling and stream lifecycle control
// - RTCP feedback support (PLI/NACK/FIR) for video recovery
// - Codec negotiation (H264 video, Opus audio, RTX retransmissions)
// - RTP timestamp mapping from source media clocks
// - Bounded synchronous jitter draining so bad media bursts cannot monopolise the Node.js event loop
// - Stream health monitoring with recovery/suppression logic
// - Startup timing and stream diagnostics logging
// - Automatic handling of packet loss, corruption, and stalled playback
// - Local vs remote stream path detection based on SDP candidates
//
// Notes:
// - WebRTC signaling and stream control are performed via the shared
// Google Home Foyer gRPC transport/client
// - ICE "connected" indicates transport readiness, not media availability
// - Stream readiness is determined by successful video frame delivery
// (first decodable keyframe), not connection state
// - Startup delays may occur due to upstream (Google) keyframe delivery behaviour
// - Audio is decoded to PCM; output pacing is owned by Streamer
// - Emitted media timestamps describe source media time derived from RTP timing
// - Output playout timing, catch-up, and live latency policy are owned by Streamer
// - Incomplete keyframes and pathological access units are dropped/recovered locally rather than blocking the plugin process
//
// Code version 2026.05.20
// Mark Hulskamp
'use strict';
// Define external module requirements
import * as werift from 'werift';
import { Decoder } from '@evan/opus';
// Define nodejs module requirements
import { Buffer } from 'node:buffer';
import { setInterval, clearInterval, setTimeout } from 'node:timers';
import path from 'node:path';
import crypto from 'node:crypto';
// Define our modules
import Streamer from './streamer.js';
import StreamTransport from './streamtransport.js';
import GrpcTransport from './grpctransport.js';
import H264 from './h264.js';
import RtpH264 from './rtph264.js';
// Define constants
import { USER_AGENT, __dirname } from './consts.js';
const EXTEND_INTERVAL = 30000; // Send extend command to Google Home Foyer every this period for active streams
const GOOGLE_HOME_FOYER_REQUEST_TIMEOUT = 15000; // Client-side timeout for Google Home Foyer gRPC requests
const GOOGLE_HOME_FOYER_BUFFER_INITIAL = 8 * 1024; // Initial 8KB buffer for gRPC responses
const GOOGLE_HOME_FOYER_BUFFER_MAX = 10 * 1024 * 1024; // Maximum 10MB buffer limit
const RTP_SEQUENCE_WRAP = 0x10000; // For wrapping sequence calculations
const RTP_SEQUENCE_MASK = 0xffff; // 16-bit RTP sequence number mask
const RTP_TIMESTAMP_MASK = 0x100000000; // 32-bit RTP timestamp wrap mask
const RTP_TIMESTAMP_MAX_DELTA = 0x7fffffff; // Max positive delta for timestamp comparison
const RTP_PACKET_HEADER_SIZE = 12; // RTP packet header size in bytes
const RTP_H264_VIDEO_PAYLOAD_TYPE = 98; // H.264 video payload type
const RTP_H264_VIDEO_RTX_PAYLOAD_TYPE = 99; // H.264 RTX payload type for retransmissions
const RTP_OPUS_AUDIO_PAYLOAD_TYPE = 111; // Opus audio payload type
const GOOGLE_HOME_FOYER_PREFIX = 'google.internal.home.foyer.v1.';
const KEYFRAME_MAX_ASSEMBLY_MS = 2500; // Drop pathological keyframes assembled too slowly
const KEYFRAME_STARTUP_MAX_ASSEMBLY_MS = 4000; // First decodable keyframe may be slow while WebRTC starts
const KEYFRAME_BLOCKING_MAX_ASSEMBLY_MS = 900; // Do not let one broken IDR block newer timestamp groups for seconds
const KEYFRAME_DEFAULT_BYTES = 2 * 1024 * 1024; // Startup allowance before source resolution/bitrate is known
const KEYFRAME_MAX_BYTES = 4 * 1024 * 1024; // Absolute bound for unusually large high-resolution keyframes
const KEYFRAME_BYTES_PER_PIXEL = 0.5; // Conservative compressed-IDR allowance derived from learned resolution
const KEYFRAME_BITRATE_SECONDS = 4; // Allow an IDR burst up to four seconds of the learned average bitrate
const DELTA_FU_SWITCH_GRACE_MS = 180; // Tiny grace before abandoning a young non-keyframe FU-A on timestamp switch
const STALLED_TIMEOUT = 10000; // Time with no playback packets before we consider stream stalled and attempt restart
const AUDIO_RTP_REORDER_DELAY_MS = 50; // Hold audio RTP briefly so reordered Opus packets can arrive before decode
const AUDIO_RTP_REORDER_MAX_PACKETS = 64; // Bound audio RTP reorder queue
const VIDEO_RTP_REORDER_DELAY_MS = 250; // Hold video RTP briefly so reordered fragments/RTX can arrive before FU-A assembly
const VIDEO_RTP_REORDER_MAX_PACKETS = 4096; // Bound video RTP queue while allowing a fragmented 4MiB keyframe plus packet overhead
const VIDEO_RTP_DRAIN_MAX_GROUPS = 8; // Bound synchronous video jitter release work per callback
const VIDEO_RTP_DRAIN_MAX_PACKETS = 128; // Bound synchronous H264 assembly work per callback
const STARTUP_KEYFRAME_PLI_INTERVAL_MS = 1500; // Retry startup keyframe requests while waiting for first decodable IDR
const STARTUP_KEYFRAME_PLI_MAX_MS = 12000; // Bound startup PLI retries so slow sources do not spam RTCP forever
// WebRTC object
export default class WebRTC extends StreamTransport {
token = undefined; // oauth2 token
fieldTest = undefined;
// Internal data only for this class
#grpcTransport = undefined; // Shared protobuf/gRPC client for Google Home Foyer APIs
#streamId = undefined; // Stream ID
#googleHomeDeviceUUID = undefined; // Normal Nest/Google protobuf device ID translated to a Google Foyer device ID
#googleHomeDeviceUUIDPromise = undefined; // Promise for in-flight HomeGraph lookup of Google Foyer device UUID
#peerConnection = undefined;
#videoTransceiver = undefined;
#audioTransceiver = undefined;
#opusDecoder = new Decoder({ channels: 2, sample_rate: 48000 });
#extendTimer = undefined; // Stream extend timer
#stalledTimer = undefined; // Interval object for no received data checks
#startupKeyframeTimer = undefined; // Interval object for bounded startup keyframe PLI retries
#startupKeyframeStartedAt = undefined; // Wall-clock time when startup keyframe retry began
#lastPacketAt = undefined; // Last playback packet receipt time in ms
#closeInProgress = false; // True while close() teardown is running to avoid re-entrant shutdown races
#reconnectPending = false; // Reconnect requested once socket closes
#tracks = { audio: {}, video: {}, talkback: {} }; // Track state for audio and video
constructor(options = {}) {
super(options);
// Setup WebRTC-specific codec defaults; StreamTransport owns the shared media shape.
this.video.codec = StreamTransport.CODEC_TYPE.H264;
this.video.clockRate = 90000;
this.audio.codec = StreamTransport.CODEC_TYPE.PCM;
this.audio.profile = 's16le';
this.audio.sampleRate = 48000;
this.audio.channels = 2;
this.audio.bitrate = this.audio.sampleRate * this.audio.channels * 16;
this.audio.frameDuration = 20;
this.talkback.codec = StreamTransport.CODEC_TYPE.OPUS;
this.talkback.sampleRate = 48000;
this.talkback.channels = 2;
this.update(options);
this.#setupGoogleHomeFoyer();
}
// Class functions
// eslint-disable-next-line no-unused-vars
async doOpen(options = {}) {
if (this.connecting === true || this.closing === true || (this.#peerConnection !== undefined && this.closed !== true)) {
return;
}
// Tell the Streamer base that we are beginning source setup.
// This is transport/control readiness only and does not mean media is flowing yet.
this.setState(StreamTransport.STATE.CONNECTING);
// Reset any previous session timers/state before attempting a new connection.
// This ensures a reconnect starts from a clean baseline rather than reusing
// timers or partially assembled media from an earlier session.
clearInterval(this.#extendTimer);
clearInterval(this.#stalledTimer);
clearInterval(this.#startupKeyframeTimer);
this.#extendTimer = undefined;
this.#stalledTimer = undefined;
this.#startupKeyframeTimer = undefined;
this.#startupKeyframeStartedAt = undefined;
this.#lastPacketAt = undefined;
this.stats.webrtc = undefined;
this.#streamId = undefined;
this.#reconnectPending = false;
this.#tracks = { audio: {}, video: {}, talkback: {} };
// Resolve Google Foyer device ID lazily so constructor prefetch failure
// does not permanently prevent future stream attempts.
if (typeof this.#googleHomeDeviceUUID !== 'string' && this.#googleHomeDeviceUUIDPromise instanceof Promise !== true) {
this.#setupGoogleHomeFoyer();
}
// Wait for any in-flight Google Home device ID lookup to finish.
if (this.#googleHomeDeviceUUIDPromise instanceof Promise) {
await this.#googleHomeDeviceUUIDPromise;
}
// open() can overlap with close() during fast stop/reopen cycles.
// Abort if teardown happened while waiting for Google Home lookup.
if (this.#closeInProgress === true || this.#peerConnection !== undefined) {
return;
}
// We still could not resolve the Google Foyer device ID.
// Without this mapping we cannot start streaming or recording.
if (typeof this.#googleHomeDeviceUUID !== 'string' || this.#googleHomeDeviceUUID === '') {
this?.log?.debug?.('Google Home device UUID not resolved for uuid "%s"', this.uuid);
this.setState(StreamTransport.STATE.CLOSED, { reason: 'google-device-id-missing' });
return;
}
let homeFoyerResponse = await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'SendCameraViewIntent', {
request: {
googleDeviceId: {
value: this.#googleHomeDeviceUUID,
},
command: 'VIEW_INTENT_START',
},
});
if (this.#closeInProgress === true || this.#peerConnection !== undefined) {
return;
}
if (homeFoyerResponse?.status !== 0) {
this?.log?.debug?.('Request to start camera viewing was not accepted for uuid "%s"', this.uuid);
this.setState(StreamTransport.STATE.CLOSED, { reason: 'view-intent-failed' });
return;
}
// Create our local WebRTC peer connection and advertise the codecs we support.
// We receive H264 video and Opus audio from the camera, then convert that into
// Streamer media items for live view and recording.
let peerConnection = new werift.RTCPeerConnection({
iceUseIpv4: true,
iceUseIpv6: false,
bundlePolicy: 'max-bundle',
codecs: {
audio: [
new werift.RTCRtpCodecParameters({
mimeType: 'audio/opus',
clockRate: 48000,
channels: 2,
rtcpFeedback: [{ type: 'nack' }],
parameters: 'minptime=10;useinbandfec=1',
payloadType: RTP_OPUS_AUDIO_PAYLOAD_TYPE,
}),
],
video: [
new werift.RTCRtpCodecParameters({
mimeType: 'video/H264',
clockRate: 90000,
rtcpFeedback: [{ type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }],
parameters: 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
payloadType: RTP_H264_VIDEO_PAYLOAD_TYPE,
}),
new werift.RTCRtpCodecParameters({
mimeType: 'video/rtx',
clockRate: 90000,
parameters: 'apt=' + RTP_H264_VIDEO_PAYLOAD_TYPE,
payloadType: RTP_H264_VIDEO_RTX_PAYLOAD_TYPE,
}),
],
},
headerExtensions: {
audio: [werift.useAudioLevelIndication()],
},
});
this.#peerConnection = peerConnection;
peerConnection.createDataChannel('webrtc-datachannel');
this.#audioTransceiver = peerConnection.addTransceiver('audio', {
direction: 'sendrecv',
});
this.#videoTransceiver = peerConnection.addTransceiver('video', {
direction: 'recvonly',
});
// Create our SDP offer and send it to Google Home Foyer.
// If accepted, we will get an SDP answer back plus a streamId for later extend/end/talkback calls.
let webRTCOffer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(webRTCOffer);
homeFoyerResponse = await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'JoinStream', {
command: 'offer',
deviceId: this.uuid,
local: false, // Request direct peer-to-peer connection if possible
streamContext: 'STREAM_CONTEXT_DEFAULT',
// Request highest possible resolution; actual delivered resolution may be lower.
requestedVideoResolution: 'VIDEO_RESOLUTION_FULL_HIGH',
sdp: webRTCOffer.sdp,
});
if (this.#peerConnection !== peerConnection) {
try {
await peerConnection?.close?.();
} catch {
// Empty
}
return;
}
if (
homeFoyerResponse?.status !== 0 ||
homeFoyerResponse?.data?.[0]?.responseType !== 'answer' ||
homeFoyerResponse?.data?.[0]?.streamId === undefined ||
homeFoyerResponse?.data?.[0]?.sdp === undefined
) {
peerConnection?.close?.();
this.#peerConnection = undefined;
this?.log?.debug?.('WebRTC offer was not agreed with remote for uuid "%s". Response: %j', this.uuid, homeFoyerResponse);
this.setState(StreamTransport.STATE.CLOSED, { reason: 'offer-rejected' });
return;
}
// If the SDP answer contains a private/local candidate, then local access was granted.
// Otherwise traffic will use the normal routed/remote path and we should continue sending
// periodic stream extension requests to keep the session alive.
let localAccessGranted =
/a=candidate:.* (10\.\d+\.\d+\.\d+|192\.168\.\d+\.\d+|172\.(1[6-9]|2\d|3[0-1])\.\d+\.\d+|fd[0-9a-f]{2}:[0-9a-f:]+)/i.test(
homeFoyerResponse.data[0].sdp || '',
) === true;
// Track subscription callbacks feed the media-specific assembly paths.
// Audio is decoded and emitted as PCM; video is assembled into complete access units.
this.#audioTransceiver?.onTrack?.subscribe?.((track) => {
this.#handlePlaybackBegin(Streamer.MEDIA_TYPE.AUDIO);
track.onReceiveRtp.subscribe((rtpPacket) => {
if (track.codec.payloadType !== RTP_OPUS_AUDIO_PAYLOAD_TYPE) {
// Not the payload type we expect for audio, so ignore
return;
}
this.#handlePlaybackAudioPacket(rtpPacket);
});
});
this.#videoTransceiver?.onTrack?.subscribe?.((track) => {
this.#handlePlaybackBegin(Streamer.MEDIA_TYPE.VIDEO);
track.onReceiveRtp.subscribe((rtpPacket) => {
if (track.codec.payloadType !== RTP_H264_VIDEO_PAYLOAD_TYPE && track.codec.payloadType !== RTP_H264_VIDEO_RTX_PAYLOAD_TYPE) {
// Not the payload types we expect for video, so ignore
return;
}
this.#handlePlaybackVideoPacket(rtpPacket);
});
});
this.#streamId = homeFoyerResponse.data[0].streamId;
// connect() can overlap with close() during fast stream stop/reopen cycles.
// If teardown replaced or cleared the active peer connection while this async
// setup was in-flight, abort this stale connect attempt safely.
if (this.#peerConnection !== peerConnection) {
try {
await peerConnection?.close?.();
} catch {
// Empty
}
return;
}
await peerConnection?.setRemoteDescription?.({
type: 'answer',
sdp: homeFoyerResponse.data[0].sdp,
});
// Monitor connection status. ICE "connected" means transport is ready,
// not that media has actually started. Actual source readiness is promoted
// later on first video packet arrival.
peerConnection.iceConnectionStateChange.subscribe(() => {
if (this.#peerConnection !== peerConnection) {
return;
}
if (this.#closeInProgress === true) {
return;
}
let state = peerConnection?.iceConnectionState;
if (state === 'connected' || state === 'completed' || state === 'checking') {
if (this.connected !== true) {
this.setState(StreamTransport.STATE.CONNECTED, { sessionId: this.#streamId });
}
return;
}
if (
(state === 'failed' || state === 'disconnected' || (state === 'closed' && this.hasConsumers() === true)) &&
this.closing !== true &&
this.closed !== true &&
this.reconnecting !== true
) {
this?.log?.debug?.('WebRTC ICE state "%s" for uuid "%s", requesting reconnect', state, this.uuid);
this.#requestReconnect('ice-' + state);
if (this.hasConsumers() === true) {
this.close();
}
}
});
// Periodically extend the active stream only when we do not have local access.
// Local streams are expected to remain valid without needing explicit extend requests.
if (localAccessGranted !== true) {
this.#extendTimer = setInterval(async () => {
if (
this.#grpcTransport !== undefined &&
this.ready === true &&
this.#streamId !== undefined &&
this.#googleHomeDeviceUUID !== undefined
) {
let extendResponse = await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'JoinStream', {
command: 'extend',
deviceId: this.uuid,
streamId: this.#streamId,
});
if (extendResponse?.data?.[0]?.streamExtensionStatus !== 'STATUS_STREAM_EXTENDED') {
this?.log?.debug?.('Error occurred while requesting stream extension for uuid "%s"', this.uuid);
this.#requestReconnect('extend-failed');
this.close();
}
}
}, EXTEND_INTERVAL);
}
}
async doClose() {
if (this.#closeInProgress === true) {
return;
}
this.#closeInProgress = true;
let closingPeerConnection = this.#peerConnection;
let closingStreamId = this.#streamId;
let talkbackActive = this.#tracks?.talkback?.active === true;
try {
// Preserve final Werift transport/media counters before closing the peer
// connection. This is especially useful when a reconnect was caused by
// consent, ICE, or packet-flow failure.
await this.refreshDiagnostics();
// Mark source as closing for a normal teardown so any in-flight playback
// callbacks stop accepting new packets while shutdown is happening.
// During reconnect we keep SOURCE_RECONNECTING so the lifecycle state
// does not bounce backwards during transport teardown.
if (this.#reconnectPending !== true) {
this.setState(StreamTransport.STATE.CLOSING, { sessionId: closingStreamId });
}
// Stop timers first so we stop producing any new work immediately.
clearInterval(this.#extendTimer);
clearInterval(this.#stalledTimer);
clearInterval(this.#startupKeyframeTimer);
this.#extendTimer = undefined;
this.#stalledTimer = undefined;
this.#startupKeyframeTimer = undefined;
this.#startupKeyframeStartedAt = undefined;
this.#lastPacketAt = undefined;
// Release any video packets that were being held briefly for RTP reordering
// before flushing the final completed access unit.
this.#drainPlaybackVideoJitterBuffer(true);
// Flush any pending video access unit before tearing state down.
// Video is emitted frame-by-frame, so the last completed frame would otherwise
// be lost if close occurs before another packet triggers a normal flush.
this.#flushPendingVideoFrame();
// Clear media/talkback track state before closing remote transport.
// This lets any in-flight callbacks naturally no-op while shutdown continues.
this.#tracks = { audio: {}, video: {}, talkback: {} };
if (closingStreamId !== undefined && talkbackActive === true) {
await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'SendTalkback', {
googleDeviceId: {
value: this.#googleHomeDeviceUUID,
},
streamId: closingStreamId,
command: 'COMMAND_STOP',
});
}
if (closingStreamId !== undefined) {
this?.log?.debug?.('Notifying remote about closing connection for uuid "%s"', this.uuid);
// Tell remote to end the stream session
await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'JoinStream', {
command: 'end',
deviceId: this.uuid,
streamId: closingStreamId,
endStreamReason: 'REASON_USER_EXITED_SESSION',
});
}
try {
await closingPeerConnection?.close?.();
} catch {
// Empty
}
// NOTE: Do NOT release the gRPC client here. It should be reused across WebRTC reconnects
// and only released during final shutdown in onShutdown(). Releasing it during
// temporary disconnects causes in-flight requests to be canceled with "pending stream has been canceled".
if (this.#streamId === closingStreamId) {
this.#streamId = undefined;
}
if (this.#peerConnection === closingPeerConnection) {
this.#peerConnection = undefined;
this.#videoTransceiver = undefined;
this.#audioTransceiver = undefined;
}
if (this.#reconnectPending === true) {
// We have a reconnect pending, so reset the flag and attempt to reconnect.
// We do this only after the current session has really closed to avoid racing
// a new stream setup against a half-torn-down old connection.
this.#reconnectPending = false;
if (this.hasConsumers() === true) {
// Defer reconnect until close() has left finally and #closeInProgress
// is false. Otherwise open() can see teardown in progress and abort
// after already moving lifecycle state back to CONNECTING.
setTimeout(() => {
this.open();
}, 0);
return;
}
}
if (this.hasConsumers() !== true && this.#reconnectPending !== true) {
this.setState(StreamTransport.STATE.CLOSED, { sessionId: closingStreamId });
}
} finally {
this.#closeInProgress = false;
}
}
async refreshDiagnostics() {
let peerConnection = this.#peerConnection;
let report = undefined;
let stats = [];
let transport = undefined;
let candidatePair = undefined;
let localCandidate = undefined;
let remoteCandidate = undefined;
let inbound = {};
let now = Date.now();
if (peerConnection === undefined || typeof peerConnection?.getStats !== 'function') {
return this.stats.webrtc;
}
if (Number.isFinite(this.stats?.webrtc?.capturedAt) === true && now - this.stats.webrtc.capturedAt < 1000) {
return this.stats.webrtc;
}
try {
report = await peerConnection.getStats();
stats = Array.from(report?.values?.() ?? []);
transport = stats.find((entry) => entry?.type === 'transport');
candidatePair =
(typeof transport?.selectedCandidatePairId === 'string' ? report.get(transport.selectedCandidatePairId) : undefined) ??
stats.find((entry) => entry?.type === 'candidate-pair' && (entry?.nominated === true || entry?.state === 'succeeded'));
localCandidate = report.get(candidatePair?.localCandidateId);
remoteCandidate = report.get(candidatePair?.remoteCandidateId);
for (let entry of stats) {
if (entry?.type === 'inbound-rtp' && (entry?.kind === 'video' || entry?.kind === 'audio')) {
inbound[entry.kind] = {
packets: entry.packetsReceived ?? 0,
bytes: entry.bytesReceived ?? 0,
lost: entry.packetsLost ?? 0,
jitterMs: Number.isFinite(entry.jitter) === true ? Math.round(entry.jitter * 100000) / 100 : undefined,
nacks: entry.nackCount ?? 0,
plis: entry.pliCount ?? 0,
};
}
}
this.stats.webrtc = {
capturedAt: now,
iceConnectionState: peerConnection.iceConnectionState,
dtlsState: transport?.dtlsState,
candidatePair: {
state: candidatePair?.state,
protocol: localCandidate?.protocol ?? remoteCandidate?.protocol,
localType: localCandidate?.candidateType,
remoteType: remoteCandidate?.candidateType,
roundTripTimeMs:
Number.isFinite(candidatePair?.currentRoundTripTime) === true
? Math.round(candidatePair.currentRoundTripTime * 100000) / 100
: undefined,
consentExpiredAt: candidatePair?.consentExpiredTimestamp,
},
inbound: inbound,
};
} catch (error) {
this?.log?.debug?.('Unable to collect WebRTC stats for uuid "%s": %s', this.uuid, error?.message || String(error));
}
return this.stats.webrtc;
}
doUpdate(options = {}) {
let newToken = undefined;
let newUuid = undefined;
let hadToken = typeof this.token === 'string' && this.token !== '';
let hadUuid = typeof this.uuid === 'string' && this.uuid !== '';
// Validate options object.
if (typeof options !== 'object' || options === null) {
return;
}
// Normalise updated values from transport options.
// Undefined means "leave existing value unchanged".
//
// Streamer now passes a shared transport update payload:
// {
// uuid,
// apiAccess,
// fieldTest
// }
//
// WebRTC derives:
// - token from apiAccess.oauth2
// - field test mode from fieldTest
newUuid = typeof options?.uuid === 'string' && options.uuid !== '' ? options.uuid : undefined;
newToken = typeof options?.apiAccess?.oauth2 === 'string' && options.apiAccess.oauth2 !== '' ? options.apiAccess.oauth2 : undefined;
// Update device UUID used for logging/signalling.
// Avoid logging on initial assignment since this may be the first update()
// call used to populate transport state from the parent device.
if (typeof newUuid === 'string' && newUuid !== this.uuid) {
if (hadUuid === true && this.hasConsumers() === true) {
this?.log?.debug?.('Google Home device UUID has changed for uuid "%s" to "%s" while WebRTC session is active.', this.uuid, newUuid);
}
this.uuid = newUuid;
// UUID changed, so cached Google Foyer mapping is no longer valid.
this.#googleHomeDeviceUUID = undefined;
this.#googleHomeDeviceUUIDPromise = undefined;
}
// Update OAuth2 access token.
// Avoid logging on initial assignment since token refreshes are normal when
// there are no active consumers.
if (typeof newToken === 'string' && newToken !== this.token) {
if (hadToken === true && this.hasConsumers() === true) {
this?.log?.debug?.('OAuth2 token has changed for uuid "%s" while WebRTC session is active. Updating stored token.', this.uuid);
}
this.token = newToken;
}
// Only update field test mode when explicitly supplied.
// This allows partial update() calls without accidentally changing behaviour.
if (typeof options?.fieldTest === 'boolean') {
this.fieldTest = options.fieldTest === true;
}
}
async doSendAudio(talkingBuffer) {
if (
Buffer.isBuffer(talkingBuffer) !== true ||
this.#googleHomeDeviceUUID === undefined ||
this.#streamId === undefined ||
typeof this.#audioTransceiver?.sender?.sendRtp !== 'function'
) {
return;
}
// Ensure talkback state exists.
if (typeof this.#tracks.talkback !== 'object' || this.#tracks.talkback === null) {
this.#tracks.talkback = {};
}
let talk = this.#tracks.talkback;
// Default RTP/codec settings for outbound Opus talkback.
if (typeof talk.id !== 'number') {
talk.id = RTP_OPUS_AUDIO_PAYLOAD_TYPE;
}
if (typeof talk.sampleRate !== 'number') {
talk.sampleRate = this.talkback.sampleRate;
}
if (typeof talk.packetTime !== 'number') {
talk.packetTime = 20;
}
if (talkingBuffer.length > 0) {
// If talkback is not active yet, ask the remote device to enable it.
if (talk.active !== true) {
// Avoid issuing duplicate async start requests if HomeKit feeds
// audio faster than Google Home Foyer responds.
if (talk.starting === true) {
return;
}
talk.starting = true;
let homeFoyerResponse = await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'SendTalkback', {
googleDeviceId: { value: this.#googleHomeDeviceUUID },
streamId: this.#streamId,
command: 'COMMAND_START',
});
talk.starting = false;
// Failed to enable talkback on the remote side.
if (homeFoyerResponse?.status !== 0) {
this?.log?.debug?.('Error starting talkback for uuid "%s"', this.uuid);
talk.active = undefined;
talk.stopPending = false;
talk.rtp = undefined;
return;
}
talk.active = true;
this?.log?.debug?.('Talking started on uuid "%s"', this.uuid);
// Edge case:
// HomeKit may stop talking while the async start request was still
// in-flight. If that happened, immediately stop cleanly.
if (talk.stopPending === true) {
talk.stopPending = false;
await this.sendAudio(Buffer.alloc(0));
return;
}
}
// Safety guard in case talkback never became active.
if (talk.active !== true) {
return;
}
// Initialise RTP packet state if not already done.
// We maintain RTP continuity so the remote endpoint accepts packets.
if (typeof talk.rtp !== 'object' || talk.rtp === null) {
talk.rtp = {};
}
if (typeof talk.rtp.sequenceNumber !== 'number') {
talk.rtp.sequenceNumber = 0;
}
if (typeof talk.rtp.timestamp !== 'number') {
// RTP timestamps are sample-clock based, not wall-clock.
talk.rtp.timestamp = Math.floor(Math.random() * 0xffffffff);
}
// Build RTP header for outbound Opus payload.
let header = new werift.RtpHeader();
header.ssrc = this.#audioTransceiver.sender.ssrc;
header.payloadType = talk.id;
header.sequenceNumber = talk.rtp.sequenceNumber++ & RTP_SEQUENCE_MASK;
header.timestamp = talk.rtp.timestamp >>> 0;
header.marker = true;
header.payloadOffset = RTP_PACKET_HEADER_SIZE;
// Send outbound RTP packet to the WebRTC sender.
let packet = new werift.RtpPacket(header, talkingBuffer);
this.#audioTransceiver.sender.sendRtp(packet.serialize());
// Advance RTP timestamp for next packet.
// Example:
// 20ms @ 48kHz = 960 samples.
talk.rtp.timestamp = (talk.rtp.timestamp + Math.round((talk.sampleRate * talk.packetTime) / 1000)) >>> 0;
return;
}
// No active or pending talkback session to stop.
if (talk.active !== true && talk.starting !== true) {
return;
}
// If the async start request is still in-flight, defer the stop until
// the start completes so we avoid racing START and STOP commands.
if (talk.starting === true) {
talk.stopPending = true;
return;
}
// Notify the remote endpoint to disable talkback.
let homeFoyerResponse = await this.#grpcTransport.command(GOOGLE_HOME_FOYER_PREFIX, 'CameraService', 'SendTalkback', {
googleDeviceId: { value: this.#googleHomeDeviceUUID },
streamId: this.#streamId,
command: 'COMMAND_STOP',
});
if (homeFoyerResponse?.status !== 0) {
this?.log?.debug?.('Error stopping talkback for uuid "%s"', this.uuid);
} else {
this?.log?.debug?.('Talking ended on uuid "%s"', this.uuid);
}
// Reset talkback state ready for next session.
talk.active = undefined;
talk.starting = false;
talk.stopPending = false;
talk.rtp = undefined;
}
#handlePlaybackBegin(mediaType) {
if (this.closing === true || this.closed === true) {
return;
}
if (mediaType === Streamer.MEDIA_TYPE.VIDEO) {
this.#ensurePlaybackVideoTrack();
this.#tracks.video.rtp = { lastSequence: undefined, lastTimestamp: undefined };
this.#tracks.video.output = { lastTimestamp: undefined };
this.#tracks.video.deltaAudit = { hasAcceptedKeyframe: false };
this.#tracks.video.h264 = this.#createH264State();
if (typeof this.#tracks.video.jitter === 'object' && this.#tracks.video.jitter !== null) {
this.clearJitterBuffer(this.#tracks.video.jitter);
}
// Start bounded startup keyframe requests. The first call may happen
// before the RTP SSRC is known, so retries continue until the first IDR.
this.#startStartupKeyframeTimer();
this.#sendVideoPLI();
this.#refreshStallTimer();
return;
}
if (mediaType === Streamer.MEDIA_TYPE.AUDIO) {
this.#ensurePlaybackAudioTrack();
this.#tracks.audio.rtp = { lastSequence: undefined, lastTimestamp: undefined };
this.#tracks.audio.output = { lastTimestamp: undefined };
if (typeof this.#tracks.audio.jitter === 'object' && this.#tracks.audio.jitter !== null) {
this.clearJitterBuffer(this.#tracks.audio.jitter);
}
this.#refreshStallTimer();
}
}
#ensurePlaybackVideoTrack() {
// Build or repair the per-session video state once, keeping packet hot paths
// focused on RTP/H264 work instead of repeated object-shape checks.
if (typeof this.#tracks !== 'object' || this.#tracks === null) {
this.#tracks = {};
}
if (typeof this.#tracks.video !== 'object' || this.#tracks.video === null) {
this.#tracks.video = {};
}
let video = this.#tracks.video;
video.id = typeof video.id === 'number' ? video.id : RTP_H264_VIDEO_PAYLOAD_TYPE;
video.rtxId = typeof video.rtxId === 'number' ? video.rtxId : RTP_H264_VIDEO_RTX_PAYLOAD_TYPE;
video.rtxSsrc = typeof video.rtxSsrc === 'number' ? video.rtxSsrc : undefined;
video.codec = typeof video.codec === 'string' ? video.codec : StreamTransport.CODEC_TYPE.H264;
video.sampleRate = typeof video.sampleRate === 'number' ? video.sampleRate : 90000;
video.lastPLITime = typeof video.lastPLITime === 'number' ? video.lastPLITime : undefined;
video.keyframeRequestInFlight = typeof video.keyframeRequestInFlight === 'boolean' ? video.keyframeRequestInFlight : false;
video.lastKeyframeEventTime = typeof video.lastKeyframeEventTime === 'number' ? video.lastKeyframeEventTime : undefined;
video.rtp = typeof video.rtp === 'object' && video.rtp !== null ? video.rtp : { lastSequence: undefined, lastTimestamp: undefined };
video.output = typeof video.output === 'object' && video.output !== null ? video.output : { lastTimestamp: undefined };
video.health = typeof video.health === 'object' && video.health !== null ? video.health : this.getMediaState('video');
video.deltaAudit =
typeof video.deltaAudit === 'object' && video.deltaAudit !== null ? video.deltaAudit : { hasAcceptedKeyframe: false };
if (typeof video.jitter !== 'object' || video.jitter === null || video.jitter.groupByTimestamp !== true) {
video.jitter = this.createJitterBuffer({
groupByTimestamp: true,
delayMs: VIDEO_RTP_REORDER_DELAY_MS,
maxPackets: VIDEO_RTP_REORDER_MAX_PACKETS,
});
}
if (typeof video.h264 !== 'object' || video.h264 === null) {
video.h264 = this.#createH264State();
}
return video;
}
#ensurePlaybackAudioTrack() {
// Build or repair the per-session audio state once so packet handling only
// deals with RTP timing, Opus decode, and decoded-frame emission.
if (typeof this.#tracks !== 'object' || this.#tracks === null) {
this.#tracks = {};
}
if (typeof this.#tracks.audio !== 'object' || this.#tracks.audio === null) {
this.#tracks.audio = {};
}
let audio = this.#tracks.audio;
audio.id = typeof audio.id === 'number' ? audio.id : RTP_OPUS_AUDIO_PAYLOAD_TYPE;
audio.codec = typeof audio.codec === 'string' ? audio.codec : StreamTransport.CODEC_TYPE.OPUS;
audio.sampleRate = typeof audio.sampleRate === 'number' ? audio.sampleRate : 48000;
audio.channels = typeof audio.channels === 'number' ? audio.channels : 2;
audio.packetTime = typeof audio.packetTime === 'number' ? audio.packetTime : 20;
audio.rtp = typeof audio.rtp === 'object' && audio.rtp !== null ? audio.rtp : { lastSequence: undefined, lastTimestamp: undefined };
audio.output = typeof audio.output === 'object' && audio.output !== null ? audio.output : { lastTimestamp: undefined };
audio.lastDecodeFallbackLogTime = typeof audio.lastDecodeFallbackLogTime === 'number' ? audio.lastDecodeFallbackLogTime : undefined;
audio.lastDecodeErrorLogTime = typeof audio.lastDecodeErrorLogTime === 'number' ? audio.lastDecodeErrorLogTime : undefined;
if (typeof audio.jitter !== 'object' || audio.jitter === null || audio.jitter.groupByTimestamp === true) {
audio.jitter = this.createJitterBuffer({
delayMs: AUDIO_RTP_REORDER_DELAY_MS,
maxPackets: AUDIO_RTP_REORDER_MAX_PACKETS,
});
}
return audio;
}
#createH264State() {
// H264 assembly state: cached parameter sets, one pending access unit,
// and one in-progress FU-A NAL reconstruction.
return {
fuParts: [],
fuBytes: 0,
fuNalType: 0,
fuRtpTimestamp: undefined,
fuFirstPacketTime: undefined,
fuLastSequence: undefined,
lastSPS: undefined,
lastPPS: undefined,
hasIDR: false,
pendingParts: [],
pendingRtpTimestamp: undefined,
pendingFirstPacketTime: undefined,
pendingKeyFrame: false,
pendingBytes: 0,
pendingHasVcl: false,
pendingMarkerSeen: false,
pendingCorrupt: false,
};
}
#keyframeByteLimit() {
let spsInfo = H264.getSPSInfo(this.#tracks?.video?.h264?.lastSPS);
let width = this.video?.width ?? spsInfo?.width ?? 0;
let height = this.video?.height ?? spsInfo?.height ?? 0;
let bitrate = Number.isFinite(this.video?.bitrate) === true ? this.video.bitrate : 0;
// The first startup keyframe may arrive before StreamTransport has learned
// resolution metadata. Use its cached SPS when available so
// high-resolution startup frames receive the same adaptive allowance.
return Math.min(
KEYFRAME_MAX_BYTES,
Math.max(KEYFRAME_DEFAULT_BYTES, width * height * KEYFRAME_BYTES_PER_PIXEL, (bitrate * KEYFRAME_BITRATE_SECONDS) / 8),
);
}
#drainPlaybackVideoJitterBuffer(force = false) {
let video = this.#tracks?.video;
let jitter = video?.jitter;
let now = Date.now();
let group = undefined;
let groupAgeMs = 0;
let keyframeAssemblyLimitMs = KEYFRAME_MAX_ASSEMBLY_MS;
let keyframeTimedOut = false;
let packetCount = 0;
let groupCount = 0;
if (
(this.closing === true && force !== true) ||
this.closed === true ||
typeof video !== 'object' ||
video === null ||
typeof jitter !== 'object' ||
jitter === null ||
jitter.groupByTimestamp !== true
) {
return;
}
for (group of this.releaseJitterGroups(jitter, {
force: force,
maxGroups: force === true ? undefined : VIDEO_RTP_DRAIN_MAX_GROUPS,
maxPackets: force === true ? undefined : VIDEO_RTP_DRAIN_MAX_PACKETS,
isComplete: (entry) => {
groupAgeMs = now - (entry?.firstReceivedAt || now);
return RtpH264.isTimestampGroupComplete(entry, this.sortJitterGroupPackets(jitter, entry));
},
canWait: (entry) => {
groupAgeMs = now - (entry?.firstReceivedAt || now);
packetCount = this.countJitterPackets(jitter);
groupCount = this.sizeJitterBuffer(jitter);
keyframeAssemblyLimitMs =
groupCount > 1
? KEYFRAME_BLOCKING_MAX_ASSEMBLY_MS
: this.ready === true
? KEYFRAME_MAX_ASSEMBLY_MS
: KEYFRAME_STARTUP_MAX_ASSEMBLY_MS;
keyframeTimedOut = entry?.hasKeyFrame === true && groupAgeMs >= keyframeAssemblyLimitMs;
if (entry?.hasKeyFrame === true && keyframeTimedOut !== true) {
return true;
}
return groupAgeMs < VIDEO_RTP_REORDER_DELAY_MS && packetCount < VIDEO_RTP_REORDER_MAX_PACKETS;
},
})) {
groupAgeMs = now - (group?.firstReceivedAt || now);
if (RtpH264.isTimestampGroupComplete(group, this.sortJitterGroupPackets(jitter, group)) !== true) {
jitter.lastReleasedTimestamp = group.rtpTimestamp;
this.recordVideoDrop(group?.hasKeyFrame === true ? 'jitter-keyframe-incomplete' : 'jitter-frame-incomplete');
if (group?.hasKeyFrame === true) {
if (typeof jitter.lastDropLogTime !== 'number' || now - jitter.lastDropLogTime >= 10000) {
jitter.lastDropLogTime = now;
this?.log?.debug?.(
'Dropping incomplete jittered WebRTC keyframe for uuid "%s": rtpTs="%s" packets="%s" ageMs="%s" ' +
'marker="%s" fuStart="%s" fuEnd="%s"',
this.uuid,
group.rtpTimestamp,
Array.isArray(group.packets) === true ? group.packets.length : 0,
Math.round(groupAgeMs),
group.markerSeen === true ? 'true' : 'false',
group.hasFragmentStart === true ? 'true' : 'false',
group.hasFragmentEnd === true ? 'true' : 'false',
);
}
this.#sendVideoPLI();
this.markMediaIssue('video', 'jitter-keyframe-incomplete');
this.clearJitterBuffer(jitter);
this.#resetFragmentedVideoFrame();
this.#resetPendingVideoFrame();
break;
}
continue;
}
this.sortJitterGroupPackets(jitter, group);
for (let packet of group.packets) {
jitter.lastReleasedSequence = packet.sequenceNumber;
this.#handlePlaybackVideoPacket(packet.packet, true);
}
jitter.lastReleasedTimestamp = group.rtpTimestamp;
}
if (force !== true && this.sizeJitterBuffer(jitter) > 0) {
this.recordVideoReorder('defers');
}
}
#appendH264NalUnit(h264, nal) {
// Append one complete H264 NAL as Annex-B and update frame/parameter-set state.
let nalType = 0;
let part = undefined;
if (typeof h264 !== 'object' || h264 === null || Buffer.isBuffer(nal) !== true || nal.length === 0) {
return false;
}
part = H264.wrapAnnexB(nal);
h264.pendingParts.push(part);
h264.pendingBytes += part.length;
nalType = nal[0] & 0x1f;
if (nalType === H264.NALUS.TYPES.SPS) {
h264.lastSPS = Buffer.from(nal);
}
if (nalType === H264.NALUS.TYPES.PPS) {
h264.lastPPS = Buffer.from(nal);
}
if (nalType === H264.NALUS.TYPES.IDR) {
h264.pendingKeyFrame = true;
h264.pendingHasVcl = true;
h264.hasIDR = true;
}
if (nalType === H264.NALUS.TYPES.SLICE_NON_IDR) {
h264.pendingHasVcl = true;
}
return true;
}
#handlePlaybackVideoPacket(rtpPacket, fromJitterBuffer = false) {
if ((this.closing === true && fromJitterBuffer !== true) || this.closed === true) {
// We are closing or closed, so ignore any incoming packets. This can happen when remote is still sending
// before we finish tearing down the connection, but we do not want to process any new packets at this point.
return;
}
let fuResult = undefined;
let stapOffset = 0;
let stapLength = 0;
let stapNal = undefined;
let seqDelta = 0;
let isRtxPacket = false;
let pendingAgeMs = undefined;
let fuAgeMs = undefined;
let pendingTsDeltaTicks = 0;
let pendingTsWrapCandidate = false;
let fuTsDeltaTicks = 0;
let fuTsWrapCandidate = false;
let incomingNalType = 0;
let incomingFuHeader = 0;
let incomingFuStart = false;
let incomingFuNalType = 0;
let incomingIsIdrFuStart = false;
let pendingPartCount = 0;
let pendingByteCount = 0;
let pendingHasContent = false;
let jitter = undefined;
let payloadInfo = undefined;
let queueSequenceNumber = 0;
let packetReceivedAt = 0;
// Ensure we have a valid RTP packet with a payload before processing video data
if (
typeof rtpPacket !== 'object' ||
rtpPacket === null ||
typeof rtpPacket?.header !== 'object' ||
rtpPacket.header === null ||
Buffer.isBuffer(rtpPacket?.payload) !== true ||
rtpPacket.payload.length === 0
) {
return;
}
// Pull out the RTP header details we use repeatedly below
let header = rtpPacket.header;
let payload = rtpPacket.payload;
let marker = header.marker === true;
let sequenceNumber = Number.isInteger(header.sequenceNumber) === true ? header.sequenceNumber : 0;
let rtpTimestamp = Number.isInteger(header.timestamp) === true ? header.timestamp >>> 0 : 0;
let payloadType = Number.isInteger(header.payloadType) === true ? header.payloadType : undefined;
let ssrc = Number.isInteger(header.ssrc) === true ? header.ssrc >>> 0 : undefined;
packetReceivedAt = Number.isFinite(rtpPacket.receivedAt) === true ? rtpPacket.receivedAt : Date.now();
let video = this.#ensurePlaybackVideoTrack();
let h264 = video.h264;
let videoRtp = video.rtp;
let deltaAudit = video.deltaAudit;
isRtxPacket = typeof payloadType === 'number' && payloadType === video.rtxId;
if (fromJitterBuffer !== true) {
jitter = video.jitter;
if (isRtxPacket === true) {
if (typeof video.rtxSsrc !== 'number' && typeof ssrc === 'number') {
video.rtxSsrc = ssrc;
}
if (typeof video.rtxSsrc === 'number' && typeof ssrc === 'number' && ssrc !== video.rtxSsrc) {
return;
}
if (Buffer.isBuffer(payload) !== true || payload.length < 3 || typeof video.ssrc !== 'number') {
return;
}
queueSequenceNumber = payload.readUInt16BE(0);
payload = payload.subarray(2);
payloadType = video.id;
ssrc = video.ssrc;
} else {
queueSequenceNumber = sequenceNumber;
}
if (typeof payloadType === 'number' && payloadType !== video.id) {
return;
}
// Learn the primary video SSRC before jitter release so startup PLI can
// request a fresh IDR even while the first timestamp group is still queued.
if (typeof video.ssrc !== 'number' && typeof ssrc === 'number') {
video.ssrc = ssrc;
}
if (typeof video.ssrc === 'number' && typeof ssrc === 'number' && ssrc !== video.ssrc) {
return;
}
payloadInfo = RtpH264.getPayloadInfo(payload);
if (
this.pushJitterPacket(jitter, {
rtpTimestamp: rtpTimestamp,
sequenceNumber: queueSequenceNumber,
receivedAt: packetReceivedAt,
marker: marker,
group: {
hasKeyFrame: payloadInfo.hasKeyFrame === true,
hasFragmentedNal: payloadInfo.hasFragmentedNal === true,
hasFragmentStart: payloadInfo.hasFragmentStart === true,
hasFragmentEnd: payloadInfo.hasFragmentEnd === true,
},
packet: {
receivedAt: packetReceivedAt,
header: {
...header,
payloadType: video.id,
sequenceNumber: queueSequenceNumber,
ssrc: ssrc,
},
payload: payload,
},
}) !== true
) {
return;
}
this.#drainPlaybackVideoJitterBuffer();
return;
}
if (typeof deltaAudit !== 'object' || deltaAudit === null) {
deltaAudit = {
hasAcceptedKeyframe: false,
};
video.deltaAudit = deltaAudit;
}
// Packets released from the jitter buffer are already normalised to the
// primary H264 payload type and SSRC. RTX is unwrapped before queueing.
if (typeof video.ssrc !== 'number' && typeof ssrc === 'number') {
video.ssrc = ssrc;
}
if (typeof video.ssrc === 'number' && typeof ssrc === 'number' && ssrc !== video.ssrc) {
return;
}
if (typeof payloadType === 'number' && payloadType !== video.id) {
return;
}
if (this.ready !== true && h264?.hasIDR !== true && typeof video.ssrc === 'number') {
this.#sendVideoPLI();
}
// Drop duplicate or clearly late/out-of-order packets before they touch assembly state.
// This mirrors the protection already used on audio and avoids duplicate fragments or
// old retransmits corruptin