homebridge-nest-accfactory/dist/system.js

Homebridge support for Nest/Google devices including HomeKit Secure Video (HKSV) support for doorbells and cameras

// Overall system communications and device management
// Part of homebridge-nest-accfactory
//
// Core platform manager for communication with Nest and Google APIs.
// Handles account authorisation, API connection lifecycle, device discovery,
// raw data aggregation, snapshot coordination, protobuf-backed observe/subscribe
// handling, and routing of updates and commands between cloud APIs and
// HomeKit device modules.
//
// Responsibilities:
// - Authorise and maintain connections to Nest and Google accounts
// - Manage Nest REST API and Google protobuf/gRPC API communication
// - Observe and subscribe to cloud updates in near real-time
// - Aggregate and maintain raw device data from multiple API sources
// - Coordinate protobuf-backed camera snapshot requests and responses
// - Discover, create, update, and remove supported device instances
// - Route HomeKit get/set requests to the correct upstream API
// - Load and coordinate device support modules
// - Build and manage protobuf-backed observe trait subscriptions
//
// Features:
// - Multi-account support (multiple Google and/or Nest accounts simultaneously)
// - Automatic reconnect and token/session refresh handling
// - Nest REST API subscribe loop and Google protobuf observe loop
// - Shared protobuf schema/type caching via protobuf.js helpers
// - Raw data merging across Nest and Google sources
// - Promise-based snapshot waiter handling for upload_live_image updates
// - Support dump generation for troubleshooting when enabled
// - Dynamic device module loading and HomeKit category selection
//
// Notes:
// - HomeKit characteristic and service management is handled by individual device modules
// - This module is responsible for platform orchestration, API communication,
//   snapshot coordination, protobuf observe handling, and device lifecycle
// - Camera, thermostat, sensor, and lock behaviour is implemented in device-specific modules
//
// Architecture:
// - Exports the main NestAccfactory platform class
// - Maintains connection state, raw data cache, snapshot waiters, and tracked devices
// - Uses shared protobuf helpers for protobuf schema/type loading and traversal
// - Creates and updates HomeKitDevice-based instances for supported device types
//
// Code version 2026.05.07
// Mark Hulskamp
'use strict';

// Define nodejs module requirements
import { Buffer } from 'node:buffer';
import { setTimeout, clearTimeout } from 'node:timers';
import path from 'node:path';
import crypto from 'node:crypto';
import process from 'node:process';
import os from 'node:os';
import { URL } from 'node:url';

// Import our modules
import GrpcTransport from './grpctransport.js';
import HomeKitDevice from './HomeKitDevice.js';
import { loadDeviceModules, getDeviceHKCategory } from './devices.js';
import { processConfig, buildConnections } from './config.js';
import { adjustTemperature, scaleValue, fetchWrapper } from './utils.js';
import { getProtoTypes } from './protobuf.js';

// Define constants
import {
  MIN_NODE_VERSION,
  USER_AGENT,
  __dirname,
  DATA_SOURCE,
  DEVICE_TYPE,
  ACCOUNT_TYPE,
  NEST_API_BUCKETS,
  PROTOBUF_RESOURCES,
} from './consts.js';

const SNAPSHOT_TIMEOUT = 7000; // Overall HomeKit snapshot timeout
const SNAPSHOT_WAIT_TIMEOUT = 3000; // Wait for Google upload_live_image observe update
const SNAPSHOT_FETCH_TIMEOUT = 3000; // HTTP fetch timeout for snapshot image

// We handle the connections to Nest/Google
// Perform device management (additions/removals/updates)
export default class NestAccfactory {
  cachedAccessories = []; // Track restored cached accessories

  // Internal data only for this class
  #connections = undefined; // Object of confirmed connections
  #rawData = {}; // Cached copy of data from both Nest and Google APIs
  #trackedDevices = new Map(); // Devices we've created, keyed by serial number
  #deviceModules = undefined; // No loaded device support modules to start

  constructor(log, config, api) {
    this.log = log;
    this.api = api;

    // Validate required version of Node.js that we're running on.
    // If less than our minimum required version, log an error and stop initialisation
    let nodeVersion = Number(process.versions.node.split('.')[0]);

    if (nodeVersion < MIN_NODE_VERSION) {
      this.log.error(
        'We no longer support running on Node.js %s. Please upgrade to Node.js %s or newer. The plugin will not be started.',
        process.versions.node,
        MIN_NODE_VERSION,
      );
      return;
    }

    // Output some basic info about the plugin starting up, which can be useful for troubleshooting
    if (config?.options?.debug === true) {
      log?.warn?.('Verbose logging enabled via configuration');
    }

    // Output some debug info about the system we're running on, which can be useful for troubleshooting
    this?.log?.debug?.('System: %s %s (%s)', os.platform(), os.release(), os.arch());
    this?.log?.debug?.('CPU: %s (%d cores)', os.cpus()?.[0]?.model, os.cpus()?.length);
    this?.log?.debug?.('Memory: %d MB total', Math.round(os.totalmem() / 1024 / 1024));
    this?.log?.debug?.('Node.js: v%s', process.versions.node);

    // Perform validation on the configuration passed into us and set defaults if not present
    this.config = processConfig(config, this.log, this.api);
    this.#connections = buildConnections(this.config);

    // Check for valid connections, either a Nest and/or Google one specified. Otherwise, return back.
    if (Object.keys(this.#connections).length === 0) {
      this?.log?.error?.('No connections have been specified in the JSON configuration. Please review');
      return;
    }

    api?.on?.('didFinishLaunching', async () => {
      // We got notified that Homebridge has finished loading

      // Load device support modules from the plugins folder if not already done
      this.#deviceModules = await loadDeviceModules(this.log, 'plugins');

      // Start reconnect loop per connection with backoff for failed tries
      // This also initiates both Nest API subscribes and Google API observes
      for (const uuid of Object.keys(this.#connections)) {
        let reconnectDelay = 15000;
        let connection = this.#connections?.[uuid];

        const reconnectLoop = async () => {
          if (connection?.authorised !== true && connection?.allowRetry !== false) {
            try {
              await this.#connect(uuid);
              this.#subscribeNestAPI(uuid);
              this.#observeGoogleAPI(uuid);
              // eslint-disable-next-line no-unused-vars
            } catch (error) {
              // Empty
            }

            reconnectDelay = connection?.authorised === true ? 15000 : Math.min(reconnectDelay * 2, 60000);
          } else {
            reconnectDelay = 15000;
          }

          if (this.#connections?.[uuid] === connection) {
            clearTimeout(connection.reconnectTimer);
            connection.reconnectTimer = setTimeout(reconnectLoop, reconnectDelay);
          }
        };

        if (connection?.exclude !== true) {
          reconnectLoop();
        } else {
          this?.log?.warn?.('Account "%s" is ignored due to it being marked as excluded', connection?.name);
        }
      }
    });

    api?.on?.('shutdown', async () => {
      // We got notified that Homebridge is shutting down
      // Perform cleanup of internal state
      for (let uuid of Object.keys(this.#connections ?? {})) {
        // Cleanup any active timers or transports for this connection
        clearTimeout(this.#connections?.[uuid]?.timer);
        clearTimeout(this.#connections?.[uuid]?.reconnectTimer);
        clearTimeout(this.#connections?.[uuid]?.subscribeTimer);
        clearTimeout(this.#connections?.[uuid]?.observeTimer);

        // If we have an active gRPC transport for this connection, release it to clean up resources and connections
        this.#connections?.[uuid]?.grpcTransport?.release?.();

        // If we have any pending snapshot waiters for this connection, trigger them
        // so any in-flight snapshot requests can finish cleanly during shutdown
        if (this.#connections?.[uuid]?.snapshotWaiters instanceof Map) {
          for (let waiter of this.#connections[uuid].snapshotWaiters.values()) {
            if (typeof waiter === 'function') {
              waiter();
            }
          }

          this.#connections[uuid].snapshotWaiters.clear();
        }
      }

      // Cleanup internal data
      this.#trackedDevices.clear();
      this.#rawData = {};
      this.#connections = undefined;
      this.#deviceModules?.clear?.();
      this.cachedAccessories = [];
    });
  }

  configureAccessory(accessory) {
    // This gets called from Homebridge each time it restores an accessory from its cache
    this?.log?.info?.('Loading accessory from cache:', accessory.displayName);

    let informationService = accessory?.getService?.(this.api.hap.Service.AccessoryInformation);
    if (informationService === undefined) {
      // Accessory is missing the required AccessoryInformation service
      // means it's not going to work and is likely a stale entry in the cache. Remove it and log an error.
      this?.log?.warn?.('Cached accessory "%s" is missing AccessoryInformation service. Removing from cache', accessory.displayName);

      try {
        this.api.unregisterPlatformAccessories(HomeKitDevice.PLUGIN_NAME, HomeKitDevice.PLATFORM_NAME, [accessory]);
        // eslint-disable-next-line no-unused-vars
      } catch (error) {
        // Empty
      }

      return;
    }

    // Accessory has the required AccessoryInformation service, so we can add the restored accessory to the accessories cache
    // This allows us to track if it has already been registered
    this.cachedAccessories.push(accessory);
  }

  async #connect(uuid) {
    if (typeof this.#connections?.[uuid] !== 'object') {
      return;
    }

    let connection = this.#connections[uuid];
    let isRetry = connection.allowRetry === true;
    let accountLabel = connection.type === ACCOUNT_TYPE.GOOGLE ? 'Google' : 'Nest';

    // If allowRetry is true, we assume this is not the first connection attempt, so we'll only use debug level logging
    // Otherwise, we'll use info level
    this?.log?.[isRetry === true ? 'debug' : 'info']?.(
      'Performing authorisation for connection "%s" %s',
      connection.name,
      connection.fieldTest === true ? 'using field test endpoints' : '',
    );

    // Cleanup runtime state before re-auth
    // Ensures no stale waiters or transports remain
    let cleanupConnectionRuntime = () => {
      if (this.#connections?.[uuid]?.snapshotWaiters instanceof Map) {
        for (let waiter of this.#connections[uuid].snapshotWaiters.values()) {
          if (typeof waiter === 'function') {
            waiter(); // resolve any pending snapshot promises
          }
        }

        this.#connections[uuid].snapshotWaiters.clear();
      }

      this.#connections?.[uuid]?.grpcTransport?.release?.();
    };

    // Create gRPC transport (protobuf backend)
    // Uses the live connection object so refreshed token values are picked up by getAuthHeader()
    let createGrpcTransport = () => {
      if (this.config?.options?.useGoogleAPI !== true) {
        return;
      }

      return new GrpcTransport({
        log: this.log,
        protoPath: path.join(__dirname, 'protobuf/root.proto'),
        endpointHost: 'https://' + this.#connections[uuid].grpcEndpointHost,
        uuid: uuid,
        userAgent: USER_AGENT,
        getAuthHeader: () => {
          let token = this.#connections?.[uuid]?.token;
          return typeof token === 'string' && token.trim() !== '' ? 'Basic ' + token : '';
        },
      });
    };

    // Apply successful auth result
    // Shared across Google and Nest flows
    let applyAuthorisedConnection = (sessionData, cameraAPI, refreshSeconds, successMessage) => {
      this?.log?.[isRetry === true ? 'debug' : 'success']?.(successMessage, connection.name);

      cleanupConnectionRuntime();

      Object.assign(this.#connections[uuid], {
        authorised: true,
        allowRetry: true,
        userID: sessionData.userid,
        transport_url: sessionData.urls.transport_url,
        weather_url: sessionData.urls.weather_url,
        token: sessionData.access_token,
        cameraAPI: cameraAPI,
        grpcTransport: createGrpcTransport(),
        snapshotWaiters: new Map(), // Keyed by resource Id for protobuf snapshot waiters
      });

      // Notify any camera related devices (camera/doorbell/floodlight) of updated auth details
      for (let [, trackedDevice] of this.#trackedDevices) {
        if (typeof trackedDevice !== 'object' || trackedDevice === null) {
          continue;
        }

        if (
          trackedDevice.type !== DEVICE_TYPE.CAMERA &&
          trackedDevice.type !== DEVICE_TYPE.DOORBELL &&
          trackedDevice.type !== DEVICE_TYPE.FLOODLIGHT
        ) {
          continue;
        }

        // Message device with updated cameraAPI details
        HomeKitDevice.message(trackedDevice.uuid, HomeKitDevice.UPDATE, {
          apiAccess: this.#connections[uuid].cameraAPI,
        });
      }

      // Schedule token refresh before expiry
      clearTimeout(this.#connections[uuid].timer);
      this.#connections[uuid].timer = setTimeout(() => {
        this?.log?.debug?.(
          'Performing periodic token refresh using %s account for connection "%s"',
          accountLabel,
          this.#connections[uuid].name,
        );
        this.#connections[uuid].allowRetry = true;
        this.#connect(uuid);
      }, refreshSeconds * 1000);
    };

    // Common error handling
    // Determines retry behaviour and logs appropriately
    let handleConnectError = (error, nonRetryableCodes, retryErrorMessage, authErrorMessage) => {
      let statusCode =
        error?.code !== undefined && error?.code !== null
          ? error.code
          : error?.status !== undefined && error?.status !== null
            ? error.status
            : undefined;

      if (nonRetryableCodes.includes(statusCode) === true) {
        this.#connections[uuid].allowRetry = false;
      }

      this.#connections[uuid].authorised = false;

      this?.log?.debug?.(
        'Failed to connect using %s credentials for connection "%s" %s: Error was "%s"',
        accountLabel,
        this.#connections[uuid].name,
        this.#connections[uuid].allowRetry === true ? 'will retry' : 'will not retry',
        typeof error?.message === 'string' ? error.message : String(error),
      );

      this?.log?.error?.(this.#connections[uuid].allowRetry === true ? retryErrorMessage : authErrorMessage, this.#connections[uuid].name);
    };

    // Google account authentication flow
    if (connection.type === ACCOUNT_TYPE.GOOGLE) {
      try {
        // Obtain OAuth token from Google using the token exchange endpoint and the cookie we already have from config
        let tokenResponse = await fetchWrapper('get', connection.issueToken, {
          headers: {
            Referer: 'https://accounts.google.com/',
            Cookie: connection.cookie,
            'User-Agent': USER_AGENT,
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'X-Requested-With': 'XmlHttpRequest',
          },
        });

        let tokenData = await tokenResponse.json();

        if (typeof tokenData?.error === 'string') {
          let error = new Error(
            (tokenData?.detail ? String(tokenData.detail) : '') + (tokenData?.error ? ' (' + String(tokenData.error) + ')' : ''),
          );
          error.name = 'GoogleAuthError';
          error.code = tokenData.error;
          error.statusText = tokenData.detail || 'OAuth error';
          throw error;
        }

        let googleOAuth2Token = tokenData.access_token.trim();

        // Obtain JWT from Google using the OAuth token we just obtained
        let jwtResponse = await fetchWrapper(
          'post',
          'https://nestauthproxyservice-pa.googleapis.com/v1/issue_jwt',
          {
            headers: {
              Referer: 'https://' + connection.referer,
              Origin: 'https://' + connection.referer,
              Authorization: tokenData.token_type + ' ' + tokenData.access_token,
              'User-Agent': USER_AGENT,
              'Content-Type': 'application/json',
              'Sec-Fetch-Mode': 'cors',
              'Sec-Fetch-Site': 'cross-site',
            },
          },
          {
            policy_id: 'authproxy-oauth-policy',
            google_oauth_access_token: tokenData.access_token,
            embed_google_oauth_access_token: true,
            expire_after: '3600s',
          },
        );

        let jwtData = await jwtResponse.json();

        if ((jwtData?.jwt?.trim?.() ?? '') === '') {
          this?.log?.debug?.('JWT response object', jwtData);
          throw new Error('Missing jwt in JWT response');
        }

        // Get Nest session using the JWT token we just obtained
        let sessionResponse = await fetchWrapper('get', new URL('/session', 'https://' + connection.restAPIHost).href, {
          headers: {
            Referer: 'https://' + connection.referer,
            Origin: 'https://' + connection.referer,
            Authorization: 'Basic ' + jwtData.jwt,
            'User-Agent': USER_AGENT,
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
          },
        });

        let sessionData = await sessionResponse.json();

        if ((sessionData?.access_token?.trim?.() ?? '') === '') {
          this?.log?.debug?.('Nest session response object', sessionData);
          throw new Error('Missing access_token in session response');
        }

        // Apply final authorised state
        applyAuthorisedConnection(
          sessionData,
          {
            key: 'Authorization',
            value: 'Basic ',
            token: sessionData.access_token,
            oauth2: googleOAuth2Token,
            fieldTest: connection.fieldTest === true,
          },
          tokenData.expires_in - 300,
          isRetry === true
            ? 'Successfully performed token refresh using Google account for connection "%s"'
            : 'Successfully authorised using Google account for connection "%s"',
        );
      } catch (error) {
        handleConnectError(
          error,
          ['USER_LOGGED_OUT', 'ERR_INVALID_URL', 401, 403],
          'Token refresh failed using Google account for connection "%s"',
          'Authorisation failed using Google account for connection "%s"',
        );
      }

      return;
    }

    // Nest account authentication flow (legacy)
    if (connection.type === ACCOUNT_TYPE.NEST) {
      try {
        // Retrieve session token from Nest login endpoint using the access token we already have from config
        let loginResponse = await fetchWrapper(
          'post',
          new URL('/api/v1/login.login_nest', 'https://webapi.' + connection.cameraAPIHost).href,
          {
            withCredentials: true,
            headers: {
              Referer: 'https://' + connection.referer,
              Origin: 'https://' + connection.referer,
              'User-Agent': USER_AGENT,
              'Content-Type': 'application/x-www-form-urlencoded',
              'Sec-Fetch-Mode': 'cors',
              'Sec-Fetch-Site': 'same-origin',
            },
          },
          Buffer.from('access_token=' + connection.access_token, 'utf8'),
        );

        let loginData = await loginResponse.json();

        if (typeof loginData?.status === 'number' && (loginData?.items?.[0]?.session_token.trim?.() ?? '') === '') {
          let error = new Error(
            (loginData?.status_detail ? String(loginData.status_detail) : '') +
              (loginData?.status_description ? ' (' + String(loginData.status_description) + ')' : '') +
              (loginData?.status_detail || loginData?.status_description ? '' : 'Nest login failed with status ' + loginData.status),
          );
          error.name = 'NestAuthError';
          error.code = loginData.status;
          error.message = loginData?.status_description || 'Error';
          throw error;
        }

        let nestToken = loginData.items[0].session_token;

        // Get Nest session details after successful login
        let sessionResponse = await fetchWrapper('get', new URL('/session', 'https://' + connection.restAPIHost).href, {
          headers: {
            Referer: 'https://' + connection.referer,
            Origin: 'https://' + connection.referer,
            Authorization: 'Basic ' + connection.access_token,
            'User-Agent': USER_AGENT,
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
          },
        });

        let sessionData = await sessionResponse.json();

        // Apply final authorised state
        applyAuthorisedConnection(
          sessionData,
          {
            key: 'cookie',
            value: connection.fieldTest === true ? 'website_ft=' : 'website_2=',
            token: nestToken,
            fieldTest: connection.fieldTest === true,
          },
          3600 * 24,
          isRetry === true
            ? 'Successfully performed token refresh using Nest account for connection "%s"'
            : 'Successfully authorised using Nest account for connection "%s"',
        );
      } catch (error) {
        handleConnectError(
          error,
          ['ERR_INVALID_URL', 401, 403],
          'Token refresh failed using Nest account for connection "%s"',
          'Authorisation failed using Nest account for connection "%s"',
        );
      }
    }
  }

  async #subscribeNestAPI(uuid, firstRun = true, fullRead = true) {
    if (
      typeof this.#connections?.[uuid] !== 'object' ||
      this.#connections?.[uuid]?.authorised !== true ||
      this.config?.options?.useNestAPI !== true
    ) {
      // Not a valid connection object and/or we're not authorised
      return;
    }

    // By default, setup for a full data read from the Nest API
    let subscribeJSONData = undefined;
    if (firstRun !== false || fullRead !== false) {
      this?.log?.debug?.('Starting Nest API subscribe for connection "%s"', this.#connections[uuid].name);
      subscribeJSONData = { known_bucket_types: NEST_API_BUCKETS, known_bucket_versions: [] };
    }

    // We have data stored from this Nest API, so setup read using known object
    // We exclude any data source other than from Nest API and also any injected data
    if (firstRun === false || fullRead === false) {
      subscribeJSONData = { objects: [] };
      subscribeJSONData.objects.push(
        ...Object.entries(this.#rawData)
          // eslint-disable-next-line no-unused-vars
          .filter(([key, value]) => value.source === DATA_SOURCE.NEST && value.connection === uuid && value?.injected !== true)
          .map(([key, value]) => ({
            object_key: key,
            object_revision: value.object_revision,
            object_timestamp: value.object_timestamp,
          })),
      );
    }

    fetchWrapper(
      'post',
      subscribeJSONData?.objects !== undefined
        ? new URL('/v5/subscribe', this.#connections[uuid].transport_url).href
        : new URL('/api/0.1/user/' + this.#connections[uuid].userID + '/app_launch', 'https://' + this.#connections[uuid].restAPIHost).href,
      {
        headers: {
          Referer: 'https://' + this.#connections[uuid].referer,
          Origin: 'https://' + this.#connections[uuid].referer,
          Authorization: 'Basic ' + this.#connections[uuid].token,
          Connection: 'keep-alive',
          'User-Agent': USER_AGENT,
          'Sec-Fetch-Mode': 'cors',
          'Sec-Fetch-Site': 'same-origin',
          'X-nl-protocol-version': 1,
          'Content-Type': 'application/json',
        },
        retry: 3,
      },
      subscribeJSONData,
    )
      .then((response) => response.json())
      .then(async (data) => {
        let changedData = new Map(); // Map of objectKey to { fields: Set(...) }
        let objects = [];

        if (Array.isArray(data?.updated_buckets) === true) {
          // Full data read response
          objects = data.updated_buckets;
        }

        if (Array.isArray(data?.objects) === true) {
          // Incremental subscribe update response
          objects = data.objects;
        }

        // Process the data we received
        fullRead = false; // Reset full refresh flag unless triggered below

        for (let object of objects) {
          let objectKey = object?.object_key;
          let incomingValue = typeof object?.value === 'object' && object.value !== null ? { ...object.value } : {};
          let existingEntry = this.#rawData?.[objectKey];
          let existingValue = typeof existingEntry?.value === 'object' && existingEntry.value !== null ? existingEntry.value : {};
          let changedFields = new Set();

          if ((objectKey?.trim?.() ?? '') === '') {
            continue;
          }

          // Detect changed top-level raw fields using shallow comparison.
          // This is intentional:
          // - nested objects/arrays are treated as changed if their reference differs
          // - avoids expensive deep comparison during frequent Nest API updates
          // - HomeKitDevice performs a deeper comparison later on the final merged device data
          //
          // NOTE: This may over-report changes for complex values, but that is acceptable here
          // because this change set is only used to guide downstream processing.
          Object.keys(incomingValue).forEach((field) => {
            if (existingValue[field] !== incomingValue[field]) {
              changedFields.add(field);
            }
          });

          if (objectKey.startsWith('structure.') === true) {
            // Add weather data based on the structure location details
            let weatherData = await this.#getLocationWeather(uuid, objectKey, incomingValue.postal_code, incomingValue.country_code);
            if (weatherData !== undefined) {
              incomingValue.weather = weatherData;
              changedFields.add('weather');
            }

            // Detect removed child objects from the swarm list and clean up local cache
            if (typeof existingValue?.swarm === 'object' && Array.isArray(incomingValue?.swarm) === true) {
              let newSwarmSet = new Set(incomingValue.swarm);

              existingValue.swarm.forEach((childObjectKey) => {
                if (newSwarmSet.has(childObjectKey) === false) {
                  delete this.#rawData[childObjectKey];
                }
              });
            }

            // Store the internal Nest structure uuid if matched to a configured home
            Object.assign(
              this.config?.homes?.find((home) => home?.name?.trim?.().toUpperCase() === incomingValue?.name?.trim?.().toUpperCase()) || {},
              { nest_home_uuid: objectKey },
            );
          }

          if (objectKey.startsWith('quartz.') === true) {
            // Retrieve additional camera/doorbell properties
            let properties = await this.#getCameraProperties(uuid, objectKey);

            incomingValue.properties =
              typeof properties === 'object' && properties.constructor === Object
                ? properties
                : typeof existingValue?.properties === 'object' && existingValue.properties.constructor === Object
                  ? existingValue.properties
                  : {};

            changedFields.add('properties');
          }

          if (objectKey.startsWith('buckets.') === true) {
            if (
              typeof existingEntry === 'object' &&
              Array.isArray(existingValue?.buckets) === true &&
              Array.isArray(incomingValue?.buckets) === true
            ) {
              // Compare previous vs incoming buckets list to detect topology changes
              let newBucketsSet = new Set(incomingValue.buckets);

              // If an existing object is missing from the new list, trigger a full refresh
              existingValue.buckets.forEach((childObjectKey) => {
                if (newBucketsSet.has(childObjectKey) === false) {
                  fullRead = true;
                }
              });

              // Detect removed objects and clean up local state
              existingValue.buckets.forEach((childObjectKey) => {
                if (newBucketsSet.has(childObjectKey) === false) {
                  // Object existed previously but is no longer referenced, so treat as removed
                  if (
                    childObjectKey.startsWith('structure.') === true ||
                    childObjectKey.startsWith('device.') === true ||
                    childObjectKey.startsWith('kryptonite.') === true ||
                    childObjectKey.startsWith('topaz.') === true ||
                    childObjectKey.startsWith('quartz.') === true
                  ) {
                    let serialNumber = this.#rawData?.[childObjectKey]?.value?.serial_number;
                    let trackedDevice = this.#trackedDevices.get(serialNumber);

                    if (trackedDevice !== undefined) {
                      // Send removed notice onto HomeKit device for it to process
                      HomeKitDevice.message(trackedDevice.uuid, HomeKitDevice.REMOVE, {});

                      // Finally, remove from tracked devices
                      this.#trackedDevices.delete(serialNumber);
                    }
                  }

                  delete this.#rawData[childObjectKey];
                }
              });
            }
          }
          // Only record this object if at least one field changed
          if (changedFields.size !== 0) {
            changedData.set(objectKey, { fields: changedFields });
          }

          // Merge incoming data into the raw data store
          this.#rawData[objectKey] = {
            object_revision: object.object_revision,
            object_timestamp: object.object_timestamp,
            connection: uuid,
            source: DATA_SOURCE.NEST,
            value: {
              ...existingValue,
              ...incomingValue,
            },
          };
        }

        await this.#processData(uuid, changedData);
      })
      .catch((error) => {
        // Attempt to extract HTTP status code from error cause or error object
        let statusCode =
          error?.code !== undefined && error?.code !== null
            ? error.code
            : error?.status !== undefined && error?.status !== null
              ? error.status
              : undefined;

        // If we get a 401 Unauthorized or 403 Forbidden and the connection was previously authorised,
        // mark it as unauthorised so the reconnect loop will handle it
        if ((statusCode === 401 || statusCode === 403) && this.#connections?.[uuid]?.authorised === true) {
          this?.log?.debug?.(
            'Connection "%s" is no longer authorised with the Nest API, will attempt to reconnect',
            this.#connections[uuid].name,
          );
          this.#connections[uuid].authorised = false;
          this.#connections[uuid].allowRetry = true;
          return;
        }

        // Log unexpected errors (excluding timeouts) for debugging
        if (
          error?.cause === undefined ||
          (error.cause?.message?.toUpperCase?.()?.includes('TIMEOUT') === false &&
            error.cause?.code?.toUpperCase?.()?.includes('TIMEOUT') === false)
        ) {
          this?.log?.debug?.(
            'Nest API had an error performing subscription with connection "%s". Error was "%s"',
            this.#connections[uuid].name,
            typeof error?.message === 'string' ? error.message : String(error),
          );
        }
      })
      .finally(() => {
        // Only continue the subscription loop if the connection is still authorised
        if (this.#connections?.[uuid]?.authorised === true) {
          clearTimeout(this.#connections[uuid].subscribeTimer);
          this.#connections[uuid].subscribeTimer = setTimeout(() => this.#subscribeNestAPI(uuid, false, fullRead), 1000);
        }
      });
  }

  async #observeGoogleAPI(uuid) {
    if (
      typeof this.#connections?.[uuid] !== 'object' ||
      this.#connections?.[uuid]?.authorised !== true ||
      this.#connections?.[uuid]?.grpcTransport === undefined ||
      this.config?.options?.useGoogleAPI !== true
    ) {
      // Not a valid connection object and/or we're not authorised
      return;
    }

    // Dynamically build the 'observe' post body data from cached protobuf message types
    let observeTraitsList = getProtoTypes(path.join(__dirname, 'protobuf/root.proto'), this.log)
      .filter((type) => {
        return (
          (this.#connections[uuid].type === ACCOUNT_TYPE.NEST &&
            type.fullName.startsWith('.nest.trait.product.camera') === false &&
            type.fullName.startsWith('.nest.trait.product.doorbell') === false &&
            (type.fullName.startsWith('.nest.trait') === true || type.fullName.startsWith('.weave.') === true)) ||
          (this.#connections[uuid].type === ACCOUNT_TYPE.GOOGLE &&
            (type.fullName.startsWith('.nest.trait') === true ||
              type.fullName.startsWith('.weave.') === true ||
              type.fullName.startsWith('.google.trait.product.camera') === true))
        );
      })
      .map((type) => ({
        traitType: type.fullName.replace(/^\.*|\.*$/g, ''),
      }));

    // Dedupe the observe traits list since there can be some overlap in the traits
    // due to the dynamic nature of the protobuf loading and trait type matching
    observeTraitsList = [...new Map(observeTraitsList.map((entry) => [entry.traitType, entry])).values()];

    // If protobuf support is unavailable or no observable traits were found,
    // do not start the observe loop. Retrying every second would only create
    // noise until the underlying protobuf load problem is fixed.
    if (observeTraitsList.length === 0) {
      this?.log?.warn?.(
        'Google API observe cannot start for connection "%s" because no observable protobuf traits were loaded',
        this.#connections[uuid].name,
      );
      return;
    }

    this.#connections[uuid].grpcTransport
      .observe(
        'nestlabs.gateway.v2.',
        'GatewayService',
        'Observe',
        { stateTypes: ['CONFIRMED', 'ACCEPTED'], traitTypeParams: observeTraitsList },
        async (message) => {
          let observeResponses = Array.isArray(message?.observeResponse) === true ? message.observeResponse : [message].filter(Boolean);
          let changedData = new Map(); // Map of resourceId to { fields: Set(...) } for processing after the loop

          // We'll use the resource status message to look for structure and/or device removals
          // We could also check for structure and/or device additions here, but we'll want to be flagged
          // that a device is 'ready' for use before we add in. This data is populated in the trait data
          for (let observeResponse of observeResponses) {
            // resourceMetas
            if (Array.isArray(observeResponse?.resourceMetas) === true) {
              for (let resource of observeResponse.resourceMetas) {
                if (
                  resource.status === 'REMOVED' &&
                  (resource.resourceId.startsWith('STRUCTURE_') === true || resource.resourceId.startsWith('DEVICE_') === true)
                ) {
                  // We have the removal of a 'home' and/or device
                  // Tidy up tracked devices since this one is removed
                  let serialNumber = this.#rawData?.[resource.resourceId]?.value?.device_identity?.serialNumber;
                  let trackedDevice = this.#trackedDevices.get(serialNumber);

                  if (trackedDevice !== undefined) {
                    // Send removed notice onto HomeKit device for it to process
                    HomeKitDevice.message(trackedDevice.uuid, HomeKitDevice.REMOVE, {});

                    // Finally, remove from tracked devices
                    this.#trackedDevices.delete(serialNumber);
                  }

                  delete this.#rawData[resource.resourceId];
                }
              }
            }

            // traitStates
            if (Array.isArray(observeResponse?.traitStates) === true) {
              // Tidy up our received trait states. This ensures we only have one status for the trait in the data we process
              // We'll favour a trait with accepted status over the same with confirmed status
              let traits = observeResponse.traitStates;
              let acceptedKeys = new Set(
                traits
                  .filter((trait) => trait.stateTypes.includes('ACCEPTED') === true)
                  .map((trait) => trait.traitId.resourceId + '/' + trait.traitId.traitLabel),
              );
              observeResponse.traitStates = [
                ...traits.filter((trait) => acceptedKeys.has(trait.traitId.resourceId + '/' + trait.traitId.traitLabel) === false),
                ...traits.filter((trait) => trait.stateTypes.includes('ACCEPTED') === true),
              ];

              for (let trait of observeResponse.traitStates) {
                let resourceId = trait.traitId.resourceId;
                let traitLabel = trait.traitId.traitLabel;
                let patchValues = trait?.patch?.values ?? {};
                let changedEntry = changedData.get(resourceId);

                // Mapped changed data for processing after the loop
                if (changedEntry === undefined) {
                  changedEntry = { fields: new Set() };
                  changedData.set(resourceId, changedEntry);
                }
                changedEntry.fields.add(traitLabel);

                // Create or update trait entry and assign latest patch values
                this.#rawData[resourceId] = {
                  connection: uuid,
                  source: DATA_SOURCE.GOOGLE,
                  value: {
                    ...this.#rawData?.[resourceId]?.value,
                    [traitLabel]: patchValues,
                  },
                };

                // Remove trait type metadata — we don't need to store it
                delete this.#rawData[resourceId]?.value?.[traitLabel]?.['@type'];

                // If we have structure location details and associated geo-location details, get the weather data for the location
                // We'll store this in the object key/value as per Nest API
                if (
                  resourceId.startsWith('STRUCTURE_') === true &&
                  traitLabel === 'structure_location' &&
                  (patchValues?.postalCode?.value?.trim?.() ?? '') !== '' &&
                  (patchValues?.countryCode?.value?.trim?.() ?? '') !== ''
                ) {
                  let weatherData = await this.#getLocationWeather(
                    uuid,
                    resourceId,
                    patchValues.postalCode.value,
                    patchValues.countryCode.value,
                  );

                  if (weatherData !== undefined && typeof this.#rawData?.[resourceId]?.value === 'object') {
                    this.#rawData[resourceId].value.weather = { ...weatherData };
                    changedEntry.fields.add('weather');
                  }
                }

                // Store the internal Nest and Google structure uuids if matched to a defined home array entry
                if (
                  resourceId.startsWith('STRUCTURE_') === true &&
                  traitLabel === 'structure_info' &&
                  (patchValues?.name?.trim?.() ?? '') !== ''
                ) {
                  Object.assign(
                    this.config?.homes?.find((home) => home?.name?.trim?.().toUpperCase() === patchValues.name?.trim?.().toUpperCase()) ||
                      {},
                    { nest_home_uuid: patchValues.rtsStructureId, google_home_uuid: resourceId },
                  );
                }

                // We have an update for a camera live image trait
                // so we'll trigger any waiting snapshot requests to process this new image data
                if (traitLabel === 'upload_live_image' && this.#connections?.[uuid]?.snapshotWaiters instanceof Map) {
                  let waiter = this.#connections[uuid].snapshotWaiters.get(resourceId);

                  this.#connections[uuid].snapshotWaiters.delete(resourceId);

                  if (typeof waiter === 'function') {
                    waiter();
                  }
                }
              }
            }
          }

          await this.#processData(uuid, changedData);
        },
      )
      .catch((error) => {
        this?.log?.debug?.(
          'Google API observe failed for connection "%s": %s',
          this.#connections?.[uuid]?.name,
          typeof error?.message === 'string' ? error.message : String(error),
        );
      })
      .finally(() => {
        // Only continue the observe loop if the connection is still authorised
        if (this.#connections?.[uuid]?.authorised === true) {
          clearTimeout(this.#connections[uuid].observeTimer);
          this.#connections[uuid].observeTimer = setTimeout(() => this.#observeGoogleAPI(uuid), 1000);
        }
      });
  }

  async #processData(uuid, changedData = undefined) {
    const dumpSupportData = (source, changedData = undefined) => {
      let sourceInfo =
        source === DATA_SOURCE.GOOGLE ? { name: 'Google API' } : source === DATA_SOURCE.NEST ? { name: 'Nest API' } : undefined;
      let didLogAny = false;
      let isDelta = changedData instanceof Map === true && changedData.size !== 0;

      // Validate we should attempt a support dump for this connection/source
      if (
        this?.config?.options?.supportDump !== true ||
        typeof uuid !== 'string' ||
        uuid.trim() === '' ||
        typeof this.#connections?.[uuid] !== 'object' ||
        typeof sourceInfo !== 'object'
      ) {
        return;
      }

      // Iterate raw data directly and decide at object/field level what to output
      Object.entries(this.#rawData).forEach(([objectKey, data]) => {
        let changedFields = isDelta === true ? changedData.get(objectKey)?.fields : undefined;
        let loggedObject = false;

        // Only process objects for this source/connection with valid value payload
        if (
          data?.source !== source ||
          data?.connection !== uuid ||
          typeof data?.value !== 'object' ||
          data.value === null ||
          Object.keys(data.value).length === 0
        ) {
          return;
        }

        // In delta mode, skip objects that were not part of this update
        if (isDelta === true && changedFields instanceof Set !== true) {
          return;
        }

        Object.entries(data.value).forEach(([key, value]) => {
          // In delta mode, only output fields that were part of this update
          if (isDelta === true && changedFields.has(key) !== true) {
            return;
          }

          // Lazily open object and print header only when we actually output something
          if (loggedObject === false) {
            if (didLogAny === false) {
              this?.log?.info?.(
                '%s support dump for %s data will be logged below for troubleshooting purposes.',
                isDelta === true ? 'Changed' : 'Full',
                sourceInfo.name,
              );
            }

            this?.log?.info?.('{');
            this?.log?.info?.('  "%s": {', objectKey);

            loggedObject = true;
            didLogAny = true;
          }

          // Pretty-print nested objects
          if (typeof value === 'object' && value !== null) {
            this?.log?.info?.('  %s:', key);

            String(JSON.stringify(value, null, 2))
              .split('\n')
              .forEach((line) => {
                this?.log?.info?.('    %s', line);
              });

            return;
          }

          // Primitive values
          this?.log?.info?.('  %s: %j', key, value);
        });

        // Close object if we logged any fields
        if (loggedObject === true) {
          this?.log?.info?.('  }');
          this?.log?.info?.('}');
        }
      });

      // Footer only if something was actually logged
      if (didLogAny === true) {
        this?.log?.info?.('End of support dump for %s data.', sourceInfo.name);
      }
    };

    // First run logs a full baseline per source/connection.
    // Later runs log only the changed fields for changed objects in this cycle.
    dumpSupportData(DATA_SOURCE.NEST, changedData);
    dumpSupportData(DATA_SOURCE.GOOGLE, changedData);

    // Process the raw data through each of the device modules to get the latest device details and states
    for (let [deviceType, deviceModule] of this.#deviceModules) {
      if (typeof deviceModule?.processRawData === 'function') {
        let devices = {};

        try {
          devices = deviceModule.processRawData(this.log, this.#rawData, this.config, deviceType, changedData);
        } catch (error) {
          this?.log?.warn?.('%s module failed to process data. Error was "%s"', deviceType, String(error));
        }

        if (typeof devices === 'object' && devices !== null) {
          for (let [serialNumber, result] of Object.entries(devices)) {
            let deviceData = result?.data;
            let isFull = result?.full === true;
            let trackedDevice = this.#trackedDevices.get(serialNumber);

            if (deviceData === null || typeof deviceData !== 'object' || deviceData?.constructor !== Object) {
              continue;
            }

            if (trackedDevice === undefined && isFull === true && deviceData?.excluded === true) {
              // We haven't tracked this device before (ie: should be a new one) and but its excluded
              let homeName =
                this.#rawData?.[deviceData.nest_google_home_uuid]?.value?.name ||
                this.#rawData?.[deviceData.nest_google_home_uuid]?.value?.structure_info?.name;

              if (deviceType !== DEVICE_TYPE.WEATHER) {
                this?.log?.warn?.(
                  'Device "%s"%s is ignored due to it being marked as excluded',
                  deviceData.description,
                  (homeName?.trim?.() ?? '') !== '' ? ' in "' + homeName + '"' : '',
                );
              }

              // Track this device even though its excluded
              this.#trackedDevices.set(serialNumber, {
                uuid: HomeKitDevice.generateUUID(HomeKitDevice.PLUGIN_NAME, this.api, serialNumber),
                nest_google_device_uuid: deviceData.nest_google_device_uuid,
                type: deviceModule.class.TYPE, // Store type of device
                source: undefined, // gets filled out later
                timers: undefined,
                exclude: true,
              });

              trackedDevice = this.#trackedDevices.get(serialNumber);

              // If the device is now marked as excluded and present in accessory cache
              // Then we'll unregister it from the Homebridge platform
              let accessory = this.cachedAccessories.find((accessory) => accessory?.UUID === trackedDevice.uuid);

              if (accessory !== undefined && typeof accessory === 'object') {
                try {
                  this.api.unregisterPlatformAccessories(HomeKitDevice.PLUGIN_NAME, HomeKitDevice.PLATFORM_NAME, [accessory]);
                  // eslint-disable-next-line no-unused-vars
                } catch (error) {
                  // Empty
                }
              }
            }

            if (trackedDevice === undefined && isFull === true && deviceData?.excluded === false) {
              // We haven't tracked this device before (ie: should be a new one) and its not excluded
              // so create the required HomeKit accessories based upon the device data
              if (
                typeof deviceModule?.class === 'function' &&
                (deviceModule.class.TYPE?.trim?.() ?? '') !== '' &&
                (deviceModule.class.VERSION?.trim?.() ?? '') !== ''
              ) {
                // We have found a device class for this device type, so we can create the device
                let accessoryName =
                  (deviceData.manufacturer?.trim() || 'Nest') +
                  ' ' +
                  deviceModule.class.TYPE.replace(/([a-z])([A-Z])/g, '$1 $2')
                    .replace(/[^a-zA-Z0-9 ]+/g, ' ')
                    .toLowerCase()
                    .replace(/\b\w/g, (character) => character.toUpperCase());

                let tempDevice = new deviceModule.class(this.cachedAccessories, this.api, this.log, deviceData);
                tempDevice.add(accessoryName, getDeviceHKCategory(deviceModule.class.TYPE), deviceData?.eveHistory === true);

                // Register per-device set/get handlers
                HomeKitDevice.message(tempDevice.uuid, HomeKitDevice.SET, async (values) => {
                  await this.#set(this.#rawData?.[values?.uuid]?.connection, values?.uuid, values);
                });

                HomeKitDevice.message(tempDevice.uuid, HomeKitDevice.GET, async (values) => {
                  return await this.#get(this.#rawData?.[values?.uuid]?.connection, values?.uuid, values);
                });

                // Track this device once created
                this.#trackedDevices.set(serialNumber, {
                  uuid: tempDevice.uuid,
                  nest_google_device_uuid: deviceData.nest_google_device_uuid,
                  type: deviceModule.class.TYPE, // Store type of