@dudousxd/nestjs-telescope
Version:
Laravel Telescope-style observability console for NestJS — core: watchers, recorder, correlation, SQLite store, headless API.
536 lines • 25.9 kB
JavaScript
// packages/core/src/alerts/telescope-alerter.ts
import { Logger } from '@nestjs/common';
import { durationToMs } from '../config/parse-duration.js';
import { EntryType } from '../entry/entry.js';
import { collectEntriesInWindow } from '../metrics/collect-window.js';
import { percentile } from '../metrics/stats.js';
import { NewExceptionTracker } from './new-exception-tracker.js';
/**
* Cap on entries scanned per windowed rule read. Alerting is a coarse "did we
* cross N" check, not analytics — a small cap keeps each tick cheap even on a
* busy store. Counts at/above the cap still cross any sane threshold, so the cap
* never hides a firing condition (it only under-counts far above threshold).
*/
const ALERT_SCAN_CAP = 10_000;
/** Stack frames carried in a `new-exception` alert (the Slack channel re-clips). */
const ALERT_STACK_FRAME_LIMIT = 12;
/** Window used only to count occurrences on an `every-exception` alert when the
* rule omits its own `window`. Firing is per-flush, so this is display-only. */
const DEFAULT_EVERY_WINDOW = '1h';
/**
* Pluggable-channel alerting (v2). Two evaluation paths:
* 1. Interval rules (`exception-rate` / `slow-request-rate` / `dropped-entries`)
* run on an unref'd timer over windowed counts.
* 2. The `new-exception` rule runs per-flush via {@link evaluateFlush} over the
* just-stored exception entries, so a brand-new error family pages within a
* flush interval rather than waiting a full evaluation interval.
*
* Every fired alert fans out to ALL configured channels concurrently; one channel
* failing never blocks the others. NEVER throws into the host: a channel failure
* is warn-logged (rate-limited per channel) and otherwise swallowed.
*/
export class TelescopeAlerter {
deps;
logger;
now;
timer = null;
/** Per-rule last-fired wall time (index-keyed; rules are a fixed array). */
lastFiredAt = new Map();
/**
* Per-family last-fired wall time for the flush-driven exception rules. Keyed
* by `${ruleType}|${familyHash}` so a `new-exception` and an `every-exception`
* rule configured together keep independent cooldown clocks.
*/
lastFiredFamily = new Map();
/** Channels we've already warned about (rate-limit failure logs by name). */
warnedChannels = new Set();
/** Previous cumulative droppedCount, for the `dropped-entries` delta. */
lastDroppedCount;
/** Bounded per-replica seen-family map backing the `new-exception` rule. */
newExceptionTracker;
/** Pre-computed `new-exception` rule (if any) so the flush path is cheap. */
newExceptionRule;
/** Pre-computed `every-exception` rule (if any) so the flush path is cheap. */
everyExceptionRule;
constructor(deps) {
this.deps = deps;
this.logger = deps.logger ?? new Logger(TelescopeAlerter.name);
this.now = deps.now ?? Date.now;
this.lastDroppedCount = deps.droppedCount();
this.newExceptionTracker = new NewExceptionTracker(deps.maxFamilies);
this.newExceptionRule =
deps.alerts.rules.find((rule) => rule.type === 'new-exception') ?? null;
this.everyExceptionRule =
deps.alerts.rules.find((rule) => rule.type === 'every-exception') ?? null;
}
/** Start the unref'd evaluation interval. Idempotent. */
start() {
if (this.timer !== null)
return;
this.timer = setInterval(() => {
this.evaluate().catch((error) => {
this.logger.warn(`Telescope alert evaluation failed: ${error.message}`);
});
}, this.deps.alerts.intervalMs);
this.timer.unref?.();
}
/** Stop the interval (shutdown). Idempotent. */
stop() {
if (this.timer !== null) {
clearInterval(this.timer);
this.timer = null;
}
}
/**
* Evaluate every INTERVAL rule once (the `new-exception` rule is handled by
* {@link evaluateFlush}, not here). Each firing rule fans out independently.
* Exposed for the timer and for deterministic tests.
*/
async evaluate() {
for (let index = 0; index < this.deps.alerts.rules.length; index++) {
const rule = this.deps.alerts.rules[index];
// `new-exception` and `every-exception` are flush-driven (see evaluateFlush),
// never evaluated on the interval timer.
if (rule === undefined || rule.type === 'new-exception' || rule.type === 'every-exception') {
continue;
}
const outcome = await this.measure(rule);
if (outcome === null)
continue;
if (this.inCooldown(index))
continue;
this.lastFiredAt.set(index, this.now());
await this.dispatch(this.buildRatePayload(rule, outcome.value, outcome.threshold));
}
}
/**
* Per-flush evaluation for the exception rules (`new-exception` /
* `every-exception`). Called with the entries a flush just stored. MUST be
* cheap: it filters to exception-type entries and does a single bounded-map
* lookup per family; the expensive batch-context fetch happens ONLY for a
* family that actually fires (and is past cooldown). No-op (zero cost beyond the
* early return) when neither rule is configured. Never throws into the host —
* failures are swallowed/logged.
*/
async evaluateFlush(storedEntries) {
const newRule = this.newExceptionRule;
const everyRule = this.everyExceptionRule;
if (newRule === null && everyRule === null)
return;
const nowMs = this.now();
try {
for (const entry of storedEntries) {
// Server exceptions AND browser-reported client_exceptions both feed the
// exception rules — a front-end error should page just like a server one.
if (entry.type !== EntryType.Exception && entry.type !== EntryType.ClientException) {
continue;
}
if (entry.familyHash === null)
continue;
// `new-exception`: fire only the FIRST time a family is seen in the window.
if (newRule !== null) {
const windowMs = durationToMs(newRule.window);
const isNew = await this.observeFamily(entry.familyHash, nowMs, windowMs);
if (isNew && !this.familyInCooldown('new-exception', entry.familyHash, nowMs)) {
this.lastFiredFamily.set(this.cooldownKey('new-exception', entry.familyHash), nowMs);
await this.dispatch(await this.buildExceptionPayload(newRule, entry, windowMs, nowMs));
}
}
// `every-exception`: fire for EVERY exception, rate-limited per family by
// the shared cooldown (independent clock from new-exception above).
if (everyRule !== null &&
!this.familyInCooldown('every-exception', entry.familyHash, nowMs)) {
const windowMs = durationToMs(everyRule.window ?? DEFAULT_EVERY_WINDOW);
this.lastFiredFamily.set(this.cooldownKey('every-exception', entry.familyHash), nowMs);
await this.dispatch(await this.buildExceptionPayload(everyRule, entry, windowMs, nowMs));
}
}
}
catch (error) {
// A bug in evaluation must never break the host's flush path.
this.logger.warn(`Telescope exception-alert evaluation failed: ${error.message}`);
}
}
/**
* "Is this a NEW error family for the window?" — using the SHARED store dedup
* when the provider implements `markFamilySeen` (so a family pages once across a
* multi-replica deployment), and the in-memory per-replica tracker otherwise.
* A shared-store failure falls back to the local tracker rather than dropping
* the alert, so a transient store hiccup degrades to once-per-pod, never to
* silent.
*/
async observeFamily(familyHash, nowMs, windowMs) {
const markFamilySeen = this.deps.storage.markFamilySeen;
if (markFamilySeen !== undefined) {
try {
return await markFamilySeen.call(this.deps.storage, familyHash, nowMs, windowMs);
}
catch (error) {
this.logger.warn(`Telescope shared new-exception dedup failed, falling back to per-replica: ${error.message}`);
}
}
return this.newExceptionTracker.observe(familyHash, nowMs, windowMs);
}
/** Returns `{ value, threshold }` when an interval rule is firing, else `null`. */
async measure(rule) {
if (rule.type === 'exception-rate') {
const value = await this.countInWindow(rule.window, EntryType.Exception);
return value >= rule.threshold ? { value, threshold: rule.threshold } : null;
}
if (rule.type === 'slow-request-rate') {
const value = await this.countSlowRequests(rule.window, rule.thresholdMs);
return value >= rule.count ? { value, threshold: rule.count } : null;
}
if (rule.type === 'metric-threshold') {
return this.measureMetric(rule);
}
// dropped-entries: delta since the previous evaluation.
const current = this.deps.droppedCount();
const delta = current - this.lastDroppedCount;
this.lastDroppedCount = current;
return delta >= rule.threshold ? { value: delta, threshold: rule.threshold } : null;
}
/** Count entries of `type` recorded in the trailing `window`. */
async countInWindow(window, type) {
const result = await this.readWindow(window, type);
return result.length;
}
/** Count request entries in the window whose `durationMs >= thresholdMs`. */
async countSlowRequests(window, thresholdMs) {
const entries = await this.readWindow(window, EntryType.Request);
let slow = 0;
for (const entry of entries) {
if (typeof entry.durationMs === 'number' && entry.durationMs >= thresholdMs) {
slow += 1;
}
}
return slow;
}
/**
* Compute a `metric-threshold` rule's metric over its window and return
* `{ value, threshold }` when it crosses in the comparator direction, else
* `null`. Latency metrics read `durationMs` only (content-less); the cache
* hit-rate needs the cache content, so it reads WITH content for that metric.
*/
async measureMetric(rule) {
const minSamples = rule.minSamples ?? 1;
const computed = await this.computeMetric(rule.metric, rule.window);
if (computed === null || computed.samples < minSamples)
return null;
const crosses = rule.comparator === 'gte'
? computed.value >= rule.threshold
: computed.value <= rule.threshold;
return crosses ? { value: computed.value, threshold: rule.threshold } : null;
}
/** Aggregate a metric over the window, or `null` when there are no samples. */
async computeMetric(metric, window) {
if (metric === 'cache-hit-rate') {
const after = new Date(this.now() - durationToMs(window));
const result = await collectEntriesInWindow(this.deps.storage, { type: EntryType.Cache, after }, { scanCap: ALERT_SCAN_CAP });
let hits = 0;
let misses = 0;
for (const entry of result.entries) {
const record = typeof entry.content === 'object' && entry.content !== null
? entry.content
: null;
if (record === null || record.operation !== 'get')
continue;
if (record.hit === true)
hits += 1;
else if (record.hit === false)
misses += 1;
}
const total = hits + misses;
if (total === 0)
return null;
return { value: hits / total, samples: total };
}
// Latency percentile metrics: read durations for the metric's type.
const type = metric.startsWith('request') ? EntryType.Request : EntryType.Query;
const entries = await this.readWindow(window, type);
const durations = [];
for (const entry of entries) {
if (typeof entry.durationMs === 'number')
durations.push(entry.durationMs);
}
if (durations.length === 0)
return null;
durations.sort((a, b) => a - b);
const q = metric.endsWith('p99-ms') ? 0.99 : 0.95;
return { value: percentile(durations, q), samples: durations.length };
}
/**
* Small windowed read reusing the analytics scan: `omitContent` (alerting only
* needs counts + `durationMs`, never payloads) and a tight `scanCap`.
*/
async readWindow(window, type) {
const after = new Date(this.now() - durationToMs(window));
const result = await collectEntriesInWindow(this.deps.storage, { type, after, omitContent: true }, { scanCap: ALERT_SCAN_CAP });
return result.entries;
}
inCooldown(index) {
const last = this.lastFiredAt.get(index);
if (last === undefined)
return false;
return this.now() - last < this.deps.alerts.cooldownMs;
}
/** Composite cooldown key so each flush rule keeps an independent per-family clock. */
cooldownKey(ruleType, familyHash) {
return `${ruleType}|${familyHash}`;
}
familyInCooldown(ruleType, familyHash, nowMs) {
const last = this.lastFiredFamily.get(this.cooldownKey(ruleType, familyHash));
if (last === undefined)
return false;
return nowMs - last < this.deps.alerts.cooldownMs;
}
/** Build the v1-compatible payload for an interval (rate) rule. */
buildRatePayload(rule, value, threshold) {
return {
rule,
value,
threshold,
firedAt: new Date(this.now()).toISOString(),
instanceId: this.deps.instanceId,
...(this.deps.alerts.dashboardUrl !== null
? { dashboardUrl: this.deps.alerts.dashboardUrl }
: {}),
};
}
/**
* Build the rich exception payload shared by `new-exception` and
* `every-exception`. Pulls the exception's own fields, then fetches its batch to
* find the sibling REQUEST entry for route/method/status/duration/user context,
* counts how many times this family appears in the trailing window, and (when a
* `geoLookup` hook is configured) resolves the client IP to a coarse location.
* This is the ONLY expensive path and runs only on a real fire.
*/
async buildExceptionPayload(rule, entry, windowMs, nowMs) {
const occurrences = await this.countFamilyInWindow(entry.type, entry.familyHash, windowMs);
const context = entry.type === EntryType.ClientException
? this.buildClientContext(entry, occurrences)
: await this.buildServerContext(entry, occurrences);
// Optional geo enrichment: resolve the client IP to a coarse location via the
// host hook. Only on a real fire, and only when an IP is present.
context.geo = await this.resolveGeo(context.clientIp);
// Auto-mode AI enrichment: briefly await a diagnosis for this family. The
// hook caps its own wait, so this never holds the alert beyond the grace.
const diagnosis = this.deps.diagnosisFor !== undefined && entry.familyHash !== null
? await this.deps.diagnosisFor(entry.familyHash).catch(() => null)
: null;
return {
rule,
value: occurrences,
threshold: 1,
firedAt: new Date(nowMs).toISOString(),
instanceId: this.deps.instanceId,
exception: context,
...(this.deps.alerts.dashboardUrl !== null
? { dashboardUrl: this.deps.alerts.dashboardUrl }
: {}),
...(diagnosis !== null ? { diagnosis } : {}),
};
}
/**
* Resolve a client IP to a coarse {@link AlertGeoLocation} via the host's
* `geoLookup` hook. Returns `null` when no hook is configured, no IP is present,
* the hook returns `null`, or the hook throws (swallowed — geo is purely
* additive and must never break or block an alert beyond the hook's own cost).
*/
async resolveGeo(clientIp) {
const hook = this.deps.alerts.geoLookup;
if (typeof hook !== 'function' || clientIp === null)
return null;
try {
return (await hook(clientIp)) ?? null;
}
catch (error) {
this.logger.warn(`Telescope alert geoLookup failed: ${error.message}`);
return null;
}
}
/**
* Build the server-exception context: the exception's own fields plus its
* sibling REQUEST entry (route/method/status/duration/user) from the same batch.
*/
async buildServerContext(entry, occurrences) {
const exceptionContent = asPartialExceptionContent(entry.content);
const request = await this.findSiblingRequest(entry.batchId);
const requestContent = request === null ? null : asPartialRequestContent(request.content);
const headers = requestContent?.headers ?? null;
return {
familyHash: entry.familyHash ?? '',
class: typeof exceptionContent.class === 'string' ? exceptionContent.class : 'Error',
message: typeof exceptionContent.message === 'string' ? exceptionContent.message : '',
stack: clipStackFrames(typeof exceptionContent.stack === 'string' ? exceptionContent.stack : null),
route: requestContent !== null && typeof requestContent.uri === 'string'
? requestContent.uri
: null,
method: requestContent !== null && typeof requestContent.method === 'string'
? requestContent.method
: null,
userAgent: headerValue(headers, 'user-agent'),
referer: headerValue(headers, 'referer') ?? headerValue(headers, 'referrer'),
componentStack: null,
extra: null,
client: false,
clientIp: requestContent !== null && typeof requestContent.ip === 'string' ? requestContent.ip : null,
geo: null,
statusCode: requestContent !== null && typeof requestContent.statusCode === 'number'
? requestContent.statusCode
: null,
durationMs: request?.durationMs ?? null,
user: userFromTags(request?.tags ?? entry.tags),
occurrences,
isNew: occurrences === 1,
entryId: entry.id,
batchId: entry.batchId,
};
}
/**
* Build the client-exception context from the browser-reported content. There's
* no sibling request (the error came straight from the browser), so `route` is
* the page `url`, `method`/`statusCode`/`durationMs` are null, and `userAgent`
* carries the reporting browser. The user comes from the entry's own
* `user:<id>` tag (the controller tagged it at record time).
*/
buildClientContext(entry, occurrences) {
const content = asPartialClientExceptionContent(entry.content);
return {
familyHash: entry.familyHash ?? '',
class: typeof content.name === 'string' ? content.name : 'Error',
message: typeof content.message === 'string' ? content.message : '',
stack: clipStackFrames(typeof content.stack === 'string' ? content.stack : null),
route: typeof content.url === 'string' ? content.url : null,
method: null,
userAgent: typeof content.userAgent === 'string' ? content.userAgent : null,
referer: null,
componentStack: typeof content.componentStack === 'string' ? content.componentStack : null,
extra: typeof content.extra === 'object' && content.extra !== null ? content.extra : null,
client: true,
clientIp: typeof content.clientIp === 'string' ? content.clientIp : null,
geo: null,
statusCode: null,
durationMs: null,
user: userFromTags(entry.tags),
occurrences,
isNew: occurrences === 1,
entryId: entry.id,
batchId: entry.batchId,
};
}
/** Find the sibling REQUEST entry in the exception's batch (or `null`). */
async findSiblingRequest(batchId) {
const batch = await this.deps.storage.batch(batchId);
return batch.find((member) => member.type === EntryType.Request) ?? null;
}
/** Count entries of this family AND type in the trailing window (>= 1). */
async countFamilyInWindow(type, familyHash, windowMs) {
if (familyHash === null)
return 1;
const after = new Date(this.now() - windowMs);
const result = await collectEntriesInWindow(this.deps.storage, { type, familyHash, after, omitContent: true }, { scanCap: ALERT_SCAN_CAP });
// The just-stored occurrence is included; never report fewer than 1.
return Math.max(1, result.entries.length);
}
/**
* Fan the payload out to every channel concurrently. Each channel's failure is
* isolated (`Promise.allSettled`) and warn-logged ONCE per channel name so a
* persistently-down destination doesn't flood the logs. Never rejects.
*/
async dispatch(payload) {
const results = await Promise.allSettled(this.deps.alerts.channels.map((channel) => channel.send(payload)));
results.forEach((result, index) => {
if (result.status === 'rejected') {
this.warnChannelFailure(this.deps.alerts.channels[index], result.reason);
}
});
}
/** Warn once per channel name; subsequent failures for that channel are silent. */
warnChannelFailure(channel, reason) {
if (channel === undefined)
return;
if (this.warnedChannels.has(channel.name))
return;
this.warnedChannels.add(channel.name);
const message = reason instanceof Error ? reason.message : String(reason);
this.logger.warn(`Telescope alert channel '${channel.name}' failed: ${message}`);
}
}
/** A non-null object narrowed to a string-keyed record (else an empty record),
* so content fields can be read without a cast. */
function asContentRecord(content) {
return typeof content === 'object' && content !== null ? { ...content } : {};
}
function asPartialExceptionContent(content) {
const record = asContentRecord(content);
return {
...(typeof record.class === 'string' ? { class: record.class } : {}),
...(typeof record.message === 'string' ? { message: record.message } : {}),
...(typeof record.stack === 'string' ? { stack: record.stack } : {}),
};
}
function asPartialRequestContent(content) {
const record = asContentRecord(content);
return {
...(typeof record.uri === 'string' ? { uri: record.uri } : {}),
...(typeof record.method === 'string' ? { method: record.method } : {}),
...(typeof record.statusCode === 'number' ? { statusCode: record.statusCode } : {}),
...(typeof record.ip === 'string' ? { ip: record.ip } : {}),
...(typeof record.headers === 'object' && record.headers !== null
? { headers: record.headers }
: {}),
};
}
/**
* Read a request header case-insensitively, returning a trimmed string or `null`.
* Express lowercases header names, but we normalize anyway so a differently-cased
* captured header (e.g. from another adapter) still resolves. A header captured as
* an array (rare, e.g. `set-cookie`) takes its first element.
*/
function headerValue(headers, name) {
if (headers === null)
return null;
const target = name.toLowerCase();
for (const key of Object.keys(headers)) {
if (key.toLowerCase() !== target)
continue;
const raw = headers[key];
const value = Array.isArray(raw) ? raw[0] : raw;
if (typeof value === 'string' && value.trim() !== '')
return value;
return null;
}
return null;
}
function asPartialClientExceptionContent(content) {
const record = asContentRecord(content);
return {
...(typeof record.name === 'string' ? { name: record.name } : {}),
...(typeof record.message === 'string' ? { message: record.message } : {}),
...(typeof record.stack === 'string' ? { stack: record.stack } : {}),
...(typeof record.url === 'string' ? { url: record.url } : {}),
...(typeof record.userAgent === 'string' ? { userAgent: record.userAgent } : {}),
...(typeof record.clientIp === 'string' ? { clientIp: record.clientIp } : {}),
...(typeof record.componentStack === 'string' ? { componentStack: record.componentStack } : {}),
...(typeof record.extra === 'object' && record.extra !== null
? { extra: record.extra }
: {}),
};
}
/** Keep at most the first N stack frames; `null` passes through. */
function clipStackFrames(stack) {
if (stack === null)
return null;
return stack.split('\n').slice(0, ALERT_STACK_FRAME_LIMIT).join('\n');
}
/** Extract the user id from a `user:<id>` tag, or `null` when none is present. */
function userFromTags(tags) {
for (const tag of tags) {
if (tag.startsWith('user:')) {
return tag.slice('user:'.length);
}
}
return null;
}
//# sourceMappingURL=telescope-alerter.js.map