@dudousxd/nestjs-telescope
Version:
Laravel Telescope-style observability console for NestJS — core: watchers, recorder, correlation, SQLite store, headless API.
553 lines • 24.6 kB
JavaScript
import { passesSampling } from '../config/sampling.js';
import { compileRedactSpec, redactBoundedWith } from '../redaction/redact.js';
import { aggregateDeltas } from '../rollup/aggregate-deltas.js';
import { isRollupStore } from '../rollup/rollup-store.js';
import { runTaggers } from '../tagging/tagger.js';
/**
* Default backoff seam: an unref'd `setTimeout` so a pending retry never keeps
* the host's event loop alive (e.g. during shutdown). Injectable for tests.
*/
function defaultUnrefDelay(ms) {
return new Promise((resolve) => {
const timer = setTimeout(resolve, ms);
timer.unref?.();
});
}
/**
* Buffers {@link Entry} objects in a fixed-capacity O(1) ring buffer and
* periodically flushes them to a {@link StorageProvider}.
*
* **Overflow policy** — overflow drops the OLDEST buffered entry (so under
* sustained overload a batch may be stored without its earliest entries);
* recent activity is preferred.
*
* **Storage failures** — when `store()` rejects, the drained batch is retried
* exactly ONCE after a bounded backoff (`retryDelayMs`, default 1000ms). Only a
* second failure drops the batch (fail-open, never grow). The retry runs inside
* the single in-flight `flushing` promise, so failed batches never pile up and
* the ring keeps absorbing/evicting meanwhile — memory stays bounded. Drops are
* surfaced via `onDrop` and the `storeFailedDropped` / `droppedCount` counters;
* each retried batch increments the `retriedFlushes` self-metric.
*/
export class Recorder {
options;
// ── Ring-buffer state ──────────────────────────────────────────────────────
ring;
/** Index of the oldest entry in the ring. */
head = 0;
/** Number of valid entries currently held. */
count = 0;
// ── Drop counters ──────────────────────────────────────────────────────────
overflowDrops = 0;
storeFailedDrops = 0;
recordErrorDrops = 0;
// ── Self-metrics counters (cheap, hot-path-safe) ───────────────────────────
recordedCount = 0;
highWaterCount = 0;
flushCount = 0;
flushedEntriesCount = 0;
/** Flush batches that failed their first store() and were retried once. */
retriedFlushCount = 0;
lastFlushDurationMs = null;
maxFlushDurationMs = null;
totalFlushDurationMs = 0;
/** Entries whose content was clipped by a redaction bound (incident guard). */
truncatedEntryCount = 0;
// ── Concurrency guard ─────────────────────────────────────────────────────
flushing = null;
// ── Overload protection ────────────────────────────────────────────────────
/**
* When paused, `record()` becomes a no-op (the entry is dropped, counted as an
* overflow drop) so a telescope under load can never amplify an incident. Set
* by the overhead guard when event-loop lag crosses its threshold; cleared
* when lag recovers. Flushing continues so the buffer still drains.
*/
paused = false;
// ── Determinism seams ─────────────────────────────────────────────────────
now;
random;
delay;
retryDelayMs;
/**
* Redaction key/path Sets compiled ONCE at boot from `options.redact`. Config
* is immutable after construction, so rebuilding these per entry (in the
* hottest function) was pure waste — they are precompiled here and reused on
* every `enrich()` via {@link redactBoundedWith}.
*/
redactSpec;
constructor(options) {
this.options = options;
this.now = options.now ?? Date.now;
this.random = options.random ?? Math.random;
this.retryDelayMs = options.retryDelayMs ?? 1_000;
this.delay = options.delay ?? defaultUnrefDelay;
this.redactSpec = compileRedactSpec(options.redact);
// Pre-size the ring so slot access is always O(1).
this.ring = new Array(options.bufferSize).fill(undefined);
}
// ── Public getters ─────────────────────────────────────────────────────────
get overflowDropped() {
return this.overflowDrops;
}
get storeFailedDropped() {
return this.storeFailedDrops;
}
/** Sum of all drop buckets (overflow + store-failed + record-error). */
get droppedCount() {
return this.overflowDrops + this.storeFailedDrops + this.recordErrorDrops;
}
/**
* Number of ring slots still holding an entry reference. After a flush this
* MUST equal `bufferUsed` (only the live, not-yet-drained entries), never the
* stale entries a previous flush left behind.
*
* @internal Test-only seam to assert `drain()` nulls drained slots so fat
* entries don't linger in the ring after a flush. Not part of the public API.
*/
get retainedSlotCount() {
let retained = 0;
for (const slot of this.ring) {
if (slot !== undefined) {
retained += 1;
}
}
return retained;
}
/**
* Snapshot of the Recorder's own behaviour. All fields are cheap integer
* counters accumulated on the hot path plus off-path flush timings — no
* per-record timing is taken, so reading this never taxes `record()`.
*/
getSelfMetrics() {
return {
recorded: this.recordedCount,
bufferSize: this.options.bufferSize,
bufferUsed: this.count,
bufferHighWater: this.highWaterCount,
flushes: this.flushCount,
flushedEntries: this.flushedEntriesCount,
retriedFlushes: this.retriedFlushCount,
lastFlushMs: this.lastFlushDurationMs,
maxFlushMs: this.maxFlushDurationMs,
totalFlushMs: this.totalFlushDurationMs,
overflowDropped: this.overflowDrops,
storeFailedDropped: this.storeFailedDrops,
droppedCount: this.droppedCount,
truncatedCount: this.truncatedEntryCount,
};
}
/**
* On-demand micro-benchmark of the synchronous capture path (sampling check +
* enrich + filter) on a representative input, WITHOUT enqueuing into the ring
* or touching storage. Returns the mean nanoseconds per call. Lives here so
* the "cost per capture" figure is honest yet never instruments live records.
*/
benchmarkRecordCost(iterations) {
if (iterations <= 0) {
return 0;
}
const sample = {
type: 'query',
content: { sql: 'select * from t where id = ?', bindings: [1], took: 1 },
};
// Warm up so JIT compilation is not charged to the measured window.
for (let warmup = 0; warmup < iterations; warmup++) {
this.measureCaptureOnce(sample);
}
const start = process.hrtime.bigint();
for (let iteration = 0; iteration < iterations; iteration++) {
this.measureCaptureOnce(sample);
}
const elapsedNanos = process.hrtime.bigint() - start;
return Number(elapsedNanos) / iterations;
}
/**
* Runs the same sampling+enrich+filter logic as `record()` but discards the
* enriched entry instead of buffering it. Used only by the benchmark.
*/
measureCaptureOnce(input) {
if (!this.passesSampling(input)) {
return;
}
const entry = this.enrich(input);
if (this.options.filter !== undefined) {
this.options.filter(entry);
}
}
// ── Core API ───────────────────────────────────────────────────────────────
/**
* Whether capture is currently paused by the overhead guard. While paused,
* `record()` drops new entries (counted as an overflow drop) but flushing
* continues so the buffer drains.
*/
get isPaused() {
return this.paused;
}
/** Pause capture: `record()` becomes a dropping no-op until {@link resume}. */
pause() {
this.paused = true;
}
/** Resume capture after a {@link pause}. */
resume() {
this.paused = false;
}
/** Synchronous, O(1), never throws into the caller. */
record(input) {
// Complete-count metrics tap: fires before pause/sampling so counters never
// under-report under overload. Isolated so a tap bug can't affect capture.
this.notifyRecorded(input);
try {
// Overload guard: while paused, drop new entries (counted as overflow) so
// a telescope under load can never amplify an incident. Flushing still
// drains whatever is already buffered.
if (this.paused) {
this.overflowDrops += 1;
this.notifyDrop(1, 'overflow');
return;
}
if (!this.passesSampling(input)) {
return;
}
const entry = this.enrich(input);
if (this.options.filter !== undefined && !this.options.filter(entry)) {
// Intentional exclusion — not a drop, no counter increment.
return;
}
this.push(entry);
}
catch {
// A telescope bug must never break the host. Swallow.
this.recordErrorDrops += 1;
this.notifyDrop(1, 'record-error');
}
}
/**
* Drains the buffer and persists via {@link StorageProvider.store}.
* Concurrent calls share the same in-flight promise — entries added
* while a flush is running are picked up on the next flush.
*/
async flush() {
if (this.flushing !== null) {
return this.flushing;
}
if (this.count === 0) {
return;
}
// Drain synchronously before first await so record() calls that arrive
// after this point are buffered for the next flush.
const drained = this.drain();
// Off the host path: timing the flush here is safe. Use the `now()` seam so
// tests stay deterministic, consistent with how the rest of the Recorder
// reads wall time.
const flushStartedAt = this.now();
const storage = this.options.storage;
this.flushing = this.storeDrained(storage, drained).finally(() => {
this.recordFlushMetrics(drained.length, this.now() - flushStartedAt);
this.flushing = null;
});
return this.flushing;
}
/**
* Persists a drained batch, chunked by `flushBatchSize` when that bounds the
* batch. Each chunk is stored sequentially (oldest→newest) with its own
* bounded retry; per-chunk rollups + the `onFlushStored` hook fire only for
* chunks that actually persisted (matching the whole-batch semantics — an
* alert never fires for a dropped chunk). A chunk's failure does not abort the
* remaining chunks.
*/
async storeDrained(storage, drained) {
const batchSize = this.options.flushBatchSize;
// Only chunk when a positive batch size actually bounds the drained count;
// otherwise store the whole batch in one call (original behaviour).
if (batchSize === undefined || batchSize <= 0 || batchSize >= drained.length) {
await this.storeChunk(storage, drained);
return;
}
for (let offset = 0; offset < drained.length; offset += batchSize) {
await this.storeChunk(storage, drained.slice(offset, offset + batchSize));
}
}
/**
* Stores one chunk with bounded retry, then fires per-chunk rollups + the
* flush hook only when it persisted. Shared by the chunked and whole-batch
* paths so the post-store side effects are identical.
*/
async storeChunk(storage, chunk) {
const stored = await this.storeWithRetry(storage, chunk);
if (stored) {
await this.recordRollupsAfterStore(storage, chunk);
await this.notifyFlushStored(chunk);
}
}
/**
* Persists `drained` with ONE bounded retry. On the first `store()` rejection
* the Recorder waits `retryDelayMs` (default 1000ms) and retries exactly once;
* a second failure drops the batch (`storeFailedDropped` + `store-failed`).
*
* Hard bounds preserved: this runs INSIDE the single in-flight `flushing`
* promise, so no second failed batch can ever be queued concurrently, and the
* ring keeps absorbing/evicting meanwhile — memory stays bounded. Returns
* whether the batch was ultimately persisted.
*/
async storeWithRetry(storage, drained) {
try {
await storage.store(drained);
return true;
}
catch {
// First failure: count the retry, back off, then try exactly once more.
this.retriedFlushCount += 1;
await this.delay(this.retryDelayMs);
try {
await storage.store(drained);
return true;
}
catch {
// Second failure: drop the batch (keep storeFailedDropped semantics).
this.storeFailedDrops += drained.length;
this.notifyDrop(drained.length, 'store-failed');
return false;
}
}
}
// ── Private helpers ────────────────────────────────────────────────────────
/**
* After a successful entry store, pre-aggregate the same batch into the
* rollup layer when the storage also implements the {@link RollupStore} SPI.
* The entries are already persisted, so a rollup failure must NOT be counted
* as a store failure — it is swallowed independently. Never throws into the
* flush chain.
*/
async recordRollupsAfterStore(storage, drained) {
if (!isRollupStore(storage))
return;
try {
await storage.recordRollups(aggregateDeltas(drained));
}
catch {
// Rollups are best-effort; entries are already stored. Swallow.
}
}
/**
* Invoke the `onFlushStored` hook with the just-persisted batch. Awaited inside
* the flush chain so the hook's work (e.g. per-flush alert evaluation) settles
* before `flush()` resolves, but wrapped so a rejection/throw is swallowed — the
* entries are already stored and a hook bug must never break the flush.
*/
async notifyFlushStored(drained) {
if (this.options.onFlushStored === undefined)
return;
try {
await this.options.onFlushStored(drained);
}
catch {
// Best-effort observability hook; never break the flush.
}
}
/**
* Invoke the `onRecorded` tap with the raw input. Isolated try/catch: a tap
* failure is swallowed and is NOT a drop — capture continues unaffected.
*/
notifyRecorded(input) {
if (this.options.onRecorded === undefined)
return;
try {
this.options.onRecorded(input);
}
catch {
// Best-effort metrics tap; never break record().
}
}
/**
* Tail-sampling decision. Delegates to the shared resolver so the same logic
* backs both the live path and the benchmark. The hot-path cost is shallow
* field reads (type, tags, durationMs, content.statusCode/failed) — no walk.
*/
passesSampling(input) {
return passesSampling(this.options.sampling, input, this.random);
}
enrich(input) {
const batch = this.options.context.current();
// Resolve batchId FIRST: an out-of-batch entry's synthetic batch id uses
// id-0, and the entry id uses id-1, keeping allocation order predictable.
const batchId = batch?.id ?? this.options.idFactory();
// Read the ambient trace context defensively: the provider contract says
// current() must not throw, but a misbehaving provider should degrade to
// null rather than drop the entry.
let trace = null;
try {
trace = this.options.traceContext?.current() ?? null;
}
catch {
trace = null;
}
// Bounded, synchronous redaction. The sync detach is load-bearing — it
// snapshots the (possibly fat, possibly live-ORM-graph) content into a plain,
// reference-free, size-capped clone at record() time (see spec §A.1). Track
// when a bound clipped content so /health can surface fat-capture pressure.
// Per-type content bounds: a rare, high-value entry (exception/
// client_exception, whose stacks are legitimately many KB) can carry a bigger
// content budget than the high-volume request/query/cache entries the global
// bound exists to guard. Only the NUMERIC bounds differ per type; the masking
// spec is shared (compiled once). Most entries have no override → the common
// path allocates nothing.
const perType = this.options.redact.perType?.[input.type];
const redactOptions = perType ? { ...this.options.redact, ...perType } : this.options.redact;
const redacted = redactBoundedWith(input.content, redactOptions, this.redactSpec);
if (redacted.truncated) {
this.truncatedEntryCount += 1;
}
// Soft-detected nestjs-context enrichment (SECONDARY to OTel). Read once,
// defensively: a misbehaving accessor degrades to no enrichment, never throws.
const ctx = this.readContextEnrichment();
// traceId precedence: an EXPLICIT `input.traceId` wins over everything — a
// watcher that already knows its own trace correlation (e.g. a span
// envelope) states it and is never second-guessed. Absent that, OTel wins;
// the context traceId is only a FALLBACK when neither yielded one — never
// clobber an OTel trace id.
const traceId = input.traceId ?? trace?.traceId ?? ctx.traceId;
const base = {
id: this.options.idFactory(),
batchId,
type: input.type,
familyHash: input.familyHash ?? null,
content: redacted.value,
// Prepend the context user/tenant tags before taggers run so taggers and
// the host `filter` can see them; runTaggers de-dupes order-preservingly.
tags: ctx.tags.length > 0 ? [...ctx.tags, ...(input.tags ?? [])] : (input.tags ?? []),
sequence: this.options.context.nextSequence(),
durationMs: input.durationMs ?? null,
origin: batch?.origin ?? 'manual',
instanceId: this.options.instanceId,
traceId,
spanId: trace?.spanId ?? null,
createdAt: input.startedAt ?? new Date(this.now()),
};
// Swap in the tagger-enriched tags IN PLACE rather than cloning the whole
// Entry just to replace one field. runTaggers reads base.tags (the original
// input/context tags) and other already-set fields; the call evaluates fully
// before the assignment, so reading base here is safe and behaviour matches
// the previous `{ ...base, tags }` exactly.
base.tags = runTaggers(base, this.options.taggers);
return base;
}
/**
* Reads the optional, soft-detected {@link ContextAccessor} once. Returns the
* context fallback `traceId` (or `null`) and any `user:`/`tenant:` tags. Every
* accessor call is wrapped so a misbehaving accessor degrades to empty
* enrichment and can never throw into `record()` (mirrors the OTel read).
*/
readContextEnrichment() {
const accessor = this.options.contextAccessor;
if (accessor === undefined) {
return { traceId: null, tags: [] };
}
const tags = [];
let traceId = null;
try {
const ctxTraceId = accessor.traceId();
if (typeof ctxTraceId === 'string' && ctxTraceId.length > 0) {
traceId = ctxTraceId;
}
}
catch {
// Degrade silently — context is a best-effort secondary source.
}
try {
const user = accessor.userRef();
if (user !== undefined && user.id !== undefined && user.id !== null) {
const id = String(user.id);
if (id.length > 0) {
tags.push(`user:${user.type}#${id}`);
}
}
}
catch {
// Degrade silently.
}
try {
const tenantId = accessor.tenantId();
if (typeof tenantId === 'string' && tenantId.length > 0) {
tags.push(`tenant:${tenantId}`);
}
}
catch {
// Degrade silently.
}
return { traceId, tags };
}
/**
* O(1) ring-buffer push. On overflow the oldest entry is evicted so that
* recent activity is always preserved.
*/
push(entry) {
const capacity = this.options.bufferSize;
if (this.count >= capacity) {
// Evict oldest (at head) to make room.
this.head = (this.head + 1) % capacity;
this.overflowDrops += 1;
this.notifyDrop(1, 'overflow');
}
else {
this.count += 1;
}
// Tail index: head + (count-1) wraps around the ring.
const tail = (this.head + this.count - 1) % capacity;
this.ring[tail] = entry;
// Cheap self-metrics: count the buffered record and track the high-water mark.
this.recordedCount += 1;
if (this.count > this.highWaterCount) {
this.highWaterCount = this.count;
}
}
/**
* Drains all entries from the ring in oldest→newest order and resets it.
* Returns a plain array for hand-off to storage.
*/
drain() {
const capacity = this.options.bufferSize;
const result = new Array(this.count);
for (let i = 0; i < this.count; i++) {
const slot = (this.head + i) % capacity;
// The slot is always defined here because we only read within count.
result[i] = this.ring[slot];
// Null the drained slot so the ring no longer retains the (potentially
// fat) entry after a flush. Without this, stale fat entries linger in
// unread slots up to capacity until overwritten — a slow memory floor
// that fed the incident's working set. Per-slot here is the cheapest
// correct clear (only the slots we actually held).
this.ring[slot] = undefined;
}
// Reset ring state.
this.head = 0;
this.count = 0;
return result;
}
/**
* Records off-path flush self-metrics. Only invoked from `flush()`, which has
* already guaranteed `drainedCount >= 1`, so every call here counts a flush
* that drained at least one entry.
*/
recordFlushMetrics(drainedCount, durationMs) {
this.flushCount += 1;
this.flushedEntriesCount += drainedCount;
this.lastFlushDurationMs = durationMs;
this.totalFlushDurationMs += durationMs;
if (this.maxFlushDurationMs === null || durationMs > this.maxFlushDurationMs) {
this.maxFlushDurationMs = durationMs;
}
}
/** Calls `onDrop` inside a try/catch so a faulty hook cannot escape. */
notifyDrop(count, reason) {
if (this.options.onDrop === undefined) {
return;
}
try {
this.options.onDrop(count, reason);
}
catch {
// A faulty hook must never break the Recorder.
}
}
}
//# sourceMappingURL=recorder.js.map