@dudousxd/nestjs-telescope
Version:
Laravel Telescope-style observability console for NestJS — core: watchers, recorder, correlation, SQLite store, headless API.
237 lines • 12.6 kB
TypeScript
import { type OnApplicationBootstrap, type OnApplicationShutdown } from '@nestjs/common';
import type { ResolvedCoreConfig } from '../config/options.js';
import type { StorageProvider } from '../storage/storage-provider.js';
/** What kicked off a prune cycle: the interval timer, or an on-demand request. */
export type PruneTrigger = 'scheduled' | 'manual';
/**
* One recorded prune cycle, kept in an in-memory ring buffer on the pruner so
* the dashboard's Prunes screen can show retention activity. Like the
* server-stats history ring this is PER-POD (each replica records its own
* cycles); prune runs are deliberately NOT stored as telescope entries — they
* would be pruned themselves and add write load to the very store retention is
* meant to shrink.
*/
export interface PruneRun {
/** ISO timestamp when the cycle started. */
at: string;
trigger: PruneTrigger;
/** Wall-clock duration of the whole cycle. */
durationMs: number;
/** Total entries deleted across the bulk delete and every per-type scope. */
deletedTotal: number;
/**
* Real per-type delete counts for the individually-handled scopes (entry
* types with a `perType` override or an archived type, each pruned in its own
* scope at its own cutoff). The global bulk delete spans every other type and
* returns a single aggregate count from the storage SPI, so it is folded into
* `deletedTotal` only — it is never attributed to a fabricated type key here.
*/
deletedByType: Record<string, number>;
/** Entries handed to the archive sink before deletion this cycle, if any. */
archivedTotal?: number;
/** First step error message captured this cycle (steps still swallow + log). */
error?: string;
}
export declare class TelescopePruner implements OnApplicationBootstrap, OnApplicationShutdown {
private readonly config;
private readonly storage;
private readonly logger;
private timer;
/**
* Latches once after the FIRST time we fall back from a missing
* `pruneScoped` to the global `prune`, so a third-party provider without
* per-type support logs the capability warning a single time, not every tick.
*/
private warnedNoScopedPrune;
/**
* The cycle currently running, or `null` when idle. A cycle can easily outlive
* its own interval on a large store — the bulk delete is one unbounded
* `DELETE`, and on a store whose retention predicate is not indexable it
* degrades to a full scan — while the timer below is fire-and-forget. Without
* this handle the ticks STACK: the process accumulates one more concurrent
* delete per interval, every one of them contending for the same rows, and
* the pile never drains. Scheduled ticks are dropped while a cycle is in
* flight; a manual {@link pruneNow} joins the in-flight cycle instead of
* adding a second one.
*/
private inFlight;
/**
* Scheduled ticks dropped during the cycle that is currently in flight. Reset
* at the end of every cycle, so it measures the CURRENT cycle's overrun, not
* a lifetime total.
*/
private skippedThisCycle;
/**
* Latches while ticks are being dropped so a persistently slow store logs the
* overlap warning once per streak rather than once per interval forever.
* Cleared by the first cycle that completes without dropping a tick behind it.
*/
private warnedOverlap;
/**
* Latches once after the first scope that falls back to an UNBOUNDED delete
* because the provider has no `pruneScopedBatch`, so a legacy/third-party
* provider says so once rather than every tick.
*/
private warnedUnboundedDelete;
/**
* Latches while cycles keep hitting `maxBatchesPerCycle`, so a store with a
* real backlog logs once per streak instead of once per cycle forever.
* Cleared by the first cycle that drains every scope inside the ceiling.
*/
private warnedBatchCeiling;
/** Batch ceilings hit during the CURRENT cycle; re-arms the warning at zero. */
private ceilingHitsThisCycle;
/** Latches while the prune lock's BACKEND is broken (not merely held). */
private warnedLockUnavailable;
/** Recent prune cycles, newest-first, capped at {@link MAX_PRUNE_RUNS}. */
private readonly runs;
/** Start time (epoch ms) of the most recent SCHEDULED cycle, for nextRunAt. */
private lastScheduledRunAtMs;
/**
* The cross-process lock, or `null` when this deployment prunes unlocked (the
* host set `prune.lock: false`, or supplied nothing and the provider has no
* lease SPI). Resolved ONCE in the constructor: the host's own implementation
* wins, else the database lease, else nothing.
*/
private readonly lock;
/**
* This process's identity as a lease holder. `instanceId` is the pod name
* under Kubernetes, which is unique per replica but NOT per process on a host
* running two of them, so the pid is appended — two pruners must never be able
* to mistake each other's lease for a re-entrant refresh of their own.
*/
private readonly lockOwner;
constructor(config: ResolvedCoreConfig, storage: StorageProvider);
onApplicationBootstrap(): void;
/**
* Run ONE prune cycle on demand (the dashboard's "Prune now" button → the
* controller's `retention/prune` route), recording it as a `manual` run.
* Returns the total number of entries deleted. Throws only if `prune` is
* unconfigured — the caller (controller) gates that and the mutation guard.
*
* When a cycle is already running this JOINS it (resolving with its deleted
* count) rather than starting a competing one, so hammering the button cannot
* pile deletes onto a store that is already struggling.
*/
pruneNow(): Promise<number>;
/** Recent prune runs (newest-first), copied so callers can't mutate the ring. */
getRuns(): PruneRun[];
/**
* Predicted next SCHEDULED prune time (epoch ms), or null when no `prune`
* window is configured. Derived from the last scheduled run's start + the
* interval, falling back to now + interval before the first cycle has run.
*/
getNextRunAtMs(): number | null;
onApplicationShutdown(): void;
/**
* Serializes prune cycles WITHIN THIS PROCESS: at most one runs at a time.
* A scheduled tick that lands on a busy pruner is dropped; a manual one joins
* the cycle already running. Without this, a cycle slower than `intervalMs`
* lets the timer queue a second, then a third, each holding write locks on
* overlapping rows in the same store, and the backlog only grows.
*
* This guard sees only THIS process. Bounding the FLEET is the job of the
* cross-process lock layered on top of it in {@link runLockedCycle}.
*/
private runGuardedCycle;
/**
* Serializes prune cycles ACROSS PROCESSES, when a lock is available.
*
* The per-process guard above bounds one pod to one cycle; a fleet of eight
* still put eight concurrent deletes on the same table, doing the same work
* eight times. This takes the advisory lease first and stands down when
* somebody else already has it.
*
* Three outcomes, all deliberate:
* - no lock configured → prune, exactly as before;
* - lease HELD by another replica → skip the cycle entirely and return 0. No
* `PruneRun` is recorded, because this pod did not prune — reporting a
* zero-deletion run would read as "nothing to delete", which is a different
* and much more alarming statement;
* - lock backend UNAVAILABLE (or it threw) → warn once per streak and prune
* ANYWAY. A broken lock must never be able to stop retention: failing open
* is at worst the behaviour that existed before the lock did, whereas
* failing closed silently lets the table grow without bound.
*/
private runLockedCycle;
/**
* One prune tick, unguarded — every caller goes through
* {@link runGuardedCycle}, so at most one of these is in flight per process.
* The retention model is:
* - Each type that needs INDIVIDUAL handling — one with a `perType` override
* OR an archived type (which must be exported before its own delete) — is
* pruned in its OWN scope, at its own cutoff (its `perType` value, else the
* global `after`), with archiving (when configured) first.
* - Every OTHER type is pruned in a single bulk delete at the global cutoff,
* with the individually-handled types carved out.
*
* Archived types are ALWAYS carved out of the bulk delete even with no `perType`
* override, so a failed sink can spare them (the bulk delete would otherwise
* wipe entries the sink never saw). With no overrides and no archive (the common
* case) the individual set is empty and this collapses to exactly one global
* `prune(cutoff, keepLast)` — identical to the historical behaviour.
*/
private runCycle;
/**
* Append a run to the newest-first ring buffer, evicting the oldest past the
* cap. Recording must NEVER throw into the prune path (a bad ISO/serialization
* would otherwise turn observability into an outage), so it is fully guarded.
*/
private recordRun;
/**
* Archives (if configured) the entries this `scope` is about to delete, then
* deletes them. When the scope targets a single archived `type` whose sink
* fails, the delete is SKIPPED (entries survive to retry next cycle) but the
* caller's other scopes are unaffected. Errors never propagate out of here.
*
* `fallbackOlderThan`/`fallbackKeepLast` are used only by the legacy global
* fallback path when the provider lacks `pruneScoped`.
*/
private pruneArchivedThenDelete;
/**
* Exports the doomed entries for an archived single-type scope to the sink in
* bounded batches. `proceed` is `true` when it is safe to delete (nothing to
* archive, archiving succeeded, or this type/scope is not archived) and
* `false` when the sink failed (skip the delete this cycle); `archived` is the
* number of entries actually handed to the sink.
*/
private archiveScope;
/**
* Deletes the scope, preferring BOUNDED BATCHES.
*
* Order of preference:
* 1. `pruneScopedBatch` — a loop of short, individually-committed deletes.
* This is the whole point: the unbounded form is a single statement that,
* on a large table, holds row locks for as long as it takes to scan the
* table, and every other writer on that database waits behind it. Batching
* deletes the same rows while giving the locks up between batches.
* 2. `pruneScoped` — one unbounded delete, per-type-aware.
* 3. the legacy global `prune` — the global cutoff for ALL types, the best a
* provider without per-type support can do. Run only for the global scope,
* to avoid deleting more than intended on a per-type scope.
*
* A `keepLast` scope always takes the unbounded path: "keep the newest N of
* the doomed rows" is a whole-set property that a bounded batch cannot express
* (hence `keepLast` is absent from {@link BoundedPruneScope}). `keepLast` is
* off by default, so the common configuration batches.
*/
private deleteScope;
/**
* Drains one scope in bounded deletes: at most `batchSize` rows per statement,
* at most `maxBatchesPerCycle` statements, pausing `batchPauseMs` between
* them, stopping as soon as the provider says the scope is drained.
*
* The ceiling is not a nicety. Without it, a table far enough behind turns one
* tick into an unbounded loop — the same "one prune runs for an hour" failure
* this replaces, only now spelled as a thousand statements instead of one. With
* it, every cycle has a known worst case and a backlog drains over several
* cycles instead of monopolising one.
*
* The pause is only ever paid BETWEEN batches, so the healthy case — one batch,
* `hasMore: false` — never waits at all. It exists for the unhealthy case,
* where a tight delete loop can starve co-tenants of a small instance's IOPS
* budget even though no individual statement holds locks for long.
*/
private deleteScopeInBatches;
}
//# sourceMappingURL=telescope-pruner.service.d.ts.map