@dudousxd/nestjs-telescope
Version:
Laravel Telescope-style observability console for NestJS — core: watchers, recorder, correlation, SQLite store, headless API.
498 lines (495 loc) • 23.4 kB
JavaScript
// packages/core/src/storage/sqlite-storage-provider.ts
import Database from 'better-sqlite3';
import { isBatchOrigin } from '../entry/entry.js';
import { mergeHistograms, normalizeHistogram } from '../rollup/rollup-store.js';
import { decodeCursor, encodeCursor } from './cursor.js';
import { safeJsonParse } from './safe-json.js';
const DEFAULT_LIMIT = 50;
/** Every persisted column EXCEPT `content`, for content-less projection scans. */
const CONTENTLESS_COLUMNS = 'id, batch_id, type, family_hash, tags, sequence, duration_ms, origin, instance_id, trace_id, span_id, created_at';
/**
* Default storage provider: embedded SQLite via better-sqlite3, JSON1 for tags.
*
* Declared against {@link BoundedPruneCapable} and {@link LeaseCapableStorage}
* (both extend `StorageProvider`) so dropping either capability is a compile
* error rather than a silent fall back to unbounded, unlocked pruning.
*/
export class SqliteStorageProvider {
db;
/** Prepared once; reused by store() and update(). */
stmtInsert;
/** Prepared once; reused by find() and update(). */
stmtFindRow;
/** Prepared once; reused by batch(). */
stmtBatch;
/** Prepared once; reused by recordRollups() to read the existing rollup row. */
stmtSelectRollup;
/** Prepared once; reused by recordRollups() — additive upsert incl. histogram. */
stmtUpsertRollup;
constructor(options = {}) {
const path = options.path ?? ':memory:';
this.db = new Database(path);
// WAL is only meaningful for file-backed databases; skip for :memory:.
if (path !== ':memory:') {
this.db.pragma('journal_mode = WAL');
}
this.db.exec(`
create table if not exists telescope_entries (
id text primary key,
batch_id text not null,
type text not null,
family_hash text,
content text not null,
tags text not null,
sequence integer not null,
duration_ms integer,
origin text not null,
instance_id text not null,
trace_id text,
span_id text,
created_at integer not null
);
create index if not exists ix_te_created on telescope_entries (created_at, id);
create index if not exists ix_te_type_created on telescope_entries (type, created_at);
create index if not exists ix_te_batch on telescope_entries (batch_id, sequence);
create index if not exists ix_te_family on telescope_entries (family_hash) where family_hash is not null;
create table if not exists telescope_rollups (
metric text not null,
bucket_start integer not null,
count integer not null,
sum integer not null,
max integer not null,
histogram text,
primary key (metric, bucket_start)
);
create index if not exists ix_tr_bucket on telescope_rollups (bucket_start);
create table if not exists telescope_seen_families (
family_hash text primary key,
last_seen integer not null
);
create table if not exists telescope_leases (
lease_key text primary key,
owner text not null,
expires_at integer not null
);
`);
// Self-heal tables created before trace_id/span_id existed (additive, no migration).
// On a freshly-created table these columns already exist, so both no-op via the swallow.
this.ensureColumn('telescope_entries', 'trace_id text');
this.ensureColumn('telescope_entries', 'span_id text');
// Self-heal a telescope_rollups table created before the histogram column
// existed. Legacy rows have a null histogram; reads normalize that to zeros.
this.ensureColumn('telescope_rollups', 'histogram text');
// Index trace_id only AFTER the column is guaranteed to exist (a legacy table
// predating the column would otherwise make the index DDL fail). The
// #/traces/:id view filters by traceId; without this it's a full scan.
this.db.exec('create index if not exists ix_te_trace on telescope_entries (trace_id) where trace_id is not null;');
// Prepare hot-path statements once, after schema is guaranteed to exist.
this.stmtInsert = this.db.prepare(`insert or replace into telescope_entries
(id, batch_id, type, family_hash, content, tags, sequence, duration_ms, origin, instance_id, trace_id, span_id, created_at)
values (, , , , , , , , , , , , )`);
this.stmtFindRow = this.db.prepare('select * from telescope_entries where id = ?');
/** Returns all entries in a batch, ordered by sequence ascending. Capped at 1000 rows. */
this.stmtBatch = this.db.prepare('select * from telescope_entries where batch_id = ? order by sequence asc limit 1000');
this.stmtSelectRollup = this.db.prepare('select metric, bucket_start, count, sum, max, histogram from telescope_rollups where metric = @metric and bucket_start = @bucket_start');
// Additive merge happens in JS (recordRollups reads the existing row, folds
// the delta in, and writes the merged totals), so the upsert REPLACES every
// aggregate column with the already-merged value. Doing the merge in JS keeps
// count/sum/max and the element-wise histogram merge on a single code path.
this.stmtUpsertRollup = this.db.prepare(`insert into telescope_rollups (metric, bucket_start, count, sum, max, histogram)
values (, , , , , )
on conflict(metric, bucket_start) do update set
count = excluded.count,
sum = excluded.sum,
max = excluded.max,
histogram = excluded.histogram`);
}
close() {
this.db.close();
}
/**
* Idempotently adds a column to `table`. SQLite has no
* `add column if not exists`, so we attempt the alter and swallow ONLY the
* duplicate-column error (the column is already present); anything else re-throws.
*/
ensureColumn(table, ddl) {
try {
this.db.exec(`alter table ${table} add column ${ddl}`);
}
catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (!message.includes('duplicate column'))
throw error;
}
}
async store(entries) {
if (entries.length === 0)
return;
const tx = this.db.transaction((rows) => {
for (const row of rows)
this.stmtInsert.run(row);
});
tx(entries.map((e) => this.toRow(e)));
}
async update(id, patch) {
// Atomic read-modify-write: the findRow and write happen in a single transaction,
// eliminating the lost-update window.
const tx = this.db.transaction(() => {
const existing = this.stmtFindRow.get(id);
if (!existing)
return;
const merged = { ...this.fromRow(existing), ...patch, id: existing.id };
this.stmtInsert.run(this.toRow(merged));
});
tx();
}
async find(id) {
const row = this.stmtFindRow.get(id);
if (!row)
return null;
const entry = this.fromRow(row);
return { ...entry, batch: await this.batch(entry.batchId) };
}
async get(query) {
const where = [];
const params = {};
if (query.ids !== undefined) {
// Empty id set ⇒ no rows. A bare `id in ()` is invalid SQLite, so return early.
if (query.ids.length === 0) {
return { data: [], nextCursor: null };
}
const placeholders = query.ids.map((_, index) => `${index}`);
where.push(`id in (${placeholders.join(', ')})`);
query.ids.forEach((id, index) => {
params[`id${index}`] = id;
});
}
if (query.type !== undefined) {
where.push('type = @type');
params.type = query.type;
}
if (query.familyHash !== undefined) {
where.push('family_hash = @familyHash');
params.familyHash = query.familyHash;
}
if (query.batchId !== undefined) {
where.push('batch_id = @batchId');
params.batchId = query.batchId;
}
if (query.traceId !== undefined) {
where.push('trace_id = @traceId');
params.traceId = query.traceId;
}
if (query.before !== undefined) {
where.push('created_at < @before');
params.before = query.before.getTime();
}
if (query.after !== undefined) {
where.push('created_at > @after');
params.after = query.after.getTime();
}
if (query.tag !== undefined) {
where.push('exists (select 1 from json_each(telescope_entries.tags) where value = @tag)');
params.tag = query.tag;
}
if (query.search !== undefined) {
// Free-text scan over the stored content JSON. SQLite LIKE is case-insensitive
// for ASCII by default. Runs in the WHERE over the `content` column, so it is
// independent of the omitContent projection.
where.push("content like '%' || @search || '%'");
params.search = query.search;
}
const cursor = query.cursor ? decodeCursor(query.cursor) : null;
if (cursor) {
where.push('(created_at < @cCreated or (created_at = @cCreated and id < @cId))');
params.cCreated = cursor.createdAt;
params.cId = cursor.id;
}
const limit = query.limit !== undefined && Number.isInteger(query.limit) && query.limit > 0
? query.limit
: DEFAULT_LIMIT;
const whereClause = where.length > 0 ? `where ${where.join(' and ')}` : '';
// omitContent: project every column EXCEPT the heavy `content` blob so the
// primary aggregate scan never reads/parses it.
const columns = query.omitContent ? CONTENTLESS_COLUMNS : '*';
const sql = `select ${columns} from telescope_entries
${whereClause}
order by created_at desc, id desc limit `;
params.limit = limit + 1; // fetch one extra to know if there is a next page
const rows = this.db.prepare(sql).all(params);
const hasMore = rows.length > limit;
const page = rows.slice(0, limit).map((row) => this.fromRow(row, query.omitContent === true));
const last = page.at(-1);
return {
data: page,
nextCursor: hasMore && last ? encodeCursor(last.createdAt.getTime(), last.id) : null,
};
}
/**
* Returns all entries belonging to `batchId`, sorted by sequence ascending.
* Capped at 1000 rows to avoid pathological memory loads.
*/
async batch(batchId) {
const rows = this.stmtBatch.all(batchId);
return rows.map((row) => this.fromRow(row));
}
async tags(prefix, query) {
const conditions = [];
const params = {};
if (prefix !== undefined) {
conditions.push(`value like || '%'`);
params.prefix = prefix;
}
// Applied in SQL, before the bound: a picker searches precisely for the values the bound cut,
// so narrowing the already-cut page could never find them.
const search = query?.search?.trim();
if (search) {
conditions.push(`lower(value) like '%' || || '%'`);
params.search = search.toLowerCase();
}
// `tag asc` is not decoration — it makes the order total, which is what lets `offset` continue a
// page instead of re-cutting an arbitrary arrangement of equal counts.
const where = conditions.length ? `where ${conditions.join(' and ')}` : '';
const page = query?.limit !== undefined ? 'limit @limit offset @offset' : '';
const sql = `select value as tag, count(*) as count
from telescope_entries, json_each(telescope_entries.tags)
${where}
group by value
order by count desc, tag asc
${page}`;
if (query?.limit !== undefined) {
params.limit = query.limit;
params.offset = query.offset ?? 0;
}
const rows = this.db.prepare(sql).all(params);
return rows.map((row) => ({ tag: row.tag, count: row.count }));
}
async prune(olderThan, keepLast) {
const cutoff = olderThan.getTime();
if (keepLast === undefined) {
return this.db.prepare('delete from telescope_entries where created_at < ?').run(cutoff)
.changes;
}
// Delete entries older than cutoff, but keep the newest `keepLast` of those old entries.
return this.db
.prepare(`delete from telescope_entries
where created_at <
and id not in (
select id from telescope_entries where created_at <
order by created_at desc, id desc limit
)`)
.run({ cutoff, keepLast }).changes;
}
async pruneScoped(input) {
const cutoff = input.before.getTime();
// Build the type predicate as a parameterised fragment so the single delete
// and the keepLast carve-out share identical row selection. `type = ?`,
// `type NOT IN (?, ?, …)`, or (neither) the whole table.
const { typeSql, typeParams } = this.scopedTypePredicate(input);
const where = `created_at < ?${typeSql ? ` and ${typeSql}` : ''}`;
if (input.keepLast === undefined) {
return this.db
.prepare(`delete from telescope_entries where ${where}`)
.run(cutoff, ...typeParams).changes;
}
// Delete the in-scope-and-old rows, but spare the newest `keepLast` of them.
return this.db
.prepare(`delete from telescope_entries
where ${where}
and id not in (
select id from telescope_entries where ${where}
order by created_at desc, id desc limit ?
)`)
.run(cutoff, ...typeParams, cutoff, ...typeParams, input.keepLast).changes;
}
async pruneScopedBatch(input) {
const cutoff = input.before.getTime();
const { typeSql, typeParams } = this.scopedTypePredicate(input);
const where = `created_at < ?${typeSql ? ` and ${typeSql}` : ''}`;
// `delete ... where id in (select ... order by ... limit ?)`, NOT
// `delete ... order by ... limit ?`: the latter needs SQLite to have been
// compiled with SQLITE_ENABLE_UPDATE_DELETE_LIMIT, which is not the default
// and is not something a library gets to assume about the host's build. The
// subquery form is plain SQL that every SQLite has, and it is the same shape
// the other providers use, so the batching semantics stay identical across
// engines. `ix_te_created` (created_at, id) serves the ordered scan.
const changes = this.db
.prepare(`delete from telescope_entries
where id in (
select id from telescope_entries where ${where}
order by created_at asc, id asc limit ?
)`)
.run(cutoff, ...typeParams, input.limit).changes;
// A single-statement delete-by-subquery deletes exactly what it selected, so
// a short batch means the scope is drained.
return { deleted: changes, hasMore: changes >= input.limit };
}
async tryAcquireLease(key, owner, ttlMs, nowMs) {
// One statement, so the check and the set cannot interleave with another
// connection on the same file. `on conflict do update ... where` applies the
// update ONLY when the existing lease is expired or already ours; when the
// predicate fails the row is untouched and `changes` is 0.
const changes = this.db
.prepare(`insert into telescope_leases (lease_key, owner, expires_at) values (?, ?, ?)
on conflict(lease_key) do update set owner = excluded.owner, expires_at = excluded.expires_at
where telescope_leases.expires_at <= ? or telescope_leases.owner = excluded.owner`)
.run(key, owner, nowMs + ttlMs, nowMs).changes;
return changes > 0;
}
async releaseLease(key, owner) {
// Owner-scoped: a holder whose lease expired and was re-granted must not
// delete the new holder's row.
this.db
.prepare('delete from telescope_leases where lease_key = ? and owner = ?')
.run(key, owner);
}
/**
* Returns the SQL fragment + ordered params for a {@link PruneScope}'s type
* selector. `type` → `type = ?`; `excludeTypes` → `type not in (?, …)` (an
* empty exclude list degenerates to "no type filter", i.e. all types); neither
* → no fragment.
*/
scopedTypePredicate(input) {
if (input.type !== undefined) {
return { typeSql: 'type = ?', typeParams: [input.type] };
}
if (input.excludeTypes !== undefined && input.excludeTypes.length > 0) {
const placeholders = input.excludeTypes.map(() => '?').join(', ');
return { typeSql: `type not in (${placeholders})`, typeParams: [...input.excludeTypes] };
}
return { typeSql: '', typeParams: [] };
}
async clear() {
this.db.exec('delete from telescope_entries; delete from telescope_rollups; delete from telescope_seen_families;');
}
/**
* SHARED new-exception dedup. Atomic check-and-update INSIDE a transaction so
* concurrent replicas writing to the SAME database file race safely: exactly
* one sees `true` for a brand-new family. Returns `true` when the family was
* never seen or last seen longer ago than `windowMs`.
*/
async markFamilySeen(familyHash, nowMs, windowMs) {
const tx = this.db.transaction(() => {
const row = this.db
.prepare('select last_seen from telescope_seen_families where family_hash = ?')
.get(familyHash);
const isNew = row === undefined || nowMs - row.last_seen >= windowMs;
this.db
.prepare(`insert into telescope_seen_families (family_hash, last_seen) values (?, ?)
on conflict(family_hash) do update set last_seen = excluded.last_seen`)
.run(familyHash, nowMs);
return isNew;
});
return tx();
}
// ── RollupStore SPI ────────────────────────────────────────────────────────
async recordRollups(deltas) {
if (deltas.length === 0)
return;
// Read-modify-write inside ONE transaction: fold each delta onto the current
// row (or zeros when absent) and write the merged totals. The histogram is
// merged element-wise; count/sum/max accumulate. Deltas in the same batch
// hitting the same bucket fold sequentially (each read sees prior writes).
const tx = this.db.transaction((batch) => {
for (const delta of batch) {
const existing = this.stmtSelectRollup.get({
metric: delta.metric,
bucket_start: delta.bucketStart,
});
const baseCount = existing?.count ?? 0;
const baseSum = existing?.sum ?? 0;
const baseMax = existing?.max ?? Number.NEGATIVE_INFINITY;
const baseHistogram = normalizeHistogram(this.parseHistogram(existing?.histogram ?? null));
const mergedHistogram = mergeHistograms(baseHistogram, delta.histogram);
this.stmtUpsertRollup.run({
metric: delta.metric,
bucket_start: delta.bucketStart,
count: baseCount + delta.count,
sum: baseSum + delta.sum,
max: Math.max(baseMax, delta.max),
histogram: JSON.stringify(mergedHistogram),
});
}
});
tx(deltas);
}
async queryRollups(metrics, fromBucket, toBucket) {
if (metrics.length === 0)
return [];
const placeholders = metrics.map((_, index) => `${index}`);
const params = { from: fromBucket, to: toBucket };
metrics.forEach((metric, index) => {
params[`m${index}`] = metric;
});
const sql = `select metric, bucket_start, count, sum, max, histogram
from telescope_rollups
where metric in (${placeholders.join(', ')})
and bucket_start between and `;
const rows = this.db.prepare(sql).all(params);
return rows.map((row) => ({
metric: row.metric,
bucketStart: row.bucket_start,
count: row.count,
sum: row.sum,
max: row.max,
histogram: normalizeHistogram(this.parseHistogram(row.histogram)),
}));
}
/**
* Parses a stored histogram JSON string back into a number[]. Legacy/null rows
* and any malformed JSON yield null, which {@link normalizeHistogram} turns
* into an all-zeros array of the canonical length.
*/
parseHistogram(raw) {
if (raw === null)
return null;
try {
const parsed = JSON.parse(raw);
if (!Array.isArray(parsed))
return null;
return parsed.map((value) => (typeof value === 'number' ? value : 0));
}
catch {
return null;
}
}
toRow(entry) {
return {
id: entry.id,
batch_id: entry.batchId,
type: entry.type,
family_hash: entry.familyHash,
content: JSON.stringify(entry.content),
tags: JSON.stringify(entry.tags),
sequence: entry.sequence,
duration_ms: entry.durationMs,
origin: entry.origin,
instance_id: entry.instanceId,
trace_id: entry.traceId,
span_id: entry.spanId,
created_at: entry.createdAt.getTime(),
};
}
fromRow(row, omitContent = false) {
// When omitContent projected the column away, `row.content` is undefined;
// skip parsing entirely and hand back null.
const rawContent = omitContent || row.content === undefined ? null : safeJsonParse(row.content, {});
const rawTags = safeJsonParse(row.tags, []);
return {
id: row.id,
batchId: row.batch_id,
type: row.type,
familyHash: row.family_hash,
content: rawContent,
tags: Array.isArray(rawTags) ? rawTags : [],
sequence: row.sequence,
durationMs: row.duration_ms,
origin: isBatchOrigin(row.origin) ? row.origin : 'manual',
instanceId: row.instance_id,
traceId: row.trace_id ?? null,
spanId: row.span_id ?? null,
createdAt: new Date(row.created_at),
};
}
}
//# sourceMappingURL=sqlite-storage-provider.js.map