@clickup/ent-framework
Version:
A PostgreSQL graph-database-alike library with microsharding and row-level security
417 lines • 18.5 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ToolScoreboard = void 0;
const chalk_1 = __importDefault(require("chalk"));
const delay_1 = __importDefault(require("delay"));
const compact_1 = __importDefault(require("lodash/compact"));
const defaults_1 = __importDefault(require("lodash/defaults"));
const first_1 = __importDefault(require("lodash/first"));
const range_1 = __importDefault(require("lodash/range"));
const sortBy_1 = __importDefault(require("lodash/sortBy"));
const takeWhile_1 = __importDefault(require("lodash/takeWhile"));
const uniqBy_1 = __importDefault(require("lodash/uniqBy"));
const p_defer_1 = __importDefault(require("p-defer"));
const table_1 = require("table");
const Client_1 = require("../abstract/Client");
const misc_1 = require("../abstract/internal/misc");
const QueryPing_1 = require("../abstract/QueryPing");
const Shard_1 = require("../abstract/Shard");
const Timeline_1 = require("../abstract/Timeline");
const DefaultMap_1 = require("../internal/DefaultMap");
const misc_2 = require("../internal/misc");
const formatTimeWithMs_1 = require("./internal/formatTimeWithMs");
const ROTATING_CHARS = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏";
const FACES = ["➀", "➁", "➂", "➃", "➄", "➅", "➆", "➇", "➈"];
/**
* A tool which plays the role of Linux `top` command, but for the Cluster.
* Tracks the state of the Cluster and Clients health.
*/
class ToolScoreboard {
/**
* Initializes the instance.
*/
constructor(options) {
this.launchedPollers = new Set();
this.renderCallCount = 0;
this.queryPollDefers = [];
/** Registry of all Islands with Clients. */
this.islands = new Map();
/** Log of queries sent (ping, discovery, tick). */
this.queries = new DefaultMap_1.DefaultMap();
/** Pool stats of Clients. */
this.poolStats = new DefaultMap_1.DefaultMap();
/** Registry of the recent swallowed errors (pings-independent). */
this.swallowedErrors = [];
/** Errors extracted from the queries log. */
this.queryErrors = [];
this.options = (0, defaults_1.default)({}, options, ToolScoreboard.DEFAULT_OPTIONS);
}
/**
* Runs an endless loop that updates the Scoreboard with the current state of
* the Cluster and yields back on every refreshMs tick.
*/
async *[Symbol.asyncIterator]() {
const cluster = this.options.cluster;
const oldLoggers = { ...cluster.options.loggers };
cluster.options.loggers.clientQueryLogger = (props) => {
for (const { islandNo, clientIdent } of this.findClients(props.address)) {
this.poolStats
.getOrAdd(islandNo, Map)
.set(clientIdent, props.poolStats);
if (props.op === misc_1.OP_SHARD_NOS) {
this.addQuery(islandNo, clientIdent, {
timestamp: Date.now(),
elapsed: props.elapsed.total,
op: "discovery",
error: props.error ? `clientQueryLogger: ${props.error}` : null,
});
}
}
};
cluster.options.loggers.swallowedErrorLogger = ({ error, elapsed }) => this.addSwallowedError({
timestamp: Date.now(),
elapsed,
message: `swallowedErrorLogger: ${error}`,
});
const tickInterval = this.options.tickMs
? setInterval(() => {
for (const [islandNo, { clients }] of this.islands) {
for (const clientIdent of clients.keys()) {
this.addQuery(islandNo, clientIdent, {
timestamp: Date.now(),
elapsed: null,
op: "tick",
error: null,
});
}
}
}, this.options.tickMs).unref()
: undefined;
const queryPollTimeouts = [];
for (const i of (0, range_1.default)(this.options.pingParallelism)) {
this.queryPollDefers[i] = (0, p_defer_1.default)();
queryPollTimeouts.push(setTimeout(() => queryPollTimeouts.push(setInterval(() => {
(0, misc_2.nullthrows)(this.queryPollDefers.at(i)).resolve();
this.queryPollDefers[i] = (0, p_defer_1.default)();
}, this.options.pingPollMs).unref()), i * (this.options.pingPollMs / this.options.pingParallelism)).unref());
}
try {
while (true) {
this.islands = new Map(await (0, misc_2.mapJoin)(cluster.islands(), async (island) => {
const res = await (0, misc_2.join)({
islandNo: island.no,
shards: island.shards().length,
clients: (0, misc_2.join)([
island.clients,
island.master(),
island.replica(),
]).then(([clients, master, replica]) => new Map([
["master", master],
["replica", replica],
...clients.entries(),
])),
});
for (const clientIdent of res.clients.keys()) {
for (const i of (0, range_1.default)(this.options.pingParallelism)) {
const key = `${island.no}:${clientIdent}:${i}`;
if (!this.launchedPollers.has(key)) {
this.launchedPollers.add(key);
(0, misc_2.runInVoid)(this.pollerLoop(island.no, clientIdent, i).finally(() => this.launchedPollers.delete(key)));
}
}
}
return [island.no, res];
}));
for (const [islandNo, statsByClientIdent] of this.poolStats.entries()) {
for (const clientIdent of statsByClientIdent.keys()) {
if (!this.islands.get(islandNo)?.clients.has(clientIdent)) {
statsByClientIdent.delete(clientIdent);
}
}
if (statsByClientIdent.size === 0) {
this.poolStats.delete(islandNo);
}
}
this.swallowedErrors = this.swallowedErrors.filter((e) => e.timestamp >
Date.now() - this.options.pingPollMs * this.options.maxQueries);
this.queryErrors = [];
for (const [islandNo, queriesByClientIdent] of this.queries.entries()) {
for (const [clientIdent, queries] of queriesByClientIdent.entries()) {
if (this.islands.get(islandNo)?.clients.has(clientIdent)) {
for (const { timestamp, elapsed, error } of queries) {
if (error) {
this.queryErrors.push({
timestamp,
elapsed,
message: error,
clientIdent,
});
}
}
}
else {
queriesByClientIdent.delete(clientIdent);
}
}
if (queriesByClientIdent.size === 0) {
this.queries.delete(islandNo);
}
}
this.queryErrors = (0, uniqBy_1.default)((0, sortBy_1.default)(this.queryErrors, ({ clientIdent }) => (typeof clientIdent === "string" ? 0 : 1), ({ message }) => message), ({ message, clientIdent }) => message + clientIdent).slice(0, this.options.maxErrors);
yield this;
await (0, delay_1.default)(this.options.refreshMs);
}
}
finally {
this.queryPollDefers.forEach((defer) => defer.resolve());
this.islands.clear();
this.queries.clear();
this.swallowedErrors = [];
this.queryErrors = [];
Object.assign(cluster.options.loggers, oldLoggers);
queryPollTimeouts.forEach((interval) => clearTimeout(interval));
clearInterval(tickInterval);
}
}
/**
* Renders the current state of the Scoreboard as a string.
*/
render() {
this.renderCallCount++;
this.renderCallFirstAt ??= Date.now();
const queriesWidth = this.options.maxQueries * 2;
const rows = [];
rows.push([
"Island",
"#",
"Client",
"Role",
"Pool Conns",
"Queries (ms or Ⓝ ×10 - pings; D - discovery; red - error)",
"Health",
]);
for (const [islandNo, { clients }] of this.islands) {
let i = 0;
for (const [clientIdent, client] of clients.entries()) {
let curQueriesWidth = 0;
const primaryColor = typeof clientIdent !== "string" ? "gray" : "white";
const connectionIssue = client.connectionIssue();
const queries = this.queries.get(islandNo)?.get(clientIdent) ?? [];
const poolStats = this.poolStats.get(islandNo)?.get(clientIdent);
rows.push([
// Island
i === 0 ? `${islandNo}` : "",
// #
chalk_1.default[primaryColor](typeof clientIdent !== "string"
? `#${clientIdent}`
: `${clientIdent}()`),
// Client
chalk_1.default[primaryColor](client.options.name),
// Role
chalk_1.default[primaryColor](client.role()),
// Pool Conns
typeof clientIdent !== "string" && poolStats
? chalk_1.default[primaryColor](`${poolStats.totalConns} (${poolStats.totalConns - poolStats.idleConns} busy)`.padEnd(13, " "))
: "",
// Queries
(0, takeWhile_1.default)((0, compact_1.default)(queries.map((query) => this.renderQuery(clientIdent, query))), ([str]) => {
if (curQueriesWidth + str.length < queriesWidth) {
curQueriesWidth += str.length + 1;
return true;
}
else {
return false;
}
})
.map(([str, color]) => chalk_1.default[color](str))
.join(" ") + " ".repeat(queriesWidth - curQueriesWidth),
// Health
connectionIssue
? chalk_1.default[clientIdentErrorColor(clientIdent)]("UNHEALTHY: " +
(0, misc_2.firstLine)(connectionIssue.cause?.message))
: chalk_1.default.green("healthy") +
(clientIdent === "replica" && client.role() === "master"
? " (but fallback to master)"
: ""),
]);
i++;
}
}
const lines = [];
lines.push("[" +
(0, formatTimeWithMs_1.formatTimeWithMs)(new Date()) +
(this.renderCallFirstAt
? `, ${((Date.now() - this.renderCallFirstAt) / 1000).toFixed(1)} sec elapsed`
: "") +
"]");
lines.push((0, table_1.table)(rows, {
drawHorizontalLine: (i, rowCount) => i === 0 || i === 1 || i === rowCount,
spanningCells: [],
}));
lines.push(...[
...this.queryErrors,
...this.swallowedErrors.map((e) => ({ ...e, clientIdent: null })),
].map(({ timestamp, message, clientIdent }) => chalk_1.default[clientIdentErrorColor(clientIdent)]((0, misc_2.indent)("- " +
(clientIdent === null
? ""
: typeof clientIdent !== "string"
? `[pinging client ${clientIdent}] `
: `[pinging ${clientIdent}()] `) +
(0, misc_2.firstLine)(message) +
` [${(0, formatTimeWithMs_1.formatTimeWithMs)(new Date(timestamp))}] `).substring(2))));
return lines.join("\n");
}
/**
* Renders a colorful cell corresponding to one query.
*/
renderQuery(clientIdent, { timestamp, elapsed, op, error }) {
const lag = Math.round(Date.now() - timestamp);
const rot = ROTATING_CHARS[this.renderCallCount % ROTATING_CHARS.length];
return op === "tick"
? [".", "gray"]
: op === "discovery"
? ["D", error ? clientIdentErrorColor(null) : "white"]
: elapsed === null && lag < this.options.pingExecTimeMs
? null
: elapsed === null && lag > this.options.pingExecTimeMs + 500
? [
rot + this.renderElapsed(lag) + rot,
error ? clientIdentErrorColor(clientIdent) : "magentaBright",
]
: [
this.renderElapsed(elapsed ?? lag),
error ? clientIdentErrorColor(clientIdent) : "green",
];
}
/**
* Renders the text value of a cell corresponding to a query.
*/
renderElapsed(elapsed) {
return elapsed > 1000
? `${Math.trunc(elapsed / 1000)}s`
: elapsed >= 10 * (FACES.length + 1)
? Math.trunc(elapsed).toString()
: elapsed < 10
? Math.trunc(elapsed).toString()
: FACES[Math.min(Math.trunc(elapsed / 10), FACES.length) - 1];
}
/**
* Runs an endless polling loop for the provided Client. The loop terminates
* if the Client disappears from the cluster.
*/
async pollerLoop(islandNo, clientIdent, i) {
const cluster = this.options.cluster;
while (true) {
const island = this.islands.get(islandNo);
if (!island) {
return;
}
const client = typeof clientIdent !== "string"
? island.clients.get(clientIdent)
: clientIdent === "master"
? Shard_1.MASTER
: Shard_1.STALE_REPLICA;
if (!client) {
return;
}
// Sync pings in all pollerLoop() loops among each other.
await (0, misc_2.nullthrows)(this.queryPollDefers.at(i)).promise;
const timestamp = Date.now();
const query = this.addQuery(islandNo, clientIdent, {
timestamp,
elapsed: null,
op: "query",
error: null,
});
const input = {
execTimeMs: this.options.pingExecTimeMs,
isWrite: clientIdent === "master",
annotation: {
trace: "scoreboard-trace",
vc: "scoreboard-vc",
debugStack: "",
whyClient: undefined,
attempt: 0,
},
};
let error = null;
try {
if (client instanceof Client_1.Client) {
await client.ping(input);
}
else {
const island = await cluster.island(islandNo);
const shard = (0, first_1.default)(island.shards());
if (!shard) {
throw Error("No Shards on this Island, or Shards discovery never succeeded.");
}
// For master() and replica(), we run queries through Shard, to
// benefit from its zero-downtime retries logic.
await shard.run(new QueryPing_1.QueryPing(input), input.annotation, new Timeline_1.Timeline(), client);
}
}
catch (e) {
error = "thrown: " + e;
}
finally {
query.elapsed = Date.now() - timestamp;
query.error = error;
}
}
}
/**
* Adds a query (probably running right now) to the Scoreboard.
*/
addQuery(islandNo, clientIdent, query) {
const slot = this.queries
.getOrAdd(islandNo, DefaultMap_1.DefaultMap)
.getOrAdd(clientIdent, Array);
slot.unshift(query);
slot.splice(this.options.maxQueries);
return query;
}
/**
* Adds an error to the Scoreboard.
*/
addSwallowedError(error) {
this.swallowedErrors = [
error,
...this.swallowedErrors.filter((e) => e.message !== error.message),
];
this.swallowedErrors.splice(this.options.maxErrors);
}
/**
* Finds all existing Client by matching their addresses to the passed one.
*/
*findClients(address) {
for (const [islandNo, { clients }] of this.islands) {
for (const [clientIdent, client] of clients.entries()) {
if (client.address() === address) {
yield { islandNo, clientIdent, client };
}
}
}
}
}
exports.ToolScoreboard = ToolScoreboard;
/** Default values for the constructor options. */
ToolScoreboard.DEFAULT_OPTIONS = {
refreshMs: 100,
pingExecTimeMs: 0,
pingParallelism: 1,
pingPollMs: 200,
tickMs: 0,
maxQueries: 30,
maxErrors: 6,
};
function clientIdentErrorColor(clientIdent) {
return clientIdent === null
? "cyan"
: typeof clientIdent !== "string"
? "yellow"
: "red";
}
//# sourceMappingURL=ToolScoreboard.js.map