@clickup/pg-microsharding
Version:
Microshards support for PostgreSQL
159 lines • 7.93 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.actionRebalance = actionRebalance;
const path_1 = require("path");
const chalk_1 = __importDefault(require("chalk"));
const compact_1 = __importDefault(require("lodash/compact"));
const first_1 = __importDefault(require("lodash/first"));
const flatten_1 = __importDefault(require("lodash/flatten"));
const sumBy_1 = __importDefault(require("lodash/sumBy"));
const prompts_1 = __importDefault(require("prompts"));
const rebalance_1 = require("../api/rebalance");
const chooseDsns_1 = require("../internal/chooseDsns");
const chunkIntoN_1 = require("../internal/chunkIntoN");
const isTrueValue_1 = require("../internal/isTrueValue");
const logging_1 = require("../internal/logging");
const names_1 = require("../internal/names");
const normalizeDsns_1 = require("../internal/normalizeDsns");
const pluralize_1 = require("../internal/pluralize");
const runInTmux_1 = require("../internal/runInTmux");
const actionList_1 = require("./actionList");
/**
* Runs a series of shard modes, concurrently, using TMUX if available.
*/
async function actionRebalance(args) {
var _a, _b;
await (0, runInTmux_1.tryReattachToTmuxSession)();
const dsns = await (0, normalizeDsns_1.normalizeDsns)(args["dsns"] || args["dsn"]);
if (dsns.length === 0) {
throw "Please provide --dsns, DB DSNs to rebalance microshards against";
}
const activateOnDestination = (0, isTrueValue_1.isTrueValue)(args["activate-on-destination"]);
if (activateOnDestination === undefined) {
throw "Please provide --activate-on-destination=yes or --activate-on-destination=no";
}
const weightSql = args["weight-sql"] || undefined;
const [decommissionDsns] = (0, chooseDsns_1.chooseDsns)({
patterns: (0, compact_1.default)((args["decommission"] || args["decomission"] || "").split(/[,\s]+/s)),
existingDsns: dsns,
requireMatch: true,
});
const maxReplicationLagSec = Number((_a = args["max-replication-lag-sec"]) !== null && _a !== void 0 ? _a : "") || undefined;
const parallelism = Number((_b = args["parallelism"]) !== null && _b !== void 0 ? _b : "") || 3;
const deactivateSQL = String(args["deactivate-sql"] || "") || undefined;
const wait = args["wait"];
const { islandNosToDsn, islands } = await (0, actionList_1.calcIslandWeights)({
dsns,
weightSql,
});
// Always exclude an island if it only has one global shard and no other
// shards. We never move it while doing rebalance, since we don't know the
// best destination fot it. The global shard can only be moved explicitly.
const specialIslands = new Map();
for (const [no, shards] of islands.entries()) {
if (shards.length === 1 && shards[0].no === 0) {
if (decommissionDsns.includes(islandNosToDsn.get(no))) {
throw (`Cannot decommission island ${(0, names_1.dsnShort)(islandNosToDsn.get(no))}, ` +
"because it has only the global microshard and no other microshards.\n" +
"Please use the move action for such an island.");
}
islands.delete(no);
specialIslands.set(no, "an island with only the global microshard; won't be touched");
}
}
logging_1.print.section("CURRENT islands weights:");
(0, logging_1.print)(inspectIslandsWeights(islandNosToDsn, islands, specialIslands));
const dsnsToIslandNo = new Map([...dsns.entries()].map(([k, v]) => [v, k]));
const decommissionIslandNos = decommissionDsns.map((dsn) => {
const islandNo = dsnsToIslandNo.get(dsn);
if (islandNo === undefined) {
throw `No such island in the list: "${(0, names_1.dsnShort)(dsn)}"`;
}
else {
return islandNo;
}
});
const groupedMoves = (0, rebalance_1.rebalance)(islands, decommissionIslandNos).map(({ from, to, shards }) => ({
fromDsn: islandNosToDsn.get(from),
toDsn: islandNosToDsn.get(to),
shards,
}));
const moves = (0, flatten_1.default)(groupedMoves.map(({ fromDsn, toDsn, shards }) => shards.map(({ no }) => ({ fromDsn, toDsn, no }))));
if (moves.length === 0) {
logging_1.print.section(`Microshards are already fairly distributed among ${(0, pluralize_1.pluralize)(islandNosToDsn.size, "island")}.`);
return true;
}
logging_1.print.section(`Proposed moves (${moves.length}):`);
(0, logging_1.print)(groupedMoves
.map(({ fromDsn, toDsn, shards }) => `- ${(0, names_1.dsnFromToShort)(fromDsn, toDsn)}: microshards [` +
shards.map((shard) => shard.no).join(", ") +
"]\n")
.join(""));
logging_1.print.section("FUTURE weights after the above moves:");
(0, logging_1.print)(inspectIslandsWeights(islandNosToDsn, islands, specialIslands));
const validResponse = "rebalance";
const response = await (0, prompts_1.default)({
type: "text",
name: "value",
message: `Type "${validResponse}" to proceed with the moves:`,
validate: (value) => value !== validResponse
? `Enter "${validResponse}" to confirm, ^C to skip`
: true,
});
if (response.value !== validResponse) {
return false;
}
const panes = (0, chunkIntoN_1.chunkIntoN)(moves, parallelism).map((sequence) => sequence.map(({ fromDsn, toDsn, no }) => ({
key: no.toString(),
title: `microshard ${no} | ${(0, names_1.dsnFromToShort)(fromDsn, toDsn)}`,
command: [
"node",
(0, path_1.resolve)(`${__dirname}/../cli`),
"move",
"--shard",
no.toString(),
"--from",
fromDsn,
"--to",
toDsn,
"--activate-on-destination",
activateOnDestination ? "yes" : "no",
...(maxReplicationLagSec
? ["--max-replication-lag-sec", maxReplicationLagSec.toString()]
: []),
...(deactivateSQL ? ["--deactivate-sql", deactivateSQL] : []),
...(wait ? ["--wait"] : []),
"--skip-config",
],
})));
await (0, runInTmux_1.runInTmux)(panes);
return true;
}
/**
* Prints weights of islands and their shards.
*/
function inspectIslandsWeights(islandNosToDsn, islands, specialIslands) {
const totalWeight = (0, sumBy_1.default)([...islands.values()], (shards) => (0, sumBy_1.default)(shards, ({ weight }) => weight)) || 1;
const unit = (0, first_1.default)((0, compact_1.default)([...islands.values()].map((shards) => { var _a; return (_a = (0, first_1.default)(shards)) === null || _a === void 0 ? void 0 : _a.unit; })));
return [...islands.entries(), ...specialIslands.entries()]
.sort(([noA], [noB]) => noA - noB)
.map(([no, shardsOrText]) => {
const prefix = `- ${(0, names_1.dsnShort)(islandNosToDsn.get(no))}: `;
if (typeof shardsOrText === "string") {
return chalk_1.default.yellow(prefix + shardsOrText) + "\n";
}
else {
const islandWeight = (0, sumBy_1.default)(shardsOrText, ({ weight }) => weight);
const shardsWithAppliedFactor = (0, sumBy_1.default)(shardsOrText, (shard) => shard.appliedFactor !== 1 ? 1 : 0);
return (prefix +
`${Math.round((islandWeight / totalWeight) * 100)}% ` +
`(${Math.round(islandWeight)}${unit ? ` ${unit}` : ""}${shardsWithAppliedFactor ? " adjusted by factor" : ""}), ` +
`${(0, pluralize_1.pluralize)(shardsOrText.length, "microshard")}${shardsWithAppliedFactor ? ` (${shardsWithAppliedFactor} with factor)` : ""}\n`);
}
})
.join("");
}
//# sourceMappingURL=actionRebalance.js.map