@clickup/pg-microsharding
Version:
Microshards support for PostgreSQL
245 lines (218 loc) • 9.85 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.weights = exports.shellQuote = exports.rebalance = exports.move = exports.install = exports.factor = exports.cleanup = exports.allocate = void 0;
exports.main = main;
exports.cli = cli;
const chalk_1 = __importDefault(require("chalk"));
const mapValues_1 = __importDefault(require("lodash/mapValues"));
const pickBy_1 = __importDefault(require("lodash/pickBy"));
const actionAllocate_1 = require("./actions/actionAllocate");
const actionCleanup_1 = require("./actions/actionCleanup");
const actionCopy_1 = require("./actions/actionCopy");
const actionFactor_1 = require("./actions/actionFactor");
const actionInstall_1 = require("./actions/actionInstall");
const actionList_1 = require("./actions/actionList");
const actionMove_1 = require("./actions/actionMove");
const actionRebalance_1 = require("./actions/actionRebalance");
const allocate_1 = require("./api/allocate");
Object.defineProperty(exports, "allocate", { enumerable: true, get: function () { return allocate_1.allocate; } });
const cleanup_1 = require("./api/cleanup");
Object.defineProperty(exports, "cleanup", { enumerable: true, get: function () { return cleanup_1.cleanup; } });
const factor_1 = require("./api/factor");
Object.defineProperty(exports, "factor", { enumerable: true, get: function () { return factor_1.factor; } });
const install_1 = require("./api/install");
Object.defineProperty(exports, "install", { enumerable: true, get: function () { return install_1.install; } });
const move_1 = require("./api/move");
Object.defineProperty(exports, "move", { enumerable: true, get: function () { return move_1.move; } });
const rebalance_1 = require("./api/rebalance");
Object.defineProperty(exports, "rebalance", { enumerable: true, get: function () { return rebalance_1.rebalance; } });
const weights_1 = require("./api/weights");
Object.defineProperty(exports, "weights", { enumerable: true, get: function () { return weights_1.weights; } });
const inspectError_1 = require("./internal/inspectError");
const logging_1 = require("./internal/logging");
const parseArgs_1 = require("./internal/parseArgs");
const quote_1 = require("./internal/quote");
Object.defineProperty(exports, "shellQuote", { enumerable: true, get: function () { return quote_1.shellQuote; } });
const readConfigs_1 = require("./internal/readConfigs");
const unindent_1 = require("./internal/unindent");
const HELP = (0, unindent_1.unindent)(`
USAGE
pg-microsharding install
[--schema-name-fmt=SCHEMA_NAME_FMT]
[--dsn=DSN | --dsns=DNS1,DSN2,...]
pg-microsharding list | ls
[--weight-sql='SELECT returning weight with optional unit']
[--verbose]
[--dsn=DSN | --dsns=DNS1,DSN2,...]
[--json]
pg-microsharding allocate
--shard=N | --shards=N-M
--migrate-cmd='shell command to run migrations'
--activate={yes | no}
[--dsn=DSN | --dsns=DNS1,DSN2,...]
pg-microsharding factor
--shard=N | --shards=N,M,... | --shards=DSN-PREFIX
--factor=P|+P.Q|-P.Q|"*P.Q"
[--dsn=DSN | --dsns=DNS1,DSN2,...]
pg-microsharding move
--shard=N
--from=DSN-OR-DSN-PREFIX-OR
--to=DSN-OR-DSN-PREFIX
--activate-on-destination={yes | no}
[--wait]
[--validate-fks]
[--max-replication-lag-sec=N]
[--dsns=DNS1,DSN2,...]
[--deactivate-sql='SQL $1 SQL']
pg-microsharding rebalance
--activate-on-destination={yes | no}
[--wait]
[--validate-fks]
[--deactivate-sql='SQL $1 SQL']
[--weight-sql='SELECT returning weight with optional unit']
[--decommission=DSN1,DSN2,...]
[--max-replication-lag-sec=N]
[--parallelism=N]
[--dsn=DSN | --dsns=DNS1,DSN2,...]
pg-microsharding cleanup
[--dsn=DSN | --dsns=DNS1,DSN2,...]
pg-microsharding copy
--schema=SCHEMA-NAME
--from=DSN-OR-DSN-PREFIX-OR
--to=DSN-OR-DSN-PREFIX
[--wait]
[--validate-fks]
[--max-replication-lag-sec=N]
[--dsns=DNS1,DSN2,...]
ENVIRONMENT VARIABLES
The tool receives parameters from command line option, but allows to set
defaults for most of them using environment variables.
Some variables are standard for psql command:
- PGUSER: default database user
- PGPASSWORD: default database password
- PGHOST: default database host
- PGPORT: default database port
- PGDATABASE: default database name
- PGSSLMODE: default SSL mode (e.g. "prefer")
- PGOPTIONS: options (like work_mem) used when creating indexes
Custom variables:
- DSNS or PGDSNS: default value for --dsns option, comma-separated
list of DSNs (see below)
- MIGRATE_CMD: default value for --migrate-cmd option
- WEIGHT_SQL: default value for --weight-sql option
- DEACTIVATE_SQL: default value for --deactivate-sql option
- PARALLELISM: default value for --parallelism option
- MAX_REPLICATION_LAG_SEC: default value for --max-replication-lag-sec
- SCHEMA_NAME_FMT: default value for --schema-name-fmt (applies only
to install action; if not passed, "sh%04d" is used)
CONFIG FILE
Unless --skip-config flag is passed, the tool also tries to find
pg-microsharding.config.js (or .ts) file in the current and parent
directories. If found, it treats its exports as environment variables
(or as a function that returns environment variables), merging them
into process.env.
DSN AND ADDRESSING DATABASES
Option --dsns, if required, should be a comma separated list of DSNs.
Also, you may pass duplicated DSNs and even DSNs of replicas: the tool
will filter them out and remain only master DSNs in the list.
DSN format examples (parts defaults are from environment variables):
- postgresql://user:pass@hostname/db?options (all parts are optional)
- hostname:port/db (all parts except the hostname are optional)
The tool also tries to find pg-microsharding.config.js (or .ts) file in
the current and parent directories and, if found, treats its exports as
environment variables, merging them into process.env.
TUNING INDEXES CREATION PERFORMANCE
When moving or rebalancing microshards, indexes creation may take a
substantial amount of time. You can tell pg-microsharding to use more
memory and run faster by passing PGOPTIONS environment variable to
the tool, similar to how you would do it in psql:
PGOPTIONS="-c maintenance_work_mem=500MB -c max_parallel_maintenance_workers=4" \\
pg-microsharding rebalance ...
REPLICATION LAG PREVENTION
The tool tries hard to not affect the replication lag of the destination
node when moving or rebalancing shards. It waits until the lag drops below
--max-replication-lag-sec seconds before running heavy operations (or
until the user presses Shift+S to force-continue).
Also, if you want the tool to pause explicitly and wait until the user
presses Shift+S before activating the shard on the destination node,
you can use the --wait option.
`);
const ACTIONS = {
allocate: actionAllocate_1.actionAllocate,
cleanup: actionCleanup_1.actionCleanup,
copy: actionCopy_1.actionCopy,
factor: actionFactor_1.actionFactor,
install: actionInstall_1.actionInstall,
list: actionList_1.actionList,
ls: actionList_1.actionList,
move: actionMove_1.actionMove,
rebalance: actionRebalance_1.actionRebalance,
};
/**
* Tool main function.
*/
async function main(argsIn) {
const args = (0, parseArgs_1.parseArgs)(argsIn);
if (!args["skip-config"]) {
for (const config of await (0, readConfigs_1.readConfigs)("pg-microsharding.config")) {
Object.assign(process.env, (0, mapValues_1.default)((0, pickBy_1.default)(config, (v) => typeof v === "string" ||
typeof v === "number" ||
typeof v === "boolean"), String));
}
}
const DSNS = process.env["PGDSNS"] || process.env["DSNS"];
if (!args["dsns"] && !args["dsn"] && DSNS) {
args["dsns"] = DSNS;
}
const MIGRATE_CMD = process.env["MIGRATE_CMD"];
if (!args["migrate-cmd"] && MIGRATE_CMD) {
args["migrate-cmd"] = MIGRATE_CMD;
}
const WEIGHT_SQL = process.env["WEIGHT_SQL"];
if (!args["weight-sql"] && WEIGHT_SQL) {
args["weight-sql"] = WEIGHT_SQL;
}
const DEACTIVATE_SQL = process.env["DEACTIVATE_SQL"];
if (!args["deactivate-sql"] && DEACTIVATE_SQL) {
args["deactivate-sql"] = DEACTIVATE_SQL;
}
const PARALLELISM = process.env["PARALLELISM"];
if (!args["parallelism"] && PARALLELISM) {
args["parallelism"] = PARALLELISM;
}
const MAX_REPLICATION_LAG_SEC = process.env["MAX_REPLICATION_LAG_SEC"];
if (!args["max-replication-lag-sec"] && MAX_REPLICATION_LAG_SEC) {
args["max-replication-lag-sec"] = MAX_REPLICATION_LAG_SEC;
}
const SCHEMA_NAME_FMT = process.env["SCHEMA_NAME_FMT"];
if (!args["schema-name-fmt"] && SCHEMA_NAME_FMT) {
args["schema-name-fmt"] = SCHEMA_NAME_FMT;
}
const action = args._[0];
if (!action) {
(0, logging_1.print)(HELP);
return true;
}
if (action in ACTIONS) {
return ACTIONS[action](args);
}
throw `Unknown action: ${action}`;
}
/**
* A wrapper around main() to call it from a bin script.
*/
function cli() {
main(process.argv.slice(2))
.then((success) => process.exit(success ? 0 : 1))
.catch((e) => {
(0, logging_1.print)(chalk_1.default.red((0, inspectError_1.inspectError)(e)));
process.exit(1);
});
}
if (require.main === module) {
cli();
}
//# sourceMappingURL=cli.js.map