@clickup/pg-microsharding
Version:
Microshards support for PostgreSQL
204 lines • 10.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.move = move;
const chalk_1 = __importDefault(require("chalk"));
const delay_1 = __importDefault(require("delay"));
const compact_1 = __importDefault(require("lodash/compact"));
const advanceSequences_1 = require("../internal/advanceSequences");
const analyzeTables_1 = require("../internal/analyzeTables");
const changeTableOwners_1 = require("../internal/changeTableOwners");
const cleanUpPubSub_1 = require("../internal/cleanUpPubSub");
const disableSubscription_1 = require("../internal/disableSubscription");
const enableSubscription_1 = require("../internal/enableSubscription");
const getDump_1 = require("../internal/getDump");
const getTablesInSchema_1 = require("../internal/getTablesInSchema");
const inspectError_1 = require("../internal/inspectError");
const logging_1 = require("../internal/logging");
const quote_1 = require("../internal/quote");
const resultAbort_1 = require("../internal/resultAbort");
const resultCommit_1 = require("../internal/resultCommit");
const runSqlProgress_1 = require("../internal/runSqlProgress");
const startCopyingTables_1 = require("../internal/startCopyingTables");
const validatePrimaryKeysFromDump_1 = require("../internal/validatePrimaryKeysFromDump");
const waitUntilBackfillCompletes_1 = require("../internal/waitUntilBackfillCompletes");
const waitUntilEverythingIsNotLaggingMuch_1 = require("../internal/waitUntilEverythingIsNotLaggingMuch");
const waitUntilIncrementalCompletes_1 = require("../internal/waitUntilIncrementalCompletes");
const wrapSigInt_1 = require("../internal/wrapSigInt");
/**
* Moves or copies a schema from one master DB to another.
*/
async function move({ schema, fromDsn, toDsn, commitAction, deactivateSQL, maxReplicationLagSec = 20, wait = false, validateFKs = false, }) {
let unlock = async () => { };
try {
const { tables, partitionedTables } = await (0, getTablesInSchema_1.getTablesInSchema)({
fromDsn,
schema,
});
const { preData, primaryKeys, postData } = await (0, getDump_1.getDump)({
fromDsn,
schema,
validateFKs,
});
(0, validatePrimaryKeysFromDump_1.validatePrimaryKeysFromDump)({
primaryKeys,
allTables: [...tables, ...partitionedTables],
});
await (0, waitUntilEverythingIsNotLaggingMuch_1.waitUntilEverythingIsNotLaggingMuch)({
when: "before starting the work",
fromDsn: undefined, // only destination physical replicas are checked
tables,
toDsn,
schema,
maxReplicationLagSec,
waitForUserToContinue: false,
});
try {
await (0, cleanUpPubSub_1.cleanUpPubSub)({ fromDsn, toDsn, schema });
await (0, runSqlProgress_1.runSqlProgressTransactional)(toDsn, preData, `Applying initial DDL (except indexes) for ${schema} to ${toDsn}`);
}
catch (e) {
// No cleanup if failed here since it's one transaction.
logging_1.log.error((0, inspectError_1.inspectError)(e));
throw ("Exited abnormally with no-op" +
(typeof e === "string" ? ` (${e})` : ""));
}
try {
await (0, wrapSigInt_1.wrapSigInt)(async (throwIfAborted) => {
if (primaryKeys.length > 0) {
await (0, runSqlProgress_1.runSqlProgressTransactional)(toDsn, (0, quote_1.join)(primaryKeys, "\n"), "Creating minimal indexes to let the logical replication replay the data");
}
// During the tables copying, change their owner to the toDsn user,
// assuming that toDsn's user may have custom values for e.g. work_mem
// or other parameters. PostgreSQL runs tablesync replication worker
// with the privileges and settings of the table's owner, so it allows
// us to have "per-tablesync" limits customization. E.g. we may want a
// way higher value for work_mem (gigabytes) to prevent excessive
// creation of temp files, and it's safe, since there are just a few
// tablesync workers running concurrently in the cluster (depends on
// max_sync_workers_per_subscription PostgreSQL setting).
const newOwner = new URL(toDsn).username;
const tablesWithOtherOwner = tables.filter(({ owner }) => owner !== newOwner || process.env["DEBUG_CHANGE_TABLE_OWNERS"]);
await (0, changeTableOwners_1.changeTableOwners)({
toDsn,
tables: tablesWithOtherOwner.map(({ schema, name }) => ({
schema,
name,
owner: newOwner,
})),
});
throwIfAborted();
await (0, startCopyingTables_1.startCopyingTables)({ fromDsn, tables, toDsn, schema });
throwIfAborted();
await (0, waitUntilBackfillCompletes_1.waitUntilBackfillCompletes)({ fromDsn, tables, toDsn, schema }, throwIfAborted);
throwIfAborted();
await (0, changeTableOwners_1.changeTableOwners)({
toDsn,
tables: tablesWithOtherOwner,
comment: "Restoring table(s) ownership back, since tablesync workers have completed",
});
throwIfAborted();
await (0, waitUntilEverythingIsNotLaggingMuch_1.waitUntilEverythingIsNotLaggingMuch)({
when: "before creating heavy indexes",
fromDsn,
tables,
toDsn,
schema,
maxReplicationLagSec,
waitForUserToContinue: wait,
}, throwIfAborted);
throwIfAborted();
await (0, disableSubscription_1.disableSubscription)({ toDsn, tables, schema });
throwIfAborted();
await (0, runSqlProgress_1.runSqlProgressNonTransactional)(toDsn, postData, "Creating heavy indexes, foreign keys etc.");
throwIfAborted();
await (0, enableSubscription_1.enableSubscription)({ toDsn, tables, schema });
throwIfAborted();
await (0, analyzeTables_1.analyzeTables)({ toDsn, tables });
throwIfAborted();
await (0, waitUntilEverythingIsNotLaggingMuch_1.waitUntilEverythingIsNotLaggingMuch)({
when: commitAction
? "before activating the destination microshard"
: "before removing the logical subscription",
fromDsn,
tables,
toDsn,
schema,
maxReplicationLagSec,
waitForUserToContinue: wait,
}, throwIfAborted);
throwIfAborted();
unlock = await (0, waitUntilIncrementalCompletes_1.waitUntilIncrementalCompletes)({ fromDsn, schema, tables }, throwIfAborted);
// This runs while the fromDsn tables are still locked.
await (0, advanceSequences_1.advanceSequences)({ fromDsn, toDsn });
});
}
catch (e) {
logging_1.log.error((0, inspectError_1.inspectError)(e));
(0, logging_1.log)("Cleaning up the half-migrated destination.");
try {
await unlock("error");
}
catch (e) {
logging_1.log.error(`Unlock failed: ${(0, inspectError_1.inspectError)(e)}`);
}
finally {
unlock = async () => { };
}
await (0, delay_1.default)(1000);
await (0, resultAbort_1.resultAbort)({ fromDsn, toDsn, schema });
throw "Exited abnormally" + (typeof e === "string" ? ` (${e})` : "");
}
try {
await (0, cleanUpPubSub_1.cleanUpPubSub)({ fromDsn, toDsn, schema });
if (commitAction) {
await (0, resultCommit_1.resultCommit)({
activateOnDestination: commitAction === "deactivate-activate",
deactivateSQL,
fromDsn,
tables,
toDsn,
schema,
});
}
try {
await (0, analyzeTables_1.analyzeTables)({ toDsn, tables });
}
catch (e) {
logging_1.log.error(`Post-move ANALYZE failed (ignoring): ${(0, inspectError_1.inspectError)(e)}`);
}
}
catch (e) {
logging_1.log.error(`Commit failed: ${(0, inspectError_1.inspectError)(e)}`);
try {
await unlock("error");
}
catch (e) {
logging_1.log.error(`Unlock failed: ${(0, inspectError_1.inspectError)(e)}`);
}
finally {
unlock = async () => { };
}
const dangerBand = chalk_1.default.bgRed(chalk_1.default.white("💀 DANGER! "
.repeat(300)
.trimEnd()
.substring(0, (logging_1.stdout.columns || 80) * 3)));
throw (0, compact_1.default)([
dangerBand,
"Exited abnormally while committing the result!",
`Schema ${schema} is in half-working state.`,
commitAction &&
`E.g. it may already be inactive on ${fromDsn}, but not yet activated on ${toDsn}.`,
"You will likely need a manual intervention.",
dangerBand,
]).join("\n");
}
}
finally {
logging_1.progress.clear();
await unlock();
}
}
//# sourceMappingURL=move.js.map