datastore-backup
Version:
Programatic Backup of Google Cloud Datastore
127 lines • 12.2 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getNamespaces = exports.dumpAllKinds = void 0;
const path_1 = __importDefault(require("path"));
const ora_1 = __importDefault(require("ora"));
const pretty_bytes_1 = __importDefault(require("pretty-bytes"));
const R = __importStar(require("ramda"));
async function dumpAllKinds(datastore, bucketName, backupName, backupDir = 'bak', spinner) {
spinner = spinner || (0, ora_1.default)({ isSilent: true });
const outputUrls = [];
backupName = backupName || (await getBackupDefaultName(datastore));
spinner.info(`backup to gs://${bucketName}/${backupDir}/${backupName}`);
spinner.info(`Dumping datastore ${await datastore.getProjectId()}` +
(datastore.namespace ? `for namespace ${datastore.namespace}` : ''));
try {
spinner.start(`Loading Kinds`);
const kindNames = await getKindNames(datastore);
spinner.succeed(`${kindNames.length} Kinds`).start();
for (const sublist of R.splitEvery(100, kindNames)) {
spinner.start();
const { outputUrl } = await dumpKinds(datastore, sublist, bucketName, backupDir, `${backupName}-${outputUrls.length}`, spinner);
outputUrls.push(outputUrl);
}
spinner.start().succeed(`written ${outputUrls}`);
}
catch (error) {
spinner.fail(error.message);
throw error;
}
return outputUrls;
}
exports.dumpAllKinds = dumpAllKinds;
async function getBackupDefaultName(datastore) {
return (new Date()
.toISOString()
.replaceAll(/[-:Z]/g, '')
.replaceAll(/\.\d+$/g, '') +
'-' +
(await datastore.getProjectId()) +
(datastore.namespace ? `:${datastore.namespace}` : ''));
}
async function dumpKinds(datastore, kindNames, backupBucket, backupDir, backupName, spinner) {
var _a, _b, _c, _d;
spinner = spinner || (0, ora_1.default)({ isSilent: true });
const infoText = kindNames.length > 1 ? `${kindNames.length} kinds ` : kindNames[0];
spinner.prefixText = infoText;
spinner.start();
const backupSubDirName = kindNames.length > 1 ? backupName : path_1.default.join(backupName, kindNames[0]);
const outputUrlPrefix = `gs://${path_1.default.join(backupBucket, backupDir, backupSubDirName)}`;
spinner.info(`Dumping ${kindNames.join(', ')}`);
spinner.info(`Dumping to ${outputUrlPrefix}`).start();
const logProgress = (status) => {
// The counters are https://github.com/dcodeIO/long.js
// const eD: number = status.progressEntities.workCompleted.toNumber();
// const eT: number = status.progressEntities.workEstimated.toNumber();
const bD = status.progressBytes.workCompleted.toNumber();
const bT = status.progressBytes.workEstimated.toNumber();
spinner.text = `${(0, pretty_bytes_1.default)(bD)} of ${(0, pretty_bytes_1.default)(bT)} ${((bD / bT) *
100).toFixed(1)}%`;
};
// https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export
try {
const [operation] = await datastore.export({
outputUrlPrefix,
kinds: kindNames,
namespaces: [datastore.namespace],
});
// see https://googleapis.github.io/gax-nodejs/classes/Operation.html#promise
// operation emits 'progress', 'error' and 'complete'
operation.on('progress', logProgress);
const response = await operation.promise();
const meta = response[1];
const timeUsed = meta.common.endTime.seconds - meta.common.startTime.seconds;
spinner.succeed(`Dumping finished ${(_b = (_a = meta === null || meta === void 0 ? void 0 : meta.progressEntities) === null || _a === void 0 ? void 0 : _a.workCompleted) === null || _b === void 0 ? void 0 : _b.toNumber()} records (${(0, pretty_bytes_1.default)((_d = (_c = meta === null || meta === void 0 ? void 0 : meta.progressBytes) === null || _c === void 0 ? void 0 : _c.workCompleted) === null || _d === void 0 ? void 0 : _d.toNumber())}) in ${timeUsed}s`);
return {
outputUrl: response[0].outputUrl,
operation,
result: response,
};
}
catch (error) {
spinner.fail(error.message);
throw error;
}
}
/** Returns a list of all Namespaces in a Datastore for the current namespace.
*/
async function getNamespaces(datastore) {
const query = datastore.createQuery('__namespace__').select('__key__');
const [entities] = await datastore.runQuery(query);
const namespaces = entities.map((entity) => entity[datastore.KEY].name);
return namespaces;
}
exports.getNamespaces = getNamespaces;
/** Returns a list of all Kinds in a Datastore for the current namespace.
*
* Kinds where the name starts with `_` are suppressed.
*/
async function getKindNames(datastore) {
const query = datastore.createQuery('__kind__').select('__key__');
const [entities] = await datastore.runQuery(query);
const kinds = entities.map((entity) => entity[datastore.KEY].name);
return kinds.filter((x) => !x.startsWith('_'));
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGF0YXN0b3JlLWJhY2t1cC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9saWIvZGF0YXN0b3JlLWJhY2t1cC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsZ0RBQXdCO0FBR3hCLDhDQUErQjtBQUMvQixnRUFBdUM7QUFDdkMseUNBQTJCO0FBRXBCLEtBQUssVUFBVSxZQUFZLENBQ2hDLFNBQW9CLEVBQ3BCLFVBQWtCLEVBQ2xCLFVBQW1CLEVBQ25CLFNBQVMsR0FBRyxLQUFLLEVBQ2pCLE9BQWE7SUFFYixPQUFPLEdBQUcsT0FBTyxJQUFJLElBQUEsYUFBRyxFQUFDLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDN0MsTUFBTSxVQUFVLEdBQWEsRUFBRSxDQUFDO0lBRWhDLFVBQVUsR0FBRyxVQUFVLElBQUksQ0FBQyxNQUFNLG9CQUFvQixDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUM7SUFDbkUsT0FBTyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsVUFBVSxJQUFJLFNBQVMsSUFBSSxVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQ3hFLE9BQU8sQ0FBQyxJQUFJLENBQ1YscUJBQXFCLE1BQU0sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFO1FBQ25ELENBQUMsU0FBUyxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsaUJBQWlCLFNBQVMsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQ3RFLENBQUM7SUFFRixJQUFJO1FBQ0YsT0FBTyxDQUFDLEtBQUssQ0FBQyxlQUFlLENBQUMsQ0FBQztRQUMvQixNQUFNLFNBQVMsR0FBRyxNQUFNLFlBQVksQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUNoRCxPQUFPLENBQUMsT0FBTyxDQUFDLEdBQUcsU0FBUyxDQUFDLE1BQU0sUUFBUSxDQUFDLENBQUMsS0FBSyxFQUFFLENBQUM7UUFFckQsS0FBSyxNQUFNLE9BQU8sSUFBSSxDQUFDLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRSxTQUFTLENBQUMsRUFBRTtZQUNsRCxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDaEIsTUFBTSxFQUFFLFNBQVMsRUFBRSxHQUFHLE1BQU0sU0FBUyxDQUNuQyxTQUFTLEVBQ1QsT0FBTyxFQUNQLFVBQVUsRUFDVixTQUFTLEVBQ1QsR0FBRyxVQUFVLElBQUksVUFBVSxDQUFDLE1BQU0sRUFBRSxFQUNwQyxPQUFPLENBQ1IsQ0FBQztZQUNGLFVBQVUsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUM7U0FDNUI7UUFDRCxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUMsT0FBTyxDQUFDLFdBQVcsVUFBVSxFQUFFLENBQUMsQ0FBQztLQUNsRDtJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsT0FBTyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDNUIsTUFBTSxLQUFLLENBQUM7S0FDYjtJQUNELE9BQU8sVUFBVSxDQUFDO0FBQ3BCLENBQUM7QUF4Q0Qsb0NBd0NDO0FBRUQsS0FBSyxVQUFVLG9CQUFvQixDQUFDLFNBQW9CO0lBQ3RELE9BQU8sQ0FDTCxJQUFJLElBQUksRUFBRTtTQUNQLFdBQVcsRUFBRTtTQUNiLFVBQVUsQ0FBQyxRQUFRLEVBQUUsRUFBRSxDQUFDO1NBQ3hCLFVBQVUsQ0FBQyxTQUFTLEVBQUUsRUFBRSxDQUFDO1FBQzVCLEdBQUc7UUFDSCxDQUFDLE1BQU0sU0FBUyxDQUFDLFlBQVksRUFBRSxDQUFDO1FBQ2hDLENBQUMsU0FBUyxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsSUFBSSxTQUFTLENBQUMsU0FBUyxFQUFFLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUN2RCxDQUFDO0FBQ0osQ0FBQztBQUVELEtBQUssVUFBVSxTQUFTLENBQ3RCLFNBQW9CLEVBQ3BCLFNBQW1CLEVBQ25CLFlBQW9CLEVBQ3BCLFNBQWlCLEVBQ2pCLFVBQWtCLEVBQ2xCLE9BQWE7O0lBRWIsT0FBTyxHQUFHLE9BQU8sSUFBSSxJQUFBLGFBQUcsRUFBQyxFQUFFLFFBQVEsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzdDLE1BQU0sUUFBUSxHQUNaLFNBQVMsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxHQUFHLFNBQVMsQ0FBQyxNQUFNLFNBQVMsQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQ3JFLE9BQU8sQ0FBQyxVQUFVLEdBQUcsUUFBUSxDQUFDO0lBQzlCLE9BQU8sQ0FBQyxLQUFLLEVBQUUsQ0FBQztJQUNoQixNQUFNLGdCQUFnQixHQUNwQixTQUFTLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxjQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsRUFBRSxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMxRSxNQUFNLGVBQWUsR0FBRyxRQUFRLGNBQUksQ0FBQyxJQUFJLENBQ3ZDLFlBQVksRUFDWixTQUFTLEVBQ1QsZ0JBQWdCLENBQ2pCLEVBQUUsQ0FBQztJQUNKLE9BQU8sQ0FBQyxJQUFJLENBQUMsV0FBVyxTQUFTLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUNoRCxPQUFPLENBQUMsSUFBSSxDQUFDLGNBQWMsZUFBZSxFQUFFLENBQUMsQ0FBQyxLQUFLLEVBQUUsQ0FBQztJQUV0RCxNQUFNLFdBQVcsR0FBRyxDQUFDLE1BQU0sRUFBRSxFQUFFO1FBQzdCLHNEQUFzRDtRQUN0RCx1RUFBdUU7UUFDdkUsdUVBQXVFO1FBQ3ZFLE1BQU0sRUFBRSxHQUFXLE1BQU0sQ0FBQyxhQUFhLENBQUMsYUFBYSxDQUFDLFFBQVEsRUFBRSxDQUFDO1FBQ2pFLE1BQU0sRUFBRSxHQUFXLE1BQU0sQ0FBQyxhQUFhLENBQUMsYUFBYSxDQUFDLFFBQVEsRUFBRSxDQUFDO1FBQ2pFLE9BQU8sQ0FBQyxJQUFJLEdBQUcsR0FBRyxJQUFBLHNCQUFXLEVBQUMsRUFBRSxDQUFDLE9BQU8sSUFBQSxzQkFBVyxFQUFDLEVBQUUsQ0FBQyxJQUFJLENBQ3pELENBQUMsRUFBRSxHQUFHLEVBQUUsQ0FBQztZQUNULEdBQUcsQ0FDSixDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDO0lBQ2xCLENBQUMsQ0FBQztJQUVGLGtGQUFrRjtJQUNsRixJQUFJO1FBQ0YsTUFBTSxDQUFDLFNBQVMsQ0FBQyxHQUFHLE1BQU0sU0FBUyxDQUFDLE1BQU0sQ0FBQztZQUN6QyxlQUFlO1lBQ2YsS0FBSyxFQUFFLFNBQVM7WUFDaEIsVUFBVSxFQUFFLENBQUMsU0FBUyxDQUFDLFNBQVMsQ0FBQztTQUNsQyxDQUFDLENBQUM7UUFDSCw2RUFBNkU7UUFDN0UscURBQXFEO1FBQ3JELFNBQVMsQ0FBQyxFQUFFLENBQUMsVUFBVSxFQUFFLFdBQVcsQ0FBQyxDQUFDO1FBQ3RDLE1BQU0sUUFBUSxHQUFHLE1BQU0sU0FBUyxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQzNDLE1BQU0sSUFBSSxHQUFHLFFBQVEsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUN6QixNQUFNLFFBQVEsR0FDWixJQUFJLENBQUMsTUFBTSxDQUFDLE9BQU8sQ0FBQyxPQUFPLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsT0FBTyxDQUFDO1FBQzlELE9BQU8sQ0FBQyxPQUFPLENBQ2Isb0JBQW9CLE1BQUEsTUFBQSxJQUFJLGFBQUosSUFBSSx1QkFBSixJQUFJLENBQUUsZ0JBQWdCLDBDQUFFLGFBQWEsMENBQUUsUUFBUSxFQUFFLGFBQWEsSUFBQSxzQkFBVyxFQUMzRixNQUFBLE1BQUEsSUFBSSxhQUFKLElBQUksdUJBQUosSUFBSSxDQUFFLGFBQWEsMENBQUUsYUFBYSwwQ0FBRSxRQUFRLEVBQUUsQ0FDL0MsUUFBUSxRQUFRLEdBQUcsQ0FDckIsQ0FBQztRQUNGLE9BQU87WUFDTCxTQUFTLEVBQUUsUUFBUSxDQUFDLENBQUMsQ0FBQyxDQUFDLFNBQVM7WUFDaEMsU0FBUztZQUNULE1BQU0sRUFBRSxRQUFRO1NBQ2pCLENBQUM7S0FDSDtJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsT0FBTyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDNUIsTUFBTSxLQUFLLENBQUM7S0FDYjtBQUNILENBQUM7QUFDRDtHQUNHO0FBRUksS0FBSyxVQUFVLGFBQWEsQ0FBQyxTQUFvQjtJQUN0RCxNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsV0FBVyxDQUFDLGVBQWUsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUN2RSxNQUFNLENBQUMsUUFBUSxDQUFDLEdBQUcsTUFBTSxTQUFTLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ25ELE1BQU0sVUFBVSxHQUFHLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDeEUsT0FBTyxVQUFVLENBQUM7QUFDcEIsQ0FBQztBQUxELHNDQUtDO0FBQ0Q7OztHQUdHO0FBQ0gsS0FBSyxVQUFVLFlBQVksQ0FBQyxTQUFvQjtJQUM5QyxNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsV0FBVyxDQUFDLFVBQVUsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUNsRSxNQUFNLENBQUMsUUFBUSxDQUFDLEdBQUcsTUFBTSxTQUFTLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ25ELE1BQU0sS0FBSyxHQUFHLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDbkUsT0FBTyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztBQUNqRCxDQUFDIn0=