datastore-backup
Version:
Programatic Backup of Google Cloud Datastore
99 lines • 10.5 kB
JavaScript
import path from 'path';
import ora from 'ora';
import prettyBytes from 'pretty-bytes';
import * as R from 'ramda';
export async function dumpAllKinds(datastore, bucketName, backupName, backupDir = 'bak', spinner) {
spinner = spinner || ora({ isSilent: true });
const outputUrls = [];
backupName = backupName || (await getBackupDefaultName(datastore));
spinner.info(`backup to gs://${bucketName}/${backupDir}/${backupName}`);
spinner.info(`Dumping datastore ${await datastore.getProjectId()}` +
(datastore.namespace ? `for namespace ${datastore.namespace}` : ''));
try {
spinner.start(`Loading Kinds`);
const kindNames = await getKindNames(datastore);
spinner.succeed(`${kindNames.length} Kinds`).start();
for (const sublist of R.splitEvery(100, kindNames)) {
spinner.start();
const { outputUrl } = await dumpKinds(datastore, sublist, bucketName, backupDir, `${backupName}-${outputUrls.length}`, spinner);
outputUrls.push(outputUrl);
}
spinner.start().succeed(`written ${outputUrls}`);
}
catch (error) {
spinner.fail(error.message);
throw error;
}
return outputUrls;
}
async function getBackupDefaultName(datastore) {
return (new Date()
.toISOString()
.replaceAll(/[-:Z]/g, '')
.replaceAll(/\.\d+$/g, '') +
'-' +
(await datastore.getProjectId()) +
(datastore.namespace ? ':' + datastore.namespace : ''));
}
async function dumpKinds(datastore, kindNames, backupBucket, backupDir, backupName, spinner) {
spinner = spinner || ora({ isSilent: true });
const infoText = kindNames.length > 1 ? `${kindNames.length} kinds ` : kindNames[0];
spinner.prefixText = infoText;
spinner.start();
const backupSubDirName = kindNames.length > 1 ? backupName : path.join(backupName, kindNames[0]);
const outputUrlPrefix = `gs://${path.join(backupBucket, backupDir, backupSubDirName)}`;
spinner.info(`Dumping ${kindNames.join(', ')}`);
spinner.info(`Dumping to ${outputUrlPrefix}`).start();
const logProgress = (status) => {
// The counters are https://github.com/dcodeIO/long.js
// const eD: number = status.progressEntities.workCompleted.toNumber();
// const eT: number = status.progressEntities.workEstimated.toNumber();
const bD = status.progressBytes.workCompleted.toNumber();
const bT = status.progressBytes.workEstimated.toNumber();
spinner.text = `${prettyBytes(bD)} of ${prettyBytes(bT)} ${((bD / bT) *
100).toFixed(1)}%`;
};
// https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export
try {
const [operation] = await datastore.export({
outputUrlPrefix,
kinds: kindNames,
namespaces: [datastore.namespace],
});
// see https://googleapis.github.io/gax-nodejs/classes/Operation.html#promise
// operation emits 'progress', 'error' and 'complete'
operation.on('progress', logProgress);
const response = await operation.promise();
const meta = response[1];
const timeUsed = meta.common.endTime.seconds - meta.common.startTime.seconds;
spinner.succeed(`Dumping finished ${meta?.progressEntities?.workCompleted?.toNumber()} records (${prettyBytes(meta?.progressBytes?.workCompleted?.toNumber())}) in ${timeUsed}s`);
return {
outputUrl: response[0].outputUrl,
operation: operation,
result: response,
};
}
catch (error) {
spinner.fail(error.message);
throw error;
}
}
/** Returns a list of all Namespaces in a Datastore for the current namespace.
*/
export async function getNamespaces(datastore) {
const query = datastore.createQuery('__namespace__').select('__key__');
const [entities] = await datastore.runQuery(query);
const namespaces = entities.map((entity) => entity[datastore.KEY].name);
return namespaces;
}
/** Returns a list of all Kinds in a Datastore for the current namespace.
*
* Kinds where the name starts with `_` are supressed.
*/
async function getKindNames(datastore) {
const query = datastore.createQuery('__kind__').select('__key__');
const [entities] = await datastore.runQuery(query);
const kinds = entities.map((entity) => entity[datastore.KEY].name);
return kinds.filter((x) => !x.startsWith('_'));
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGF0YXN0b3JlLWJhY2t1cC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9saWIvZGF0YXN0b3JlLWJhY2t1cC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLElBQUksTUFBTSxNQUFNLENBQUM7QUFHeEIsT0FBTyxHQUFZLE1BQU0sS0FBSyxDQUFDO0FBQy9CLE9BQU8sV0FBVyxNQUFNLGNBQWMsQ0FBQztBQUN2QyxPQUFPLEtBQUssQ0FBQyxNQUFNLE9BQU8sQ0FBQztBQUUzQixNQUFNLENBQUMsS0FBSyxVQUFVLFlBQVksQ0FDaEMsU0FBb0IsRUFDcEIsVUFBa0IsRUFDbEIsVUFBbUIsRUFDbkIsU0FBUyxHQUFHLEtBQUssRUFDakIsT0FBYTtJQUViLE9BQU8sR0FBRyxPQUFPLElBQUksR0FBRyxDQUFDLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDN0MsTUFBTSxVQUFVLEdBQWEsRUFBRSxDQUFDO0lBRWhDLFVBQVUsR0FBRyxVQUFVLElBQUksQ0FBQyxNQUFNLG9CQUFvQixDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUM7SUFDbkUsT0FBTyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsVUFBVSxJQUFJLFNBQVMsSUFBSSxVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQ3hFLE9BQU8sQ0FBQyxJQUFJLENBQ1YscUJBQXFCLE1BQU0sU0FBUyxDQUFDLFlBQVksRUFBRSxFQUFFO1FBQ25ELENBQUMsU0FBUyxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsaUJBQWlCLFNBQVMsQ0FBQyxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQ3RFLENBQUM7SUFFRixJQUFJO1FBQ0YsT0FBTyxDQUFDLEtBQUssQ0FBQyxlQUFlLENBQUMsQ0FBQztRQUMvQixNQUFNLFNBQVMsR0FBRyxNQUFNLFlBQVksQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUNoRCxPQUFPLENBQUMsT0FBTyxDQUFDLEdBQUcsU0FBUyxDQUFDLE1BQU0sUUFBUSxDQUFDLENBQUMsS0FBSyxFQUFFLENBQUM7UUFFckQsS0FBSyxNQUFNLE9BQU8sSUFBSSxDQUFDLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRSxTQUFTLENBQUMsRUFBRTtZQUNsRCxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDaEIsTUFBTSxFQUFFLFNBQVMsRUFBRSxHQUFHLE1BQU0sU0FBUyxDQUNuQyxTQUFTLEVBQ1QsT0FBTyxFQUNQLFVBQVUsRUFDVixTQUFTLEVBQ1QsR0FBRyxVQUFVLElBQUksVUFBVSxDQUFDLE1BQU0sRUFBRSxFQUNwQyxPQUFPLENBQ1IsQ0FBQztZQUNGLFVBQVUsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUM7U0FDNUI7UUFDRCxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUMsT0FBTyxDQUFDLFdBQVcsVUFBVSxFQUFFLENBQUMsQ0FBQztLQUNsRDtJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsT0FBTyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDNUIsTUFBTSxLQUFLLENBQUM7S0FDYjtJQUNELE9BQU8sVUFBVSxDQUFDO0FBQ3BCLENBQUM7QUFFRCxLQUFLLFVBQVUsb0JBQW9CLENBQUMsU0FBb0I7SUFDdEQsT0FBTyxDQUNMLElBQUksSUFBSSxFQUFFO1NBQ1AsV0FBVyxFQUFFO1NBQ2IsVUFBVSxDQUFDLFFBQVEsRUFBRSxFQUFFLENBQUM7U0FDeEIsVUFBVSxDQUFDLFNBQVMsRUFBRSxFQUFFLENBQUM7UUFDNUIsR0FBRztRQUNILENBQUMsTUFBTSxTQUFTLENBQUMsWUFBWSxFQUFFLENBQUM7UUFDaEMsQ0FBQyxTQUFTLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxHQUFHLEdBQUcsU0FBUyxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQ3ZELENBQUM7QUFDSixDQUFDO0FBRUQsS0FBSyxVQUFVLFNBQVMsQ0FDdEIsU0FBb0IsRUFDcEIsU0FBbUIsRUFDbkIsWUFBb0IsRUFDcEIsU0FBaUIsRUFDakIsVUFBa0IsRUFDbEIsT0FBYTtJQUViLE9BQU8sR0FBRyxPQUFPLElBQUksR0FBRyxDQUFDLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDN0MsTUFBTSxRQUFRLEdBQ1osU0FBUyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLEdBQUcsU0FBUyxDQUFDLE1BQU0sU0FBUyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDckUsT0FBTyxDQUFDLFVBQVUsR0FBRyxRQUFRLENBQUM7SUFDOUIsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO0lBQ2hCLE1BQU0sZ0JBQWdCLEdBQ3BCLFNBQVMsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsVUFBVSxFQUFFLFNBQVMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQzFFLE1BQU0sZUFBZSxHQUFHLFFBQVEsSUFBSSxDQUFDLElBQUksQ0FDdkMsWUFBWSxFQUNaLFNBQVMsRUFDVCxnQkFBZ0IsQ0FDakIsRUFBRSxDQUFDO0lBQ0osT0FBTyxDQUFDLElBQUksQ0FBQyxXQUFXLFNBQVMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ2hELE9BQU8sQ0FBQyxJQUFJLENBQUMsY0FBYyxlQUFlLEVBQUUsQ0FBQyxDQUFDLEtBQUssRUFBRSxDQUFDO0lBRXRELE1BQU0sV0FBVyxHQUFHLENBQUMsTUFBTSxFQUFFLEVBQUU7UUFDN0Isc0RBQXNEO1FBQ3RELHVFQUF1RTtRQUN2RSx1RUFBdUU7UUFDdkUsTUFBTSxFQUFFLEdBQVcsTUFBTSxDQUFDLGFBQWEsQ0FBQyxhQUFhLENBQUMsUUFBUSxFQUFFLENBQUM7UUFDakUsTUFBTSxFQUFFLEdBQVcsTUFBTSxDQUFDLGFBQWEsQ0FBQyxhQUFhLENBQUMsUUFBUSxFQUFFLENBQUM7UUFDakUsT0FBTyxDQUFDLElBQUksR0FBRyxHQUFHLFdBQVcsQ0FBQyxFQUFFLENBQUMsT0FBTyxXQUFXLENBQUMsRUFBRSxDQUFDLElBQUksQ0FDekQsQ0FBQyxFQUFFLEdBQUcsRUFBRSxDQUFDO1lBQ1QsR0FBRyxDQUNKLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUM7SUFDbEIsQ0FBQyxDQUFDO0lBRUYsa0ZBQWtGO0lBQ2xGLElBQUk7UUFDRixNQUFNLENBQUMsU0FBUyxDQUFDLEdBQUcsTUFBTSxTQUFTLENBQUMsTUFBTSxDQUFDO1lBQ3pDLGVBQWU7WUFDZixLQUFLLEVBQUUsU0FBUztZQUNoQixVQUFVLEVBQUUsQ0FBQyxTQUFTLENBQUMsU0FBUyxDQUFDO1NBQ2xDLENBQUMsQ0FBQztRQUNILDZFQUE2RTtRQUM3RSxxREFBcUQ7UUFDckQsU0FBUyxDQUFDLEVBQUUsQ0FBQyxVQUFVLEVBQUUsV0FBVyxDQUFDLENBQUM7UUFDdEMsTUFBTSxRQUFRLEdBQUcsTUFBTSxTQUFTLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDM0MsTUFBTSxJQUFJLEdBQUcsUUFBUSxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQ3pCLE1BQU0sUUFBUSxHQUNaLElBQUksQ0FBQyxNQUFNLENBQUMsT0FBTyxDQUFDLE9BQU8sR0FBRyxJQUFJLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxPQUFPLENBQUM7UUFDOUQsT0FBTyxDQUFDLE9BQU8sQ0FDYixvQkFBb0IsSUFBSSxFQUFFLGdCQUFnQixFQUFFLGFBQWEsRUFBRSxRQUFRLEVBQUUsYUFBYSxXQUFXLENBQzNGLElBQUksRUFBRSxhQUFhLEVBQUUsYUFBYSxFQUFFLFFBQVEsRUFBRSxDQUMvQyxRQUFRLFFBQVEsR0FBRyxDQUNyQixDQUFDO1FBQ0YsT0FBTztZQUNMLFNBQVMsRUFBRSxRQUFRLENBQUMsQ0FBQyxDQUFDLENBQUMsU0FBUztZQUNoQyxTQUFTLEVBQUUsU0FBUztZQUNwQixNQUFNLEVBQUUsUUFBUTtTQUNqQixDQUFDO0tBQ0g7SUFBQyxPQUFPLEtBQUssRUFBRTtRQUNkLE9BQU8sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQzVCLE1BQU0sS0FBSyxDQUFDO0tBQ2I7QUFDSCxDQUFDO0FBQ0Q7R0FDRztBQUVILE1BQU0sQ0FBQyxLQUFLLFVBQVUsYUFBYSxDQUFDLFNBQW9CO0lBQ3RELE1BQU0sS0FBSyxHQUFHLFNBQVMsQ0FBQyxXQUFXLENBQUMsZUFBZSxDQUFDLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDO0lBQ3ZFLE1BQU0sQ0FBQyxRQUFRLENBQUMsR0FBRyxNQUFNLFNBQVMsQ0FBQyxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDbkQsTUFBTSxVQUFVLEdBQUcsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUN4RSxPQUFPLFVBQVUsQ0FBQztBQUNwQixDQUFDO0FBQ0Q7OztHQUdHO0FBQ0gsS0FBSyxVQUFVLFlBQVksQ0FBQyxTQUFvQjtJQUM5QyxNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsV0FBVyxDQUFDLFVBQVUsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUNsRSxNQUFNLENBQUMsUUFBUSxDQUFDLEdBQUcsTUFBTSxTQUFTLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ25ELE1BQU0sS0FBSyxHQUFHLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDbkUsT0FBTyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztBQUNqRCxDQUFDIn0=