datastore-backup
Version:
Programatic Backup of Google Cloud Datastore
105 lines • 6.94 kB
JavaScript
;
/*
* backup.ts
*
* Created by Dr. Maximillian Dornseif 2021-12-20 in datastore-backup 1.0.0
* Copyright (c) 2021 Dr. Maximillian Dornseif
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDataset = void 0;
const datastore_1 = require("@google-cloud/datastore");
// import { Storage } from '@google-cloud/storage';
const ora_1 = __importDefault(require("ora"));
const datastore_backup_1 = require("./lib/datastore-backup");
async function main() {
const datastore = new datastore_1.Datastore(); // { namespace: 'production' }); //{ projectId: 'huwawi2' });
// const datasetName = `${await datastore.getProjectId()}_test`;
// const bigquery = new BigQuery({ projectId: 'huwawisql' });
// await getDataset(bigquery, datasetName, ora('BigQuery'));
const spinner = (0, ora_1.default)().start();
await (0, datastore_backup_1.dumpAllKinds)(datastore, 'hudora-tmp',
// const backupBucket = 'appengine-backups-us';
undefined, undefined, spinner);
// const spinner2 = ora('Startup').start()
// const dir = path.parse(outputUrl).dir
// spinner2.text = `Reading ${dir}`
// // we now have to search the actual kind data
// // gs://hudora-tmp/test8/2021-12-21T10:00:18.725Z/NumberingAncestor/namespace_production/kind_NumberingAncestor/namespace_production_kind_NumberingAncestor.export_metadata
// const storage = new Storage()
// const dirWithoutBucket = dir.split('/').slice(3).join('/')
// const kindFiles = (
// await storage.bucket(backupBucket).getFiles({
// prefix: dirWithoutBucket,
// })
// )[0]
// .map((x) => x.name)
// .filter((x) => x.endsWith('.export_metadata'))
// for (const fname of kindFiles) {
// const newKindName = fname.match(/_kind_(.+)\.export_metadata$/)[1]
// spinner2.text = `Loading ${newKindName}`
// const metadataUrl = `gs://${path.join(backupBucket, fname)}`
// // console.log(meta);
// // console.log('operation', operation);
// const jobOptions = {
// configuration: {
// jobType: 'LOAD',
// load: {
// sourceUris: [metadataUrl],
// destinationTable: {
// projectId: 'huwawisql',
// datasetId: datasetName,
// tableId: newKindName,
// },
// autodetect: true,
// ignoreUnknownValues: false,
// maxBadRecords: 0,
// sourceFormat: 'DATASTORE_BACKUP',
// writeDisposition: 'WRITE_TRUNCATE',
// },
// },
// jobReference: {
// projectId: 'huwawisql',
// jobId: `import-${newKindName}-${datasetName}-${new Date().getTime()}`,
// },
// }
// try {
// const response: JobResponse = await bigquery.createJob(jobOptions)
// const job = response[0]
// const r2 = await job.promise()
// const timeUsed = (r2[0].statistics.endTime - r2[0].statistics.startTime) / 1000
// spinner2.succeed(`Loading ${newKindName} done in ${timeUsed}s`)
// } catch (error) {
// spinner.fail(error.message)
// throw error
// }
// }
}
main().then(console.log).catch(console.error);
async function getDataset(bigquery, datasetName, spinner) {
let dataset;
spinner = spinner || (0, ora_1.default)({ isSilent: true });
spinner.text = `checking BigQuery Dataset ${bigquery.projectId}${datasetName}`;
try {
dataset = bigquery.dataset(datasetName);
await dataset.getTables();
spinner.success(` BigQuery Dataset ${datasetName} exists`);
}
catch (error) {
if (!error.message.startsWith('Not found: Dataset ')) {
throw error;
}
// Create the dataset
const [reply] = await bigquery.createDataset(datasetName, {
location: 'EU',
});
console.log(reply);
dataset = reply[0];
spinner.warn(` BigQuery Dataset ${dataset.id} created`);
}
return dataset;
}
exports.getDataset = getDataset;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYmFja3VwLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2JhY2t1cC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUE7Ozs7O0dBS0c7Ozs7OztBQUdILHVEQUFvRDtBQUNwRCxtREFBbUQ7QUFDbkQsOENBQXNCO0FBQ3RCLDZEQUFzRDtBQUV0RCxLQUFLLFVBQVUsSUFBSTtJQUNqQixNQUFNLFNBQVMsR0FBRyxJQUFJLHFCQUFTLEVBQUUsQ0FBQyxDQUFDLDZEQUE2RDtJQUNoRyxnRUFBZ0U7SUFFaEUsNkRBQTZEO0lBQzdELDREQUE0RDtJQUU1RCxNQUFNLE9BQU8sR0FBRyxJQUFBLGFBQUcsR0FBRSxDQUFDLEtBQUssRUFBRSxDQUFDO0lBRTlCLE1BQU0sSUFBQSwrQkFBWSxFQUNoQixTQUFTLEVBQ1QsWUFBWTtJQUNaLCtDQUErQztJQUMvQyxTQUFTLEVBQ1QsU0FBUyxFQUNULE9BQU8sQ0FDUixDQUFDO0lBRUYsMENBQTBDO0lBRTFDLHdDQUF3QztJQUN4QyxtQ0FBbUM7SUFDbkMsZ0RBQWdEO0lBQ2hELDhLQUE4SztJQUM5SyxnQ0FBZ0M7SUFDaEMsNkRBQTZEO0lBQzdELHNCQUFzQjtJQUN0QixrREFBa0Q7SUFDbEQsZ0NBQWdDO0lBQ2hDLE9BQU87SUFDUCxPQUFPO0lBQ1Asd0JBQXdCO0lBQ3hCLG1EQUFtRDtJQUVuRCxtQ0FBbUM7SUFDbkMsdUVBQXVFO0lBQ3ZFLDZDQUE2QztJQUM3QyxpRUFBaUU7SUFDakUsMEJBQTBCO0lBQzFCLDRDQUE0QztJQUM1Qyx5QkFBeUI7SUFDekIsdUJBQXVCO0lBQ3ZCLHlCQUF5QjtJQUN6QixnQkFBZ0I7SUFDaEIscUNBQXFDO0lBQ3JDLDhCQUE4QjtJQUM5QixvQ0FBb0M7SUFDcEMsb0NBQW9DO0lBQ3BDLGtDQUFrQztJQUNsQyxhQUFhO0lBQ2IsNEJBQTRCO0lBQzVCLHNDQUFzQztJQUN0Qyw0QkFBNEI7SUFDNUIsNENBQTRDO0lBQzVDLDhDQUE4QztJQUM5QyxXQUFXO0lBQ1gsU0FBUztJQUNULHNCQUFzQjtJQUN0QixnQ0FBZ0M7SUFDaEMsK0VBQStFO0lBQy9FLFNBQVM7SUFDVCxNQUFNO0lBRU4sVUFBVTtJQUNWLHlFQUF5RTtJQUN6RSw4QkFBOEI7SUFDOUIscUNBQXFDO0lBQ3JDLHNGQUFzRjtJQUN0RixzRUFBc0U7SUFDdEUsc0JBQXNCO0lBQ3RCLGtDQUFrQztJQUNsQyxrQkFBa0I7SUFDbEIsTUFBTTtJQUNOLElBQUk7QUFDTixDQUFDO0FBRUQsSUFBSSxFQUFFLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBRXZDLEtBQUssVUFBVSxVQUFVLENBQzlCLFFBQWtCLEVBQ2xCLFdBQW1CLEVBQ25CLE9BQWE7SUFFYixJQUFJLE9BQWdCLENBQUM7SUFDckIsT0FBTyxHQUFHLE9BQU8sSUFBSSxJQUFBLGFBQUcsRUFBQyxFQUFFLFFBQVEsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQzdDLE9BQU8sQ0FBQyxJQUFJLEdBQUcsNkJBQTZCLFFBQVEsQ0FBQyxTQUFTLEdBQUcsV0FBVyxFQUFFLENBQUM7SUFDL0UsSUFBSTtRQUNGLE9BQU8sR0FBRyxRQUFRLENBQUMsT0FBTyxDQUFDLFdBQVcsQ0FBQyxDQUFDO1FBQ3hDLE1BQU0sT0FBTyxDQUFDLFNBQVMsRUFBRSxDQUFDO1FBQzFCLE9BQU8sQ0FBQyxPQUFPLENBQUMscUJBQXFCLFdBQVcsU0FBUyxDQUFDLENBQUM7S0FDNUQ7SUFBQyxPQUFPLEtBQUssRUFBRTtRQUNkLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLFVBQVUsQ0FBQyxxQkFBcUIsQ0FBQyxFQUFFO1lBQ3BELE1BQU0sS0FBSyxDQUFDO1NBQ2I7UUFDRCxxQkFBcUI7UUFDckIsTUFBTSxDQUFDLEtBQUssQ0FBQyxHQUFHLE1BQU0sUUFBUSxDQUFDLGFBQWEsQ0FBQyxXQUFXLEVBQUU7WUFDeEQsUUFBUSxFQUFFLElBQUk7U0FDZixDQUFDLENBQUM7UUFDSCxPQUFPLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQ25CLE9BQU8sR0FBRyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7UUFDbkIsT0FBTyxDQUFDLElBQUksQ0FBQyxxQkFBcUIsT0FBTyxDQUFDLEVBQUUsVUFBVSxDQUFDLENBQUM7S0FDekQ7SUFDRCxPQUFPLE9BQU8sQ0FBQztBQUNqQixDQUFDO0FBekJELGdDQXlCQyJ9