datastore-backup
Version:
Programatic Backup of Google Cloud Datastore
98 lines • 6.64 kB
JavaScript
/*
* backup.ts
*
* Created by Dr. Maximillian Dornseif 2021-12-20 in datastore-backup 1.0.0
* Copyright (c) 2021 Dr. Maximillian Dornseif
*/
import { Datastore } from '@google-cloud/datastore';
// import { Storage } from '@google-cloud/storage';
import ora from 'ora';
import { dumpAllKinds } from './lib/datastore-backup';
async function main() {
const datastore = new Datastore(); // { namespace: 'production' }); //{ projectId: 'huwawi2' });
// const datasetName = `${await datastore.getProjectId()}_test`;
// const bigquery = new BigQuery({ projectId: 'huwawisql' });
// await getDataset(bigquery, datasetName, ora('BigQuery'));
const spinner = ora().start();
await dumpAllKinds(datastore, 'hudora-tmp',
// const backupBucket = 'appengine-backups-us';
undefined, undefined, spinner);
// const spinner2 = ora('Startup').start()
// const dir = path.parse(outputUrl).dir
// spinner2.text = `Reading ${dir}`
// // we now have to search the actual kind data
// // gs://hudora-tmp/test8/2021-12-21T10:00:18.725Z/NumberingAncestor/namespace_production/kind_NumberingAncestor/namespace_production_kind_NumberingAncestor.export_metadata
// const storage = new Storage()
// const dirWithoutBucket = dir.split('/').slice(3).join('/')
// const kindFiles = (
// await storage.bucket(backupBucket).getFiles({
// prefix: dirWithoutBucket,
// })
// )[0]
// .map((x) => x.name)
// .filter((x) => x.endsWith('.export_metadata'))
// for (const fname of kindFiles) {
// const newKindName = fname.match(/_kind_(.+)\.export_metadata$/)[1]
// spinner2.text = `Loading ${newKindName}`
// const metadataUrl = `gs://${path.join(backupBucket, fname)}`
// // console.log(meta);
// // console.log('operation', operation);
// const jobOptions = {
// configuration: {
// jobType: 'LOAD',
// load: {
// sourceUris: [metadataUrl],
// destinationTable: {
// projectId: 'huwawisql',
// datasetId: datasetName,
// tableId: newKindName,
// },
// autodetect: true,
// ignoreUnknownValues: false,
// maxBadRecords: 0,
// sourceFormat: 'DATASTORE_BACKUP',
// writeDisposition: 'WRITE_TRUNCATE',
// },
// },
// jobReference: {
// projectId: 'huwawisql',
// jobId: `import-${newKindName}-${datasetName}-${new Date().getTime()}`,
// },
// }
// try {
// const response: JobResponse = await bigquery.createJob(jobOptions)
// const job = response[0]
// const r2 = await job.promise()
// const timeUsed = (r2[0].statistics.endTime - r2[0].statistics.startTime) / 1000
// spinner2.succeed(`Loading ${newKindName} done in ${timeUsed}s`)
// } catch (error) {
// spinner.fail(error.message)
// throw error
// }
// }
}
main().then(console.log).catch(console.error);
export async function getDataset(bigquery, datasetName, spinner) {
let dataset;
spinner = spinner || ora({ isSilent: true });
spinner.text = `checking BigQuery Dataset ${bigquery.projectId}${datasetName}`;
try {
dataset = bigquery.dataset(datasetName);
await dataset.getTables();
spinner.success(` BigQuery Dataset ${datasetName} exists`);
}
catch (error) {
if (!error.message.startsWith('Not found: Dataset ')) {
throw error;
}
// Create the dataset
const [reply] = await bigquery.createDataset(datasetName, {
location: 'EU',
});
console.log(reply);
dataset = reply[0];
spinner.warn(` BigQuery Dataset ${dataset.id} created`);
}
return dataset;
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYmFja3VwLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2JhY2t1cC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7R0FLRztBQUdILE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUNwRCxtREFBbUQ7QUFDbkQsT0FBTyxHQUFHLE1BQU0sS0FBSyxDQUFDO0FBQ3RCLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSx3QkFBd0IsQ0FBQztBQUV0RCxLQUFLLFVBQVUsSUFBSTtJQUNqQixNQUFNLFNBQVMsR0FBRyxJQUFJLFNBQVMsRUFBRSxDQUFDLENBQUMsNkRBQTZEO0lBQ2hHLGdFQUFnRTtJQUVoRSw2REFBNkQ7SUFDN0QsNERBQTREO0lBRTVELE1BQU0sT0FBTyxHQUFHLEdBQUcsRUFBRSxDQUFDLEtBQUssRUFBRSxDQUFDO0lBRTlCLE1BQU0sWUFBWSxDQUNoQixTQUFTLEVBQ1QsWUFBWTtJQUNaLCtDQUErQztJQUMvQyxTQUFTLEVBQ1QsU0FBUyxFQUNULE9BQU8sQ0FDUixDQUFDO0lBRUYsMENBQTBDO0lBRTFDLHdDQUF3QztJQUN4QyxtQ0FBbUM7SUFDbkMsZ0RBQWdEO0lBQ2hELDhLQUE4SztJQUM5SyxnQ0FBZ0M7SUFDaEMsNkRBQTZEO0lBQzdELHNCQUFzQjtJQUN0QixrREFBa0Q7SUFDbEQsZ0NBQWdDO0lBQ2hDLE9BQU87SUFDUCxPQUFPO0lBQ1Asd0JBQXdCO0lBQ3hCLG1EQUFtRDtJQUVuRCxtQ0FBbUM7SUFDbkMsdUVBQXVFO0lBQ3ZFLDZDQUE2QztJQUM3QyxpRUFBaUU7SUFDakUsMEJBQTBCO0lBQzFCLDRDQUE0QztJQUM1Qyx5QkFBeUI7SUFDekIsdUJBQXVCO0lBQ3ZCLHlCQUF5QjtJQUN6QixnQkFBZ0I7SUFDaEIscUNBQXFDO0lBQ3JDLDhCQUE4QjtJQUM5QixvQ0FBb0M7SUFDcEMsb0NBQW9DO0lBQ3BDLGtDQUFrQztJQUNsQyxhQUFhO0lBQ2IsNEJBQTRCO0lBQzVCLHNDQUFzQztJQUN0Qyw0QkFBNEI7SUFDNUIsNENBQTRDO0lBQzVDLDhDQUE4QztJQUM5QyxXQUFXO0lBQ1gsU0FBUztJQUNULHNCQUFzQjtJQUN0QixnQ0FBZ0M7SUFDaEMsK0VBQStFO0lBQy9FLFNBQVM7SUFDVCxNQUFNO0lBRU4sVUFBVTtJQUNWLHlFQUF5RTtJQUN6RSw4QkFBOEI7SUFDOUIscUNBQXFDO0lBQ3JDLHNGQUFzRjtJQUN0RixzRUFBc0U7SUFDdEUsc0JBQXNCO0lBQ3RCLGtDQUFrQztJQUNsQyxrQkFBa0I7SUFDbEIsTUFBTTtJQUNOLElBQUk7QUFDTixDQUFDO0FBRUQsSUFBSSxFQUFFLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBRTlDLE1BQU0sQ0FBQyxLQUFLLFVBQVUsVUFBVSxDQUM5QixRQUFrQixFQUNsQixXQUFtQixFQUNuQixPQUFhO0lBRWIsSUFBSSxPQUFnQixDQUFDO0lBQ3JCLE9BQU8sR0FBRyxPQUFPLElBQUksR0FBRyxDQUFDLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDN0MsT0FBTyxDQUFDLElBQUksR0FBRyw2QkFBNkIsUUFBUSxDQUFDLFNBQVMsR0FBRyxXQUFXLEVBQUUsQ0FBQztJQUMvRSxJQUFJO1FBQ0YsT0FBTyxHQUFHLFFBQVEsQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLENBQUM7UUFDeEMsTUFBTSxPQUFPLENBQUMsU0FBUyxFQUFFLENBQUM7UUFDMUIsT0FBTyxDQUFDLE9BQU8sQ0FBQyxxQkFBcUIsV0FBVyxTQUFTLENBQUMsQ0FBQztLQUM1RDtJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsVUFBVSxDQUFDLHFCQUFxQixDQUFDLEVBQUU7WUFDcEQsTUFBTSxLQUFLLENBQUM7U0FDYjtRQUNELHFCQUFxQjtRQUNyQixNQUFNLENBQUMsS0FBSyxDQUFDLEdBQUcsTUFBTSxRQUFRLENBQUMsYUFBYSxDQUFDLFdBQVcsRUFBRTtZQUN4RCxRQUFRLEVBQUUsSUFBSTtTQUNmLENBQUMsQ0FBQztRQUNILE9BQU8sQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDbkIsT0FBTyxHQUFHLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUNuQixPQUFPLENBQUMsSUFBSSxDQUFDLHFCQUFxQixPQUFPLENBQUMsRUFBRSxVQUFVLENBQUMsQ0FBQztLQUN6RDtJQUNELE9BQU8sT0FBTyxDQUFDO0FBQ2pCLENBQUMifQ==