dynoport
Version:
Dynoport is a CLI tool that allows you to easily import and export data from a specified DynamoDB table. It provides a convenient way to transfer data between DynamoDB and JSON files
520 lines (519 loc) • 26.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/* eslint-disable @typescript-eslint/no-unsafe-call */
const commander_1 = require("commander");
// eslint-disable-next-line node/no-extraneous-import
const aws_sdk_1 = __importDefault(require("aws-sdk"));
const fs_1 = __importDefault(require("fs"));
const ndjson = __importStar(require("ndjson"));
const chunk_1 = __importDefault(require("lodash/chunk"));
const bluebird = __importStar(require("bluebird"));
const spinner_helper_1 = require("./spinner-helper");
const chalk_1 = __importDefault(require("chalk"));
const inquirer_1 = __importDefault(require("inquirer"));
const path_1 = __importDefault(require("path"));
const packageJson = require('../package.json');
const version = packageJson.version;
const program = new commander_1.Command();
program
.version(version)
.name('dynamo-tool')
.option('-t, --table <tableName>', 'DynamoDB table name')
.option('-f, --filePath <filePath>', 'Output JSON file path')
.option('-r, --region <region>', 'AWS region to use')
.option('-m, --mode <mode>', 'Mode [export|import|wizard]', 'wizard')
.parse(process.argv);
// Helper function to format elapsed time
function formatElapsedTime(milliseconds) {
const seconds = Math.floor(milliseconds / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
if (hours > 0) {
return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
}
else if (minutes > 0) {
return `${minutes}m ${seconds % 60}s`;
}
else {
return `${seconds}s`;
}
}
// Display ASCII art logo
function displayLogo() {
console.log(chalk_1.default.cyan(`
╔═══════════════════════════════════════════╗
║ ║
║ ║
║ ║
║ DYNOPORT ║
║ ║
║ ║
║ ║
║ CLI Tool v${version.padEnd(27)}║
╚═══════════════════════════════════════════╝
`));
}
function exportTableToJson(tableName, outputFilePath, region) {
var _a, _b, _c;
return __awaiter(this, void 0, void 0, function* () {
const mainSpinner = (0, spinner_helper_1.createSpinner)(`Preparing to export table '${tableName}'...`).start();
const startTime = Date.now();
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const dynamodb = new aws_sdk_1.default.DynamoDB.DocumentClient({ region });
// Get table info for reporting
try {
const dynamodbRaw = new aws_sdk_1.default.DynamoDB({ region });
const tableInfo = yield dynamodbRaw
.describeTable({ TableName: tableName })
.promise();
const itemCount = ((_a = tableInfo.Table) === null || _a === void 0 ? void 0 : _a.ItemCount) || 0;
const tableSizeBytes = ((_b = tableInfo.Table) === null || _b === void 0 ? void 0 : _b.TableSizeBytes) || 0;
const tableSizeMB = (tableSizeBytes / (1024 * 1024)).toFixed(2);
mainSpinner.info(`Table info: ~${itemCount.toLocaleString()} items, ${tableSizeMB} MB`);
}
catch (error) {
mainSpinner.info('Could not retrieve table size information');
}
mainSpinner.text = `Exporting table '${tableName}' to '${outputFilePath}'...`;
const scanParams = {
TableName: tableName,
};
let totalCount = 0;
const outputStream = fs_1.default.createWriteStream(outputFilePath, { flags: 'a' });
let progressSpinner = (0, spinner_helper_1.createSpinner)('Scanning items...').start();
try {
let batchNumber = 1;
do {
progressSpinner.text = `Batch #${batchNumber}: Scanning items...`;
const scanResult = yield dynamodb.scan(scanParams).promise();
scanParams.ExclusiveStartKey = scanResult.LastEvaluatedKey;
const batchCount = ((_c = scanResult.Items) === null || _c === void 0 ? void 0 : _c.length) || 0;
totalCount += batchCount;
progressSpinner.succeed(`Batch #${batchNumber}: Retrieved ${batchCount} items (total: ${totalCount})`);
if (scanResult.Items && scanResult.Items.length > 0) {
const writeSpinner = (0, spinner_helper_1.createSpinner)(`Writing ${batchCount} items to file...`).start();
scanResult.Items.forEach(item => {
const timestampedObj = Object.assign({}, item);
const jsonString = JSON.stringify(timestampedObj);
outputStream.write(jsonString + '\n');
});
writeSpinner.succeed(`Wrote ${batchCount} items to file`);
}
if (scanParams.ExclusiveStartKey) {
batchNumber++;
progressSpinner = (0, spinner_helper_1.createSpinner)(`Batch #${batchNumber}: Scanning items...`).start();
}
} while (scanParams.ExclusiveStartKey);
outputStream.end();
const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(2);
const elapsedTimeFormatted = formatElapsedTime(Date.now() - startTime);
mainSpinner.succeed(chalk_1.default.green('✨ Export completed successfully!'));
// Summary box
console.log(chalk_1.default.cyan('\n┌─────────────────────────────────────────┐'));
console.log(chalk_1.default.cyan('│ EXPORT SUMMARY │'));
console.log(chalk_1.default.cyan('├─────────────────────────────────────────┤'));
console.log(chalk_1.default.cyan(`│ Table: ${tableName.substring(0, 27).padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Items: ${totalCount.toString().padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Output: ${path_1.default
.basename(outputFilePath)
.substring(0, 27)
.padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Time taken: ${elapsedTimeFormatted.padEnd(27)} │`));
console.log(chalk_1.default.cyan('└─────────────────────────────────────────┘\n'));
}
catch (error) {
progressSpinner.fail('Error exporting DynamoDB table:');
console.error(chalk_1.default.red(error));
mainSpinner.fail(chalk_1.default.red('Export failed!'));
}
});
}
// eslint-disable-next-line @typescript-eslint/require-await
function importJsonToTable(tableName, filePath, region) {
return __awaiter(this, void 0, void 0, function* () {
const mainSpinner = (0, spinner_helper_1.createSpinner)(`Preparing to import data to '${tableName}'...`).start();
const startTime = Date.now();
// Get file size for reporting
try {
const stats = fs_1.default.statSync(filePath);
const fileSizeMB = (stats.size / (1024 * 1024)).toFixed(2);
mainSpinner.info(`File info: ${fileSizeMB} MB`);
}
catch (error) {
mainSpinner.info('Could not retrieve file size information');
}
mainSpinner.text = `Importing JSON from '${filePath}' to table '${tableName}'...`;
// Create an AWS DynamoDB client
const dynamoDB = new aws_sdk_1.default.DynamoDB.DocumentClient({ region });
// Read the NDJSON file
const fileStream = fs_1.default.createReadStream(filePath);
// Parse the NDJSON data
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const parser = ndjson.parse();
// Chunk size for parallel requests
const chunkSize = 25; // Adjust this as needed
// Array to hold batch write requests
const writeRequests = [];
let itemCount = 0;
let successCount = 0;
let errorCount = 0;
// Event handler for each parsed item
// eslint-disable-next-line @typescript-eslint/no-misused-promises, @typescript-eslint/require-await
parser.on('data', (item) => __awaiter(this, void 0, void 0, function* () {
itemCount++;
// Create a new batch write request for each item
writeRequests.push({
PutRequest: {
Item: item,
},
});
if (itemCount % 1000 === 0) {
mainSpinner.text = `Reading data: ${itemCount.toLocaleString()} items processed...`;
}
}));
// Event handler for the end of the file
// eslint-disable-next-line @typescript-eslint/no-misused-promises
parser.on('end', () => __awaiter(this, void 0, void 0, function* () {
var _a, e_1, _b, _c;
mainSpinner.succeed(`Read ${itemCount.toLocaleString()} items from file`);
let progressSpinner = (0, spinner_helper_1.createSpinner)(`Preparing to write ${itemCount} items to DynamoDB...`).start();
// Execute any remaining batch write requests
let batch = 0;
try {
const totalChunks = Math.ceil(writeRequests.length / 500);
try {
for (var _d = true, _e = __asyncValues((0, chunk_1.default)(writeRequests, 500)), _f; _f = yield _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const chunkedWriteRequest = _c;
batch++;
progressSpinner.text = `Processing batch ${batch}/${totalChunks} (${chunkedWriteRequest.length} items)`;
const writeRequestsPrime = (0, chunk_1.default)(chunkedWriteRequest, 25);
const totalBatches = writeRequestsPrime.length;
let processedBatch = 0;
yield bluebird.Promise.map(writeRequestsPrime, (writeRequestPrime) => __awaiter(this, void 0, void 0, function* () {
var _g, _h;
processedBatch++;
if (processedBatch % 5 === 0) {
progressSpinner.text = `Batch ${batch}/${totalChunks}: Writing sub-batch ${processedBatch}/${totalBatches}`;
}
const params = {
RequestItems: {
[tableName]: writeRequestPrime,
},
};
try {
const result = yield dynamoDB.batchWrite(params).promise();
// Count successful items
const unprocessedItems = ((_h = (_g = result.UnprocessedItems) === null || _g === void 0 ? void 0 : _g[tableName]) === null || _h === void 0 ? void 0 : _h.length) || 0;
successCount += writeRequestPrime.length - unprocessedItems;
errorCount += unprocessedItems;
}
catch (err) {
errorCount += writeRequestPrime.length;
console.error(chalk_1.default.red(`Error in batch: ${err.message}`));
}
}), { concurrency: 5 });
progressSpinner.succeed(`Completed batch ${batch}/${totalChunks}`);
if (batch < totalChunks) {
progressSpinner = (0, spinner_helper_1.createSpinner)(`Processing batch ${batch + 1}/${totalChunks}...`).start();
}
}
finally {
_d = true;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
}
finally { if (e_1) throw e_1.error; }
}
const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(2);
const elapsedTimeFormatted = formatElapsedTime(Date.now() - startTime);
mainSpinner.succeed(chalk_1.default.green('✨ Import completed!'));
// Summary box
console.log(chalk_1.default.cyan('\n┌─────────────────────────────────────────┐'));
console.log(chalk_1.default.cyan('│ IMPORT SUMMARY │'));
console.log(chalk_1.default.cyan('├─────────────────────────────────────────┤'));
console.log(chalk_1.default.cyan(`│ Table: ${tableName.substring(0, 27).padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Total items: ${itemCount.toString().padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Successful: ${successCount.toString().padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Failed: ${errorCount.toString().padEnd(27)} │`));
console.log(chalk_1.default.cyan(`│ Time taken: ${elapsedTimeFormatted.padEnd(27)} │`));
console.log(chalk_1.default.cyan('└─────────────────────────────────────────┘\n'));
}
catch (err) {
progressSpinner.fail(`Bulk write failed: ${err}`);
mainSpinner.fail(chalk_1.default.red('Import failed!'));
}
}));
// Pipe the file stream through the NDJSON parser
fileStream.pipe(parser);
});
}
// eslint-disable-next-line @typescript-eslint/require-await
function listAwsRegions() {
return __awaiter(this, void 0, void 0, function* () {
// You could fetch this from AWS SDK or use a predefined list
return [
'us-east-1',
'us-east-2',
'us-west-1',
'us-west-2',
'eu-west-1',
'eu-central-1',
'ap-northeast-1',
'ap-southeast-1',
'ap-southeast-2',
];
});
}
function listTablesInRegion(region) {
return __awaiter(this, void 0, void 0, function* () {
const spinner = (0, spinner_helper_1.createSpinner)('Fetching tables from selected region...').start();
try {
const dynamodb = new aws_sdk_1.default.DynamoDB({ region });
const result = yield dynamodb.listTables({}).promise();
spinner.succeed('Tables fetched successfully');
return result.TableNames || [];
}
catch (error) {
spinner.fail('Failed to fetch tables');
console.error(error);
return [];
}
});
}
function runWizard() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
console.log(chalk_1.default.blue('=== DynamoDB Backup Wizard ==='));
// Step 1: Choose operation
const { operation } = yield inquirer_1.default.prompt([
{
type: 'list',
name: 'operation',
message: 'What would you like to do?',
choices: [
{ name: 'Export a table to JSON file', value: 'export' },
{ name: 'Import a JSON file to a table', value: 'import' },
],
},
]);
const ec2 = new aws_sdk_1.default.EC2({ region: 'us-east-1' }); // Use any region to initialize
const response = yield ec2.describeRegions({}).promise();
// Step 2: Choose region
const regions = ((_a = response.Regions) === null || _a === void 0 ? void 0 : _a.map(region => region.RegionName || '')) || [];
const { selectedRegion } = yield inquirer_1.default.prompt([
{
type: 'list',
name: 'selectedRegion',
message: 'Select AWS region:',
choices: regions,
},
]);
if (operation === 'export') {
// Export workflow
// Step 3: Choose table
const spinner = (0, spinner_helper_1.createSpinner)('Connecting to AWS...').start();
const tables = yield listTablesInRegion(selectedRegion);
spinner.stop();
if (tables.length === 0) {
console.log(chalk_1.default.yellow('No tables found in the selected region.'));
return;
}
const { selectedTable } = yield inquirer_1.default.prompt([
{
type: 'list',
name: 'selectedTable',
message: 'Select DynamoDB table to export:',
choices: tables,
},
]);
// Step 4: Choose output file
const { outputFilePath } = yield inquirer_1.default.prompt([
{
type: 'input',
name: 'outputFilePath',
message: 'Enter output file path:',
default: `./${selectedTable}-backup-${new Date().toISOString().split('T')[0]}.json`,
},
]);
// Step 5: Confirm and execute
const { confirm } = yield inquirer_1.default.prompt([
{
type: 'confirm',
name: 'confirm',
message: `Ready to export table '${selectedTable}' to '${outputFilePath}'?`,
default: true,
},
]);
if (confirm) {
yield exportTableToJson(selectedTable, outputFilePath, selectedRegion);
}
else {
console.log(chalk_1.default.yellow('Export cancelled.'));
}
}
else {
// Import workflow
// Step 3: Choose input file
const { inputFilePath } = yield inquirer_1.default.prompt([
{
type: 'input',
name: 'inputFilePath',
message: 'Enter the path to the JSON file to import:',
validate: input => {
if (!fs_1.default.existsSync(input)) {
return 'File does not exist. Please enter a valid file path.';
}
return true;
},
},
]);
// Step 4: Choose target table
const spinner = (0, spinner_helper_1.createSpinner)('Connecting to AWS...').start();
const tables = yield listTablesInRegion(selectedRegion);
spinner.stop();
let selectedTable;
if (tables.length === 0) {
const { newTableName } = yield inquirer_1.default.prompt([
{
type: 'input',
name: 'newTableName',
message: 'No existing tables found. Enter the name of a new table to create:',
validate: input => {
if (!input.trim()) {
return 'Table name cannot be empty';
}
return true;
},
},
]);
selectedTable = newTableName;
console.log(chalk_1.default.yellow(`Note: Table '${selectedTable}' does not exist yet. You may need to create it first.`));
}
else {
const { tableChoice } = yield inquirer_1.default.prompt([
{
type: 'list',
name: 'tableChoice',
message: 'Select target DynamoDB table:',
choices: [
...tables,
new inquirer_1.default.Separator(),
{ name: 'Enter a different table name', value: 'custom' },
],
},
]);
if (tableChoice === 'custom') {
const { customTableName } = yield inquirer_1.default.prompt([
{
type: 'input',
name: 'customTableName',
message: 'Enter the table name:',
validate: input => {
if (!input.trim()) {
return 'Table name cannot be empty';
}
return true;
},
},
]);
selectedTable = customTableName;
}
else {
selectedTable = tableChoice;
}
}
// Step 5: Confirm and execute
const { confirm } = yield inquirer_1.default.prompt([
{
type: 'confirm',
name: 'confirm',
message: `Ready to import data from '${inputFilePath}' to table '${selectedTable}'?`,
default: true,
},
]);
if (confirm) {
yield importJsonToTable(selectedTable, inputFilePath, selectedRegion);
}
else {
console.log(chalk_1.default.yellow('Import cancelled.'));
}
}
});
}
// Display the logo at the start
displayLogo();
const { table, filePath, mode, region } = program.opts();
if (mode === 'wizard' || (!mode && !table && !filePath)) {
// Run in wizard mode if explicitly selected or if no required parameters are provided
void runWizard();
}
else if (!table || !filePath) {
console.error(chalk_1.default.red('Please provide the required options.'));
program.help();
}
else {
if (mode === 'export') {
// Export mode
console.log(chalk_1.default.blue('Export mode selected.'));
void exportTableToJson(table, filePath, region || 'us-east-1');
}
else if (mode === 'import') {
// Import mode
console.log(chalk_1.default.blue('Import mode selected.'));
void importJsonToTable(table, filePath, region || 'us-east-1');
}
else {
console.error(chalk_1.default.red('Invalid mode. Please select either "export", "import", or "wizard".'));
program.help();
}
}