serverless-dynamodb
Version:
Serverless plugin to run DynamoDB local
124 lines (123 loc) • 4.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.locateSeeds = exports.writeSeeds = void 0;
const node_path_1 = __importDefault(require("node:path"));
const node_fs_1 = __importDefault(require("node:fs"));
// DynamoDB has a 25 item limit in batch requests
// https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
const MAX_MIGRATION_CHUNK = 25;
/**
* Writes a batch chunk of migration seeds to DynamoDB. DynamoDB has a limit on the number of
* items that may be written in a batch operation.
* @param {function} dynamodbWriteFunction The DynamoDB DocumentClient.batchWrite or DynamoDB.batchWriteItem function
*/
function writeSeedBatch(dynamodbWriteFunction, tableName, seedValues) {
const params = {
RequestItems: {
[tableName]: seedValues.map((seed) => ({
PutRequest: {
Item: seed,
},
})),
},
};
return new Promise((resolve, reject) => {
// interval lets us know how much time we have burnt so far. This lets us have a backoff mechanism to try
// again a few times in case the Database resources are in the middle of provisioning.
function execute(interval) {
setTimeout(async () => {
try {
await dynamodbWriteFunction(params);
resolve();
}
catch (err) {
if (err) {
if (err instanceof Error && 'code' in err && err.code === 'ResourceNotFoundException' && interval <= 5000) {
execute(interval + 1000);
}
else if (err instanceof TypeError && err.message === "Cannot read properties of undefined (reading '0')") {
reject(new Error(`Failed to seed items for the ${tableName} table because of an AWS library error. This usually means your \`rawsources\` seed files are invalid.`, { cause: err }));
}
else {
reject(err);
}
}
}
}, interval);
}
execute(0);
});
}
/**
* Writes a seed corpus to the given database table
* @param dynamodbWriteFunction The DynamoDB DocumentClient.batchWrite or DynamoDB.batchWriteItem function
*/
async function writeSeeds(dynamodbWriteFunction, tableName, seedValues) {
if (!dynamodbWriteFunction) {
throw new Error('dynamodbWriteFunction argument must be provided');
}
if (!tableName) {
throw new Error('table name argument must be provided');
}
if (!seedValues) {
throw new Error('seeds argument must be provided');
}
const seedChunks = chunk(seedValues, MAX_MIGRATION_CHUNK);
await Promise.all(seedChunks.map((chunk) => writeSeedBatch(dynamodbWriteFunction, tableName, chunk)));
}
exports.writeSeeds = writeSeeds;
const chunk = (input, size) => {
return input.reduce((arr, item, idx) => {
return idx % size === 0
? [...arr, [item]]
: [...arr.slice(0, -1), [...(arr.slice(-1)[0] ?? []), item]];
}, []);
};
/**
* Scrapes seed files out of a given location. This file may contain
* either a simple json object, or an array of simple json objects. An array
* of json objects is returned.
*
* @param {string} location the filename to read seeds from.
* @returns {object[]} json
*/
function getSeedsAtLocation(location) {
// load the file
const rawFile = node_fs_1.default.readFileSync(location, { encoding: 'utf-8' });
// parse as JSON
const parsed = safeParse(rawFile);
if (parsed instanceof Error) {
throw new Error(`Failed to parse JSON when reading seed file '${location}'`, parsed);
}
// ensure the output is an array
const array = Array.isArray(parsed) ? parsed : [parsed];
return array;
}
function safeParse(json) {
try {
return JSON.parse(json);
}
catch (err) {
if (err instanceof Error)
return err;
return new Error(String(err));
}
}
/**
* Locates seeds given a set of files to scrape
* @param {string[]} sources The filenames to scrape for seeds
* @returns {object[]} The items to seed
*/
function locateSeeds(sources, cwd = process.cwd()) {
const locations = sources.map((source) => node_path_1.default.join(cwd, source));
return locations.map((location) => {
if (!node_fs_1.default.existsSync(location)) {
throw new Error(`source file ${location} does not exist`);
}
return getSeedsAtLocation(location);
}).flat(1);
}
exports.locateSeeds = locateSeeds;