@cumulus/aws-client
Version:
Utilities for working with AWS
178 lines • 6.98 kB
JavaScript
;
/**
* @module DynamoDb
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.deleteAndWaitForDynamoDbTableNotExists = exports.createAndWaitForDynamoDbTable = exports.parallelScan = exports.scan = exports.get = void 0;
const p_map_1 = __importDefault(require("p-map"));
const p_retry_1 = __importDefault(require("p-retry"));
const range_1 = __importDefault(require("lodash/range"));
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
const errors_1 = require("@cumulus/errors");
const services_1 = require("./services");
const utils_1 = require("./utils");
/**
* Call DynamoDb client get
*
* See [DocumentClient.get()](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/modules/_aws_sdk_lib_dynamodb.html)
* for descriptions of `params` and the return data.
*
* @param {Object} params
* @param {string} params.tableName - Table name to read
* @param {GetCommandInput.Key} params.item - Key identifying object to get
* @param {DynamoDBDocument} params.client - Instance of a DynamoDb DocumentClient
* @param {Object} params.getParams - Additional parameters for DocumentClient.get()
* @returns {Promise<Object>}
* @throws {RecordDoesNotExist} if a record cannot be found
*/
const get = async (params) => {
const { client, getParams = {}, item, tableName, } = params;
const getResponse = await client.get({
...getParams,
TableName: tableName,
Key: item,
});
if (getResponse.Item)
return getResponse.Item;
throw new errors_1.RecordDoesNotExist(`No record found for ${JSON.stringify(item)} in ${tableName}`);
};
exports.get = get;
/**
* Call DynamoDb client scan
*
* See [DocumentClient.scan()](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/modules/_aws_sdk_lib_dynamodb.html)
* for descriptions of `params` and the return data.
*
* @param {Object} params
* @returns {Promise<Object>}
*/
exports.scan = (0, utils_1.improveStackTrace)(async (params) => {
const { client, fields, limit, query, select, startKey, tableName, } = params;
const scanParams = {
TableName: tableName,
};
if (query) {
if (query.filter && query.values) {
scanParams.FilterExpression = query.filter;
scanParams.ExpressionAttributeValues = query.values;
}
if (query.names) {
scanParams.ExpressionAttributeNames = query.names;
}
}
if (fields) {
scanParams.ProjectionExpression = fields;
}
if (limit) {
scanParams.Limit = limit;
}
if (select) {
scanParams.Select = select;
}
if (startKey) {
scanParams.ExclusiveStartKey = startKey;
}
const response = await client.scan(scanParams);
// recursively go through all the records
if (response.LastEvaluatedKey) {
const more = await (0, exports.scan)({
tableName,
client,
query,
fields,
limit,
select,
startKey: response.LastEvaluatedKey,
});
if (more.Items) {
response.Items = (response.Items || []).concat(more.Items);
}
if (typeof response.Count === 'number' && typeof more.Count === 'number') {
response.Count += more.Count;
}
}
return response;
});
/**
* Do a parallel scan of DynamoDB table using a document client.
*
* See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.ParallelScan.
* See [DocumentClient.scan()](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/modules/_aws_sdk_lib_dynamodb.html).
*
* @param {Object} params
* @param {number} params.totalSegments
* Total number of segments to divide table into for parallel scanning
* @param {ScanInput} params.scanParams
* Params for the DynamoDB client scan operation
* See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
* @param {function} params.processItemsFunc - Function used to process returned items by scan
* @param {DynamoDBDocument} [params.dynamoDbClient] - Instance of Dynamo DB document client
* @param {pRetry.Options} [params.retryOptions] - Retry options for scan operations
* @returns {Promise}
*/
const parallelScan = async (params) => {
const { totalSegments, scanParams, processItemsFunc, dynamoDbClient, retryOptions, } = params;
return await (0, p_map_1.default)((0, range_1.default)(totalSegments), async (_, segmentIndex) => {
let exclusiveStartKey;
const segmentScanParams = {
...scanParams,
TotalSegments: totalSegments,
Segment: segmentIndex,
};
/* eslint-disable no-await-in-loop */
do {
const { Items = [], LastEvaluatedKey, } = await (0, p_retry_1.default)(() => dynamoDbClient.scan(segmentScanParams), retryOptions);
exclusiveStartKey = LastEvaluatedKey;
segmentScanParams.ExclusiveStartKey = exclusiveStartKey;
await processItemsFunc(Items);
} while (exclusiveStartKey);
/* eslint-enable no-await-in-loop */
return Promise.resolve();
}, {
stopOnError: false,
});
};
exports.parallelScan = parallelScan;
/**
* Create a DynamoDB table and then wait for the table to exist
*
* @param {Object} params - the same params that you would pass to AWS.createTable
* See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/classes/dynamodb.html#createtable
* @returns {Promise<Object>} the output of the createTable call
*
* @static
*/
async function createAndWaitForDynamoDbTable(params) {
const dynamoDbClient = (0, services_1.dynamodb)();
const createTableResult = await dynamoDbClient.createTable(params);
await (0, client_dynamodb_1.waitUntilTableExists)({
client: dynamoDbClient,
minDelay: 1,
maxWaitTime: 3,
}, { TableName: params.TableName });
return createTableResult;
}
exports.createAndWaitForDynamoDbTable = createAndWaitForDynamoDbTable;
/**
* Delete a DynamoDB table and then wait for the table to not exist
*
* @param {Object} params - the same params that you would pass to AWS.deleteTable
* See https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb/classes/dynamodb.html#deletetable
* @returns {Promise}
*
* @static
*/
async function deleteAndWaitForDynamoDbTableNotExists(params) {
const dynamoDbClient = (0, services_1.dynamodb)();
await dynamoDbClient.deleteTable(params);
await (0, client_dynamodb_1.waitUntilTableNotExists)({
client: dynamoDbClient,
minDelay: 1,
maxWaitTime: 3,
}, { TableName: params.TableName });
}
exports.deleteAndWaitForDynamoDbTableNotExists = deleteAndWaitForDynamoDbTableNotExists;
//# sourceMappingURL=DynamoDb.js.map