@webda/aws
Version:
Webda AWS Services implementation
710 lines • 25.4 kB
JavaScript
import { ConditionalCheckFailedException, DynamoDB, DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { DynamoDBDocument } from "@aws-sdk/lib-dynamodb";
import { CoreModel, Store, StoreNotFoundError, StoreParameters, UpdateConditionFailError, WebdaError, WebdaQL } from "@webda/core";
import { AWSServiceParameters } from "./index.js";
/**
* Define DynamoDB parameters
*/
export class DynamoStoreParameters extends AWSServiceParameters(StoreParameters) {
constructor(params, service) {
super(params, service);
if (this.table === undefined) {
throw new WebdaError.CodeError("DYNAMODB_TABLE_PARAMETER_REQUIRED", "Need to define a table at least");
}
this.globalIndexes ?? (this.globalIndexes = {});
}
}
/**
* DynamoStore handles the DynamoDB
*
* Parameters:
* accessKeyId: '' // try WEBDA_AWS_KEY env variable if not found
* secretAccessKey: '' // try WEBDA_AWS_SECRET env variable if not found
* table: ''
* region: ''
*
* @WebdaModda
*/
export default class DynamoStore extends Store {
/**
* Load the parameters
*
* @param params
*/
loadParameters(params) {
return new DynamoStoreParameters(params, this);
}
/**
* Create the AWS client
*/
resolve() {
super.resolve();
this._client = DynamoDBDocument.from(new DynamoDBClient(this.parameters), {
marshallOptions: {
removeUndefinedValues: true
}
});
return this;
}
/**
* Copy one DynamoDB table to another
*
* @param output
* @param source
* @param target
*/
static async copyTable(output, source, target) {
let db = new DynamoDB({});
let ExclusiveStartKey;
let props = await db.describeTable({
TableName: source
});
output.startProgress("copyTable", props.Table.ItemCount, `Copying ${source} to ${target}`);
do {
let info = await db.scan({
TableName: source,
ExclusiveStartKey
});
do {
let items = [];
while (info.Items.length && items.length < 25) {
items.push(info.Items.shift());
}
if (!items.length) {
break;
}
let params = {
RequestItems: {}
};
params.RequestItems[target] = items.map(Item => ({
PutRequest: { Item }
}));
await db.batchWriteItem(params);
output.incrementProgress(items.length, "copyTable");
} while (true);
ExclusiveStartKey = info.LastEvaluatedKey;
} while (ExclusiveStartKey);
}
/**
* @inheritdoc
*/
async _exists(uid) {
// Should use find + limit 1
return (await this._get(uid)) !== undefined;
}
/**
* @inheritdoc
*/
async _save(object, uid = object.uuid) {
return this._update(object, uid);
}
/**
* @override
*
* @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html
* @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html
*/
async find(query) {
let scan = true;
let IndexName = undefined;
let result;
let primaryKeys = { uuid: null };
Object.keys(this.parameters.globalIndexes).forEach(name => {
primaryKeys[this.parameters.globalIndexes[name].key] = name;
});
// We could use PartQL but localstack is not compatible
let filter = new WebdaQL.AndExpression([]);
let KeyConditionExpression = "";
let ExpressionAttributeValues = {};
let FilterExpression = [];
let ExpressionAttributeNames = {};
let indexNode;
let sortOrder = "ASC";
let processingExpression;
if (!(query.filter instanceof WebdaQL.AndExpression)) {
processingExpression = new WebdaQL.AndExpression([query.filter]);
}
else {
processingExpression = query.filter;
}
// Search for the index
processingExpression.children.some(child => {
if (child instanceof WebdaQL.ComparisonExpression) {
// Primary key requires equal operator
if (child.operator === "=" && primaryKeys[child.attribute[0]] !== undefined) {
// Query not scan
scan = false;
IndexName = primaryKeys[child.attribute[0]];
indexNode = child;
KeyConditionExpression = `#${child.attribute[0]} = :${child.attribute[0]}`;
ExpressionAttributeNames[`#${child.attribute[0]}`] = child.attribute[0];
ExpressionAttributeValues[`:${child.attribute[0]}`] = child.value;
if (primaryKeys[child.attribute[0]] !== null &&
query.orderBy &&
this.parameters.globalIndexes[primaryKeys[child.attribute[0]]].sort) {
const sortKey = this.parameters.globalIndexes[primaryKeys[child.attribute[0]]].sort;
// Might update sort order
query.orderBy.some(order => {
if (order.field === sortKey) {
sortOrder = order.direction;
}
});
}
return true;
}
}
});
let count = 1;
// Build the Query/Filter
processingExpression.children.forEach(child => {
if (child === indexNode) {
return;
}
// Only work on Comparison Node for now
if (child instanceof WebdaQL.ComparisonExpression) {
let operator = child.operator;
// DynamoDB does not manage LIKE
if (child.operator === "LIKE") {
filter.children.push(child);
return;
}
// CONTAINS -> contains (category, :category1)
// != is <> in Dynamo
if (operator === "!=") {
operator = "<>";
}
// DynamoDB does not allow more than 100 items in IN
if (child.operator === "IN" && child.value.length > 100) {
filter.children.push(child);
return;
}
// Subfields like team.id needs to be #a1.#a2
let attr = `a${count++}`;
let fullAttr = `#${attr}`;
child.attribute.slice(1).forEach(v => {
let subAttr = `#a${count++}`;
fullAttr += `.${subAttr}`;
ExpressionAttributeNames[subAttr] = v;
});
ExpressionAttributeNames[`#${attr}`] = child.attribute[0];
let valueExpression = `:${attr}`;
if (child.operator === "IN") {
// Need to follow `a IN (b, c, d)
valueExpression = "(" + valueExpression;
ExpressionAttributeValues[`:${attr}`] = child.value[0];
child.value.splice(1).forEach(v => {
let subAttr = `:a${count++}`;
valueExpression += `, ${subAttr}`;
ExpressionAttributeValues[subAttr] = v;
});
valueExpression += ")";
}
else {
// For all other just give the value
ExpressionAttributeValues[`:${attr}`] = child.value;
}
// Manage CONTAINS
if (child.operator === "CONTAINS") {
FilterExpression.push(`contains(${fullAttr}, :${attr})`);
return;
}
// If this is a sort key
if (IndexName && this.parameters.globalIndexes[IndexName].sort === child.attribute[0]) {
// Sort key
KeyConditionExpression += ` AND ${fullAttr} ${operator} ${valueExpression}`;
return;
}
// Otherwise fallback to normal Filter
FilterExpression.push(`${fullAttr} ${operator} ${valueExpression}`);
}
else {
filter.children.push(child);
}
});
if (!Object.keys(ExpressionAttributeNames).length) {
ExpressionAttributeNames = undefined;
}
if (!Object.keys(ExpressionAttributeValues).length) {
ExpressionAttributeValues = undefined;
}
let ExclusiveStartKey = query.continuationToken
? JSON.parse(Buffer.from(query.continuationToken, "base64").toString())
: undefined;
// Scan if not primary key was provided
if (scan) {
// SHould log bad query
const Items = [];
do {
result = await this._client.scan({
TableName: this.parameters.table,
FilterExpression: FilterExpression.length ? FilterExpression.join(" AND ") : undefined,
ExclusiveStartKey,
ExpressionAttributeNames,
ExpressionAttributeValues,
Limit: query.limit - Items.length
});
Items.push(...result.Items);
result.Items = Items;
ExclusiveStartKey = result.LastEvaluatedKey;
} while (Items.length < query.limit && result.LastEvaluatedKey);
}
else {
result = await this._client.query({
TableName: this.parameters.table,
IndexName,
ExclusiveStartKey,
KeyConditionExpression,
FilterExpression: FilterExpression.length ? FilterExpression.join(" AND ") : undefined,
ExpressionAttributeNames,
ExpressionAttributeValues,
Limit: query.limit,
ScanIndexForward: sortOrder === "ASC" ? true : false
});
}
return {
results: result.Items.map(c => this.initModel(c)),
filter,
continuationToken: result.Items.length >= query.limit
? Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString("base64")
: undefined
};
}
/**
* Serialize a date for DynamoDB
* @param date
* @returns
*/
_serializeDate(date) {
return JSON.stringify(date).replace(/"/g, "");
}
/**
* Clean object to store in DynamoDB
*
* @param object
* @returns
*/
_cleanObject(object) {
if (typeof object !== "object")
return object;
if (object instanceof Date) {
return this._serializeDate(object);
}
if (object instanceof CoreModel) {
object = object.toStoredJSON();
}
let res;
if (object instanceof Array) {
res = [];
}
else {
res = {};
}
for (let i in object) {
if (object[i] === "" || i.startsWith("__store")) {
continue;
}
res[i] = this._cleanObject(object[i]);
}
return res;
}
/**
* @inheritdoc
*/
async _removeAttribute(uuid, attribute, itemWriteCondition, itemWriteConditionField) {
let params = {
TableName: this.parameters.table,
Key: {
uuid
}
};
let attrs = {};
attrs["#attr"] = attribute;
attrs["#lastUpdate"] = "_lastUpdate";
params.ExpressionAttributeNames = attrs;
params.ConditionExpression = "attribute_exists(#uu)";
params.ExpressionAttributeValues = {
":lastUpdate": this._serializeDate(new Date())
};
params.UpdateExpression = "REMOVE #attr SET #lastUpdate = :lastUpdate";
if (itemWriteCondition) {
this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
}
else {
attrs["#uu"] = "uuid";
params.ConditionExpression = "attribute_exists(#uu)";
}
try {
await this._client.update(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
if (!itemWriteConditionField) {
throw new StoreNotFoundError(uuid, this.getName());
}
throw new UpdateConditionFailError(uuid, itemWriteConditionField, itemWriteCondition);
}
throw err;
}
}
/**
* @inheritdoc
*/
async _deleteItemFromCollection(uid, prop, index, itemWriteCondition, itemWriteConditionField, updateDate) {
let params = {
TableName: this.parameters.table,
Key: {
uuid: uid
}
};
let attrs = {};
attrs["#" + prop] = prop;
attrs["#lastUpdate"] = "_lastUpdate";
params.ExpressionAttributeNames = attrs;
params.ExpressionAttributeValues = {
":lastUpdate": this._serializeDate(updateDate)
};
params.UpdateExpression = "REMOVE #" + prop + "[" + index + "] SET #lastUpdate = :lastUpdate";
if (itemWriteCondition) {
params.ExpressionAttributeValues[":condValue"] = itemWriteCondition;
attrs["#condName"] = prop;
attrs["#field"] = itemWriteConditionField;
params.ConditionExpression = "#condName[" + index + "].#field = :condValue";
}
try {
await this._client.update(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
}
else if (err.name === "ValidationException") {
throw new StoreNotFoundError(uid, this.getName());
}
throw err;
}
}
/**
* @inheritdoc
*/
async _upsertItemToCollection(uuid, prop, item, index, itemWriteCondition, itemWriteConditionField, updateDate) {
let params = {
TableName: this.parameters.table,
Key: {
uuid
}
};
let attrValues = {};
let attrs = {};
attrs["#" + prop] = prop;
attrs["#lastUpdate"] = "_lastUpdate";
attrValues[":" + prop] = this._cleanObject(item);
attrValues[":lastUpdate"] = this._serializeDate(updateDate);
attrValues[":uuid"] = uuid;
attrs["#uuid"] = this._uuidField;
params.ConditionExpression = "#uuid = :uuid";
params.ExpressionAttributeValues = attrValues;
params.ExpressionAttributeNames = attrs;
if (index === undefined) {
params.UpdateExpression =
"SET #" +
prop +
"= list_append(if_not_exists (#" +
prop +
", :empty_list),:" +
prop +
"), #lastUpdate = :lastUpdate";
attrValues[":" + prop] = [attrValues[":" + prop]];
attrValues[":empty_list"] = [];
}
else {
params.UpdateExpression = "SET #" + prop + "[" + index + "] = :" + prop + ", #lastUpdate = :lastUpdate";
if (itemWriteCondition) {
attrValues[":condValue"] = itemWriteCondition;
attrs["#condName"] = prop;
attrs["#field"] = itemWriteConditionField;
params.ConditionExpression = "#condName[" + index + "].#field = :condValue and #uuid = :uuid";
}
}
try {
await this._client.update(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new UpdateConditionFailError(uuid, itemWriteConditionField, itemWriteCondition);
}
else if (err.name === "ValidationException") {
throw new StoreNotFoundError(uuid, this.getName());
}
throw err;
}
}
/**
* REturn the write condition as string
* @param writeCondition
* @param field
* @returns
*/
setWriteCondition(params, writeCondition, field = "_lastUpdate") {
params.ExpressionAttributeNames ?? (params.ExpressionAttributeNames = {});
params.ExpressionAttributeValues ?? (params.ExpressionAttributeValues = {});
params.ExpressionAttributeNames["#cf"] = field;
if (writeCondition instanceof Date) {
writeCondition = this._serializeDate(writeCondition);
}
params.ExpressionAttributeValues[":cf"] = writeCondition;
params.ConditionExpression = "#cf = :cf";
}
/**
* @inheritdoc
*/
async _delete(uid, writeCondition, itemWriteConditionField) {
let params = {
TableName: this.parameters.table,
Key: {
uuid: uid
}
};
if (writeCondition) {
this.setWriteCondition(params, writeCondition, itemWriteConditionField);
}
try {
await this._client.delete(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new UpdateConditionFailError(uid, itemWriteConditionField, writeCondition);
}
throw err;
}
}
/**
* @inheritdoc
*/
async _patch(object, uid, itemWriteCondition, itemWriteConditionField) {
object = this._cleanObject(object);
let expr = "SET ";
let sep = "";
let attrValues = {};
let attrs = {};
let skipUpdate = true;
let i = 1;
for (let attr in object) {
if (attr === "uuid" || object[attr] === undefined) {
continue;
}
skipUpdate = false;
expr += sep + "#a" + i + " = :v" + i;
attrValues[":v" + i] = object[attr];
attrs["#a" + i] = attr;
sep = ",";
i++;
}
if (skipUpdate) {
return;
}
let params = {
TableName: this.parameters.table,
Key: {
uuid: uid
},
UpdateExpression: expr,
ExpressionAttributeValues: attrValues,
ExpressionAttributeNames: attrs
};
// The Write Condition checks the value before writing
if (itemWriteCondition) {
this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
}
try {
await this._client.update(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
}
throw err;
}
}
/**
* @inheritdoc
*/
async _update(object, uid, itemWriteCondition, itemWriteConditionField) {
object = this._cleanObject(object);
object.uuid = uid;
let params = {
TableName: this.parameters.table,
Item: object
};
// The Write Condition checks the value before writing
if (itemWriteCondition) {
this.setWriteCondition(params, itemWriteCondition, itemWriteConditionField);
}
try {
await this._client.put(params);
return object;
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new UpdateConditionFailError(uid, itemWriteConditionField, itemWriteCondition);
}
throw err;
}
}
/**
* @inheritdoc
*/
async _scan(items = [], paging = undefined) {
let data = await this._client.scan({
TableName: this.parameters.table,
Limit: this.parameters.scanPage,
ExclusiveStartKey: paging
});
for (let i in data.Items) {
items.push(this.initModel(data.Items[i]));
}
if (data.LastEvaluatedKey) {
return this._scan(items, data.LastEvaluatedKey);
}
return items;
}
/**
* @inheritdoc
*/
async getAll(uids) {
if (!uids) {
return this._scan([]);
}
let params = {
RequestItems: {}
};
params["RequestItems"][this.parameters.table] = {
Keys: uids.map(value => {
return {
uuid: value
};
})
};
let result = await this._client.batchGet(params);
return result.Responses[this.parameters.table].map(this.initModel, this);
}
/**
* @inheritdoc
*/
async _get(uid, raiseIfNotFound = false) {
let params = {
TableName: this.parameters.table,
Key: {
uuid: uid
}
};
let item = (await this._client.get(params)).Item;
if (!item) {
if (raiseIfNotFound) {
throw new StoreNotFoundError(uid, this.getName());
}
return undefined;
}
return this.initModel(item);
}
/**
* @inheritdoc
*/
async _incrementAttributes(uid, parameters, updateDate) {
let params = {
TableName: this.parameters.table,
Key: {
uuid: uid
},
UpdateExpression: "SET #ud = :ud ADD ",
ConditionExpression: "attribute_exists(#uu)",
ExpressionAttributeValues: {
":ud": this._serializeDate(updateDate)
},
ExpressionAttributeNames: {
"#ud": "_lastUpdate",
"#uu": "uuid"
}
};
parameters.forEach((p, i) => {
params.ExpressionAttributeValues[`:v${i}`] = p.value;
params.ExpressionAttributeNames[`#a${i}`] = p.property;
});
params.UpdateExpression += parameters.map((_, i) => `#a${i} :v${i}`).join(",");
try {
return await this._client.update(params);
}
catch (err) {
if (err instanceof ConditionalCheckFailedException) {
throw new StoreNotFoundError(uid, this.getName());
}
throw err;
}
}
/**
* @inheritdoc
*/
getARNPolicy(accountId) {
let region = this.parameters.region || "us-east-1";
return {
Sid: this.constructor.name + this._name,
Effect: "Allow",
Action: [
"dynamodb:BatchGetItem",
"dynamodb:BatchWriteItem",
"dynamodb:DeleteItem",
"dynamodb:GetItem",
"dynamodb:GetRecords",
"dynamodb:GetShardIterator",
"dynamodb:PutItem",
"dynamodb:Query",
"dynamodb:Scan",
"dynamodb:UpdateItem"
],
Resource: ["arn:aws:dynamodb:" + region + ":" + accountId + ":table/" + this.parameters.table]
};
}
/**
* @inheritdoc
*/
async __clean() {
let params = {
TableName: this.parameters.table
};
let result = await this._client.scan(params);
let promises = [];
for (let i in result.Items) {
promises.push(this._delete(result.Items[i].uuid));
}
await Promise.all(promises);
}
/**
* @inheritdoc
*/
getCloudFormation(deployer) {
if (this.parameters.CloudFormationSkip) {
return {};
}
let resources = {};
this.parameters.CloudFormation = this.parameters.CloudFormation || {};
this.parameters.CloudFormation.Table = this.parameters.CloudFormation.Table || {};
let KeySchema = this.parameters.CloudFormation.KeySchema || [{ KeyType: "HASH", AttributeName: "uuid" }];
let AttributeDefinitions = this.parameters.CloudFormation.AttributeDefinitions || [];
this.parameters.CloudFormation.Table.BillingMode =
this.parameters.CloudFormation.Table.BillingMode || "PAY_PER_REQUEST";
AttributeDefinitions.push({ AttributeName: "uuid", AttributeType: "S" });
resources[this._name + "DynamoTable"] = {
Type: "AWS::DynamoDB::Table",
Properties: {
...this.parameters.CloudFormation.Table,
TableName: this.parameters.table,
KeySchema,
AttributeDefinitions,
Tags: deployer.getDefaultTags(this.parameters.CloudFormation.Table.Tags)
}
};
// Add any Other resources with prefix of the service
return resources;
}
}
export { DynamoStore };
//# sourceMappingURL=dynamodb.js.map