@aws-lambda-powertools/idempotency
Version:
The idempotency package for the Powertools for AWS Lambda (TypeScript) library. It provides options to make your Lambda functions idempotent and safe to retry.
288 lines (287 loc) • 12.1 kB
JavaScript
import { createHash } from 'node:crypto';
import { getStringFromEnv } from '@aws-lambda-powertools/commons/utils/env';
import { LRUCache } from '@aws-lambda-powertools/commons/utils/lru-cache';
import { search } from '@aws-lambda-powertools/jmespath';
import { IdempotencyRecordStatus } from '../constants.js';
import { deepSort } from '../deepSort.js';
import { IdempotencyItemAlreadyExistsError, IdempotencyKeyError, IdempotencyValidationError, } from '../errors.js';
import { IdempotencyRecord } from './IdempotencyRecord.js';
/**
* Base class for all persistence layers. This class provides the basic functionality for
* saving, retrieving, and deleting idempotency records. It also provides the ability to
* configure the persistence layer from the idempotency config.
* @class
*/
class BasePersistenceLayer {
idempotencyKeyPrefix;
cache;
configured = false;
eventKeyJmesPath;
expiresAfterSeconds = 60 * 60; // 1 hour default
hashFunction = 'md5';
payloadValidationEnabled = false;
throwOnNoIdempotencyKey = false;
useLocalCache = false;
validationKeyJmesPath;
#jmesPathOptions;
constructor() {
this.idempotencyKeyPrefix = getStringFromEnv({
key: 'AWS_LAMBDA_FUNCTION_NAME',
defaultValue: '',
});
}
/**
* Initialize the base persistence layer from the configuration settings
*
* @param {BasePersistenceLayerConfigureOptions} options - configuration object for the persistence layer
*/
configure(options) {
const { config: idempotencyConfig, keyPrefix, functionName } = options;
if (keyPrefix?.trim()) {
this.idempotencyKeyPrefix = keyPrefix.trim();
}
else if (functionName?.trim()) {
this.idempotencyKeyPrefix = `${this.idempotencyKeyPrefix}.${functionName.trim()}`;
}
// Prevent reconfiguration
if (this.configured) {
return;
}
this.configured = true;
this.eventKeyJmesPath = idempotencyConfig?.eventKeyJmesPath;
this.validationKeyJmesPath = idempotencyConfig?.payloadValidationJmesPath;
this.#jmesPathOptions = idempotencyConfig.jmesPathOptions;
this.payloadValidationEnabled =
this.validationKeyJmesPath !== undefined || false;
this.throwOnNoIdempotencyKey =
idempotencyConfig?.throwOnNoIdempotencyKey || false;
this.eventKeyJmesPath = idempotencyConfig.eventKeyJmesPath;
this.expiresAfterSeconds = idempotencyConfig.expiresAfterSeconds; // 1 hour default
this.useLocalCache = idempotencyConfig.useLocalCache;
if (this.useLocalCache) {
this.cache = new LRUCache({
maxSize: idempotencyConfig.maxLocalCacheSize,
});
}
this.hashFunction = idempotencyConfig.hashFunction;
}
/**
* Deletes a record from the persistence store for the persistence key generated from the data passed in.
*
* @param data - the data payload that will be hashed to create the hash portion of the idempotency key
*/
async deleteRecord(data) {
const idempotencyRecord = new IdempotencyRecord({
idempotencyKey: this.getHashedIdempotencyKey(data),
status: IdempotencyRecordStatus.EXPIRED,
});
await this._deleteRecord(idempotencyRecord);
this.deleteFromCache(idempotencyRecord.idempotencyKey);
}
/**
* Retrieve the number of seconds that records will be kept in the persistence store
*/
getExpiresAfterSeconds() {
return this.expiresAfterSeconds;
}
/**
* Retrieves idempotency key for the provided data and fetches data for that key from the persistence store
*
* @param data - the data payload that will be hashed to create the hash portion of the idempotency key
*/
async getRecord(data) {
const idempotencyKey = this.getHashedIdempotencyKey(data);
const cachedRecord = this.getFromCache(idempotencyKey);
if (cachedRecord) {
this.validatePayload(data, cachedRecord);
return cachedRecord;
}
const record = await this._getRecord(idempotencyKey);
this.processExistingRecord(record, data);
return record;
}
/**
* Check whether payload validation is enabled or not
*/
isPayloadValidationEnabled() {
return this.payloadValidationEnabled;
}
/**
* Validates an existing record against the data payload being processed.
* If the payload does not match the stored record, an `IdempotencyValidationError` error is thrown.
*
* Whenever a record is retrieved from the persistence layer, it should be validated against the data payload
* being processed. This is to ensure that the data payload being processed is the same as the one that was
* used to create the record in the first place.
*
* The record is also saved to the local cache if local caching is enabled.
*
* @param storedDataRecord - the stored record to validate against
* @param processedData - the data payload being processed and to be validated against the stored record
*/
processExistingRecord(storedDataRecord, processedData) {
this.validatePayload(processedData, storedDataRecord);
this.saveToCache(storedDataRecord);
return storedDataRecord;
}
/**
* Saves a record indicating that the function's execution is currently in progress
*
* @param data - the data payload that will be hashed to create the hash portion of the idempotency key
* @param remainingTimeInMillis - the remaining time left in the lambda execution context
*/
async saveInProgress(data, remainingTimeInMillis) {
const idempotencyRecord = new IdempotencyRecord({
idempotencyKey: this.getHashedIdempotencyKey(data),
status: IdempotencyRecordStatus.INPROGRESS,
expiryTimestamp: this.getExpiryTimestamp(),
payloadHash: this.getHashedPayload(data),
});
if (remainingTimeInMillis) {
idempotencyRecord.inProgressExpiryTimestamp =
Date.now() + remainingTimeInMillis;
}
else {
console.warn('Could not determine remaining time left. Did you call registerLambdaContext on IdempotencyConfig?');
}
const cachedRecord = this.getFromCache(idempotencyRecord.idempotencyKey);
if (cachedRecord) {
throw new IdempotencyItemAlreadyExistsError(`Failed to put record for already existing idempotency key: ${idempotencyRecord.idempotencyKey}`, cachedRecord);
}
await this._putRecord(idempotencyRecord);
}
/**
* Saves a record of the function completing successfully. This will create a record with a COMPLETED status
* and will save the result of the completed function in the idempotency record.
*
* @param data - the data payload that will be hashed to create the hash portion of the idempotency key
* @param result - the result of the successfully completed function
*/
async saveSuccess(data, result) {
const idempotencyRecord = new IdempotencyRecord({
idempotencyKey: this.getHashedIdempotencyKey(data),
status: IdempotencyRecordStatus.COMPLETED,
expiryTimestamp: this.getExpiryTimestamp(),
responseData: result,
payloadHash: this.getHashedPayload(data),
});
await this._updateRecord(idempotencyRecord);
this.saveToCache(idempotencyRecord);
}
deleteFromCache(idempotencyKey) {
if (!this.useLocalCache)
return;
// Delete from local cache if it exists
if (this.cache?.has(idempotencyKey)) {
this.cache?.remove(idempotencyKey);
}
}
/**
* Generates a hash of the data and returns the digest of that hash
*
* @param data the data payload that will generate the hash
* @returns the digest of the generated hash
*/
generateHash(data) {
const hash = createHash(this.hashFunction);
hash.update(data);
return hash.digest('base64');
}
/**
* Creates the expiry timestamp for the idempotency record
*
* @returns the expiry time for the record expressed as number of seconds past the UNIX epoch
*/
getExpiryTimestamp() {
const currentTime = Date.now() / 1000;
return Math.round(currentTime + this.expiresAfterSeconds);
}
getFromCache(idempotencyKey) {
if (!this.useLocalCache)
return undefined;
const cachedRecord = this.cache?.get(idempotencyKey);
if (cachedRecord) {
// if record is not expired, return it
if (!cachedRecord.isExpired())
return cachedRecord;
// if record is expired, delete it from cache
this.deleteFromCache(idempotencyKey);
}
}
/**
* Generates the idempotency key used to identify records in the persistence store.
*
* @param data the data payload that will be hashed to create the hash portion of the idempotency key
* @returns the idempotency key
*/
getHashedIdempotencyKey(data) {
const payload = this.eventKeyJmesPath
? search(this.eventKeyJmesPath, data, this.#jmesPathOptions)
: data;
if (BasePersistenceLayer.isMissingIdempotencyKey(payload)) {
if (this.throwOnNoIdempotencyKey) {
throw new IdempotencyKeyError('No data found to create a hashed idempotency_key');
}
console.warn(`No value found for idempotency_key. jmespath: ${this.eventKeyJmesPath}`);
}
return `${this.idempotencyKeyPrefix}#${this.generateHash(JSON.stringify(deepSort(payload)))}`;
}
/**
* Extract payload using validation key jmespath and return a hashed representation
*
* @param data payload
*/
getHashedPayload(data) {
if (this.isPayloadValidationEnabled() && this.validationKeyJmesPath) {
return this.generateHash(JSON.stringify(deepSort(search(this.validationKeyJmesPath, data, this.#jmesPathOptions))));
}
return '';
}
static isMissingIdempotencyKey(data) {
if (Array.isArray(data) || typeof data === 'object') {
if (data === null)
return true;
for (const value of Object.values(data)) {
if (value) {
return false;
}
}
return true;
}
return !data;
}
/**
* Save record to local cache except for when status is `INPROGRESS`.
*
* Records with `INPROGRESS` status are not cached because we have no way to
* reflect updates that might happen to the record outside the execution
* context of the function.
*
* @param record - record to save
*/
saveToCache(record) {
if (!this.useLocalCache)
return;
if (record.getStatus() === IdempotencyRecordStatus.INPROGRESS)
return;
this.cache?.add(record.idempotencyKey, record);
}
/**
* Validates the payload against the stored record. If the payload does not match the stored record,
* an `IdempotencyValidationError` error is thrown.
*
* @param data - The data payload to validate against the stored record
* @param storedDataRecord - The stored record to validate against
*/
validatePayload(data, storedDataRecord) {
if (this.payloadValidationEnabled) {
const hashedPayload = data instanceof IdempotencyRecord
? data.payloadHash
: this.getHashedPayload(data);
if (hashedPayload !== storedDataRecord.payloadHash) {
throw new IdempotencyValidationError('Payload does not match stored record for this event key', storedDataRecord);
}
}
}
}
export { BasePersistenceLayer };