cloudrx
Version:
TypeScript library for streaming cloud provider events using RxJS with automatic persistence and replay capabilities
356 lines • 17 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.DynamoDB = void 0;
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
const base_1 = require("../base");
const rxjs_1 = require("rxjs");
const client_dynamodb_streams_1 = require("@aws-sdk/client-dynamodb-streams");
const util_dynamodb_1 = require("@aws-sdk/util-dynamodb");
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
class DynamoDB extends base_1.CloudProvider {
constructor(id, opts) {
super(id, opts);
this.translation = {
marshallOptions: {},
unmarshallOptions: {
convertWithoutMapWrapper: false,
},
};
this._client = opts?.client || new client_dynamodb_1.DynamoDBClient();
this._hashKey = opts?.hashKey || 'hashKey';
this._rangeKey = opts?.rangeKey || 'rangeKey';
this._ttlAttribute = opts?.ttlAttribute || 'expires';
this._pollInterval = opts?.pollInterval || 5000;
}
get client() {
return this._client;
}
get tableName() {
return `cloudrx-${this.id}`;
}
get tableArn() {
if (!this._tableArn) {
throw new base_1.FatalError('Table ARN is not yet available');
}
return this._tableArn;
}
get hashKey() {
return this._hashKey;
}
get rangeKey() {
return this._rangeKey;
}
get ttlAttribute() {
return this._ttlAttribute;
}
get pollInterval() {
return this._pollInterval;
}
get streamClient() {
if (!this._streamClient) {
throw new base_1.FatalError('Stream client is not yet available');
}
return this._streamClient;
}
get streamArn() {
if (!this._streamArn) {
throw new base_1.FatalError('Stream ARN is not yet available');
}
return this._streamArn;
}
get shards() {
const { id } = this;
if (DynamoDB.shards[id]) {
this.logger.debug?.(`[${this.id}] Returning existing shard observable`);
return DynamoDB.shards[id];
}
DynamoDB.shards[id] = (0, rxjs_1.timer)(0, this.pollInterval || 5000).pipe((0, rxjs_1.takeUntil)((0, rxjs_1.fromEvent)(this.signal, 'abort')), (0, rxjs_1.switchMap)((tick) => {
this.logger.debug?.(`[${this.id}] [iter:${tick}] Stream refresh...`);
return (0, rxjs_1.from)(this.streamClient
.send(new client_dynamodb_streams_1.DescribeStreamCommand({ StreamArn: this.streamArn }))
.then((response) => {
return response.StreamDescription?.Shards || [];
})
.catch((error) => {
this.logger.error?.(`[${this.id}] Failed to describe stream:`, error);
return [];
}));
}), (0, rxjs_1.switchMap)((shards) => (0, rxjs_1.from)(shards)), (0, rxjs_1.scan)((acc, shard) => {
if (shard.ShardId && !acc.seenShardIds.has(shard.ShardId)) {
acc.seenShardIds.add(shard.ShardId);
return { seenShardIds: acc.seenShardIds, newShard: shard };
}
return { seenShardIds: acc.seenShardIds, newShard: null };
}, { seenShardIds: new Set(), newShard: null }), (0, rxjs_1.filter)(({ newShard }) => newShard !== null), (0, rxjs_1.map)(({ newShard }) => newShard), (0, rxjs_1.shareReplay)(1));
return DynamoDB.shards[id];
}
_init() {
this.logger.debug?.(`[${this.id}] Initializing DynamoDB provider...`);
const describe$ = (0, rxjs_1.defer)(() => {
this.logger.debug?.(`[${this.id}] Describing existing table and TTL...`);
return (0, rxjs_1.forkJoin)([
this.client.send(new client_dynamodb_1.DescribeTableCommand({ TableName: this.tableName })),
this.client.send(new client_dynamodb_1.DescribeTimeToLiveCommand({ TableName: this.tableName })),
]).pipe((0, rxjs_1.map)(([table, ttl]) => {
this.logger.debug?.(`[${this.id}] Successfully described table and TTL`);
return {
table: table.Table,
ttl: ttl.TimeToLiveDescription,
};
}));
});
const create$ = (0, rxjs_1.defer)(() => {
this.logger.debug?.(`[${this.id}] Creating new table and TTL...`);
return (0, rxjs_1.forkJoin)([
this.client.send(new client_dynamodb_1.CreateTableCommand({
TableName: this.tableName,
KeySchema: [
{ AttributeName: this.hashKey, KeyType: 'HASH' },
{ AttributeName: this.rangeKey, KeyType: 'RANGE' },
],
AttributeDefinitions: [
{ AttributeName: this.hashKey, AttributeType: 'S' },
{ AttributeName: this.rangeKey, AttributeType: 'S' },
],
BillingMode: 'PAY_PER_REQUEST',
StreamSpecification: {
StreamEnabled: true,
StreamViewType: 'NEW_AND_OLD_IMAGES',
},
})),
this.client.send(new client_dynamodb_1.UpdateTimeToLiveCommand({
TableName: this.tableName,
TimeToLiveSpecification: {
AttributeName: this.ttlAttribute,
Enabled: true,
},
})),
]).pipe((0, rxjs_1.map)(([table, ttl]) => {
this.logger.debug?.(`[${this.id}] Successfully created table and TTL`);
return {
table: table.TableDescription,
ttl: ttl.TimeToLiveSpecification,
};
}));
});
const assert = (table, ttl, error) => {
if (this.signal.aborted) {
throw new base_1.FatalError('Aborted');
}
if (error) {
if (error instanceof base_1.FatalError || error instanceof base_1.RetryError) {
throw error;
}
if (error &&
typeof error === 'object' &&
'name' in error &&
error.name === 'AggregateError' &&
'errors' in error &&
Array.isArray(error.errors)) {
const firstError = error.errors[0];
if (firstError &&
typeof firstError === 'object' &&
'name' in firstError) {
return assert(table, ttl, firstError);
}
throw new base_1.FatalError(`AggregateError: ${error.message}`);
}
if ('code' in error && error.code === 'ECONNREFUSED') {
throw new base_1.RetryError('Connection refused');
}
if (error.name === 'ResourceNotFoundException') {
throw new base_1.RetryError('Resource not found');
}
if (error.name === 'ResourceInUseException') {
throw new base_1.RetryError('Resource in use');
}
if (error.name === 'ValidationException') {
throw new base_1.RetryError('Validation error');
}
throw new base_1.FatalError(`${error.name}: ${error.message}`);
}
if (!table || !ttl) {
throw new base_1.FatalError('Table or TTL is not yet available');
}
if (table.TableStatus !== 'ACTIVE') {
throw new base_1.RetryError('Table is not yet active');
}
if (ttl.TimeToLiveStatus !== 'ENABLED' &&
ttl.TimeToLiveStatus !== 'ENABLING') {
throw new base_1.RetryError(`TTL is not yet enabled, current status: ${ttl.TimeToLiveStatus}`);
}
if (table.KeySchema?.find((key) => key.KeyType === 'HASH' && key.AttributeName !== this.hashKey)) {
throw new base_1.FatalError(`Hash key does not match desired name of \`${this.hashKey}\`: ${JSON.stringify(table.KeySchema)}`);
}
if (table.KeySchema?.find((key) => key.KeyType === 'RANGE' && key.AttributeName !== this.rangeKey)) {
throw new base_1.FatalError(`Range key does not match desired name of \`${this.rangeKey}\`: ${JSON.stringify(table.KeySchema)}`);
}
if (table.AttributeDefinitions?.find((key) => key.AttributeName === this.hashKey && key.AttributeType !== 'S')) {
throw new base_1.FatalError(`Hash Key needs to be a string type: ${JSON.stringify(table.AttributeDefinitions)}`);
}
if (table.AttributeDefinitions?.find((key) => key.AttributeName === this.rangeKey && key.AttributeType !== 'S')) {
throw new base_1.FatalError(`Range Key needs to be a string type: ${JSON.stringify(table.AttributeDefinitions)}`);
}
if (table.AttributeDefinitions?.find((key) => key.AttributeName === this.ttlAttribute && key.AttributeType !== 'N')) {
throw new base_1.FatalError(`Table needs a TTL attribute named "${this.ttlAttribute}" of type number`);
}
if (table.StreamSpecification?.StreamEnabled !== true) {
throw new base_1.FatalError(`Table needs to have streams enabled`);
}
if (table.StreamSpecification?.StreamViewType !== 'NEW_AND_OLD_IMAGES') {
throw new base_1.FatalError(`Table needs to have streams configured to emit new and old images`);
}
if (ttl?.AttributeName !== this.ttlAttribute) {
throw new base_1.FatalError(`TTL attribute needs to be named ${this.ttlAttribute}`);
}
return { table, ttl };
};
const check = (delay) => (0, rxjs_1.timer)(delay).pipe((0, rxjs_1.takeUntil)((0, rxjs_1.fromEvent)(this.signal, 'abort')), (0, rxjs_1.switchMap)(() => describe$.pipe((0, rxjs_1.switchMap)(({ table, ttl }) => (0, rxjs_1.of)(assert(table, ttl))))), (0, rxjs_1.catchError)((err) => {
if (err instanceof base_1.FatalError) {
return (0, rxjs_1.throwError)(() => err);
}
return create$.pipe((0, rxjs_1.switchMap)(({ table, ttl }) => (0, rxjs_1.of)(assert(table, ttl))));
}), (0, rxjs_1.catchError)((err) => {
if (err instanceof base_1.FatalError) {
return (0, rxjs_1.throwError)(() => err);
}
return (0, rxjs_1.of)(assert(undefined, undefined, err));
}), (0, rxjs_1.catchError)((err) => {
if (err instanceof base_1.RetryError) {
return check(1000);
}
return (0, rxjs_1.throwError)(() => err);
}));
return check(0).pipe((0, rxjs_1.map)(({ table }) => {
this.logger.debug?.(`[${this.id}] Setting table ARN and stream ARN...`);
this._tableArn = `${table.TableArn}`;
this._streamArn = `${table.LatestStreamArn}`;
}), (0, rxjs_1.switchMap)(() => (0, rxjs_1.combineLatest)([
this.client.config.region(),
this.client.config.credentials(),
this.client.config.endpoint
? this.client.config.endpoint()
: Promise.resolve(undefined),
])), (0, rxjs_1.map)(([region, credentials, endpoint]) => {
this.logger.debug?.(`[${this.id}] Creating stream client with endpoint:`, endpoint);
this._streamClient = endpoint
? new client_dynamodb_streams_1.DynamoDBStreamsClient({
region,
credentials,
endpoint,
})
: new client_dynamodb_streams_1.DynamoDBStreamsClient({
region,
credentials,
});
}), (0, rxjs_1.map)(() => {
this.logger.info?.(`[${this.id}] ${this._tableArn}`);
return this;
}));
}
_stream(all) {
return new rxjs_1.Observable((subscriber) => {
const shardIteratorType = all ? 'TRIM_HORIZON' : 'LATEST';
this.logger.debug?.(`[${this.id}] Starting _stream with ${shardIteratorType}, streamArn: ${this.streamArn}`);
const iterator = new rxjs_1.Subject();
const subscriptions = [];
subscriptions.push(this.shards
.pipe((0, rxjs_1.concatMap)((shard) => (0, rxjs_1.from)(this.streamClient.send(new client_dynamodb_streams_1.GetShardIteratorCommand({
StreamArn: this.streamArn,
ShardId: shard.ShardId,
ShardIteratorType: shardIteratorType,
})))))
.subscribe({
next: ({ ShardIterator }) => {
if (ShardIterator) {
iterator.next(ShardIterator);
}
},
error: (error) => {
this.logger.warn?.(`[${this.id}] Failed to get shard iterator:`, error);
},
}));
subscriptions.push(iterator
.pipe((0, rxjs_1.takeUntil)((0, rxjs_1.fromEvent)(this.signal, 'abort')), (0, rxjs_1.concatMap)((position) => (0, rxjs_1.from)(this.streamClient.send(new client_dynamodb_streams_1.GetRecordsCommand({ ShardIterator: position })))))
.subscribe({
next: ({ Records = [], NextShardIterator }) => {
subscriber.next(Records);
if (NextShardIterator) {
subscriptions.push(rxjs_1.asyncScheduler.schedule(() => {
iterator.next(NextShardIterator);
}, !Records.length ? 100 : 0));
}
},
error: (_error) => {
},
complete: () => {
subscriber.next([]);
subscriber.complete();
},
}));
return () => {
this.logger.debug?.(`[${this.id}] Stream cleanup`);
subscriptions.forEach((sub) => sub.unsubscribe());
iterator.complete();
};
});
}
_store(item) {
return new rxjs_1.Observable((subscriber) => {
this.logger.debug?.(`[${this.id}] Storing item:`, item);
const timestamp = performance.now();
const hashKeyValue = `item-${timestamp}`;
const rangeKeyValue = `${timestamp}`;
const record = {
[this.hashKey]: hashKeyValue,
[this.rangeKey]: rangeKeyValue,
data: item,
timestamp,
};
const matcher = (event) => {
const dynamoRecord = event.dynamodb;
if (!dynamoRecord?.Keys)
return false;
const eventHashKey = dynamoRecord.Keys[this.hashKey]?.S;
const eventRangeKey = dynamoRecord.Keys[this.rangeKey]?.S;
return eventHashKey === hashKeyValue && eventRangeKey === rangeKeyValue;
};
const subscription = (0, rxjs_1.from)(this.client.send(new lib_dynamodb_1.PutCommand({
TableName: this.tableName,
Item: record,
}))).subscribe({
error: (error) => {
this.logger.error?.(`[${this.id}] Failed to store item:`, error);
subscriber.error(new base_1.FatalError(`Failed to store item: ${error.message}`));
},
complete: () => {
this.logger.debug?.(`[${this.id}] Successfully stored item with hashKey: ${hashKeyValue}, rangeKey: ${rangeKeyValue}`);
subscriber.next(matcher);
subscriber.complete();
},
});
return () => {
subscription.unsubscribe();
};
});
}
_unmarshall(event) {
const marker = event.dynamodb?.SequenceNumber;
if (!marker) {
throw new base_1.FatalError('Invalid DynamoDB record: missing SequenceNumber');
}
const image = event.dynamodb?.NewImage || event.dynamodb?.OldImage;
if (!image) {
throw new base_1.FatalError('Invalid DynamoDB record: missing NewImage or OldImage');
}
const storedData = (0, util_dynamodb_1.unmarshall)(image, this.translation.unmarshallOptions);
const { data } = storedData;
return {
...data,
__marker__: marker,
};
}
}
exports.DynamoDB = DynamoDB;
DynamoDB.shards = {};
//# sourceMappingURL=provider.js.map