dynamodb-time-series-manager
Version:
Managing Time Series Statistics with TimeSeriesStatisticsManager and AWS DynamoDB
247 lines (246 loc) • 9.02 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.TimeSeriesStatisticsManager = void 0;
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
const moment_1 = __importDefault(require("moment"));
const timePartitions = [
{
name: "minute",
format: "YYYY-MM-DDTHH:mm:00.000Z",
interval: 60
},
{
name: "hour",
format: "YYYY-MM-DDTHH:00:00.000Z",
interval: 60 * 60
},
{
name: "day",
format: "YYYY-MM-DDT00:00:00.000Z",
interval: 60 * 60 * 24
},
{
name: "month",
format: "YYYY-MM-01T00:00:00.000Z",
interval: 60 * 60 * 24 * 30
},
{
name: "year",
format: "YYYY-01-01T00:00:00.000Z",
interval: 60 * 60 * 24 * 365
}
];
class TimeSeriesStatisticsManager {
constructor(options) {
if (options.client) {
this.client = lib_dynamodb_1.DynamoDBDocument.from(options.client);
}
else {
this.client = lib_dynamodb_1.DynamoDBDocument.from(new client_dynamodb_1.DynamoDBClient({}));
}
if (options.timePartitions) {
this.timePartitions = [...options.timePartitions];
}
else {
this.timePartitions = [...timePartitions];
}
if (options.table) {
this.table = options.table;
}
else {
this.table = "statistics";
}
this.state = 'INITIALIZING';
this.onReadyPromises = [];
Promise.resolve()
.then(() => {
return this.createTableIfNotExists();
})
.then(() => {
this.state = 'INITIALIZED';
this.resolveReadyPromises();
})
.catch((error) => {
this.state = "FAIL";
this.rejectReadyPromises(error);
});
}
onReady() {
return new Promise((resolve, reject) => {
if (this.state === 'INITIALIZED') {
resolve();
}
else if (this.state === 'FAIL') {
reject();
}
else {
this.onReadyPromises.push((error) => {
if (error) {
reject(error);
}
else {
resolve();
}
});
}
});
}
resolveReadyPromises() {
for (const resolve of this.onReadyPromises) {
resolve();
}
this.onReadyPromises = [];
}
rejectReadyPromises(error) {
for (const resolve of this.onReadyPromises) {
resolve(error);
}
this.onReadyPromises = [];
}
// Wait until the table exists
async waitUntilTableExists(timeout = 6000) {
const command = { TableName: this.table };
const startTime = Date.now();
const endTime = startTime + timeout;
while (Date.now() < endTime) {
try {
let result = await this.client.send(new client_dynamodb_1.DescribeTableCommand(command));
if (result.Table.TableStatus == client_dynamodb_1.TableStatus.ACTIVE) {
return;
}
else if (result.Table.TableStatus == client_dynamodb_1.TableStatus.DELETING || result.Table.TableStatus == client_dynamodb_1.TableStatus.INACCESSIBLE_ENCRYPTION_CREDENTIALS) {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000));
}
catch (e) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
throw new Error(`Timed out waiting for table ${this.table} to exist`);
}
async createTableIfNotExists() {
try {
await this.client.send(new client_dynamodb_1.DescribeTableCommand({ TableName: this.table }));
}
catch (error) {
if (error.name === "ResourceNotFoundException") {
const params = {
AttributeDefinitions: [
{ AttributeName: "topic_period", AttributeType: "S" },
{ AttributeName: "time_partition", AttributeType: "N" }
],
KeySchema: [
{ AttributeName: "topic_period", KeyType: "HASH" },
{ AttributeName: "time_partition", KeyType: "RANGE" }
],
ProvisionedThroughput: {
ReadCapacityUnits: 5,
WriteCapacityUnits: 5
},
TableName: this.table
};
await this.client.send(new client_dynamodb_1.CreateTableCommand(params));
// Wait until table is active
await this.waitUntilTableExists();
}
else {
console.error("Error checking for the existence of the DynamoDB table:", error);
throw error;
}
}
}
createTopicPeriod(topic, period) {
return `${topic}#${period}`;
}
getTimePartition(period) {
return this.timePartitions.find((tp) => tp.name === period);
}
getTimePartitionValue(timestamp, partition) {
const date = (0, moment_1.default)(timestamp);
const formattedDate = date.utc().format(partition.format);
return formattedDate;
}
async addStatistic(topic, timestamp, amount) {
await this.onReady();
for (const partition of this.timePartitions) {
const timePartitionValue = this.getTimePartitionValue(timestamp, partition);
const timePartition = new Date(timePartitionValue);
const updateParams = {
TableName: this.table,
Key: {
topic_period: this.createTopicPeriod(topic, partition.name),
time_partition: timePartition.getTime()
},
UpdateExpression: "ADD #count :incr SET #timestamp = :time",
ExpressionAttributeNames: {
"#count": "count",
"#timestamp": "time"
},
ExpressionAttributeValues: {
":incr": amount ? amount : 1,
":time": timePartition.toISOString()
},
ReturnValues: "UPDATED_NEW"
};
try {
const updateCommand = new lib_dynamodb_1.UpdateCommand(updateParams);
await this.client.send(updateCommand);
}
catch (error) {
console.error(`Error updating ${partition.name} statistic:`, JSON.stringify(error, null, 2));
throw error;
}
}
}
async getStatisticsPeriod(topic, period, startTime, endTime) {
await this.onReady();
const periodTopic = this.createTopicPeriod(topic, period);
const partition = this.getTimePartition(period);
startTime = Math.floor(startTime / partition.interval) * partition.interval;
endTime = Math.floor(endTime / partition.interval) * partition.interval;
const queryParams = {
TableName: this.table,
KeyConditionExpression: "topic_period = :topic_period AND time_partition BETWEEN :startTime AND :endTime",
ExpressionAttributeValues: {
":topic_period": periodTopic,
":startTime": startTime,
":endTime": endTime
}
};
try {
const response = await this.client.send(new lib_dynamodb_1.QueryCommand(queryParams));
if (response.Items) {
return response.Items.map((item) => {
const [topic, period] = item.topic_period.split("#");
return {
topic,
period,
count: item.count,
time_partition: item.time_partition,
};
});
}
else {
return [];
}
}
catch (error) {
console.error("Error getting statistic:", JSON.stringify(error, null, 2));
throw error;
}
}
async getStatistics(topic, startTime, endTime) {
await this.onReady();
const results = await Promise.all(this.timePartitions.map(async (partition) => {
const result = await this.getStatisticsPeriod(topic, partition.name, startTime, endTime);
return result;
}));
return results.flat();
}
}
exports.TimeSeriesStatisticsManager = TimeSeriesStatisticsManager;