pink-bears
Version:
Intelligent rate limiting middleware with MongoDB integration and caching for Node.js applications
291 lines (276 loc) • 11.1 kB
JavaScript
const { DynamoDBClient, PutItemCommand, UpdateItemCommand, QueryCommand } = require("@aws-sdk/client-dynamodb");
const { unmarshall } = require("@aws-sdk/util-dynamodb");
const ddbClient = new DynamoDBClient({ region: process.env.region });
const File = require('./file');
const {errorConstants} = require('../../constants');
const {emitEvent} = require('../../errorEmitter');
const ttlInSeconds = 24 * 60 * 60; // One day in seconds
const batchSize = 20;
class Status {
async setTotalCount(totalCount, traceId) {
try {
const batchCount = Math.ceil(totalCount / batchSize);
const object = {
traceId: { S: traceId },
row: {N: `0`},
totalCount: { N: `${totalCount}` },
batchCount: { N: `${batchCount}` },
status: { S: "Processing" },
successCount: {N: `0`},
errorCount: {N: `0`}
}
await this.started(object);
} catch (error) {
console.log("Error while setting batch size", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async updateCount(count, traceId){
try {
const batchCount = Math.ceil(count / batchSize);
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: { S: traceId },
row: { N: `0` }
},
UpdateExpression: "set #attrName1 = :attrValue1, #attrName2 = :attrValue2",
ExpressionAttributeNames: { "#attrName1": "totalCount", "#attrName2": "batchCount"},
ExpressionAttributeValues: { ":attrValue1": { N: `${count.toString()}` }, ":attrValue2": {N: `${batchCount.toString()}`} },
ReturnValues: "UPDATED_NEW",
};
await this.updateItem(params);
} catch (error) {
console.log("Error while updating total Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async started(attributes) {
try {
const tableObject = {
TableName: process.env.dynamoStatusTable,
Item: attributes
};
await this.putItem(tableObject);
} catch (error) {
console.log("Error in Status started ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async success(traceId){
try{
const incrementAmount = 1;
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: { S: traceId },
row: { N: `0` }
},
UpdateExpression: `SET successCount = successCount + :inc`,
ExpressionAttributeValues: { ":inc": { N: `${incrementAmount.toString()}` } },
ReturnValues: "UPDATED_NEW",
};
await this.updateItem(params);
}catch(error){
console.log("Error while incrementing the success Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async writeFailure(row, contact, error, traceId, file, prevCount) {
try {
//writing error record
const attributes = {
traceId: { S: traceId },
row: { N: `${row}`},
status: { S: "Failure" },
error: { S: `${file}, ${prevCount?row-prevCount:row}, ${contact}, ${error}`},
ttl: { N: `${Math.floor(Date.now() / 1000) + ttlInSeconds}` }, // Set TTL to current time + 24 hours
}
const tableObject = {
TableName: process.env.dynamoStatusTable,
Item: attributes
}
await this.putItem(tableObject);
//incrementing error count in first record
try{
const incrementAmount = 1;
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: { S: traceId },
row: { N: `0` }
},
UpdateExpression: `SET errorCount = errorCount + :inc`,
ExpressionAttributeValues: { ":inc": { N: `${incrementAmount.toString()}` } },
ReturnValues: "UPDATED_NEW",
};
await this.updateItem(params);
}catch(error){
console.log("Error while incrementing the error Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
} catch (error) {
console.log("Error in writeFailure where error = ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async changeStatus(error, traceId) {
try {
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: { S: traceId },
row: { N: `0` }
},
UpdateExpression: "set #attrName1 = :attrValue1, #attrName2 = :attrValue2",
ExpressionAttributeNames: { "#attrName1": "status" , "#attrName2": "error"},
ExpressionAttributeValues: { ":attrValue1": { S: "Completed" }, ":attrValue2": {S: error} },
ReturnValues: "UPDATED_NEW",
};
await this.updateItem(params);
} catch (error) {
console.log("Error while changing failed status", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async status(response, count, batch, traceId, contact, file, prevCount){
try {
if (isNaN(count)) {
throw { name: "ValidationError", message: "row should be a number" };
}
if(!response){
response ={
status:500,
data: "Internal Server Error: No Reply from Server"
};
}
const success = new RegExp("20[0-9]");
if(success.exec(response.status)){
await this.success(traceId);
}else{
let row;
if(!prevCount && prevCount!==0){
row = (batch * batchSize)+count;
}else {
if(batch>1){
row = prevCount + (batch*batchSize) + count;
} else {
row = prevCount+count;
}
}
await this.writeFailure(row, JSON.stringify(contact), JSON.stringify(response?.data), traceId, file, prevCount);
}
}
catch (error) {
console.log("Error while checking status ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async putItem(params) {
try {
const data = await ddbClient.send(new PutItemCommand(params));
return data;
} catch (error) {
console.log("Error while putting items in dynamo db", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async updateItem(params) {
try {
const data = await ddbClient.send(new UpdateItemCommand(params));
return data;
} catch (error) {
console.log("Error while updating item in dynamo db", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async getErrorURL(key, type) {
const traceId = key.split('/')[0];
try {
const file = new File();
const objectExists = await file.findObject(key);
if (!objectExists) {
const params = {
TableName: process.env.dynamoStatusTable,
KeyConditionExpression: '#pk = :pk and #sk > :sk',
ExpressionAttributeNames: {
'#pk': 'traceId',
'#sk': 'row'
},
ExpressionAttributeValues: {
':pk': { S: traceId },
':sk': { N: '0' }
},
};
const data = await ddbClient.send(new QueryCommand(params));
if (data.Count) {
const dataArray = data.Items ? data.Items.map((item) => unmarshall(item)) : [];
const errors = [];
for (const error of dataArray) {
errors.push(error.error);
}
const stringContent = errors.join('\n');
try {
await file.uploadToS3(key, stringContent, type);
} catch (error) {
console.log('uploading data into s3', error);
throw error;
}
} else {
return false;
}
}
return true;
} catch (error) {
console.log(`Error while querying for in dynamoDb for traceId : ${traceId}`, error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async getStatus(traceId) {
try {
const params = {
TableName: process.env.dynamoStatusTable,
KeyConditionExpression: '#pk = :pk and #sk = :sk',
ExpressionAttributeNames: {
'#pk': 'traceId',
'#sk': 'row'
},
ExpressionAttributeValues: {
':pk': { S: traceId },
':sk': { N: '0' }
},
};
const data = await ddbClient.send(new QueryCommand(params));
if (data.Count) {
const statusArray = data.Items ? data.Items.map((item) => unmarshall(item)) : [];
const status = statusArray[0];
if (status.totalCount != status.successCount + status.errorCount) {
if (status?.error) {
status.status = "Failed";
}
} else {
status.status = "Completed";
}
return status;
} else {
return { "status": "Processing" };
}
} catch (error) {
console.log(`Error while querying for status in dynamoDb for traceId fn. Name getStatus : ${traceId}`, error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
}
module.exports = Status;