pink-bears
Version:
Intelligent rate limiting middleware with MongoDB integration and caching for Node.js applications
369 lines • 10.1 kB
JavaScript
const {
DynamoDBClient,
PutItemCommand,
UpdateItemCommand,
QueryCommand
} = require("@aws-sdk/client-dynamodb");
const {
unmarshall
} = require("@aws-sdk/util-dynamodb");
const ddbClient = new DynamoDBClient({
region: process.env.region
});
const File = require('./file');
const {
errorConstants
} = require('../../constants');
const {
emitEvent
} = require('../../errorEmitter');
const ttlInSeconds = 24 * 60 * 60; // One day in seconds
const batchSize = 20;
class Status {
async setTotalCount(totalCount, traceId) {
try {
const batchCount = Math.ceil(totalCount / batchSize);
const object = {
traceId: {
S: traceId
},
row: {
N: `0`
},
totalCount: {
N: `${totalCount}`
},
batchCount: {
N: `${batchCount}`
},
status: {
S: "Processing"
},
successCount: {
N: `0`
},
errorCount: {
N: `0`
}
};
await this.started(object);
} catch (error) {
console.log("Error while setting batch size", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async updateCount(count, traceId) {
try {
const batchCount = Math.ceil(count / batchSize);
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: {
S: traceId
},
row: {
N: `0`
}
},
UpdateExpression: "set #attrName1 = :attrValue1, #attrName2 = :attrValue2",
ExpressionAttributeNames: {
"#attrName1": "totalCount",
"#attrName2": "batchCount"
},
ExpressionAttributeValues: {
":attrValue1": {
N: `${count.toString()}`
},
":attrValue2": {
N: `${batchCount.toString()}`
}
},
ReturnValues: "UPDATED_NEW"
};
await this.updateItem(params);
} catch (error) {
console.log("Error while updating total Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async started(attributes) {
try {
const tableObject = {
TableName: process.env.dynamoStatusTable,
Item: attributes
};
await this.putItem(tableObject);
} catch (error) {
console.log("Error in Status started ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async success(traceId) {
try {
const incrementAmount = 1;
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: {
S: traceId
},
row: {
N: `0`
}
},
UpdateExpression: `SET successCount = successCount + :inc`,
ExpressionAttributeValues: {
":inc": {
N: `${incrementAmount.toString()}`
}
},
ReturnValues: "UPDATED_NEW"
};
await this.updateItem(params);
} catch (error) {
console.log("Error while incrementing the success Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async writeFailure(row, contact, error, traceId, file, prevCount) {
try {
//writing error record
const attributes = {
traceId: {
S: traceId
},
row: {
N: `${row}`
},
status: {
S: "Failure"
},
error: {
S: `${file}, ${prevCount ? row - prevCount : row}, ${contact}, ${error}`
},
ttl: {
N: `${Math.floor(Date.now() / 1000) + ttlInSeconds}`
} // Set TTL to current time + 24 hours
};
const tableObject = {
TableName: process.env.dynamoStatusTable,
Item: attributes
};
await this.putItem(tableObject);
//incrementing error count in first record
try {
const incrementAmount = 1;
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: {
S: traceId
},
row: {
N: `0`
}
},
UpdateExpression: `SET errorCount = errorCount + :inc`,
ExpressionAttributeValues: {
":inc": {
N: `${incrementAmount.toString()}`
}
},
ReturnValues: "UPDATED_NEW"
};
await this.updateItem(params);
} catch (error) {
console.log("Error while incrementing the error Count", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
} catch (error) {
console.log("Error in writeFailure where error = ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async changeStatus(error, traceId) {
try {
const params = {
TableName: process.env.dynamoStatusTable,
Key: {
traceId: {
S: traceId
},
row: {
N: `0`
}
},
UpdateExpression: "set #attrName1 = :attrValue1, #attrName2 = :attrValue2",
ExpressionAttributeNames: {
"#attrName1": "status",
"#attrName2": "error"
},
ExpressionAttributeValues: {
":attrValue1": {
S: "Completed"
},
":attrValue2": {
S: error
}
},
ReturnValues: "UPDATED_NEW"
};
await this.updateItem(params);
} catch (error) {
console.log("Error while changing failed status", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async status(response, count, batch, traceId, contact, file, prevCount) {
try {
if (isNaN(count)) {
throw {
name: "ValidationError",
message: "row should be a number"
};
}
if (!response) {
response = {
status: 500,
data: "Internal Server Error: No Reply from Server"
};
}
const success = new RegExp("20[0-9]");
if (success.exec(response.status)) {
await this.success(traceId);
} else {
let row;
if (!prevCount && prevCount !== 0) {
row = batch * batchSize + count;
} else {
if (batch > 1) {
row = prevCount + batch * batchSize + count;
} else {
row = prevCount + count;
}
}
await this.writeFailure(row, JSON.stringify(contact), JSON.stringify(response?.data), traceId, file, prevCount);
}
} catch (error) {
console.log("Error while checking status ", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async putItem(params) {
try {
const data = await ddbClient.send(new PutItemCommand(params));
return data;
} catch (error) {
console.log("Error while putting items in dynamo db", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async updateItem(params) {
try {
const data = await ddbClient.send(new UpdateItemCommand(params));
return data;
} catch (error) {
console.log("Error while updating item in dynamo db", error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async getErrorURL(key, type) {
const traceId = key.split('/')[0];
try {
const file = new File();
const objectExists = await file.findObject(key);
if (!objectExists) {
const params = {
TableName: process.env.dynamoStatusTable,
KeyConditionExpression: '#pk = :pk and #sk > :sk',
ExpressionAttributeNames: {
'#pk': 'traceId',
'#sk': 'row'
},
ExpressionAttributeValues: {
':pk': {
S: traceId
},
':sk': {
N: '0'
}
}
};
const data = await ddbClient.send(new QueryCommand(params));
if (data.Count) {
const dataArray = data.Items ? data.Items.map(item => unmarshall(item)) : [];
const errors = [];
for (const error of dataArray) {
errors.push(error.error);
}
const stringContent = errors.join('\n');
try {
await file.uploadToS3(key, stringContent, type);
} catch (error) {
console.log('uploading data into s3', error);
throw error;
}
} else {
return false;
}
}
return true;
} catch (error) {
console.log(`Error while querying for in dynamoDb for traceId : ${traceId}`, error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
async getStatus(traceId) {
try {
const params = {
TableName: process.env.dynamoStatusTable,
KeyConditionExpression: '#pk = :pk and #sk = :sk',
ExpressionAttributeNames: {
'#pk': 'traceId',
'#sk': 'row'
},
ExpressionAttributeValues: {
':pk': {
S: traceId
},
':sk': {
N: '0'
}
}
};
const data = await ddbClient.send(new QueryCommand(params));
if (data.Count) {
const statusArray = data.Items ? data.Items.map(item => unmarshall(item)) : [];
const status = statusArray[0];
if (status.totalCount != status.successCount + status.errorCount) {
if (status?.error) {
status.status = "Failed";
}
} else {
status.status = "Completed";
}
return status;
} else {
return {
"status": "Processing"
};
}
} catch (error) {
console.log(`Error while querying for status in dynamoDb for traceId fn. Name getStatus : ${traceId}`, error);
await emitEvent(errorConstants.dynamo, error);
throw error;
}
}
}
module.exports = Status;