elastic-firehose-logger/src/index.js

Logger to send logs to Firehose and Elasticsearch in prod

const { FirehoseClient, PutRecordBatchCommand } = require('@aws-sdk/client-firehose');
const TimeStampHelper = require('./TimeStampHelper');
const { Client } = require('@elastic/elasticsearch');

class FirehoseElasticLogger {
    constructor({ streamName, elasticUrl, region}){
        this.firehoseClient = new FirehoseClient({ region: region});
        this.streamName = streamName;
        this.logQueue = [];
        this.maxBatchSize = 500;
        this.flushInterval = 5000;  // 5 seconds

        this.elasticUrl = elasticUrl
        this.index = 'application-logs';
        this.elasticClient = new Client({ node: this.elasticUrl });

        setInterval(() => this.flushFirehoseLogs(), this.flushInterval);
    }

    async logRequest(req, res,next, appName) {
        //Exempt HealthCheck API from logging
        if (/\/healthCheck/.test(req.url)) {
            return next();
        }

        const start = Date.now();
        const url = req.url;

        res.on('finish', async () => {
            const responseTime = Date.now();
            let logData;
            try{
                logData = {
                    timestamp: TimeStampHelper(Date.now()),
                    method: req.method,
                    url: url,
                    statuscode: res.statusCode,
                    headers: req.headers,
                    body: req.body,
                    requesttimestamp: start,
                    responsetimestamp: responseTime,
                    duration: responseTime - start,
                    application: appName,
                    level: res.statusCode < 400 ? 'info' : 'error',
                    requestid: req.requestId,
                    username: res?.req?.user === undefined ?  req?.body?.username: res?.req?.user?.Username,
                    resbody: res?.locals?.response,
                    sourceip: req?.socket?.remoteAddress ? req.socket.remoteAddress : 'sourceip'
                };
            } catch(error){
                console.error('Error in log data structure:', error);
            }

            await this.logToFirehose(logData);
            await this.logToElasticsearch(logData,appName);
        });
        
        next();
    }

    async logToFirehose(logData) {
        this.logQueue.push({
            Data: Buffer.from(JSON.stringify(logData) + '\n')
        });
        if(this.logQueue.length >= this.maxBatchSize) {
            this.flushFirehoseLogs();
        }
    }

    async flushFirehoseLogs() {
        if(this.logQueue.length === 0) return;

        const batch = this.logQueue.splice(0, this.maxBatchSize);
        const params = {
            DeliveryStreamName: this.streamName,
            Records: batch
        };

        try {
            const response = await this.firehoseClient.send(new PutRecordBatchCommand(params));
            if(response.FailedPutCount > 0){
                console.error(`Firehose failed to deliver ${response.FailedPutCount} records`);
            } 
        } catch (error) {
            // console.error(`Firehose batch log error:`, error);
            this.logQueue.push(...batch); //Re-add failed logs 
        }
    }

    async logToElasticsearch(logData,appName) {
        const date = new Date().toISOString().split('T')[0];
        const appIndex = `${this.index}-${appName.toLowerCase()}-${date}`;

        try {
            await this.elasticClient.index({
                index: appIndex,
                document: logData,
            });

            // console.log(`Log successfully sent to Elasticsearch for ${appName}:`, logData);
        } catch(error){
            console.error(`Error sending logs to Elasticsearch for ${appName}:`, error);
        }
    }

}

module.exports = FirehoseElasticLogger;