atatus-nodejs/lib/collector/aggregator/index.js

Atatus APM agent for Node.js

'use strict'
const util = require('util')
const LimitedPriorityQueue = require('../lib/priorityqueue')
const Queue = require('../lib/queue')

const ERROR_EVENTS_LIMIT = 20
const ERROR_REQUESTS_LIMIT = 20
const TRACE_EVENTS_LIMIT = 5
let ANALYTICS_EVENTS_LIMIT = 10000

const KIND_TRANSACTION = 'transaction'
const TYPE_NODEJS = 'Node.js'

const TIMEOUT_DURATION = 5 * 60 * 1000    // 5 minutes

module.exports = Aggregator

function Aggregator(agent) {
    this.agent = agent
    this.transactionSpans = {}
    this.transactions = {}
    this.errorMetrics = {}
    this.errorEvents = new Queue(ERROR_EVENTS_LIMIT)
    this.errorRequests = new Queue(ERROR_REQUESTS_LIMIT)
    this.traceEvents = new LimitedPriorityQueue(TRACE_EVENTS_LIMIT)
    this.metrics = []

    ANALYTICS_EVENTS_LIMIT = agent._conf.maxAnalyticsEvents || ANALYTICS_EVENTS_LIMIT
    this.analyticsEvents = new LimitedPriorityQueue(ANALYTICS_EVENTS_LIMIT)
    this.analyticsEventsSize = 0
}

function _getSpanType (spanType, subtype, action) {

    if (spanType === TYPE_NODEJS) {
        return {
            kind: TYPE_NODEJS,
            type: TYPE_NODEJS,
            knownKind: false
        }
    }
    // let spanTypes = spanType.split('.')

    let availableKinds = {
      'db': 'Database',
      'cache': 'Database',
      'ext': 'Remote',
      'external': 'Remote',
      'websocket': 'Remote',
      'template': 'Template'
    }

    let availableTypes = {
      'mysql': 'MySQL',
      'postgresql': 'Postgres',
      'mssql': 'MS SQL',
      'mongodb': 'MongoDB',
      'redis': 'Redis',
      'graphql': 'GraphQL',
      'elasticsearch': 'Elasticsearch',
      'cassandra': 'Cassandra',
      'http': 'External Requests',
      'https': 'External Requests',
      'http2': 'External Requests'
    }

    let kind = availableKinds[spanType] || spanType
    let type = availableTypes[subtype] || subtype

    if (spanType === 'websocket') {
        type = 'WebSocket'
    } else if (spanType === 'template') {
        type = 'Template'
    }

    return {
        kind: kind,
        type: type || kind,
        knownKind: !!availableKinds[spanType]
    }
}

// let debugTxnMap = {}

Aggregator.prototype.addTxn = function (item) {
    let txnName = item.name;
    if (!txnName) {
      return;
    }

    // debugTxnMap[item.id] = new Date().getTime();

    item.duration = item.duration || 0
    let txn = this.transactions[txnName]
    if (!txn) {
        txn = {
            name: txnName,
            background: (item.type === 'background'),
            // [count, total, min, max ]
            durations: [ 0, 0, item.duration, item.duration ],
            kind: KIND_TRANSACTION,
            type: this.agent._conf.frameworkName,
            spansObject: {}
        }
    }
    this.transactions[txnName] = txn

    // [count, total, min, max ]
    txn.durations[0]++
    txn.durations[1] += item.duration
    txn.durations[2] = Math.min(item.duration, txn.durations[2])
    txn.durations[3] = Math.max(item.duration, txn.durations[3])

    if (!this.transactionSpans[item.id]) {
        this.transactionSpans[item.id] = {
          spansArray: []
        }
    }

    if (this.transactionSpans[item.id].timeoutId) {
      clearTimeout(this.transactionSpans[item.id].timeoutId)
    }

    let totalSpanTime = 0
    let analytics = this.agent._conf.analytics && !!(this.agent._conf.backendConfig && this.agent._conf.backendConfig.analytics)
    let analyticsCaptureOutgoing = this.agent._conf.analyticsCaptureOutgoing
    let analyticsSpan = { databaseCount: 0, remoteCount: 0, databaseDur: 0, remoteDur: 0 }
    let aggregatorInstance = this

    this.transactionSpans[item.id].spansArray.forEach(function(span) {
        let spanName = span.name
        if (!spanName) {
            return;
        }

        span.duration = span.duration || 0

        let spanObj = txn.spansObject[spanName]
        if (!spanObj) {
            spanObj = {
                name: spanName,
                // durations: [count, total, min, max ]
                durations: [ 0, 0, span.duration, span.duration ]
            }
        }
        txn.spansObject[spanName] = spanObj

        let spanTypes = _getSpanType(span.type, span.subtype, span.action)
        spanObj.type = spanTypes.type
        spanObj.kind = spanTypes.kind

        if (analytics && analyticsCaptureOutgoing && (spanObj.kind === "Remote") && (spanObj.type !== "WebSocket")) {
            addAnalyticsOutgoingData(span, item, aggregatorInstance)
        }

        // Only if analytics is enabled
        if (analytics) {
            if (spanObj.kind === "Database") {
                analyticsSpan.databaseCount++
                analyticsSpan.databaseDur += span.duration
            }
            if (spanObj.kind === "Remote") {
                analyticsSpan.remoteCount++
                analyticsSpan.remoteDur += span.duration
            }
        }

        // To avoid adding custom span duration in total span time.
        if (spanTypes.knownKind) {
            totalSpanTime += span.duration
        }

        // [count, total, min, max ]
        spanObj.durations[0]++
        spanObj.durations[1] += span.duration
        spanObj.durations[2] = Math.min(span.duration, spanObj.durations[2])
        spanObj.durations[3] = Math.max(span.duration, spanObj.durations[3])
    })


    // Calculate nodejs span
    if (totalSpanTime < item.duration) {
        let duration = item.duration - totalSpanTime
        let spanName = this.agent._conf.frameworkName || TYPE_NODEJS

        let spanObj = txn.spansObject[spanName]
        if (!spanObj) {
            spanObj = {
                name: spanName,
                // durations: [count, total, min, max ]
                durations: [ 0, 0, duration, duration ]
            }
        }
        txn.spansObject[spanName] = spanObj

        if (spanObj.durations[0] === 0) {
            spanObj.type = TYPE_NODEJS
            spanObj.kind = TYPE_NODEJS
        }

        // [count, total, min, max ]
        spanObj.durations[0]++
        spanObj.durations[1] += duration
        spanObj.durations[2] = Math.min(duration, spanObj.durations[2])
        spanObj.durations[3] = Math.max(duration, spanObj.durations[3])
    }

    if (item.duration >= this.agent._conf.traceThreshold) {
        item.spansArray = this.transactionSpans[item.id].spansArray

        if (totalSpanTime < item.duration) {
            item.spansArray.unshift({
                transaction_id: item.id,
                parent_id: item.id,
                trace_id: item.trace_id,
                name: this.agent._conf.frameworkName || TYPE_NODEJS,
                type: TYPE_NODEJS,
                duration: item.duration - totalSpanTime,
            })
        }

        this.traceEvents.add(item)
    }

    // Only if analytics is enabled
    if (analytics) {
        addAnalyticsData(item, analyticsSpan, this)
    }

    delete this.transactionSpans[item.id]
}

// Add analytics outgoing request payload
function addAnalyticsOutgoingData (span, currentTransaction, aggregatorInstance) {
    currentTransaction = currentTransaction || {}
    let txnContext = currentTransaction.context || {}
    let user = txnContext.user || {}
    let company = txnContext.company || {}
    let txnRequest = txnContext.request || {}
    let txnRequestHeader = txnRequest.headers || {}
    let ip = txnRequestHeader['x-forwarded-for'] || txnRequestHeader['X-Forwarded-For'] ||
            (txnRequest.socket && txnRequest.socket.remote_address) || ''
    ip = ip.split(',')[0] || ''

    let context = (span.context && span.context.http) || {}
    let request = context.request || {}
    let response = context.response || {}
    let requestBody = request.body || ''
    let responseBody = response.body || ''
    let requestHeaders = request.headers || {}
    let responseHeaders = context.responseHeaders || {}
    let bodyLength = ((requestBody.length || 0) + (responseBody.length || 0)) || 0

    let payload = {
        timestamp: currentTransaction.timestamp ? Math.floor(currentTransaction.timestamp / 1000) : new Date().getTime(),
        txnId: span.transaction_id || span.id,
        trace_id: span.trace_id,
        name: span.name || span.transaction_name,
        duration: span.duration,
        requestHeaders: requestHeaders,
        responseHeaders: responseHeaders,
        requestBody: requestBody,
        responseBody: responseBody,
        bodyLength: bodyLength,
        customData: txnContext.custom || {},
        url: context.url || '',
        statusCode: context.statusCode || 200,
        method: context.method || 'GET',
        userAgent: requestHeaders['user-agent'],
        ip: ip,
        direction: 'outgoing',
        userId: user.id,
        userName: user.name,
        userEmail: user.email,
        companyId: company.id,
    }

    try {
        if (aggregatorInstance.agent._conf.analyticsSkip && aggregatorInstance.agent._conf.analyticsSkip(payload)) {
          aggregatorInstance.agent.logger.debug('Skipped analytics outgoing event: ' + payload.name)
          return
        }
    } catch (e) {
        aggregatorInstance.agent.logger.error('Analytics skip function failed with exception: %s', e.message)
    }

    try {
        if (aggregatorInstance.agent._conf.analyticsMask) {
          aggregatorInstance.agent._conf.analyticsMask(payload)
        }
    } catch (e) {
        aggregatorInstance.agent.logger.error('Analytics mask function failed with exception: %s', e.message)
    }

    // Add payload to events
    aggregatorInstance.analyticsEvents.add(payload)
    aggregatorInstance.analyticsEventsSize += (payload.bodyLength || 0)

}

function addAnalyticsData (item, analyticsSpan, aggregatorInstance) {
    let request = _getRequestPayload(item.context)
    let user = (item.context && item.context.user) || {}
    let company = (item.context && item.context.company) || {}

    // For custom start transaction, there wont be any status code and url.
    if (!request.statusCode && !request.url) {
        return;
    }

    let payload = {
        timestamp: item.timestamp ? Math.floor(item.timestamp / 1000) : new Date().getTime(),
        txnId: item.transaction_id || item.id,
        // kind: KIND_TRANSACTION,
        // type: TYPE_NODEJS,
        name: item.name || item.transaction_name,
        duration: item.duration,
        dbCalls: analyticsSpan.databaseCount,
        dbDuration: analyticsSpan.databaseDur,
        extCalls: analyticsSpan.remoteCount,
        extDuration: analyticsSpan.remoteDur,
        errorClass: analyticsSpan.errClass || '',
        errorMessage: analyticsSpan.errMsg || '',
        customData: item.context && item.context.custom,
        // tags: item.context && item.context.tags && Object.values(item.context.tags),
        direction: 'incoming',
        userId: user.id,
        userName: user.name,
        userEmail: user.email,
        companyId: company.id,
        ...request
    }

    try {
        if (aggregatorInstance.agent._conf.analyticsSkip && aggregatorInstance.agent._conf.analyticsSkip(payload)) {
          aggregatorInstance.agent.logger.debug('Skipped analytics event: ' + payload.name)
          return
        }
    } catch (e) {
        aggregatorInstance.agent.logger.error('Analytics skip function failed with exception: %s', e.message)
    }

    try {
        if (aggregatorInstance.agent._conf.analyticsMask) {
          aggregatorInstance.agent._conf.analyticsMask(payload)
        }
    } catch (e) {
        aggregatorInstance.agent.logger.error('Analytics mask function failed with exception: %s', e.message)
    }

    // Add payload to events
    aggregatorInstance.analyticsEvents.add(payload)
    aggregatorInstance.analyticsEventsSize += (payload.bodyLength || 0)
}

Aggregator.prototype.addSpan = function (item) {
    // if (debugTxnMap[item.transaction_id]) {
    //   this.agent.logger.debug(`Span late - ${item.name} - ${new Date().getTime() - debugTxnMap[item.transaction_id]} ms`)
    // }
    let spanName = item.name
    let transaction_id = item.transaction_id

    if (!spanName || (this.agent._conf.serverHosts && this.agent._conf.serverHosts.indexOf(spanName) !== -1)) {
        return;
    }

    this.transactionSpans[transaction_id] =  this.transactionSpans[transaction_id] || { spansArray: null };
    let spansArray = this.transactionSpans[transaction_id].spansArray;

    if (spansArray) {
        spansArray.push(item)
    } else {
        this.transactionSpans[transaction_id].spansArray = [ item ]

        // Clear spans after certain duration
        let timeoutId = setTimeout( () => {
            if (this.transactionSpans[transaction_id]) {
                delete this.transactionSpans[transaction_id];
            }
        }, TIMEOUT_DURATION);

        this.transactionSpans[transaction_id].timeoutId = timeoutId
    }
}

Aggregator.prototype.addErrorEvents = function (item) {
    let exception = item.exception || item.log
    if (!exception) {
        return;
    }

    let error = {
        timestamp: new Date().getTime(),
        transaction: item.transaction_name,
        request: _getRequestHeader(item.context),
        customData: item.context && item.context.custom,
        tags: item.context && item.context.tags && Object.values(item.context.tags),
        user: item.context && item.context.user
    }

    error.exceptions = [{
        class: exception.type || 'Error',
        message: exception.message,
        stacktrace: []
    }]

    // let errSpan = {
    //     errClass: error.exceptions[0].class,
    //     errMsg: error.exceptions[0].message
    // }


    exception.stacktrace = exception.stacktrace || []
    exception.stacktrace.forEach(function(callSite) {
        // Ignore if file name is containing atatus nodejs, anonymous or no file name
        let fileName = callSite.filename
        if (!fileName || fileName.indexOf("/atatus-nodejs/") !== -1 || fileName === "<anonymous>") {
            return;
        }

        let frame = {
           f: fileName,
           p: callSite.abs_path,
           m: callSite.function || "none",
           ln: callSite.lineno,
           inp: !callSite.library_frame
        };
        error.exceptions[0].stacktrace.push(frame)
    });

    // Only if analytics is enabled
    // if (this.agent._conf.analytics) {
    //     addAnalyticsData(item, errSpan, this)
    // }

    this.errorEvents.add(error)
}

Aggregator.prototype.addErrorMetrics = function (statusCode, item) {
    let txnName = item.name;
    if (!txnName || !statusCode) {
      return;
    }

    let errorMetric = this.errorMetrics[item.name]
    if (!errorMetric) {
        errorMetric = {
            name: txnName,
            kind: "transaction",
            type: this.agent._conf.frameworkName,
            statusCodes: {}
        }
    }
    let requestHeader = item.context &&
                   item.context.request &&
                   item.context.request.headers || {}

    this.errorMetrics[item.name] = errorMetric
    errorMetric.statusCodes[statusCode] = (errorMetric.statusCodes[statusCode] || 0) + 1;

    let errorRequests = {
        name: txnName,
        type: this.agent._conf.frameworkName,
        kind: KIND_TRANSACTION,
        request: _getRequestHeader(item.context),
        customData: item.context && item.context.custom,
        tags: item.context && item.context.tags && Object.values(item.context.tags),
        user: item.context && item.context.user
    }

    this.errorRequests.add(errorRequests)
}

Aggregator.prototype.addMetric = function (item) {

    if (!item || !item.samples) {
        return
    }

    let samples = item.samples
    let metric = {
        cpu: {
            total: (samples['system.cpu.total.norm.pct'] && samples['system.cpu.total.norm.pct'].value) || 0
        },
        memory: {
            total: (samples['system.memory.total'] && samples['system.memory.total'].value) || 0,
            actualFree: (samples['system.memory.actual.free'] && samples['system.memory.actual.free'].value) || 0
        },
        process: {
            cpu: {
                total: (samples['system.process.cpu.total.norm.pct'] && samples['system.process.cpu.total.norm.pct'].value) || 0,
                system: (samples['system.process.cpu.system.norm.pct'] && samples['system.process.cpu.system.norm.pct'].value) || 0,
                user: (samples['system.process.cpu.user.norm.pct'] && samples['system.process.cpu.user.norm.pct'].value) || 0
            },
            memory: {
                rssBytes: (samples['system.process.memory.rss.bytes'] && samples['system.process.memory.rss.bytes'].value) || 0
            }
        },
        nodejs: {
            handlesActive: (samples['nodejs.handles.active'] && samples['nodejs.handles.active'].value) || 0,
            requestsActive: (samples['nodejs.requests.active'] && samples['nodejs.requests.active'].value) || 0,
            eventLoopDelay: (samples['nodejs.eventloop.delay.avg.ms'] && samples['nodejs.eventloop.delay.avg.ms'].value) || 0,
            memoryHeapAllocated: (samples['nodejs.memory.heap.allocated.bytes'] && samples['nodejs.memory.heap.allocated.bytes'].value) || 0,
            memoryHeapUsed: (samples['nodejs.memory.heap.used.bytes'] && samples['nodejs.memory.heap.used.bytes'].value) || 0
        },
        timestamp: item.timestamp ? Math.floor(item.timestamp / 1000) : new Date().getTime()
    }

    // drop memory value is 0
    if (metric.memory.total === 0) {
        return
    }

    this.metrics.push(metric)
}

// Get request and response payload for analytics
function _getRequestPayload (context) {
    let request = (context && context.request) || {}
    let response = (context && context.response) || {}
    let requestBody = request.body || ''
    let responseBody = (context && context.responseBody) || response.body || ''
    let requestHeader = request.headers || {}
    let ip = requestHeader['x-forwarded-for'] || requestHeader['X-Forwarded-For'] ||
            (request.socket && request.socket.remote_address) || ''
    ip = ip.split(',')[0] || ''

    let bodyLength = ((requestBody.length || 0) + (responseBody.length || 0)) || 0

    return {
        statusCode: response.status_code,
        method: request.method,
        url: (request.url && (request.url.full || request.url.raw)) || '',
        userAgent: requestHeader['user-agent'],
        ip: ip,
        requestHeaders: request.headers || {},
        responseHeaders: response.headers || {},
        requestBody: requestBody,
        responseBody: responseBody,
        bodyLength: bodyLength
        // host: request.url && request.url.hostname,
        // port: (request.url && request.url.port && parseInt(request.url.port)) || 0
    }
}

function _getRequestHeader (context) {
    let request = context && context.request || {}
    let requestHeader = request && request.headers || {}
    let response = context && context.response || {}
    let ip = requestHeader['x-forwarded-for'] || requestHeader['X-Forwarded-For'] ||
            (request.socket && request.socket.remote_address) || ''

    ip = ip.split(',')[0] || ''

    return {
        accept: requestHeader.accept,
        'accept-encoding': requestHeader['accept-encoding'],
        'accept-language': requestHeader['accept-language'],
        ip: ip,
        referer: requestHeader.referer,
        host: requestHeader.host,
        port: (request.url && request.url.port && parseInt(request.url.port)) || 0,
        method: request.method,
        userAgent: requestHeader['user-agent'],
        path: (request.url && request.url.raw) || '',
        statusCode: response.status_code
    }
}

Aggregator.prototype.getAndResetTracesPayload = function getTracesPayload () {

    let traceEvents = this.traceEvents.toArray() || []
    this.resetTraces()

    let tracesPayload = []

    traceEvents.forEach((event) => {

        let txnName = event.name;
        if (!txnName) {
          return;
        }

        let trace = {
            name: txnName,
            id: event.id,
            type: this.agent._conf.frameworkName,
            kind: KIND_TRANSACTION,
            // start: 389882784298,
            duration: event.duration,
            entries: [],
            funcs: [],
            request: _getRequestHeader(event.context),
            customData: event.context && event.context.custom,
            tags: event.context && event.context.tags && Object.values(event.context.tags),
            user: event.context && event.context.user
        }

        event.spansArray.forEach((span, index) => {
            let spanTypes = _getSpanType(span.type, span.subtype, span.action)
            let traceEntry = {
                i: index,
                lv: 1, //span.level ||
                so: span.startOffset || 0,
                du: span.duration,
                ly: {
                    name: span.name,
                    type: spanTypes.type,
                    kind: spanTypes.kind
                },
            }

            if (span.context && span.context.db) {
                traceEntry.dt = {
                    // dns: "",
                    query: span.context.db.statement
                }
            } else if (span.context && span.context.http) {
                traceEntry.dt = {
                    url: span.context.http.url
                }
            }
            trace.entries.push(traceEntry)
            trace.funcs.push(span.name)
        });

        // let idFinderObject = {}
        // let multiDimensionArray = []
        // trace.entries.forEach((span, index) => {
        //     let indexToPush = idFinderObject[span.parent_id]
        //     if (span.level === 1) {
        //         multiDimensionArray.push( [ span ] )
        //         idFinderObject[span.parent_id] = span.parent_id
        //     } else {

        //     }
        // }

        // let transaction_id = event.transaction_id
        // for (let i = event.spansArray.length - 1; i >= 0; i--) {
        //     if (event.spansArray[i].parent_id != transaction_id && event.spansArray[i + 1]) {
        //         event.spansArray[i].lv = event.spansArray[i + 1].lv + 1
        //     }
        // }

        tracesPayload.push(trace)
    });

    return {
        startTime: new Date().getTime() - (60*1000),
        endTime: new Date().getTime(),
        traces: tracesPayload
    }
}

Aggregator.prototype.getAndResetTransactionsPayload = function () {

    let transactions = this.transactions
    this.resetTransactions()

    let txnsPayload = []
    for (let txnName of Object.keys(transactions)) {

        let txn = transactions[txnName]
        txn.traces = Object.values(txn.spansObject)

        delete txn.spansObject
        txnsPayload.push(txn)
    }

    return {
        startTime: new Date().getTime() - (60*1000),
        endTime: new Date().getTime(),
        transactions: txnsPayload
    }
}

Aggregator.prototype.getAndResetAnalyticsPayload = function () {

    let analyticsEvents = this.analyticsEvents.toArray() || []
    let analyticsEventsSize = this.analyticsEventsSize || 0
    this.resetAnalyticsEvents()

    return {
        startTime: new Date().getTime() - (60*1000),
        endTime: new Date().getTime(),
        requests: analyticsEvents,
        size: analyticsEventsSize
    }
}

Aggregator.prototype.getAndResetErrorEventsPayload = function () {
    let errorEvents = this.errorEvents.toArray()

    this.resetErrorEvents()

    return {
        errors: errorEvents
    }
}

Aggregator.prototype.getAndResetErrorMetricsPayload = function () {

    let errorMetrics = Object.values(this.errorMetrics)
    let errorRequests = this.errorRequests.toArray()

    this.resetErrorMetrics()

    return {
        startTime: new Date().getTime() - (60*1000),
        endTime: new Date().getTime(),
        errorMetrics: errorMetrics,
        errorRequests: errorRequests
    }
}

Aggregator.prototype.getAndResetMetrics = function () {

    let metrics = this.metrics
    this.resetMetrics()
    return {
        startTime: new Date().getTime() - (60*1000),
        endTime: new Date().getTime(),
        metrics: {
            nodejs: metrics,
        }
    }
}

Aggregator.prototype.getAndResetHostInfoPayload = function () {
    return {
    }
}

Aggregator.prototype.resetTransactions = function () {
    this.transactions = {}
}

Aggregator.prototype.resetAnalyticsEvents = function () {
    this.analyticsEvents = new LimitedPriorityQueue(ANALYTICS_EVENTS_LIMIT)
    this.analyticsEventsSize = 0
}

Aggregator.prototype.resetTraces = function () {
    this.traceEvents = new LimitedPriorityQueue(TRACE_EVENTS_LIMIT)
}

Aggregator.prototype.resetErrorEvents = function () {
    this.errorEvents = new Queue(ERROR_EVENTS_LIMIT)
}

Aggregator.prototype.resetErrorMetrics = function () {
    this.errorMetrics = {}
    this.errorRequests = new Queue(ERROR_REQUESTS_LIMIT)
}

Aggregator.prototype.resetMetrics = function () {
    this.metrics = []
}

Aggregator.prototype.mergeTransactions = function(payload) {

}
Aggregator.prototype.mergeAnalyticsEvents = function(payload) {

}
Aggregator.prototype.mergeTraces = function(payload) {

}
Aggregator.prototype.mergeErrorEventsPayload = function(payload) {

}
Aggregator.prototype.mergeErrorMetrics = function(payload) {

}
Aggregator.prototype.mergeMetrics = function(payload) {

}


Aggregator.prototype.isTransactionsEmpty = function () {
    return (Object.keys(this.transactions).length < 1)
}

Aggregator.prototype.isAnalyticsEmpty = function () {
    return (this.analyticsEvents.length < 1)
}

Aggregator.prototype.isTracesEmpty = function () {
    return (this.traceEvents.length < 1)
}

Aggregator.prototype.isErrorEventsEmpty = function () {
    return (this.errorEvents.length < 1)
}

Aggregator.prototype.isErrorMetricsEmpty = function () {
    return (this.errorRequests.length < 1)
}

Aggregator.prototype.isMetricsEmpty = function () {
    return (this.metrics.length < 1)
}