ibmapm-restclient/lib/restclient/sender-queue.js

Node.js rest client for ICAM.

// Copyright IBM Corp. 2017. All Rights Reserved.
// Node module: ibmapm
// This file is licensed under the Apache License 2.0.
// License text available at https://opensource.org/licenses/Apache-2.0
'use strict';
var url = require('url');
var http = require('http');
var https = require('https');
var HttpsProxyAgent = require('https-proxy-agent');
var request = require('request');
var zlib = require('zlib');
var logger = require('../plugins/logutil').getLogger('restclient_sender-queue.js');
var resourceRegistryDumper = require('../plugins/logutil').getResourceRegisterDumper();
var util = require('../plugins/util');
var pluginconfig = require('./config').pluginConfig;
var uuid = require('uuid');
var fs = require('fs');

var httpProxy;
var proxyAgent;

var queues = {};

var senderQueueLoop = [];

const keepAliveAgentHttp = new http.Agent({
    keepAlive: process.env.KNJ_KEEPALIVE ? util.isTrue(process.env.KNJ_KEEPALIVE) : true,
    maxSockets: process.env.KNJ_MAXSOCKETS ? process.env.KNJ_MAXSOCKETS : 1,
    keepAliveMsecs: process.env.KNJ_KEEPALIVEMSECS ? process.env.KNJ_KEEPALIVEMSECS : 120000
});
const keepAliveAgentHttps = new https.Agent({
    keepAlive: process.env.KNJ_KEEPALIVE ? util.isTrue(process.env.KNJ_KEEPALIVE) : true,
    maxSockets: process.env.KNJ_MAXSOCKETS ? process.env.KNJ_MAXSOCKETS : 1,
    keepAliveMsecs: process.env.KNJ_KEEPALIVEMSECS ? process.env.KNJ_KEEPALIVEMSECS : 120000
});


SenderQueue.prototype.send = function(task, quequeName) {
    // special for cloudnative register
    if (task.success || task.empty) {
        return;
    }
    if (process.env.DC_TEST_MODE) {
        task.success = true;
        return;
    }
    let payloadString = task.payload ? task.payload : '';
    if (typeof payloadString === 'object' && payloadString.length === 0) {
        return;
    }

    let urlMap = url.parse(task.url);
    let sendmethod = urlMap.protocol === 'https:' ? https : http;
    if (!task.additionalHeader ||
        !task.additionalHeader['Content-Type'] ||
        task.additionalHeader['Content-Type'] === 'application/json') {
        payloadString = (typeof task.payload === 'object') ?
            JSON.stringify(task.payload) : task.payload ? task.payload : '';
    }

    let header = {
        'Content-Type': 'application/json',
        'Content-Length': payloadString.length,
        'X-TransactionID': uuid.v1()
    };

    if (task.type === 'amui:') {
        var content = fs.readFileSync(payloadString.toString());
        header['Content-Length'] = content.length;
    }

    if (task.gzipped) {
        header['Content-Encoding'] = 'gzip';
        payloadString = zlib.gzipSync(payloadString);
        header['Content-Length'] = payloadString.length;
    }
    let key;
    if (task.additionalHeader) { // mixin additionalHeader and header
        for (key in task.additionalHeader) {
            header[key] = task.additionalHeader[key];
        }
    }
    let options = {
        hostname: urlMap.hostname,
        host: urlMap.host,
        path: urlMap.path,
        method: task.GET ? 'GET' : (task.PUT ? 'PUT' : 'POST'),
        agent: keepAliveAgentHttp,
        port: urlMap.port,
        rejectUnauthorized: false,
        headers: header,
        protocol: urlMap.protocol,
        query: urlMap.query
    };
    if (urlMap.auth) {
        options.auth = urlMap.auth;
    }

    if (urlMap.protocol === 'https:') {
        options.agent = keepAliveAgentHttps;
    }

    if (!urlMap.port) {
        options.port = urlMap.protocol === 'https:' ? 443 : 80;
    }
    if (task.addtionalOptions) {
        for (key in task.addtionalOptions) {
            options[key] = task.addtionalOptions[key];
        }
    }
    if (task.type.indexOf('amui:') === 0) {
        logger.debug('Post payload to ', task.url, ' ; header: ',
            JSON.stringify(options.headers));
    } else {
        if (task.GET) {
            logger.debug('Get from ' + task.url, ' ; header: ',
                JSON.stringify(options.headers));
        } else {
            logger.debug((task.PUT ? 'Put' : 'Post') + ' payload to ', task.url, ' ; header: ',
                JSON.stringify(options.headers), ' ; payload: ', payloadString);
        }
    }

    if (task.type.indexOf('resources: ') === 0 || task.type.indexOf('dc:') === 0 ||
        task.type.indexOf('dcconfig: ') === 0 || task.type.indexOf('updateResources: ') === 0) {

        if (task.GET) {
            resourceRegistryDumper.info('Get from ' + task.url + ' ; header: ' +
                JSON.stringify(options.headers) + ' ;');
        } else {
            resourceRegistryDumper.info((task.PUT ? 'Put' : 'Post') + ' payload to ' + task.url + ' ; header: ' +
                JSON.stringify(options.headers) + ' ; payload: ' + payloadString);
        }
    }
    if (process.env.KNJ_PROXY && urlMap.protocol === 'http:') {
        sendThrouHTTPProxy(options, task, payloadString);

    } else {
        if (process.env.KNJ_PROXY && urlMap.protocol === 'https:') {
            if (!proxyAgent) {
                proxyAgent = new HttpsProxyAgent(process.env.KNJ_PROXY);
                proxyAgent = new HttpsProxyAgent(process.env.KNJ_PROXY);
            }
            options.agent = proxyAgent;
        }
        if (process.env.KDE_GATEWAY) {
            options.path = options.protocol + '//' + options.host + options.path;
            options.host = process.env.KDE_GATEWAY;
            options.hostname = process.env.KDE_GATEWAY;
        }
        sendData(options, sendmethod, task, quequeName, payloadString);
    };

};

function sendData(options, sendmethod, task, quequeName, payloadString) {
    util.tlsFix8(options);
    var req = sendmethod.request(options, function(res) {
        if (res && (res.statusCode >= 200 && res.statusCode < 300 || res.statusCode === 409)) {
            logger.debug('Request ', task.type, 'To', task.url,
                ' Response statusCode: ', res.statusCode);
            task.success = true;
        } else {
            if (res) {
                logger.warn('Request ', task.type, 'To', task.url,
                    ' Response statusCode: ', res.statusCode);
            }
        }
        res.on('data', function(d) {
            logger.debug('Request ' + task.type + ' response: ' + d.toString());
        });
        res.on('error', function(error) {
            logger.error('Request ' + task.type + ' response error: ' + error);
        });
        if (task.callback) {
            task.callback(null, res);
        }
    });
    req.on('error', function(error) {
        logger.error('Request ' + task.type + ' request error:' + error);
        if (task.callback) {
            task.callback(error);
        }
    });
    // logger.debug(payloadString);
    if (task.type === 'amui:') {
        var content = fs.readFileSync(payloadString.toString());
        req.write(content);
    } else {
        req.write(payloadString);
    }
    req.end();
}

function sendThrouHTTPProxy(options, task, payloadString) {
    httpProxy = httpProxy || request.defaults({
        proxy: process.env.KNJ_PROXY,
        rejectUnauthorized: false
    });
    var protocol = 'http://';
    try {
        if (options.method === 'POST') {
            if (task.type === 'amui:') {
                payloadString = fs.readFileSync(payloadString.toString());
            }
            httpProxy.post(protocol + options.hostname + ':' + options.port + options.path, {
                body: payloadString,
                headers: options.headers
            }, function(err, resp, body) {
                if (err) {
                    logger.error('Request ' + task.type + ' through proxy, Error:', err);
                }
                if (resp) {
                    if (resp.statusCode >= 200 && resp.statusCode < 300 || resp.statusCode === 409) {
                        logger.debug('Request ' + task.type + ' through proxy, response statusCode: ' +
                            resp.statusCode);
                    } else {
                        logger.warn('Request ' + task.type + ' through proxy, response statusCode: ' +
                            resp.statusCode);
                    }
                }
            });
        } else {
            httpProxy.get(protocol + options.hostname + ':' + options.port + options.path, {
                // body: payloadString,
                headers: options.headers
            }, function(err, resp, body) {
                if (task.callback) {
                    task.callback(err, resp);
                }
                if (err) {
                    logger.error('Request ' + task.type + ' through proxy, Error:', err);
                }
                if (resp) {
                    logger.debug('Request ' + task.type + ' through proxy, response statusCode: ' +
                        resp.statusCode);
                }
            });
        }
    } catch (e) {
        logger.error('Request ' + task.type + ' through proxy, Error:', e.message);
        logger.error('Request ' + task.type + ' through proxy, Error:', e.stack);
    }
}

function SenderQueue(name) {
    this.name = name;
    this.dcQueue = [];
    this.resourceQueue = [];
    this.metricQueue = [];
    this.aarQueue = [];
    this.adrQueue = [];
    this.jsoQueue = [];
    this.metaQueue = [];
    this.amuiQueue = [];
    this.amsQueue = [];
    this.queryQueue = [];
}

SenderQueue.prototype.addTask = function(task) {

    if (task.type === 'dc:') {
        if (pluginconfig.dcconsumers.length === 0) {
            logger.debug('No consumers on dc queue. give up the payload!');
            return; // no one monitor dc queue, give up the payload directly
        }
        if (this.dcQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.dcQueue.push(task);
        }
        if (pluginconfig.dcqueue.isBatch === false ||
            pluginconfig.dcqueue.batchSize <= this.dcQueue.length) {
            queues[this.name].consumeDC();
        }

    } else if (task.type && (task.type.indexOf('resources:') === 0 || task.type.indexOf('updateResources:') === 0)) {
        if (pluginconfig.resourceconsumers.length === 0) {
            logger.debug('No consumers on resource queue. give up the payload!');
            return; // no one monitor resource queue, give up the payload directly
        }
        if (this.resourceQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.resourceQueue.push(task);
        }
        if (pluginconfig.resourcequeue.isBatch === false ||
            pluginconfig.resourcequeue.batchSize <= this.resourceQueue.length) {
            queues[this.name].consumeResource();
        }
    } else if (task.type && task.type.indexOf('metrics:') === 0) {
        if (pluginconfig.metricconsumers.length === 0) {
            logger.debug('No consumers on metric queue. give up the payload!');
            return; // no one monitor metric queue, give up the payload directly
        }
        if (this.metricQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.metricQueue.push(task);
        }
        if (pluginconfig.metricqueue.isBatch === false ||
            pluginconfig.metricqueue.batchSize <= this.metricQueue.length) {
            queues[this.name].consumeMetric();
        }
    } else if (task.type && task.type.indexOf('aar:') === 0) {
        if (pluginconfig.aarconsumers.length === 0) {
            logger.debug('No consumers on aar queue. give up the payload!');
            return; // no one monitor aar queue, give up the payload directly
        }
        if (this.aarQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.aarQueue.push(task);
        }
        if (pluginconfig.aarqueue.isBatch === false ||
            pluginconfig.aarqueue.batchSize <= this.aarQueue.length) {
            queues[this.name].consumeAAR();
        }
    } else if (task.type && task.type.indexOf('adr:') === 0) {
        if (pluginconfig.adrconsumers.length === 0) {
            logger.debug('No consumers on adr queue. give up the payload!');
            return; // no one monitor adr queue, give up the payload directly
        }
        if (this.adrQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.adrQueue.push(task);
        }
        if (pluginconfig.adrqueue.isBatch === false ||
            pluginconfig.adrqueue.batchSize <= this.adrQueue.length) {
            queues[this.name].consumeADR();
        }
    } else if (task.type && task.type.indexOf('jso:') === 0) {
        if (pluginconfig.jsoconsumers.length === 0) {
            logger.debug('No consumers on jso queue. give up the payload!');
            return; // no one monitor jso queue, give up the payload directly
        }
        if (this.jsoQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.jsoQueue.push(task);
        }
        if (pluginconfig.jsoqueue.isBatch === false ||
            pluginconfig.jsoqueue.batchSize <= this.jsoQueue.length) {
            queues[this.name].consumeJSO();
        }
    } else if (task.type && task.type.indexOf('metadata:') === 0) {
        if (pluginconfig.metaconsumers.length === 0) {
            logger.debug('No consumers on meta queue. give up the payload!');
            return; // no one monitor jso queue, give up the payload directly
        }
        if (this.metaQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.metaQueue.push(task);
        }
        if (pluginconfig.metaqueue.isBatch === false ||
            pluginconfig.metaqueue.batchSize <= this.metaQueue.length) {
            queues[this.name].consumeMETA();
        }
    } else if (task.type && task.type.indexOf('amui:') === 0) {
        if (pluginconfig.amuiconsumers.length === 0) {
            logger.debug('No consumers on amui queue. give up the payload!');
            return; // no one monitor jso queue, give up the payload directly
        }
        if (this.amuiQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.amuiQueue.push(task);
        }
        if (pluginconfig.amuiqueue.isBatch === false ||
            pluginconfig.amuiqueue.batchSize <= this.amuiQueue.length) {
            queues[this.name].consumeAMUI();
        }
    } else if (task.type &&
        (task.type.indexOf('dcconfig:') === 0 || task.type.indexOf('situation:') === 0
            || task.type.indexOf('rteNotify:') === 0)) {
        if (pluginconfig.amsconsumers.length === 0) {
            logger.debug('No consumers on ams queue. give up the payload!');
            return; // no one monitor jso queue, give up the payload directly
        }
        if (this.amsQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.amsQueue.push(task);
        }
        if (pluginconfig.amsqueue.isBatch === false ||
            pluginconfig.amsqueue.batchSize <= this.amsQueue.length) {
            queues[this.name].consumeAms();
        }
    } else if (task.type && task.type.indexOf('query') === 0) {
        if (pluginconfig.queryconsumers.length === 0) {
            logger.debug('No consumers on query queue. give up the payload!');
            return; // no one monitor jso queue, give up the payload directly
        }
        if (this.queryQueue.length >= pluginconfig.queueMaxSize) {
            logger.debug('The', task.type, 'queue length is bigger than', pluginconfig.queueMaxSize);
        } else {
            this.queryQueue.push(task);
        }
        if (pluginconfig.queryqueue.isBatch === false ||
            pluginconfig.queryqueue.batchSize <= this.queryQueue.length) {
            queues[this.name].consumeQuery();
        }
    }
    // else {
    //     this.other.push(task);
    // }
};

var EventEmitter = require('events').EventEmitter;
var event = new EventEmitter();


SenderQueue.prototype.consumeDC = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeDC');
    if (
        this.dcQueue.length) {
        logger.debug('dcQueue.length=', this.dcQueue.length);
    }

    if (this.dcQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.dcqueue.batchSize > 0) {
            var batchsize = pluginconfig.dcqueue.batchSize > this.dcQueue.length ?
                this.dcQueue.length : pluginconfig.dcqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.dcQueue.shift());
            }
        } else {
            tasks.push(this.dcQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.DC);
        }
    }

};

SenderQueue.prototype.consumeMetric = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeMetric');
    if (this.metricQueue.length) {
        logger.debug('metricQueue.length=', this.metricQueue.length);
    }

    if (this.metricQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.metricqueue.batchSize > 0) {
            var batchsize = pluginconfig.metricqueue.batchSize > this.metricQueue.length ?
                this.metricQueue.length : pluginconfig.metricqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.metricQueue.shift());
            }
        } else {
            tasks.push(this.metricQueue.shift());
        }

        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.METRICS);
        }
    }

};

SenderQueue.prototype.consumeResource = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeResource');
    if (this.resourceQueue.length) {
        logger.debug('resourceQueue.length=', this.resourceQueue.length);
    }

    if (this.resourceQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.resourcequeue.batchSize > 0) {
            var batchsize = pluginconfig.resourcequeue.batchSize > this.resourceQueue.length ?
                this.resourceQueue.length : pluginconfig.resourcequeue.batchSize;

            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.resourceQueue.shift());
            }
        } else {
            tasks.push(this.resourceQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.RESOURCE);
        }
    }

};


SenderQueue.prototype.consumeAms = function() {
    // sequence: dc -> resource -> Ams -> Query -> metric -> aar/adr
    logger.debug('consumeAms');
    if (this.amsQueue.length) {
        logger.debug('amsQueue.length=', this.amsQueue.length);
    }

    if (this.amsQueue.length > 0) {
        var tasks = [];
        tasks.push(this.amsQueue.shift());
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.AMS);
        }
    }

};

SenderQueue.prototype.consumeQuery = function() {
    // sequence: dc -> resource -> Ams ->  Query -> metric -> aar/adr
    logger.debug('consumeQuery');
    if (this.queryQueue.length) {
        logger.debug('queryQueue.length=', this.queryQueue.length);
    }

    if (this.queryQueue.length > 0) {
        var tasks = [];
        tasks.push(this.queryQueue.shift());
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.QUERY);
        }
    }

};

SenderQueue.prototype.consumeAAR = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeAAR');
    if (this.aarQueue.length) {
        logger.debug('aarQueue.length=', this.aarQueue.length);
    }

    if (this.aarQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.aarqueue.batchSize > 0) {
            var batchsize = pluginconfig.aarqueue.batchSize > this.aarQueue.length ?
                this.aarQueue.length : pluginconfig.aarqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.aarQueue.shift());
            }
        } else {
            tasks.push(this.aarQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.AAR);
        }
    }

};

SenderQueue.prototype.consumeADR = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeADR');
    if (this.adrQueue.length) {
        logger.debug('adrQueue.length=', this.adrQueue.length);
    }

    if (this.adrQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.adrqueue.batchSize > 0) {
            var batchsize = pluginconfig.adrqueue.batchSize > this.adrQueue.length ?
                this.adrQueue.length : pluginconfig.adrqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.adrQueue.shift());
            }
        } else {
            tasks.push(this.adrQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.ADR);
        }
    }

};


SenderQueue.prototype.consumeJSO = function() {
    // sequence: dc -> resource -> metric -> aar/adr
    logger.debug('consumeJSO');
    if (this.jsoQueue.length) {
        logger.debug('jsoQueue.length=', this.jsoQueue.length);
    }

    if (this.jsoQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.jsoqueue.batchSize > 0) {
            var batchsize = pluginconfig.jsoqueue.batchSize > this.jsoQueue.length ?
                this.jsoQueue.length : pluginconfig.jsoqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.jsoQueue.shift());
            }
        } else {
            tasks.push(this.jsoQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.JSO);
        }
    }

};

SenderQueue.prototype.consumeMETA = function() {
    logger.debug('consumeMETA');
    if (this.metaQueue.length) {
        logger.debug('metaQueue.length=', this.metaQueue.length);
    }

    if (this.metaQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.metaqueue.batchSize > 0) {
            var batchsize = pluginconfig.metaqueue.batchSize > this.metaQueue.length ?
                this.metaQueue.length : pluginconfig.metaqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.metaQueue.shift());
            }
        } else {
            tasks.push(this.metaQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.META);
        }
    }

};
SenderQueue.prototype.consumeAMUI = function() {
    logger.debug('consumeAMUI');
    if (this.amuiQueue.length) {
        logger.debug('amuiQueue.length=', this.amuiQueue.length);
    }

    if (this.amuiQueue.length > 0) {
        var tasks = [];
        if (pluginconfig.amuiqueue.batchSize > 0) {
            var batchsize = pluginconfig.amuiqueue.batchSize > this.amuiQueue.length ?
                this.amuiQueue.length : pluginconfig.amuiqueue.batchSize;
            for (var index = 0; index < batchsize; index++) {
                tasks.push(this.amuiQueue.shift());
            }
        } else {
            tasks.push(this.amuiQueue.shift());
        }
        if (tasks) {
            event.emit('httpsend', tasks, this.name, pluginconfig.queuetypes.AMUI);
        }
    }

};
event.on('httpsend', queueListener);

function cloneTask(task, callbacks) {
    var i = 0;
    if (task instanceof Array) {
        logger.debug('cloneTask', 'is array');
        let newTask = [];
        task.forEach(function(element) {
            var t = JSON.parse(JSON.stringify(element));
            t.callback = callbacks[i];
            i += 1;
            newTask.push(t);
        });
        return newTask;
    } else {
        logger.debug('cloneTask', 'is not array');
        var t1 = JSON.parse(JSON.stringify(task));
        t1.callback = callbacks[0];
        return [t1];
    }
}

function queueListener(task, quequeName, type) {
    var res = {
        statusCode: 202
    };
    var consumersArr = [];
    logger.debug('queueListener', 'queue type', type);
    if (pluginconfig.queuetypes.DC === type) {
        consumersArr = pluginconfig.dcconsumers;
    } else if (pluginconfig.queuetypes.AAR === type) {
        consumersArr = pluginconfig.aarconsumers;
    } else if (pluginconfig.queuetypes.ADR === type) {
        consumersArr = pluginconfig.adrconsumers;
    } else if (pluginconfig.queuetypes.RESOURCE === type) {
        consumersArr = pluginconfig.resourceconsumers;
    } else if (pluginconfig.queuetypes.METRICS === type) {
        consumersArr = pluginconfig.metricconsumers;
    } else if (pluginconfig.queuetypes.JSO === type) {
        consumersArr = pluginconfig.jsoconsumers;
    } else if (pluginconfig.queuetypes.META === type) {
        consumersArr = pluginconfig.metaconsumers;
    } else if (pluginconfig.queuetypes.AMUI === type) {
        consumersArr = pluginconfig.amuiconsumers;
    } else if (pluginconfig.queuetypes.AMS === type) {
        consumersArr = pluginconfig.amsconsumers;
    } else if (pluginconfig.queuetypes.QUERY === type) {
        consumersArr = pluginconfig.queryconsumers;
    }
    var theCallbacks = [];
    if (Array.isArray(task)) {
        task.forEach(function(t) {
            theCallbacks.push(t.callback);
        });
    } else {
        theCallbacks.push(task.callback);
    }


    consumersArr.forEach(function(element) {
        try {
            let newTask = cloneTask(task, theCallbacks);
            element.send(newTask);
        } catch (error) {
            logger.error('failed to call customized consumer, Error:', error);
            res = {};
            res.statusCode = 400;
            res.error = error;
        }
    }, this);
};


exports.getQueue = function(name) {

    if (!queues[name]) {
        queues[name] = new SenderQueue(name);

        var intervalTopo = setInterval(function() { queues[name].consumeResource(); },
            pluginconfig.resourcequeue.freq);
        intervalTopo.unref();
        senderQueueLoop.push(intervalTopo);

        var intervalDC = setInterval(function() { queues[name].consumeDC(); },
            pluginconfig.dcqueue.freq);
        intervalDC.unref();
        senderQueueLoop.push(intervalDC);

        var intervalMetric = setInterval(function() { queues[name].consumeMetric(); },
            pluginconfig.metricqueue.freq);
        intervalMetric.unref();
        senderQueueLoop.push(intervalMetric);

        var intervalAAR = setInterval(function() { queues[name].consumeAAR(); },
            pluginconfig.aarqueue.freq);
        intervalAAR.unref();
        senderQueueLoop.push(intervalAAR);

        var intervalADR = setInterval(function() { queues[name].consumeADR(); },
            pluginconfig.adrqueue.freq);
        intervalADR.unref();
        senderQueueLoop.push(intervalADR);

        var intervalJSO = setInterval(function() { queues[name].consumeJSO(); },
            pluginconfig.jsoqueue.freq);
        intervalJSO.unref();
        senderQueueLoop.push(intervalJSO);

        var intervalMeta = setInterval(function() { queues[name].consumeMETA(); },
            pluginconfig.metaqueue.freq);
        intervalMeta.unref();
        senderQueueLoop.push(intervalMeta);

        var intervalAMUI = setInterval(function() { queues[name].consumeAMUI(); },
            pluginconfig.amuiqueue.freq);
        intervalAMUI.unref();
        senderQueueLoop.push(intervalAMUI);

        var intervalAMS = setInterval(function() { queues[name].consumeAms(); },
            pluginconfig.amsqueue.freq);
        intervalAMS.unref();
        senderQueueLoop.push(intervalAMS);

        var intervalQUERY = setInterval(function() { queues[name].consumeQuery(); },
            pluginconfig.queryqueue.freq);
        intervalQUERY.unref();
        senderQueueLoop.push(intervalQUERY);
    }
    return queues[name];
};

exports.stopQueue = function() {
    senderQueueLoop.forEach(function(item) {
        clearInterval(item);
    });
};