@itentialopensource/adapter-kafka
Version:
[Deprecated] Itential adapter to connect to kafka
708 lines (653 loc) • 23.5 kB
JavaScript
/* @copyright Itential, LLC 2019 (pre-modifications) */
// Set globals
/* global describe it log pronghornProps */
/* eslint global-require:warn */
/* eslint no-unused-vars: warn */
// include required items for testing & logging
const assert = require('assert');
const fs = require('fs-extra');
const mocha = require('mocha');
const path = require('path');
const util = require('util');
const winston = require('winston');
const execute = require('child_process').execSync;
const { expect } = require('chai');
const { use } = require('chai');
const td = require('testdouble');
const Ajv = require('ajv');
const ajv = new Ajv({ strictSchema: false, allErrors: true, allowUnionTypes: true });
const anything = td.matchers.anything();
// stub and attemptTimeout are used throughout the code so set them here
let logLevel = 'none';
const stub = true;
const isRapidFail = false;
const attemptTimeout = 120000;
// these variables can be changed to run in integrated mode so easier to set them here
// always check these in with bogus data!!!
const host = 'replace.hostorip.here';
const username = 'username';
const password = 'password';
const port = 80;
const sslenable = false;
const sslinvalid = false;
// these are the adapter properties. You generally should not need to alter
// any of these after they are initially set up
global.pronghornProps = {
pathProps: {
encrypted: false
},
adapterProps: {
adapters: [{
id: 'Test-kafka',
type: 'Kafka',
interval_time: 5000,
stub: true,
properties: {
topics: []
}
}]
}
};
global.$HOME = `${__dirname}/../..`;
// set the log levels that Pronghorn uses, spam and trace are not defaulted in so without
// this you may error on log.trace calls.
const myCustomLevels = {
levels: {
spam: 6,
trace: 5,
debug: 4,
info: 3,
warn: 2,
error: 1,
none: 0
}
};
// need to see if there is a log level passed in
process.argv.forEach((val) => {
// is there a log level defined to be passed in?
if (val.indexOf('--LOG') === 0) {
// get the desired log level
const inputVal = val.split('=')[1];
// validate the log level is supported, if so set it
if (Object.hasOwnProperty.call(myCustomLevels.levels, inputVal)) {
logLevel = inputVal;
}
}
});
// need to set global logging
global.log = winston.createLogger({
level: logLevel,
levels: myCustomLevels.levels,
transports: [
new winston.transports.Console()
]
});
/**
* Runs the error asserts for the test
*/
function runErrorAsserts(data, error, code, origin, displayStr) {
assert.equal(null, data);
assert.notEqual(undefined, error);
assert.notEqual(null, error);
assert.notEqual(undefined, error.IAPerror);
assert.notEqual(null, error.IAPerror);
assert.notEqual(undefined, error.IAPerror.displayString);
assert.notEqual(null, error.IAPerror.displayString);
assert.equal(code, error.icode);
assert.equal(origin, error.IAPerror.origin);
assert.equal(displayStr, error.IAPerror.displayString);
}
// require the adapter that we are going to be using
const Kafka = require('../../adapter');
// delete the .DS_Store directory in entities -- otherwise this will cause errors
const dirPath = path.join(__dirname, '../../entities/.DS_Store');
if (fs.existsSync(dirPath)) {
try {
fs.removeSync(dirPath);
console.log('.DS_Store deleted');
} catch (e) {
console.log('Error when deleting .DS_Store:', e);
}
}
// begin the testing - these should be pretty well defined between the describe and the it!
describe('[unit] Kafka Adapter Test', () => {
describe('Kafka Class Tests', () => {
const a = new Kafka(
pronghornProps.adapterProps.adapters[0].id,
pronghornProps.adapterProps.adapters[0].properties
);
if (isRapidFail) {
const state = {};
state.passed = true;
mocha.afterEach(function x() {
state.passed = state.passed
&& (this.currentTest.state === 'passed');
});
mocha.beforeEach(function x() {
if (!state.passed) {
return this.currentTest.skip();
}
return true;
});
}
describe('#class instance created', () => {
it('should be a class with properties', (done) => {
assert.notEqual(null, a);
assert.notEqual(undefined, a);
const check = global.pronghornProps.adapterProps.adapters[0].id;
assert.equal(check, a.id);
done();
}).timeout(attemptTimeout);
});
let wffunctions = [];
describe('#getWorkflowFunctions', () => {
it('should retrieve workflow functions', (done) => {
wffunctions = a.getWorkflowFunctions();
assert.notEqual(0, wffunctions.length);
done();
}).timeout(attemptTimeout);
});
describe('package.json', () => {
it('should have a package.json', (done) => {
fs.exists('package.json', (val) => {
assert.equal(true, val);
done();
});
});
it('package.json should be validated', (done) => {
try {
const packageDotJson = require('../../package.json');
// Define the JSON schema for package.json
const packageJsonSchema = {
type: 'object',
properties: {
name: { type: 'string' },
version: { type: 'string' }
// May need to add more properties as needed
},
required: ['name', 'version']
};
const validate = ajv.compile(packageJsonSchema);
const isValid = validate(packageDotJson);
if (isValid === false) {
log.error('The package.json contains errors');
assert.equal(true, isValid);
} else {
assert.equal(true, isValid);
}
done();
} catch (error) {
log.error(`Test Failure: ${error}`);
done(error);
}
});
it('package.json should be customized', (done) => {
const packageDotJson = require('../../package.json');
assert.notEqual(-1, packageDotJson.name.indexOf('kafka'));
assert.notEqual(undefined, packageDotJson.version);
assert.notEqual(null, packageDotJson.version);
assert.notEqual('', packageDotJson.version);
done();
});
});
describe('pronghorn.json', () => {
it('should have a pronghorn.json', (done) => {
fs.exists('pronghorn.json', (val) => {
assert.equal(true, val);
done();
});
});
it('pronghorn.json should be customized', (done) => {
const pronghornDotJson = require('../../pronghorn.json');
assert.notEqual(-1, pronghornDotJson.id.indexOf('kafka'));
assert.equal('Kafka', pronghornDotJson.export);
assert.equal('Kafka', pronghornDotJson.displayName);
assert.equal('Kafka', pronghornDotJson.title);
done();
});
it('pronghorn.json should only expose workflow functions', (done) => {
const pronghornDotJson = require('../../pronghorn.json');
for (let m = 0; m < pronghornDotJson.methods.length; m += 1) {
let found = false;
let paramissue = false;
for (let w = 0; w < wffunctions.length; w += 1) {
if (pronghornDotJson.methods[m].name === wffunctions[w]) {
found = true;
const methLine = execute(`grep "${wffunctions[w]}(" adapter.js | grep "callback) {"`).toString();
let wfparams = [];
if (methLine.indexOf('(') >= 0 && methLine.indexOf(')') >= 0) {
const temp = methLine.substring(methLine.indexOf('(') + 1, methLine.indexOf(')'));
wfparams = temp.split(',');
for (let t = 0; t < wfparams.length; t += 1) {
// remove default value from the parameter name
wfparams[t] = wfparams[t].substring(0, wfparams[t].search(/=/) > 0 ? wfparams[t].search(/#|\?|=/) : wfparams[t].length);
// remove spaces
wfparams[t] = wfparams[t].trim();
if (wfparams[t] === 'callback') {
wfparams.splice(t, 1);
}
}
}
// if there are inputs defined but not on the method line
if (wfparams.length === 0 && (pronghornDotJson.methods[m].input
&& pronghornDotJson.methods[m].input.length > 0)) {
paramissue = true;
} else if (wfparams.length > 0 && (!pronghornDotJson.methods[m].input
|| pronghornDotJson.methods[m].input.length === 0)) {
// if there are no inputs defined but there are on the method line
paramissue = true;
} else {
for (let p = 0; p < pronghornDotJson.methods[m].input.length; p += 1) {
let pfound = false;
for (let wfp = 0; wfp < wfparams.length; wfp += 1) {
if (pronghornDotJson.methods[m].input[p].name.toUpperCase() === wfparams[wfp].toUpperCase()) {
pfound = true;
}
}
if (!pfound) {
paramissue = true;
}
}
for (let wfp = 0; wfp < wfparams.length; wfp += 1) {
let pfound = false;
for (let p = 0; p < pronghornDotJson.methods[m].input.length; p += 1) {
if (pronghornDotJson.methods[m].input[p].name.toUpperCase() === wfparams[wfp].toUpperCase()) {
pfound = true;
}
}
if (!pfound) {
paramissue = true;
}
}
}
break;
}
}
if (!found) {
// this is the reason to go through both loops - log which ones are not found so
// they can be worked
log.error(`${pronghornDotJson.methods[m].name} not found in workflow functions`);
}
if (paramissue) {
// this is the reason to go through both loops - log which ones are not found so
// they can be worked
log.error(`${pronghornDotJson.methods[m].name} has a parameter mismatch`);
}
assert.equal(true, found);
assert.equal(false, paramissue);
}
done();
}).timeout(attemptTimeout);
it('pronghorn.json should expose all workflow functions', (done) => {
const pronghornDotJson = require('../../pronghorn.json');
for (let w = 0; w < wffunctions.length; w += 1) {
let found = false;
for (let m = 0; m < pronghornDotJson.methods.length; m += 1) {
if (pronghornDotJson.methods[m].name === wffunctions[w]) {
found = true;
break;
}
}
if (!found) {
// this is the reason to go through both loops - log which ones are not found so
// they can be worked
log.error(`${wffunctions[w]} not found in pronghorn.json`);
}
assert.equal(true, found);
}
done();
});
});
describe('propertiesSchema.json', () => {
it('should have a propertiesSchema.json', (done) => {
fs.exists('propertiesSchema.json', (val) => {
assert.equal(true, val);
done();
});
});
it('propertiesSchema.json should be customized', (done) => {
const propertiesDotJson = require('../../propertiesSchema.json');
assert.equal('adapter-kafka', propertiesDotJson.$id);
done();
});
});
describe('error.json', () => {
it('should have an error.json', (done) => {
fs.exists('error.json', (val) => {
assert.equal(true, val);
done();
});
});
});
describe('README.md', () => {
it('should have a README', (done) => {
fs.exists('README.md', (val) => {
assert.equal(true, val);
done();
});
});
it('README.md should be customized', (done) => {
fs.readFile('README.md', 'utf8', (err, data) => {
assert.equal(-1, data.indexOf('[System]'));
assert.equal(-1, data.indexOf('[system]'));
assert.equal(-1, data.indexOf('[version]'));
assert.equal(-1, data.indexOf('[namespace]'));
done();
});
});
});
describe('#connect', () => {
it('should have a connect function', (done) => {
assert.equal(true, typeof a.connect === 'function');
done();
});
});
describe('#healthCheck', () => {
it('should have a healthCheck function', (done) => {
assert.equal(true, typeof a.healthCheck === 'function');
done();
});
});
/*
-----------------------------------------------------------------------
-----------------------------------------------------------------------
*** All code above this comment will be replaced during a migration ***
******************* DO NOT REMOVE THIS COMMENT BLOCK ******************
-----------------------------------------------------------------------
-----------------------------------------------------------------------
*/
describe('#createTopics - errors', () => {
it('should have a createTopics function', (done) => {
assert.equal(true, typeof a.createTopics === 'function');
done();
});
it('should error on createTopics - no topics', (done) => {
try {
a.createTopics(null, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-createTopics', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#sendMessage - errors', () => {
it('should have a sendMessage function', (done) => {
assert.equal(true, typeof a.sendMessage === 'function');
done();
});
it('should error on sendMessage - no payloads', (done) => {
try {
a.sendMessage(null, (data, error) => {
try {
const displayE = 'payloads is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-sendMessage', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#subscribe - errors', () => {
it('should have a subscribe function', (done) => {
assert.equal(true, typeof a.subscribe === 'function');
done();
});
it('should error on subscribe - no topics', (done) => {
try {
a.subscribe(null, 0, 0, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-subscribe', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
it('should error on subscribe - no topics', (done) => {
try {
a.subscribe([], 0, 0, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-subscribe', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#subscribeAvro - errors', () => {
it('should have a subscribeAvro function', (done) => {
assert.equal(true, typeof a.subscribeAvro === 'function');
done();
});
it('should error on subscribeAvro - no topics', (done) => {
try {
a.subscribeAvro(null, 0, 0, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-subscribeAvro', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
it('should error on subscribeAvro - no topics', (done) => {
try {
a.subscribeAvro([], 0, 0, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-subscribeAvro', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#setOffset - errors', () => {
it('should have a setOffset function', (done) => {
assert.equal(true, typeof a.setOffset === 'function');
done();
});
it('should error on setOffset - no topic', (done) => {
try {
a.setOffset(null, null, null, (data, error) => {
try {
const displayE = 'topic is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-setOffset', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
it('should error on setOffset - no partition', (done) => {
try {
a.setOffset('fakedata', null, null, (data, error) => {
try {
const displayE = 'partition is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-setOffset', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
it('should error on setOffset - no partition', (done) => {
try {
a.setOffset('fakedata', 'fakedata', null, (data, error) => {
try {
const displayE = 'offset is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-setOffset', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#pauseTopics - errors', () => {
it('should have a pauseTopics function', (done) => {
assert.equal(true, typeof a.pauseTopics === 'function');
done();
});
it('should error on pauseTopics - no topics', (done) => {
try {
a.pauseTopics(null, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-pauseTopics', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#resumeTopics - errors', () => {
it('should have a resumeTopics function', (done) => {
assert.equal(true, typeof a.resumeTopics === 'function');
done();
});
it('should error on resumeTopics - no topics', (done) => {
try {
a.resumeTopics(null, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-resumeTopics', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#pauseConsumer - errors', () => {
it('should have a pauseConsumer function', (done) => {
assert.equal(true, typeof a.pauseConsumer === 'function');
done();
});
});
describe('#resumeConsumer - errors', () => {
it('should have a resumeConsumer function', (done) => {
assert.equal(true, typeof a.resumeConsumer === 'function');
done();
});
});
describe('#commitOffset - errors', () => {
it('should have a commitOffset function', (done) => {
assert.equal(true, typeof a.commitOffset === 'function');
done();
});
});
describe('#unsubscribe - errors', () => {
it('should have a unsubscribe function', (done) => {
assert.equal(true, typeof a.unsubscribe === 'function');
done();
});
it('should error on unsubscribe - no topics', (done) => {
try {
a.unsubscribe(null, (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-unsubscribe', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
it('should error on unsubscribe - no topics', (done) => {
try {
a.unsubscribe([], (data, error) => {
try {
const displayE = 'topics is required';
runErrorAsserts(data, error, 'AD.300', 'Test-kafka-adapter-unsubscribe', displayE);
done();
} catch (ex) {
log.error(`Test Failure: ${ex}`);
done(ex);
}
});
} catch (exc) {
log.error(`Adapter Exception: ${exc}`);
done(exc);
}
}).timeout(attemptTimeout);
});
describe('#closeConsumer - errors', () => {
it('should have a closeConsumer function', (done) => {
assert.equal(true, typeof a.closeConsumer === 'function');
done();
});
});
});
});