node-test-bed-adapter
Version:
An adapter to connect a node.js application to the Test-bed's Common Information Space or Common Simulation Space.
670 lines • 24.6 kB
JavaScript
import * as fs from 'fs';
import * as path from 'path';
import { Kafka, Partitioners, } from 'kafkajs';
import { EventEmitter } from 'events';
import { SchemaRegistry, SchemaPublisher, avroHelperFactory, CorePublishTopics, AccessRemoveTopic, TimeTopic, AccessInviteTopic, AdminHeartbeatTopic, HeartbeatTopic, CoreSubscribeTopics, } from './avro/index.mjs';
import { clone, uuid4 } from './utils/index.mjs';
import { Logger, FileLogger, KafkaLogger, ConsoleLogger, } from './logger/index.mjs';
import { LargeFileUploadService, TimeService, computerInfo, } from './services/index.mjs';
export class TestBedAdapter extends EventEmitter {
static HeartbeatInterval = 10000;
isConnected = false;
groupId;
schemaPublisher;
schemaRegistry;
largeFileUploadService;
log = Logger.instance;
client;
producer;
consumer;
config;
/** Map of all initialized topics, i.e. with validators/encoders/decoders */
consumerTopics = {};
producerTopics = {};
/** Location of the configuration file */
configFile = 'config/test-bed-config.json';
timeService = new TimeService();
origin;
constructor(config) {
super();
if (!config) {
config = this.loadOptionsFromFile();
}
else if (typeof config === 'string') {
config = this.loadOptionsFromFile(config);
}
this.validateOptions(config);
this.groupId = config.groupId || config.clientId || '';
this.config = this.setDefaultOptions(config);
this.schemaPublisher = new SchemaPublisher(this.config);
this.schemaRegistry = new SchemaRegistry(this.config);
this.largeFileUploadService = new LargeFileUploadService(this.config);
computerInfo(config.externalIP, (info, err) => {
if (err) {
return console.error(err);
}
this.origin = JSON.stringify(info);
});
}
async connect() {
try {
await this.initLogger();
await this.schemaPublisher.init();
this.client = new Kafka(this.config);
await this.initialize();
return true;
}
catch (e) {
return false;
}
}
disconnect() {
this.removeAllListeners();
this.consumer?.disconnect();
this.producer?.disconnect();
}
/**
* Get the underlying KafkaJS client, if defined.
* Should be available after the 'ready' event is emitted.
*/
getClient() {
return this.client;
}
/**
* A dictionary containing a clone of all the key schemas with key the bare topic name and
* value the instance of the AVRO schema and schema ID.
*/
get keySchemas() {
return this.schemaRegistry.keySchemas;
}
/**
* A dictionary containing a clone of all the value schemas with key the bare topic name and
* value the instance of the AVRO schema and schema ID.
*/
get valueSchemas() {
return this.schemaRegistry.valueSchemas;
}
/** Pause all topics when no topics are specified, or only the ones that are specified */
pause(topics) {
if (!this.consumer) {
return this.emitErrorMsg('Consumer not ready!');
}
if (typeof topics === 'undefined') {
topics = Object.keys(this.consumerTopics).map((topic) => ({ topic }));
}
this.consumer.pause(topics);
}
/** Resume all topics when no topics are specified, or only the ones that are specified */
resume(topics) {
if (!this.consumer) {
return this.emitErrorMsg('Consumer not ready!');
}
if (typeof topics === 'undefined') {
topics = Object.keys(this.consumerTopics).map((topic) => ({ topic }));
}
this.consumer.resume(topics);
}
/** Deprecated: use disconnect */
close() {
this.removeAllListeners();
this.disconnect();
}
keyFactory = () => ({
distributionID: uuid4(),
senderID: this.config.clientId || this.config.groupId,
dateTimeSent: Date.now(),
dateTimeExpires: 0,
distributionStatus: 'Exercise',
distributionKind: 'Unknown',
});
async send(payload, cb) {
if (!this.producer) {
return this.emitErrorMsg('Producer not ready!');
}
const topic = this.producerTopics[payload.topic];
if (!topic) {
return cb(`Topic ${payload.topic} not found: please register first! ${JSON.stringify(payload)}`, undefined);
}
// const keyFactory = () => ({
// distributionID: uuid4(),
// senderID: this.config.clientId || this.config.groupId,
// dateTimeSent: Date.now(),
// dateTimeExpires: 0,
// distributionStatus: 'Exercise',
// distributionKind: 'Unknown',
// });
// const key = topic.encodeKey(keyFactory());
const plMessages = payload.messages.map((m) => ({
...m,
key: m.key || topic.encodeKey(this.keyFactory()),
}));
if (topic.isValid(plMessages)) {
const messages = plMessages.map((message) => topic.encode(message));
const recordMetadata = this.producer &&
(await this.producer.send({ ...payload, messages }).catch((e) => {
cb(JSON.stringify(e));
}));
recordMetadata && cb(undefined, recordMetadata);
}
else {
return cb('Error validating message', undefined);
}
}
/**
* Returns (a clone of) the configuration options.
*/
get configuration() {
return clone(this.config);
}
/**
* Create topics by requesting their metadata.
* It only works when `auto.create.topics.enable = true`.
*/
async createTopics(topics) {
if (!this.client || topics.length === 0) {
return false;
}
const admin = this.client.admin();
await admin.connect();
const metadata = await admin.fetchTopicMetadata();
const existingTopics = metadata.topics.map((md) => md.name);
const newTopics = topics.filter((t) => existingTopics.indexOf(t.topic) < 0);
if (newTopics.length === 0) {
return true;
}
const success = await admin.createTopics({ topics: newTopics });
if (!success) {
console.warn(`Could not create the following topics: ${JSON.stringify(topics)}`);
}
await admin.disconnect();
return success;
}
/**
* Add topics (encoding utf8)
*
* @param topics Array of topics to add
*/
async addConsumerTopics(topics) {
if (!topics || topics.length === 0 || !this.consumer) {
return;
}
const myTopics = topics instanceof Array ? topics : [...topics];
const newTopics = this.initializeConsumerTopics(myTopics);
const consumer = this.consumer;
const run = async () => {
const fromBeginning = this.config.fromOffset === 'earliest';
if (this.client && fromBeginning) {
const admin = this.client.admin();
await admin.connect();
consumer.on('consumer.connect', async () => {
for (const topic of newTopics) {
const offsets = await admin.fetchTopicOffsets(topic);
offsets.forEach((offset) => consumer.seek({
topic,
partition: offset.partition,
offset: offset.low,
}));
}
});
}
consumer.on('consumer.disconnect', (e) => {
console.log(e);
});
await consumer.connect();
await consumer.subscribe({
topics: newTopics,
fromBeginning,
});
consumer.run({
eachMessage: async (message) => this.handleMessage(message),
});
};
run().catch((e) => {
console.error(e);
});
}
async addProducerTopics(topics) {
if (!topics || topics.length === 0) {
return;
}
const newTopics = await this.initializeProducerTopics(typeof topics === 'string' ? [topics] : topics);
if (this.config.autoCreateTopics === true) {
await this.createTopics(newTopics.map((topic) => ({
topic,
numPartitions: this.config.partitions,
})));
}
else
this.log.info('AutoCreateTopics is disabled, so not creating any topics.');
}
/**
* Load the metadata for all topics (in case of an empty array), or specific ones.
*
* @param topics If topics is an empty array, retreive the metadata of all topics
* @param cb callback function to return the metadata results
*/
async loadMetadataForTopics(topics, cb) {
if (!this.client || !this.isConnected) {
cb('Client is not connected');
return;
}
const admin = this.client.admin();
await admin.connect();
const metadata = await admin.fetchTopicMetadata({ topics });
await admin.disconnect();
cb(undefined, metadata.topics);
}
/**
* Get the simulation time as UTC Date.
*/
get simulationTime() {
return this.timeService.simulationTime;
}
/**
* Get the simulation state.
*/
get timeState() {
return this.timeService.timeState;
}
/**
* Positive number, indicating how fast the simulation / trial time moves with respect
* to the actual time. A value of 0 means a pause, 1 is as fast as real-time.
*/
get simulationSpeed() {
return this.timeService.simulationSpeed;
}
/**
* Get elapsed time in msec.
*/
get timeElapsed() {
return this.timeService.timeElapsed;
}
/**
* Upload a file to the large file service, if enabled.
* NOTE: the configuration needs to specify the URL of the large file service, e.g. http://localhost:9090
*/
uploadFile(file, isPrivate, cb) {
this.largeFileUploadService.upload(file, isPrivate, cb);
}
/** List of the uploaded schemas, if any */
get uploadedSchemas() {
return this.schemaPublisher ? this.schemaPublisher.uploadedSchemas : [];
}
// PRIVATE METHODS
/** After the Kafka client is connected, initialize the other services too, starting with the schema registry. */
initialize() {
return new Promise(async (resolve, reject) => {
try {
await this.schemaRegistry.init();
await this.initProducer();
await this.addProducerTopics(this.config.produce);
await this.addKafkaLogger();
await this.initConsumer(this.config.consume);
await this.addConsumerTopics(this.config.consume);
this.isConnected = true;
if (typeof this.config.heartbeatInterval !== 'undefined' &&
this.config.heartbeatInterval > 0) {
await this.startHeartbeat();
}
this.emit('ready');
}
catch (err) {
return this.emitErrorMsg(`Error initializing kafka services: ${err}`, reject);
}
resolve();
});
}
async initProducer() {
if (!this.client) {
return this.emitErrorMsg('Client not ready!');
}
this.producer = this.client.producer({
allowAutoTopicCreation: true,
createPartitioner: Partitioners.DefaultPartitioner,
});
await this.producer.connect();
}
async initLogger() {
const loggers = [];
const logOptions = this.config.logging;
if (logOptions) {
if (logOptions.logToConsole) {
loggers.push({
logger: new ConsoleLogger(),
minLevel: logOptions.logToConsole,
});
}
if (logOptions.logToFile) {
loggers.push({
logger: new FileLogger(logOptions.logFile || 'log.txt'),
minLevel: logOptions.logToFile,
});
}
this.log.initialize(loggers);
}
}
/** If required, add the Kafka logger too (after the producer has been initialised). */
addKafkaLogger() {
return new Promise((resolve) => {
if (!this.producer) {
return resolve();
}
const logOptions = this.config.logging;
if (logOptions && logOptions.logToKafka) {
this.log.addLogger({
logger: new KafkaLogger({
adapter: this,
clientId: this.groupId,
stringBasedKey: typeof this.config.stringBasedKey === 'undefined'
? true
: this.config.stringBasedKey,
}),
minLevel: logOptions.logToKafka,
});
}
resolve();
});
}
async initConsumer(topics) {
if (!topics || topics.length === 0) {
return;
}
if (!this.client) {
return this.emitErrorMsg('initConsumer() - Client not ready!');
}
const consumer = this.client.consumer({
groupId: this.groupId,
minBytes: this.config.fetchMinBytes,
maxBytes: this.config.fetchMaxBytes,
sessionTimeout: this.config.sessionTimeout,
rebalanceTimeout: this.config.rebalanceTimeout,
allowAutoTopicCreation: true,
});
// Wrap events emitted by the consumer
for (const event of [
'consumer.heartbeat',
'consumer.commit_offsets',
'consumer.group_join',
'consumer.fetch_start',
'consumer.fetch',
'consumer.start_batch_process',
'consumer.end_batch_process',
'consumer.connect',
'consumer.disconnect',
'consumer.stop',
'consumer.crash',
'consumer.rebalancing',
'consumer.received_unsubscribed_topics',
'consumer.network.request',
'consumer.network.request_timeout',
'consumer.network.request_queue_size',
]) {
consumer.on(event, (ev) => {
this.emit(event, ev);
});
}
// consumer.on('consumer.crash', (ev) =>
// this.emitErrorMsg(JSON.stringify(ev))
// );
this.consumer = consumer;
}
handleMessage(payload) {
const { topic, partition, message } = payload;
const consumerTopic = this.consumerTopics[topic];
if (!consumerTopic || !consumerTopic.decode) {
this.emit('raw', payload);
return;
}
const { key, value } = consumerTopic.decode(message);
switch (topic) {
default:
this.emit('message', {
topic,
partition,
key,
value,
});
break;
case TimeTopic:
const timeMessage = value;
if (timeMessage) {
this.timeService.setSimTime(timeMessage);
this.emit('time', timeMessage);
}
else {
this.log.warn(`Could not decode topic ${TimeTopic}. Is the schema correct?`);
}
break;
case AccessInviteTopic:
const invitation = value;
if (invitation.id.toLowerCase() === this.groupId.toLowerCase()) {
this.registerTopic(invitation);
}
break;
case AccessRemoveTopic:
const remove = value;
this.unregisterTopic(remove);
break;
case AdminHeartbeatTopic:
const ahb = value;
// console.log(`Admin heartbeat received`);
// console.log(JSON.stringify(ahb));
this.emit('heartbeat', ahb);
break;
}
}
async registerTopic(invitation) {
if (invitation.subscribeAllowed &&
(await this.schemaRegistry.registerNewTopic(invitation.topicName))) {
await this.addConsumerTopics(invitation.topicName);
}
if (invitation.publishAllowed &&
(await this.schemaRegistry.registerNewTopic(invitation.topicName))) {
this.addProducerTopics(invitation.topicName);
}
console.log(`Invitation received`);
console.log(JSON.stringify(invitation));
}
async unregisterTopic(remove) {
this.schemaRegistry.unregisterTopic(remove.topicName);
}
/**
* Add the topics to the configuration and initialize the decoders.
* @param topics topics to add
*/
initializeConsumerTopics(topics) {
if (!topics || topics.length === 0) {
return [];
}
const newTopics = topics.reduce((acc, topic) => {
if (this.consumerTopics.hasOwnProperty(topic)) {
return acc;
}
if (!this.schemaRegistry.valueSchemas.hasOwnProperty(topic)) {
this.log.error(`initializeConsumerTopics - no schema registered for topic ${topic}`);
return acc;
}
acc.push(topic);
const initializedTopic = { topic };
const avro = avroHelperFactory(this.schemaRegistry, topic);
initializedTopic.decode = avro.decode;
this.consumerTopics[topic] = initializedTopic;
return acc;
}, []);
return newTopics;
}
/**
* Add the topics to the configuration and initialize the encoders/validators.
* @param topics topics to add
*/
async initializeProducerTopics(topics) {
if (!topics || topics.length === 0) {
return [];
}
const newTopics = [];
for (const topic of topics) {
if (this.producerTopics.hasOwnProperty(topic))
continue;
await this.schemaRegistry.registerNewTopic(topic);
if (!(this.schemaRegistry.valueSchemas.hasOwnProperty(topic) ||
this.schemaRegistry.valueSchemas.hasOwnProperty(topic + '-value'))) {
this.log.error(`initializeProducerTopics - no schema registered for topic ${topic}`);
continue;
}
newTopics.push(topic);
if (this.config.produce &&
this.config.produce instanceof Array &&
this.config.produce.indexOf(topic) < 0) {
this.config.produce.push(topic);
}
const initializedTopic = { topic: topic };
const avro = avroHelperFactory(this.schemaRegistry, topic);
initializedTopic.encode = avro.encode;
initializedTopic.encodeKey = avro.encodeKey;
initializedTopic.isValid = avro.isValid;
this.producerTopics[topic] = initializedTopic;
}
return newTopics;
}
/**
* Start transmitting a heartbeat message.
*/
async startHeartbeat() {
if (this.isConnected) {
return;
}
const sendHeartbeat = async () => {
if (!this.isConnected ||
!this.producerTopics.hasOwnProperty(HeartbeatTopic)) {
return;
}
if (!this.producer) {
return this.emitErrorMsg('Producer not ready!');
}
await this.send({
topic: HeartbeatTopic,
messages: [
{
key: this.config.clientId,
value: {
id: this.config.clientId,
alive: Date.now(),
origin: this.origin,
},
},
],
}, (error) => {
if (error) {
this.log.error(error);
}
setTimeout(sendHeartbeat, this.config.heartbeatInterval || TestBedAdapter.HeartbeatInterval);
});
};
this.log.info('Started heartbeat');
await sendHeartbeat();
}
/**
* Set the default options of the configuration.
* @param options current configuration
*/
setDefaultOptions(options) {
const opt = Object.assign({
kafkaHost: 'broker:3501',
brokers: [],
schemaRegistry: 'schema_registry:3502',
clientId: '',
groupId: '',
autoConnect: true,
wrapUnions: 'auto',
stringBasedKey: true,
fromOffset: 'latest',
heartbeatInterval: TestBedAdapter.HeartbeatInterval,
produce: [],
logging: {},
encoding: 'buffer',
keyEncoding: 'buffer',
protocol: ['roundrobin'],
sessionTimeout: 200000,
maxConnectionRetries: 10,
autoRegisterDefaultSchemas: true,
connectTimeout: 5000,
partitionerType: 2,
partitions: 1,
autoCreateTopics: false,
}, options);
if (opt.brokers.length === 0 &&
opt.brokers instanceof Array &&
opt.kafkaHost) {
opt.brokers.push(...opt.kafkaHost.split(',').map((b) => b.trim()));
}
if (!opt.groupId && opt.clientId) {
opt.groupId = opt.clientId;
if (!opt.groupId)
throw Error('Missing option: groupId or clientId must be specified!');
}
opt.clientId = opt.groupId;
const consume = !opt.consume
? []
: typeof opt.consume === 'string'
? [opt.consume]
: opt.consume;
opt.consume = consume;
const produce = !opt.produce
? []
: typeof opt.produce === 'string'
? [opt.produce]
: opt.produce;
opt.produce = produce;
if (opt.autoRegisterDefaultSchemas) {
CoreSubscribeTopics.filter((t) => consume.indexOf(t) < 0).forEach((t) => consume.push(t));
CorePublishTopics(opt.largeFileService ? true : false)
.filter((t) => produce.indexOf(t) < 0)
.forEach((t) => produce.push(t));
}
if (!/\/$/.test(opt.schemaRegistry)) {
opt.schemaRegistry = `${opt.schemaRegistry}/`;
}
return opt;
}
/**
* Validate that all required options are set, or throw an error if not.
* @param options current configuration
*/
validateOptions(options) {
if (!options.clientId && !options.groupId) {
throw new Error('No clientId or groupId specified!');
}
if (!options.kafkaHost) {
throw new Error('No kafkaHost specified!');
}
if (!options.schemaRegistry) {
throw new Error('No schema registry specified!');
}
if (!options.schemaRegistry.match(/^http/)) {
options.schemaRegistry = 'http://' + options.schemaRegistry;
}
if (options.heartbeatInterval && options.heartbeatInterval < 0) {
throw new Error('Heartbeat interval must be positive!');
}
}
/**
* Load the configuration options from file.
* @param configFile configuration file path
*/
loadOptionsFromFile(configFile = this.configFile) {
configFile = path.resolve(configFile);
// this.log(configFile);
if (fs.existsSync(configFile)) {
return JSON.parse(fs.readFileSync(configFile, { encoding: 'utf8' }));
}
throw new Error(`Error loading options! Either supply them as parameter or as a configuration file at ${configFile}.`);
}
emitErrorMsg(msg, cb) {
this.log.error(msg);
this.emit('error', msg);
if (cb) {
cb(msg);
}
}
}
//# sourceMappingURL=test-bed-adapter.mjs.map