@energyweb/node-red-contrib-green-proof-worker
Version:
## Peer dependencies
158 lines (157 loc) • 6.43 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SourceKafka = void 0;
const tslib_1 = require("tslib");
const promises_1 = require("timers/promises");
const z = tslib_1.__importStar(require("zod"));
const errors_1 = require("../errors");
const node_1 = require("../node");
const nodes_config_env_1 = require("../nodes-config-env");
const InputMessage = z.union([
z.object({
topic: z.literal('finished-processing'),
payload: z.object({
kafkaMessageOffset: z.string(),
})
}),
z.object({
topic: z.literal('force-reset')
})
]);
const Config = z.object({
kafkaConfig: z.string(),
topic: z.string(),
});
const SourceKafka = (api) => class SourceKafka extends node_1.Node {
constructor(config) {
super(api, config, InputMessage);
/** Used to orchestrate consumer retry */
this.isDestroying = false;
this.kafkaPendingProcessing = null;
const { kafkaConfig, topic } = Config.parse({
...config,
topic: config.topic || this.getNodeEnvConfig().EWX_SOLUTION_ID || nodes_config_env_1.nodesGlobalEnvConfig.VOTING_SERVICE_CONFIG_SOLUTION_NAMESPACE
});
const configNode = this.api.getNode(kafkaConfig);
if (!configNode) {
throw new errors_1.GGPError(errors_1.ErrorCode.SourceKafkaConfigNotFound, {});
}
this.consumer = configNode.data.then(({ kafka, clientId }) => {
return kafka.consumer({
groupId: clientId,
retry: {
initialRetryTime: 1500,
maxRetryTime: 15000,
},
sessionTimeout: 120000, // This is max time the processing one message may take
});
});
this.runConsumerWithRetry({ topic });
}
runConsumerWithRetry({ topic }) {
this.runConsumer({ topic }).catch((error) => {
if (this.isDestroying) {
// if we are destroying the node, then we should not retry, it is expected
return;
}
const delay = 5000;
this.api.error(error);
this.api.log(`Retrying to connect in ${delay} ms`);
return (0, promises_1.setTimeout)(delay).then(() => this.runConsumerWithRetry({ topic }));
});
}
async runConsumer({ topic }) {
this.setStatus('connecting');
const consumer = await this.consumer;
await consumer.connect();
await consumer.subscribe({
topic,
fromBeginning: true,
});
this.setStatus('connected');
await consumer.run({
// We do the autocommit, however it auto-commits after `eachMessage` finishes processing
// `eachMessage` will save promise to be resolved when the input is back.
eachMessage: ({ message }) => {
const stringifiedMessage = message.value?.toString();
if (!stringifiedMessage) {
this.api.log('Received message with no value');
return Promise.resolve();
}
const parsedMessage = (() => {
try {
return JSON.parse(stringifiedMessage);
}
catch (e) {
this.api.warn(`Cannot parse message as error. Message is skipped. Error: ${e.message}`);
return Promise.resolve();
}
})();
if (!parsedMessage) {
return Promise.resolve();
}
this.sendBuilder({})
.addPayload({
...parsedMessage,
kafkaMessageOffset: message.offset,
})
.sendToOutput(0);
return new Promise((resolve, reject) => {
this.kafkaPendingProcessing = {
// Let's treat offset as message id (https://github.com/open-telemetry/opentelemetry-specification/issues/2971)
// We could also use txId but that would require understanding message structure from that node
// which I would like to avoid
messageOffset: message.offset,
reject,
resolve,
};
this.setStatus('waiting');
}).finally(() => this.setStatus('connected'));
},
});
}
setStatus(status) {
switch (status) {
case 'connected':
this.api.status({ fill: 'green', shape: 'ring', text: 'Connected and subscribed' });
break;
case 'waiting':
this.api.status({ fill: 'yellow', shape: 'dot', text: `Waiting for message (offset ${this.kafkaPendingProcessing?.messageOffset}) to be processed` });
break;
case 'connecting':
this.api.status({ fill: 'yellow', shape: 'ring', text: 'Trying to connect to broker' });
break;
}
}
/**
* Input is responsible for resolving promise that kafka consumer is waiting for
*/
onInput(message) {
if (!this.kafkaPendingProcessing) {
return;
}
if (message.topic === 'force-reset') {
this.kafkaPendingProcessing.reject(new Error('Kafka: Force resetted'));
this.kafkaPendingProcessing = null;
return;
}
if (this.kafkaPendingProcessing.messageOffset !== message.payload.kafkaMessageOffset) {
throw new errors_1.GGPError(errors_1.ErrorCode.SourceKafkaUnexpectedMessage, {
expected: this.kafkaPendingProcessing.messageOffset,
received: message.payload.kafkaMessageOffset
});
}
this.api.log(`Message processed: ${message.payload.kafkaMessageOffset}`);
this.kafkaPendingProcessing.resolve();
this.kafkaPendingProcessing = null;
}
async onDestroy() {
this.isDestroying = true;
if (this.kafkaPendingProcessing) {
this.kafkaPendingProcessing.reject(new Error('Kafka: Destroying node'));
}
const consumer = await this.consumer;
await consumer.disconnect();
}
};
exports.SourceKafka = SourceKafka;