kafka-ts
Version:
**KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.
119 lines (118 loc) • 4.81 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProducerBuffer = void 0;
const api_1 = require("../api");
class ProducerBuffer {
options;
buffer = [];
head = 0;
isFlushing = false;
constructor(options) {
this.options = options;
}
enqueue(messages) {
return new Promise((resolve, reject) => {
this.buffer.push({ messages, resolve, reject });
this.flush();
});
}
async flush() {
if (this.isFlushing)
return;
this.isFlushing = true;
const { cluster, state, nodeId, maxBatchSize } = this.options;
while (true) {
const batch = [];
const resolvers = [];
const rejecters = [];
while (this.head < this.buffer.length) {
const entry = this.buffer[this.head++];
batch.push(...entry.messages);
resolvers.push(entry.resolve);
rejecters.push(entry.reject);
const nextLength = this.buffer[this.head]?.messages.length ?? 0;
if (batch.length + nextLength > maxBatchSize) {
break;
}
}
if (!batch.length)
break;
this.compactBuffer();
const topicPartitionMessages = {};
batch.forEach((message) => {
topicPartitionMessages[message.topic] ??= {};
topicPartitionMessages[message.topic][message.partition] ??= [];
topicPartitionMessages[message.topic][message.partition].push(message);
});
const defaultTimestamp = BigInt(Date.now());
const topicData = Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
name: topic,
partitionData: Object.entries(partitionMessages).map(([partition, messages]) => {
const partitionIndex = parseInt(partition);
let baseTimestamp;
let maxTimestamp;
messages.forEach(({ timestamp = defaultTimestamp }) => {
if (!baseTimestamp || timestamp < baseTimestamp) {
baseTimestamp = timestamp;
}
if (!maxTimestamp || timestamp > maxTimestamp) {
maxTimestamp = timestamp;
}
});
return {
index: partitionIndex,
baseOffset: 0n,
partitionLeaderEpoch: -1,
attributes: 0,
lastOffsetDelta: messages.length - 1,
baseTimestamp: baseTimestamp ?? 0n,
maxTimestamp: maxTimestamp ?? 0n,
producerId: state.producerId,
producerEpoch: 0,
baseSequence: state.getSequence(topic, partitionIndex),
records: messages.map((message, index) => ({
attributes: 0,
timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
offsetDelta: index,
key: message.key ?? null,
value: message.value,
headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
key,
value,
})),
})),
};
}),
}));
try {
await cluster.sendRequestToNode(nodeId)(api_1.API.PRODUCE, {
transactionalId: null,
acks: -1,
timeoutMs: 30000,
topicData,
});
topicData.forEach(({ name, partitionData }) => {
partitionData.forEach(({ index, records }) => {
state.updateSequence(name, index, records.length);
});
});
resolvers.forEach((resolve) => resolve());
}
catch (error) {
rejecters.forEach((reject) => reject(error));
}
}
this.isFlushing = false;
}
compactBuffer() {
if (this.head >= this.buffer.length) {
this.buffer = [];
this.head = 0;
}
else if (this.head > 1000 && this.head > this.buffer.length / 2) {
this.buffer = this.buffer.slice(this.head);
this.head = 0;
}
}
}
exports.ProducerBuffer = ProducerBuffer;