@resonatehq/kafka
Version:
Resonate Kafka Transport
491 lines • 17.5 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Kafka = void 0;
const kafka_javascript_1 = require("@confluentinc/kafka-javascript");
const exceptions_1 = __importDefault(require("@resonatehq/sdk/dist/src/exceptions"));
const util = __importStar(require("@resonatehq/sdk/dist/src/util"));
class Kafka {
pid;
group;
unicast;
anycast;
requestPartition;
responsePartition;
requestTopic;
responseTopic;
producer;
consumerResponse;
consumerMessages;
subscriptions = { invoke: [], resume: [], notify: [] };
pendingRequests;
constructor({ brokers = ["localhost:9092"], group = "default", pid = crypto.randomUUID().replace(/-/g, ""), requestPartition = undefined, requestTopic = "resonate", responsePartition = undefined, responseTopic = "resonate", kafka = undefined, } = {}) {
kafka =
kafka ??
new kafka_javascript_1.KafkaJS.Kafka({
"allow.auto.create.topics": true,
"client.id": pid,
log_level: 3,
"metadata.broker.list": brokers.join(", "),
"metadata.max.age.ms": 1000,
});
this.group = group;
this.pid = pid;
this.requestPartition = requestPartition;
this.responsePartition = responsePartition;
this.requestTopic = requestTopic;
this.responseTopic = responseTopic;
this.unicast = `kafka://${this.group}`;
this.anycast = `kafka://${this.group}`;
this.pendingRequests = new Map();
this.producer = kafka.producer();
this.consumerResponse = kafka.consumer({
"auto.offset.reset": "latest",
"enable.auto.commit": true,
"group.id": this.pid,
"session.timeout.ms": 6000,
});
this.consumerMessages = kafka.consumer({
"auto.offset.reset": "earliest",
"enable.auto.commit": true,
"group.id": this.group,
"session.timeout.ms": 6000,
});
}
async start() {
await this.consumerMessages.connect();
await this.consumerMessages.subscribe({ topic: this.group });
await this.producer.connect();
await this.consumerResponse.connect();
await this.consumerResponse.subscribe({ topic: this.responseTopic });
await this.consumerResponse.run({
eachMessage: async ({ message }) => {
if (message.value === undefined) {
return;
}
util.assertDefined(message.value);
const res = JSON.parse(message.value.toString());
if (res.target !== this.pid) {
return;
}
const callback = this.pendingRequests.get(res.correlationId);
if (!callback) {
return;
}
if (res.error) {
callback(exceptions_1.default.SERVER_ERROR(res.error.message, true, res.error));
}
else {
callback(undefined, mapKafkaResponseToResponse(res));
}
this.pendingRequests.delete(res.correlationId);
},
});
await this.consumerMessages.run({
eachMessage: async ({ message }) => {
let msg;
try {
const data = JSON.parse(message.value?.toString() ?? "{}");
if ((data?.type === "invoke" || data?.type === "resume") && util.isTaskRecord(data?.task)) {
msg = {
type: data.type,
task: data.task,
headers: data.head ?? {},
};
}
else if (data?.type === "notify" && util.isDurablePromiseRecord(data?.promise)) {
msg = {
type: data.type,
promise: convertPromise(data.promise),
headers: data.head ?? {},
};
}
else {
throw new Error("invalid message");
}
}
catch {
console.warn("Networking. Received invalid message. Will continue.");
return;
}
this.recv(msg);
},
});
// Block until Kafka assigns to both consumers.
// This prevents a potential deadlock where
// subsequent requests are sent but cannot
// be received due to unassigned topics.
while (this.consumerResponse.assignment().length === 0) {
await new Promise((r) => setTimeout(r, 100));
}
while (this.consumerMessages.assignment().length === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
recv(msg) {
for (const callback of this.subscriptions[msg.type]) {
callback(msg);
}
}
async send(req, callback, _headers = {}) {
const correlationId = crypto.randomUUID();
this.pendingRequests.set(correlationId, callback);
const { op, payload } = mapRequestToKafkaRequest(req);
const kafkaRequest = {
target: "resonate.server",
replyTo: {
topic: this.responseTopic,
target: this.pid,
partition: this.responsePartition,
},
correlationId: correlationId,
operation: op,
payload: payload,
};
try {
await this.producer.send({
topic: this.requestTopic,
messages: [
{
value: JSON.stringify(kafkaRequest),
partition: this.requestPartition,
},
],
});
}
catch (e) {
console.log(e);
}
}
async stop() {
await this.producer.disconnect();
await this.consumerResponse.disconnect();
await this.consumerMessages.disconnect();
}
subscribe(type, callback) {
this.subscriptions[type].push(callback);
}
match(target) {
return `kafka://${target}`;
}
}
exports.Kafka = Kafka;
function mapRequestToKafkaRequest(req) {
switch (req.kind) {
case "createPromise":
return {
op: "promises.create",
payload: {
id: req.id,
timeout: req.timeout,
param: req.param,
tags: req.tags,
iKey: req.iKey,
strict: req.strict,
},
};
case "createPromiseAndTask":
return {
op: "promises.createtask",
payload: {
promise: {
id: req.promise.id,
timeout: req.promise.timeout,
param: req.promise.param,
tags: req.promise.tags,
},
task: { processId: req.task.processId, ttl: req.task.ttl },
iKey: req.iKey,
strict: req.strict,
},
};
case "readPromise":
return { op: "promises.read", payload: { id: req.id } };
case "completePromise":
return {
op: "promises.complete",
payload: {
id: req.id,
state: req.state,
value: req.value,
iKey: req.iKey,
strict: req.strict,
},
};
case "createCallback":
return {
op: "promises.callback",
payload: {
promiseId: req.promiseId,
rootPromiseId: req.rootPromiseId,
timeout: req.timeout,
recv: req.recv,
},
};
case "searchPromises":
return {
op: "promises.search",
payload: {
id: req.id,
state: req.state,
limit: req.limit,
cursor: req.cursor,
},
};
case "createSubscription":
return {
op: "promises.subscribe",
payload: {
id: req.id,
promiseId: req.promiseId,
timeout: req.timeout,
recv: req.recv,
},
};
case "createSchedule":
return {
op: "schedules.create",
payload: {
id: req.id,
description: req.description,
cron: req.cron,
tags: req.tags,
promiseId: req.promiseId,
promiseTimeout: req.promiseTimeout,
promiseParam: req.promiseParam,
promiseTags: req.promiseTags,
iKey: req.iKey,
},
};
case "readSchedule":
return { op: "schedules.read", payload: { id: req.id } };
case "searchSchedules":
return {
op: "schedules.search",
payload: { id: req.id, limit: req.limit, cursor: req.cursor },
};
case "deleteSchedule":
return { op: "schedules.delete", payload: { id: req.id } };
case "claimTask":
return {
op: "tasks.claim",
payload: {
id: req.id,
counter: req.counter,
processId: req.processId,
ttl: req.ttl,
},
};
case "completeTask":
return {
op: "tasks.complete",
payload: { id: req.id, counter: req.counter },
};
case "dropTask":
return {
op: "tasks.drop",
payload: { id: req.id, counter: req.counter },
};
case "heartbeatTasks":
return { op: "tasks.heartbeat", payload: { processId: req.processId } };
}
}
function mapKafkaResponseToResponse({ operation, response }) {
switch (operation) {
case "promises.create":
return {
kind: "createPromise",
promise: convertPromise(response),
};
case "promises.createtask":
return {
kind: "createPromiseAndTask",
promise: convertPromise(response.promise),
task: response.task ? convertTask(response.task) : undefined,
};
case "promises.read":
return {
kind: "readPromise",
promise: convertPromise(response),
};
case "promises.search":
return {
kind: "searchPromises",
promises: (response.promises ?? []).map(convertPromise),
cursor: response.cursor,
};
case "promises.complete":
return {
kind: "completePromise",
promise: convertPromise(response),
};
case "promises.callback":
return {
kind: "createCallback",
callback: response.callback ? convertCallback(response.callback) : undefined,
promise: convertPromise(response.promise),
};
case "promises.subscribe":
return {
kind: "createSubscription",
callback: response.callback ? convertCallback(response.callback) : undefined,
promise: convertPromise(response.promise),
};
case "schedules.create":
return {
kind: "createSchedule",
schedule: convertSchedule(response),
};
case "schedules.read":
return {
kind: "readSchedule",
schedule: convertSchedule(response),
};
case "schedules.search":
return {
kind: "searchSchedules",
schedules: (response.schedules ?? []).map(convertSchedule),
cursor: response.cursor,
};
case "schedules.delete":
return {
kind: "deleteSchedule",
};
case "tasks.claim":
return {
kind: "claimTask",
message: {
kind: response.type,
promises: {
root: response.promises.root
? {
id: response.promises.root.id,
data: convertPromise(response.promises.root.data),
}
: undefined,
leaf: response.promises.leaf
? {
id: response.promises.leaf.id,
data: convertPromise(response.promises.leaf.data),
}
: undefined,
},
},
};
case "tasks.complete":
return {
kind: "completeTask",
task: convertTask(response),
};
case "tasks.drop":
return {
kind: "dropTask",
};
case "tasks.heartbeat":
return {
kind: "heartbeatTasks",
tasksAffected: response.tasksAffected,
};
}
}
function convertPromise(promise) {
return {
id: promise.id,
state: convertState(promise.state),
timeout: promise.timeout,
param: promise.param,
value: promise.value,
tags: promise.tags || {},
iKeyForCreate: promise.idempotencyKeyForCreate,
iKeyForComplete: promise.idempotencyKeyForComplete,
createdOn: promise.createdOn,
completedOn: promise.completedOn,
};
}
function convertState(state) {
switch (state) {
case "PENDING":
return "pending";
case "RESOLVED":
return "resolved";
case "REJECTED":
return "rejected";
case "REJECTED_CANCELED":
return "rejected_canceled";
case "REJECTED_TIMEDOUT":
return "rejected_timedout";
default:
throw new Error(`Unknown API state: ${state}`);
}
}
function convertSchedule(schedule) {
return {
id: schedule.id,
description: schedule.description,
cron: schedule.cron,
tags: schedule.tags || {},
promiseId: schedule.promiseId,
promiseTimeout: schedule.promiseTimeout,
promiseParam: schedule.promiseParam,
promiseTags: schedule.promiseTags || {},
iKey: schedule.idempotencyKey,
lastRunTime: schedule.lastRunTime,
nextRunTime: schedule.nextRunTime,
createdOn: schedule.createdOn,
};
}
function convertCallback(callback) {
return {
id: callback.id,
promiseId: callback.promiseId,
timeout: callback.timeout,
createdOn: callback.createdOn,
};
}
function convertTask(task) {
return {
id: task.id,
rootPromiseId: task.rootPromiseId,
counter: task.counter,
timeout: task.timeout,
processId: task.processId,
createdOn: task.createdOn,
completedOn: task.completedOn,
};
}
//# sourceMappingURL=index.js.map