schematic-kafka
Version:
Encode and decode kafka messages with Confluent Schema Registry (pure typescript implementation)
185 lines • 10.3 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaRegistryHelper = void 0;
const tslib_1 = require("tslib");
const kafka_helper_1 = require("./kafka-helper");
const schema_registry_client_1 = require("./schema-registry-client");
const function_cacher_1 = require("./function-cacher");
/**
* Helper class to cache calls to the SchemaRegistryClient
*/
class CachedSchemaRegistryClient extends schema_registry_client_1.SchemaRegistryClient {
constructor(...schemaRegistryClientOptions) {
super(...schemaRegistryClientOptions);
this.cacher = new function_cacher_1.FunctionCacher();
this.checkSchema = this.cacher.createCachedFunction(super.checkSchema, [true, true], this);
this.getLatestVersionForSubject = this.cacher.createCachedFunction(super.getLatestVersionForSubject, [true], this);
this.getSchemaById = this.cacher.createCachedFunction(super.getSchemaById, [true], this);
this.listVersionsForId = this.cacher.createCachedFunction(super.listVersionsForId, [true], this);
}
}
class KafkaRegistryHelper {
constructor(...schemaRegistryOptions) {
this.schemaHandlers = new Map();
this.schemaRegistryClient = new CachedSchemaRegistryClient(...schemaRegistryOptions);
}
/**
* Helper function to add a schema Handler in a syntactic sugary way
* Note: The registry helper will call the factory every time a message is encoded/decoded
* It is your responsibility to handle caching of the schema handlers
* @param schemaType
* @param schemaHandlerFactory
* @returns the KafkaRegistryHelper instance
*/
withSchemaHandler(schemaType, schemaHandlerFactory) {
this.schemaHandlers.set(schemaType, schemaHandlerFactory);
return this;
}
/**
* Checks whether a schema is available in the schema registry, if not tries to register it
* @param subject
* @param schemaType
* @param schema the registry needs the schema to be a string
* @param references
* @returns schema string and schemaId
*/
makeSureSchemaIsRegistered(subject_1) {
return tslib_1.__awaiter(this, arguments, void 0, function* (subject, schemaType = schema_registry_client_1.SchemaType.AVRO, schema, references) {
var _a, _b, _c;
// register schema or fetch schema
let registrySchemaId = undefined;
let registrySchema = undefined;
let registrySchemaType = undefined;
if (schema) {
try {
const checkSchemaResult = yield this.schemaRegistryClient.checkSchema(subject, {
schemaType,
schema,
references,
});
registrySchemaId = checkSchemaResult.id;
registrySchema = (_a = checkSchemaResult.schema) !== null && _a !== void 0 ? _a : schema;
registrySchemaType = checkSchemaResult.schemaType;
}
catch (e) {
if (e instanceof schema_registry_client_1.SchemaRegistryError && e.errorCode === 404) {
// schema does not exist, need to create it
const registerSchemaResult = yield this.schemaRegistryClient.registerSchema(subject, {
schemaType,
schema,
references,
});
registrySchemaId = registerSchemaResult.id;
// According to https://docs.confluent.io/platform/current/schema-registry/develop/api.html#post--subjects-(string-%20subject)-versions
// the register post should return schema, schemaType, references and id. However, the response only contains the id.
registrySchema = (_b = registerSchemaResult.schema) !== null && _b !== void 0 ? _b : schema;
registrySchemaType = (_c = registerSchemaResult.schemaType) !== null && _c !== void 0 ? _c : schemaType;
}
else {
throw e;
}
}
}
else {
const { id, schema, schemaType } = yield this.schemaRegistryClient.getLatestVersionForSubject(subject);
registrySchemaId = id;
registrySchema = schema;
registrySchemaType = schemaType;
}
return { id: registrySchemaId, schema: registrySchema, schemaType: registrySchemaType !== null && registrySchemaType !== void 0 ? registrySchemaType : schemaType };
});
}
/**
* Encode a message with a given schema id
* @param schemaId id of the schema
* @param message formatted in whatever type SchemaHandler works with
* @param schemaType (optional, default = AVRO) schema type can be given, mostly for safety reason
* @returns message encoded with schema associated with schema id
*/
encodeForId(schemaId_1, message_1) {
return tslib_1.__awaiter(this, arguments, void 0, function* (schemaId, message, schemaType = schema_registry_client_1.SchemaType.AVRO) {
if (!this.schemaHandlers.get(schemaType)) {
throw new Error(`No protocol handler for protocol ${schemaType}`);
}
const { schema: registrySchema, schemaType: registrySchemaType = schema_registry_client_1.SchemaType.AVRO } = yield this.schemaRegistryClient.getSchemaById(schemaId);
if (schemaType !== registrySchemaType) {
throw new Error("Mismatch between schemaType argument and schema registry schemaType");
}
return this.encodeMessage(schemaId, message, registrySchemaType, registrySchema);
});
}
/**
* Encode a message by subject name
* @param subject subject name (for kafka messages, don't forget the -key/-value postfix)
* @param message formatted in whatever type SchemaHandler works with
* @param schemaType (optional, default = AVRO) type of schema (e.g. PROTOBUG)
* @param schema serialized representation of the message schema (this will be registered with registry if it doesn't exist there yet)
* @param references additional schema references
* @returns message encoded for given subject / schema (schema id will be determined automatically)
*/
encodeForSubject(subject_1, message_1) {
return tslib_1.__awaiter(this, arguments, void 0, function* (subject, message, schemaType = schema_registry_client_1.SchemaType.AVRO, schema, references) {
if (!this.schemaHandlers.get(schemaType)) {
throw new Error(`No protocol handler for protocol ${schemaType}`);
}
const { id, schema: registrySchema, schemaType: registrySchemaType = schema_registry_client_1.SchemaType.AVRO, } = yield this.makeSureSchemaIsRegistered(subject, schemaType, schema, references);
if (schemaType !== registrySchemaType) {
throw new Error("Mismatch between schemaType argument and schema registry schemaType");
}
return this.encodeMessage(id, message, registrySchemaType, registrySchema);
});
}
/**
* decode a kafka event message
* @param message message with or without schema preamble. Note: If the preamble is missing the payload will be passed through
* @returns message in whatever format the SchemaHandler for the schemaType produces
*/
decode(message) {
return tslib_1.__awaiter(this, void 0, void 0, function* () {
const { schemaId, payload } = (0, kafka_helper_1.kafkaDecode)(message);
if (schemaId) {
const { schema, schemaType } = yield this.schemaRegistryClient.getSchemaById(schemaId);
// if schemaType isn't provided, use avro (it's the default)
return this.schemaHandlers
.get(schemaType !== null && schemaType !== void 0 ? schemaType : schema_registry_client_1.SchemaType.AVRO)(schema)
.decode(payload);
}
else {
return payload;
}
});
}
/**
* decode a kafka event message, but queries the schema for possible subject name and versions
* @param message message with or without schema preamble. Note: If the preamble is missing the payload will be passed through
* @returns decoded message in whatever format the SchemaHandler for the schemaType produces plus a list of possible subject names and versions matching the schema id used for encoding
*/
decodeWithSubjectAndVersionInformation(message) {
return tslib_1.__awaiter(this, void 0, void 0, function* () {
const { schemaId, payload } = (0, kafka_helper_1.kafkaDecode)(message);
const result = { message: payload };
if (schemaId) {
// if schemaType isn't provided, use avro (it's the default)
const { schema, schemaType = schema_registry_client_1.SchemaType.AVRO } = yield this.schemaRegistryClient.getSchemaById(schemaId);
result.subjects = yield this.schemaRegistryClient.listVersionsForId(schemaId);
result.message = this.schemaHandlers.get(schemaType)(schema).decode(payload);
}
return result;
});
}
/**
* handle message encoding with schemaHandler
* @param schemaId id of the schema
* @param message formatted in whatever type SchemaHandler works with
* @param schemaType schema type, schema handler for this type has to be registered
* @param schema the schema as provided by the protocol handler
* @returns message encoded with handler corresponding to the schemaType
*/
encodeMessage(schemaId, message, schemaType, schema) {
const schemaHandler = this.schemaHandlers.get(schemaType)(schema);
const encodedMessage = schemaHandler.encode(message);
return (0, kafka_helper_1.kafkaEncode)(schemaId, encodedMessage);
}
}
exports.KafkaRegistryHelper = KafkaRegistryHelper;
//# sourceMappingURL=kafka-registry-helper.js.map