@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
526 lines (525 loc) • 22.6 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProtobufDeserializer = exports.ProtobufSerializer = exports.PROTOBUF_TYPE = void 0;
const serde_1 = require("./serde");
const schemaregistry_client_1 = require("../schemaregistry-client");
const protobuf_1 = require("@bufbuild/protobuf");
const wkt_1 = require("@bufbuild/protobuf/wkt");
const lru_cache_1 = require("lru-cache");
const meta_pb_1 = require("../confluent/meta_pb");
const json_stringify_deterministic_1 = __importDefault(require("json-stringify-deterministic"));
const decimal_pb_1 = require("../confluent/types/decimal_pb");
const calendar_period_pb_1 = require("../google/type/calendar_period_pb");
const color_pb_1 = require("../google/type/color_pb");
const date_pb_1 = require("../google/type/date_pb");
const datetime_pb_1 = require("../google/type/datetime_pb");
const dayofweek_pb_1 = require("../google/type/dayofweek_pb");
const fraction_pb_1 = require("../google/type/fraction_pb");
const expr_pb_1 = require("../google/type/expr_pb");
const latlng_pb_1 = require("../google/type/latlng_pb");
const money_pb_1 = require("../google/type/money_pb");
const postal_address_pb_1 = require("../google/type/postal_address_pb");
const quaternion_pb_1 = require("../google/type/quaternion_pb");
const timeofday_pb_1 = require("../google/type/timeofday_pb");
const month_pb_1 = require("../google/type/month_pb");
exports.PROTOBUF_TYPE = "PROTOBUF";
const builtinDeps = new Map([
['confluent/meta.proto', meta_pb_1.file_confluent_meta],
['confluent/type/decimal.proto', decimal_pb_1.file_confluent_types_decimal],
['google/type/calendar_period.proto', calendar_period_pb_1.file_google_type_calendar_period],
['google/type/color.proto', color_pb_1.file_google_type_color],
['google/type/date.proto', date_pb_1.file_google_type_date],
['google/type/datetime.proto', datetime_pb_1.file_google_type_datetime],
['google/type/dayofweek.proto', dayofweek_pb_1.file_google_type_dayofweek],
['google/type/expr.proto', expr_pb_1.file_google_type_expr],
['google/type/fraction.proto', fraction_pb_1.file_google_type_fraction],
['google/type/latlng.proto', latlng_pb_1.file_google_type_latlng],
['google/type/money.proto', money_pb_1.file_google_type_money],
['google/type/month.proto', month_pb_1.file_google_type_month],
['google/type/postal_address.proto', postal_address_pb_1.file_google_type_postal_address],
['google/type/quaternion.proto', quaternion_pb_1.file_google_type_quaternion],
['google/type/timeofday.proto', timeofday_pb_1.file_google_type_timeofday],
['google/protobuf/any.proto', wkt_1.file_google_protobuf_any],
['google/protobuf/api.proto', wkt_1.file_google_protobuf_api],
['google/protobuf/descriptor.proto', wkt_1.file_google_protobuf_descriptor],
['google/protobuf/duration.proto', wkt_1.file_google_protobuf_duration],
['google/protobuf/empty.proto', wkt_1.file_google_protobuf_empty],
['google/protobuf/field_mask.proto', wkt_1.file_google_protobuf_field_mask],
['google/protobuf/source_context.proto', wkt_1.file_google_protobuf_source_context],
['google/protobuf/struct.proto', wkt_1.file_google_protobuf_struct],
['google/protobuf/timestamp.proto', wkt_1.file_google_protobuf_timestamp],
['google/protobuf/type.proto', wkt_1.file_google_protobuf_type],
['google/protobuf/wrappers.proto', wkt_1.file_google_protobuf_wrappers],
]);
/**
* ProtobufSerializer is a serializer for Protobuf messages.
*/
class ProtobufSerializer extends serde_1.Serializer {
/**
* Creates a new ProtobufSerializer.
* @param client - the schema registry client
* @param serdeType - the serializer type
* @param conf - the serializer configuration
* @param ruleRegistry - the rule registry
*/
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
this.registry = conf.registry ?? (0, protobuf_1.createMutableRegistry)();
this.fileRegistry = (0, protobuf_1.createFileRegistry)();
this.schemaToDescCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.descToSchemaCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.fieldTransformer = async (ctx, fieldTransform, msg) => {
return await this.fieldTransform(ctx, fieldTransform, msg);
};
for (const rule of this.ruleRegistry.getExecutors()) {
rule.configure(client.config(), new Map(Object.entries(conf.ruleConfig ?? {})));
}
}
/**
* Serializes a message.
* @param topic - the topic
* @param msg - the message
* @param headers - optional headers
*/
async serialize(topic, msg, headers) {
if (this.client == null) {
throw new Error('client is not initialized');
}
if (msg == null) {
throw new Error('message is empty');
}
const typeName = msg.$typeName;
if (typeName == null) {
throw new serde_1.SerializationError('message type name is empty');
}
const messageDesc = this.registry.getMessage(typeName);
if (messageDesc == null) {
throw new serde_1.SerializationError('message descriptor not in registry');
}
let schema = undefined;
// Don't derive the schema if it is being looked up in the following ways
if (this.config().useSchemaId == null &&
!this.config().useLatestVersion &&
this.config().useLatestWithMetadata == null) {
const fileDesc = messageDesc.file;
schema = await this.getSchemaInfo(fileDesc);
}
const [schemaId, info] = await this.getSchemaId(exports.PROTOBUF_TYPE, topic, msg, schema, 'serialized');
const subject = this.subjectName(topic, info);
msg = await this.executeRules(subject, topic, schemaregistry_client_1.RuleMode.WRITE, null, info, msg, null);
schemaId.messageIndexes = this.toMessageIndexArray(messageDesc);
let msgBytes = Buffer.from((0, protobuf_1.toBinary)(messageDesc, msg));
msgBytes = await this.executeRulesWithPhase(subject, topic, schemaregistry_client_1.RulePhase.ENCODING, schemaregistry_client_1.RuleMode.WRITE, null, info, msgBytes, null);
return this.serializeSchemaId(topic, msgBytes, schemaId, headers);
}
async getSchemaInfo(fileDesc) {
const value = this.descToSchemaCache.get(fileDesc.name);
if (value != null) {
return value;
}
const deps = this.toProtobufSchema(fileDesc);
const autoRegister = this.config().autoRegisterSchemas;
const normalize = this.config().normalizeSchemas;
const metadata = await this.resolveDependencies(fileDesc, deps, "", Boolean(autoRegister), Boolean(normalize));
const info = {
schema: metadata.schema,
schemaType: metadata.schemaType,
references: metadata.references,
metadata: metadata.metadata,
ruleSet: metadata.ruleSet,
};
this.descToSchemaCache.set(fileDesc.name, info);
return info;
}
toProtobufSchema(fileDesc) {
const deps = new Map();
this.toDependencies(fileDesc, deps);
return deps;
}
toDependencies(fileDesc, deps) {
deps.set(fileDesc.name, Buffer.from((0, protobuf_1.toBinary)(wkt_1.FileDescriptorProtoSchema, fileDesc.proto)).toString('base64'));
fileDesc.dependencies.forEach((dep) => {
if (!isBuiltin(dep.name)) {
this.toDependencies(dep, deps);
}
});
}
async resolveDependencies(fileDesc, deps, subject, autoRegister, normalize) {
const refs = [];
for (let i = 0; i < fileDesc.dependencies.length; i++) {
const dep = fileDesc.dependencies[i];
const depName = dep.name + '.proto';
if (isBuiltin(depName)) {
continue;
}
const ref = await this.resolveDependencies(dep, deps, depName, autoRegister, normalize);
if (ref == null) {
throw new serde_1.SerializationError('dependency not found');
}
refs.push({ name: depName, subject: ref.subject, version: ref.version });
}
const info = {
schema: deps.get(fileDesc.name),
schemaType: 'PROTOBUF',
references: refs
};
let id = -1;
let version = 0;
if (subject !== '') {
if (autoRegister) {
id = await this.client.register(subject, info, normalize);
}
else {
id = await this.client.getId(subject, info, normalize);
}
version = await this.client.getVersion(subject, info, normalize, false);
}
return {
id: id,
// TODO verify that guid is not required
guid: "",
subject: subject,
version: version,
schema: info.schema,
schemaType: info.schemaType,
references: info.references,
metadata: info.metadata,
ruleSet: info.ruleSet,
};
}
toMessageIndexArray(messageDesc) {
return this.toMessageIndexes(messageDesc, 0);
}
toMessageIndexes(messageDesc, count) {
const index = this.toIndex(messageDesc);
const parent = messageDesc.parent;
if (parent == null) {
// parent is FileDescriptor, we reached the top of the stack, so we are
// done. Allocate an array large enough to hold count+1 entries and
// populate first value with index
const msgIndexes = [];
msgIndexes.push(index);
return msgIndexes;
}
else {
const msgIndexes = this.toMessageIndexes(parent, count + 1);
msgIndexes.push(index);
return msgIndexes;
}
}
toIndex(messageDesc) {
const parent = messageDesc.parent;
if (parent == null) {
const fileDesc = messageDesc.file;
for (let i = 0; i < fileDesc.messages.length; i++) {
if (fileDesc.messages[i] === messageDesc) {
return i;
}
}
}
else {
for (let i = 0; i < parent.nestedMessages.length; i++) {
if (parent.nestedMessages[i] === messageDesc) {
return i;
}
}
}
throw new serde_1.SerializationError('message descriptor not found in file descriptor');
}
async fieldTransform(ctx, fieldTransform, msg) {
const fileDesc = await this.toFileDesc(this.client, ctx.target);
const typeName = msg.$typeName;
const messageDesc = this.toMessageDescFromName(fileDesc, typeName);
return await transform(ctx, messageDesc, msg, fieldTransform);
}
async toFileDesc(client, info) {
const value = this.schemaToDescCache.get((0, json_stringify_deterministic_1.default)(info.schema));
if (value != null) {
return value;
}
const fileDesc = await this.parseFileDesc(client, info);
if (fileDesc == null) {
throw new serde_1.SerializationError('file descriptor not found');
}
this.schemaToDescCache.set((0, json_stringify_deterministic_1.default)(info.schema), fileDesc);
return fileDesc;
}
async parseFileDesc(client, info) {
const deps = new Map();
await this.resolveReferences(client, info, deps, 'serialized');
const fileDesc = (0, protobuf_1.fromBinary)(wkt_1.FileDescriptorProtoSchema, Buffer.from(info.schema, 'base64'));
const fileRegistry = newFileRegistry(fileDesc, deps);
this.fileRegistry = (0, protobuf_1.createFileRegistry)(this.fileRegistry, fileRegistry);
return this.fileRegistry.getFile(fileDesc.name);
}
toMessageDescFromName(fd, msgName) {
for (let i = 0; i < fd.messages.length; i++) {
if (fd.messages[i].typeName === msgName) {
return fd.messages[i];
}
}
throw new serde_1.SerializationError('message descriptor not found');
}
}
exports.ProtobufSerializer = ProtobufSerializer;
/**
* ProtobufDeserializer is a deserializer for Protobuf messages.
*/
class ProtobufDeserializer extends serde_1.Deserializer {
/**
* Creates a new ProtobufDeserializer.
* @param client - the schema registry client
* @param serdeType - the deserializer type
* @param conf - the deserializer configuration
* @param ruleRegistry - the rule registry
*/
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
this.fileRegistry = (0, protobuf_1.createFileRegistry)();
this.schemaToDescCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.fieldTransformer = async (ctx, fieldTransform, msg) => {
return await this.fieldTransform(ctx, fieldTransform, msg);
};
for (const rule of this.ruleRegistry.getExecutors()) {
rule.configure(client.config(), new Map(Object.entries(conf.ruleConfig ?? {})));
}
}
/**
* Deserializes a message.
* @param topic - the topic
* @param payload - the message payload
* @param headers - optional headers
*/
async deserialize(topic, payload, headers) {
if (!Buffer.isBuffer(payload)) {
throw new Error('Invalid buffer');
}
if (payload.length === 0) {
return null;
}
const schemaId = new serde_1.SchemaId(exports.PROTOBUF_TYPE);
const [info, bytesRead] = await this.getWriterSchema(topic, payload, schemaId, headers, 'serialized');
payload = payload.subarray(bytesRead);
const fd = await this.toFileDesc(this.client, info);
const messageDesc = this.toMessageDescFromIndexes(fd, schemaId.messageIndexes);
const subject = this.subjectName(topic, info);
payload = await this.executeRulesWithPhase(subject, topic, schemaregistry_client_1.RulePhase.ENCODING, schemaregistry_client_1.RuleMode.READ, null, info, payload, null);
const readerMeta = await this.getReaderSchema(subject, 'serialized');
const msgBytes = payload;
let msg = (0, protobuf_1.fromBinary)(messageDesc, msgBytes);
// Currently JavaScript does not support migration rules
// because of lack of support for DynamicMessage
let target;
if (readerMeta != null) {
target = readerMeta;
}
else {
target = info;
}
msg = await this.executeRules(subject, topic, schemaregistry_client_1.RuleMode.READ, null, target, msg, null);
return msg;
}
async fieldTransform(ctx, fieldTransform, msg) {
const fileDesc = await this.toFileDesc(this.client, ctx.target);
const typeName = msg.$typeName;
const messageDesc = this.toMessageDescFromName(fileDesc, typeName);
return await transform(ctx, messageDesc, msg, fieldTransform);
}
async toFileDesc(client, info) {
const value = this.schemaToDescCache.get((0, json_stringify_deterministic_1.default)(info.schema));
if (value != null) {
return value;
}
const fileDesc = await this.parseFileDesc(client, info);
if (fileDesc == null) {
throw new serde_1.SerializationError('file descriptor not found');
}
this.schemaToDescCache.set((0, json_stringify_deterministic_1.default)(info.schema), fileDesc);
return fileDesc;
}
async parseFileDesc(client, info) {
const deps = new Map();
await this.resolveReferences(client, info, deps, 'serialized');
const fileDesc = (0, protobuf_1.fromBinary)(wkt_1.FileDescriptorProtoSchema, Buffer.from(info.schema, 'base64'));
const fileRegistry = newFileRegistry(fileDesc, deps);
this.fileRegistry = (0, protobuf_1.createFileRegistry)(this.fileRegistry, fileRegistry);
return this.fileRegistry.getFile(fileDesc.name);
}
toMessageDescFromName(fd, msgName) {
for (let i = 0; i < fd.messages.length; i++) {
if (fd.messages[i].typeName === msgName) {
return fd.messages[i];
}
}
throw new serde_1.SerializationError('message descriptor not found');
}
toMessageDescFromIndexes(fd, msgIndexes) {
let index = msgIndexes[0];
if (msgIndexes.length === 1) {
return fd.messages[index];
}
return this.toNestedMessageDesc(fd.messages[index], msgIndexes.slice(1));
}
toNestedMessageDesc(parent, msgIndexes) {
let index = msgIndexes[0];
if (msgIndexes.length === 1) {
return parent.nestedMessages[index];
}
return this.toNestedMessageDesc(parent.nestedMessages[index], msgIndexes.slice(1));
}
}
exports.ProtobufDeserializer = ProtobufDeserializer;
function newFileRegistry(fileDesc, deps) {
const resolve = (depName) => {
if (isBuiltin(depName)) {
const dep = builtinDeps.get(depName);
if (dep == null) {
throw new serde_1.SerializationError(`dependency ${depName} not found`);
}
return dep;
}
else {
const dep = deps.get(depName);
if (dep == null) {
throw new serde_1.SerializationError(`dependency ${depName} not found`);
}
const fileDesc = (0, protobuf_1.fromBinary)(wkt_1.FileDescriptorProtoSchema, Buffer.from(dep, 'base64'));
fileDesc.name = depName;
return fileDesc;
}
};
return (0, protobuf_1.createFileRegistry)(fileDesc, resolve);
}
async function transform(ctx, descriptor, msg, fieldTransform) {
if (msg == null || descriptor == null) {
return msg;
}
if (Array.isArray(msg)) {
for (let i = 0; i < msg.length; i++) {
msg[i] = await transform(ctx, descriptor, msg[i], fieldTransform);
}
}
if (msg instanceof Map) {
return msg;
}
const typeName = msg.$typeName;
if (typeName != null) {
const fields = descriptor.fields;
for (let i = 0; i < fields.length; i++) {
const fd = fields[i];
await transformField(ctx, fd, descriptor, msg, fieldTransform);
}
return msg;
}
const fieldCtx = ctx.currentField();
if (fieldCtx != null) {
const ruleTags = ctx.rule.tags ?? [];
if (ruleTags == null || ruleTags.length === 0 || !disjoint(new Set(ruleTags), fieldCtx.tags)) {
return await fieldTransform.transform(ctx, fieldCtx, msg);
}
}
return msg;
}
async function transformField(ctx, fd, desc, msg, fieldTransform) {
try {
ctx.enterField(msg, desc.typeName + '.' + fd.name, fd.name, getType(fd), getInlineTags(fd));
let value = null;
if (fd.oneof != null) {
let oneof = msg[fd.oneof.localName];
if (oneof != null && oneof.case === fd.localName) {
value = oneof.value;
}
else {
// skip oneof fields that are not set
return;
}
}
else {
value = msg[fd.localName];
}
const newValue = await transform(ctx, desc, value, fieldTransform);
if (ctx.rule.kind === 'CONDITION') {
if (newValue === false) {
throw new serde_1.RuleConditionError(ctx.rule);
}
}
else {
if (fd.oneof != null) {
msg[fd.oneof.localName] = { case: fd.localName, value: newValue };
}
else {
msg[fd.localName] = newValue;
}
}
}
finally {
ctx.leaveField();
}
}
function getType(fd) {
let kind = fd.fieldKind;
if (fd.fieldKind === 'list') {
kind = fd.listKind;
}
switch (kind) {
case 'map':
return serde_1.FieldType.MAP;
case 'message':
return serde_1.FieldType.RECORD;
case 'enum':
return serde_1.FieldType.ENUM;
case 'scalar':
switch (fd.scalar) {
case protobuf_1.ScalarType.STRING:
return serde_1.FieldType.STRING;
case protobuf_1.ScalarType.BYTES:
return serde_1.FieldType.BYTES;
case protobuf_1.ScalarType.INT32:
case protobuf_1.ScalarType.SINT32:
case protobuf_1.ScalarType.UINT32:
case protobuf_1.ScalarType.FIXED32:
case protobuf_1.ScalarType.SFIXED32:
return serde_1.FieldType.INT;
case protobuf_1.ScalarType.INT64:
case protobuf_1.ScalarType.SINT64:
case protobuf_1.ScalarType.UINT64:
case protobuf_1.ScalarType.FIXED64:
case protobuf_1.ScalarType.SFIXED64:
return serde_1.FieldType.LONG;
case protobuf_1.ScalarType.FLOAT:
return serde_1.FieldType.FLOAT;
case protobuf_1.ScalarType.DOUBLE:
return serde_1.FieldType.DOUBLE;
case protobuf_1.ScalarType.BOOL:
return serde_1.FieldType.BOOLEAN;
default:
return serde_1.FieldType.NULL;
}
default:
return serde_1.FieldType.NULL;
}
}
function getInlineTags(fd) {
const options = fd.proto.options;
if (options != null && (0, protobuf_1.hasExtension)(options, meta_pb_1.field_meta)) {
const option = (0, protobuf_1.getExtension)(options, meta_pb_1.field_meta);
return new Set(option.tags);
}
return new Set();
}
function disjoint(tags1, tags2) {
for (let tag of tags1) {
if (tags2.has(tag)) {
return false;
}
}
return true;
}
function isBuiltin(name) {
return name.startsWith('confluent/') ||
name.startsWith('google/protobuf/') ||
name.startsWith('google/type/');
}