@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
625 lines (624 loc) • 22.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RuleConditionError = exports.RuleError = exports.NoneAction = exports.ErrorAction = exports.FieldType = exports.FieldContext = exports.FieldRuleExecutor = exports.RuleContext = exports.TopicNameStrategy = exports.Deserializer = exports.Serializer = exports.Serde = exports.SerializationError = exports.MAGIC_BYTE = exports.SerdeType = void 0;
const wildcard_matcher_1 = require("./wildcard-matcher");
const schemaregistry_client_1 = require("../schemaregistry-client");
const rule_registry_1 = require("./rule-registry");
var SerdeType;
(function (SerdeType) {
SerdeType["KEY"] = "KEY";
SerdeType["VALUE"] = "VALUE";
})(SerdeType || (exports.SerdeType = SerdeType = {}));
exports.MAGIC_BYTE = Buffer.alloc(1);
/**
* SerializationError represents a serialization error
*/
class SerializationError extends Error {
constructor(message) {
super(message);
}
}
exports.SerializationError = SerializationError;
/**
* Serde represents a serializer/deserializer
*/
class Serde {
constructor(client, serdeType, conf, ruleRegistry) {
this.fieldTransformer = null;
this.client = client;
this.serdeType = serdeType;
this.conf = conf;
this.ruleRegistry = ruleRegistry ?? rule_registry_1.RuleRegistry.getGlobalInstance();
}
close() {
return;
}
subjectName(topic, info) {
const strategy = this.conf.subjectNameStrategy ?? exports.TopicNameStrategy;
return strategy(topic, this.serdeType, info);
}
async resolveReferences(client, schema, deps, format) {
let references = schema.references;
if (references == null) {
return;
}
for (let ref of references) {
let metadata = await client.getSchemaMetadata(ref.subject, ref.version, true, format);
deps.set(ref.name, metadata.schema);
await this.resolveReferences(client, metadata, deps);
}
}
async executeRules(subject, topic, ruleMode, source, target, msg, inlineTags) {
if (msg == null || target == null) {
return msg;
}
let rules;
switch (ruleMode) {
case schemaregistry_client_1.RuleMode.UPGRADE:
rules = target.ruleSet?.migrationRules;
break;
case schemaregistry_client_1.RuleMode.DOWNGRADE:
rules = source?.ruleSet?.migrationRules?.map(x => x).reverse();
break;
default:
rules = target.ruleSet?.domainRules;
if (ruleMode === schemaregistry_client_1.RuleMode.READ) {
// Execute read rules in reverse order for symmetry
rules = rules?.map(x => x).reverse();
}
break;
}
if (rules == null) {
return msg;
}
for (let i = 0; i < rules.length; i++) {
let rule = rules[i];
if (this.isDisabled(rule)) {
continue;
}
let mode = rule.mode;
switch (mode) {
case schemaregistry_client_1.RuleMode.WRITEREAD:
if (ruleMode !== schemaregistry_client_1.RuleMode.WRITE && ruleMode !== schemaregistry_client_1.RuleMode.READ) {
continue;
}
break;
case schemaregistry_client_1.RuleMode.UPDOWN:
if (ruleMode !== schemaregistry_client_1.RuleMode.UPGRADE && ruleMode !== schemaregistry_client_1.RuleMode.DOWNGRADE) {
continue;
}
break;
default:
if (mode !== ruleMode) {
continue;
}
break;
}
let ctx = new RuleContext(source, target, subject, topic, this.serdeType === SerdeType.KEY, ruleMode, rule, i, rules, inlineTags, this.fieldTransformer);
let ruleExecutor = this.ruleRegistry.getExecutor(rule.type);
if (ruleExecutor == null) {
await this.runAction(ctx, ruleMode, rule, this.getOnFailure(rule), msg, new Error(`could not find rule executor of type ${rule.type}`), 'ERROR');
return msg;
}
try {
let result = await ruleExecutor.transform(ctx, msg);
switch (rule.kind) {
case 'CONDITION':
if (result === false) {
throw new RuleConditionError(rule);
}
break;
case 'TRANSFORM':
msg = result;
break;
}
await this.runAction(ctx, ruleMode, rule, msg != null
? this.getOnSuccess(rule) : this.getOnFailure(rule), msg, null, msg != null ? 'NONE' : 'ERROR');
}
catch (error) {
if (error instanceof SerializationError) {
throw error;
}
await this.runAction(ctx, ruleMode, rule, this.getOnFailure(rule), msg, error, 'ERROR');
}
}
return msg;
}
getOnSuccess(rule) {
let override = this.ruleRegistry.getOverride(rule.type);
if (override != null && override.onSuccess != null) {
return override.onSuccess;
}
return rule.onSuccess;
}
getOnFailure(rule) {
let override = this.ruleRegistry.getOverride(rule.type);
if (override != null && override.onFailure != null) {
return override.onFailure;
}
return rule.onFailure;
}
isDisabled(rule) {
let override = this.ruleRegistry.getOverride(rule.type);
if (override != null && override.disabled != null) {
return override.disabled;
}
return rule.disabled;
}
async runAction(ctx, ruleMode, rule, action, msg, err, defaultAction) {
let actionName = this.getRuleActionName(rule, ruleMode, action);
if (actionName == null) {
actionName = defaultAction;
}
let ruleAction = this.getRuleAction(ctx, actionName);
if (ruleAction == null) {
throw new RuleError(`Could not find rule action of type ${actionName}`);
}
try {
await ruleAction.run(ctx, msg, err);
}
catch (error) {
if (error instanceof SerializationError) {
throw error;
}
console.warn("could not run post-rule action %s: %s", actionName, error);
}
}
getRuleActionName(rule, ruleMode, actionName) {
if (actionName == null || actionName === '') {
return null;
}
if ((rule.mode === schemaregistry_client_1.RuleMode.WRITEREAD || rule.mode === schemaregistry_client_1.RuleMode.UPDOWN) && actionName.includes(',')) {
let parts = actionName.split(',');
switch (ruleMode) {
case schemaregistry_client_1.RuleMode.WRITE:
case schemaregistry_client_1.RuleMode.UPGRADE:
return parts[0];
case schemaregistry_client_1.RuleMode.READ:
case schemaregistry_client_1.RuleMode.DOWNGRADE:
return parts[1];
}
}
return actionName;
}
getRuleAction(ctx, actionName) {
if (actionName === 'ERROR') {
return new ErrorAction();
}
else if (actionName === 'NONE') {
return new NoneAction();
}
return this.ruleRegistry.getAction(actionName);
}
}
exports.Serde = Serde;
/**
* Serializer represents a serializer
*/
class Serializer extends Serde {
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
}
config() {
return this.conf;
}
// GetID returns a schema ID for the given schema
async getId(topic, msg, info, format) {
let autoRegister = this.config().autoRegisterSchemas;
let useSchemaId = this.config().useSchemaId;
let useLatestWithMetadata = this.conf.useLatestWithMetadata;
let useLatest = this.config().useLatestVersion;
let normalizeSchema = this.config().normalizeSchemas;
let id = -1;
let subject = this.subjectName(topic, info);
if (autoRegister) {
id = await this.client.register(subject, info, Boolean(normalizeSchema));
}
else if (useSchemaId != null && useSchemaId >= 0) {
info = await this.client.getBySubjectAndId(subject, useSchemaId, format);
id = useSchemaId;
}
else if (useLatestWithMetadata != null && Object.keys(useLatestWithMetadata).length !== 0) {
let metadata = await this.client.getLatestWithMetadata(subject, useLatestWithMetadata, true, format);
info = metadata;
id = metadata.id;
}
else if (useLatest) {
let metadata = await this.client.getLatestSchemaMetadata(subject, format);
info = metadata;
id = metadata.id;
}
else {
id = await this.client.getId(subject, info, Boolean(normalizeSchema));
}
return [id, info];
}
writeBytes(id, msgBytes) {
const idBuffer = Buffer.alloc(4);
idBuffer.writeInt32BE(id, 0);
return Buffer.concat([exports.MAGIC_BYTE, idBuffer, msgBytes]);
}
}
exports.Serializer = Serializer;
/**
* Deserializer represents a deserializer
*/
class Deserializer extends Serde {
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
}
config() {
return this.conf;
}
async getSchema(topic, payload, format) {
const magicByte = payload.subarray(0, 1);
if (!magicByte.equals(exports.MAGIC_BYTE)) {
throw new SerializationError(`Message encoded with magic byte ${JSON.stringify(magicByte)}, expected ${JSON.stringify(exports.MAGIC_BYTE)}`);
}
const id = payload.subarray(1, 5).readInt32BE(0);
let subject = this.subjectName(topic);
return await this.client.getBySubjectAndId(subject, id, format);
}
async getReaderSchema(subject, format) {
let useLatestWithMetadata = this.config().useLatestWithMetadata;
let useLatest = this.config().useLatestVersion;
if (useLatestWithMetadata != null && Object.keys(useLatestWithMetadata).length !== 0) {
return await this.client.getLatestWithMetadata(subject, useLatestWithMetadata, true, format);
}
if (useLatest) {
return await this.client.getLatestSchemaMetadata(subject, format);
}
return null;
}
hasRules(ruleSet, mode) {
switch (mode) {
case schemaregistry_client_1.RuleMode.UPGRADE:
case schemaregistry_client_1.RuleMode.DOWNGRADE:
return this.checkRules(ruleSet?.migrationRules, (ruleMode) => ruleMode === mode || ruleMode === schemaregistry_client_1.RuleMode.UPDOWN);
case schemaregistry_client_1.RuleMode.UPDOWN:
return this.checkRules(ruleSet?.migrationRules, (ruleMode) => ruleMode === mode);
case schemaregistry_client_1.RuleMode.WRITE:
case schemaregistry_client_1.RuleMode.READ:
return this.checkRules(ruleSet?.domainRules, (ruleMode) => ruleMode === mode || ruleMode === schemaregistry_client_1.RuleMode.WRITEREAD);
case schemaregistry_client_1.RuleMode.WRITEREAD:
return this.checkRules(ruleSet?.domainRules, (ruleMode) => ruleMode === mode);
}
}
checkRules(rules, filter) {
if (rules == null) {
return false;
}
for (let rule of rules) {
let ruleMode = rule.mode;
if (ruleMode && filter(ruleMode)) {
return true;
}
}
return false;
}
async getMigrations(subject, sourceInfo, target, format) {
let version = await this.client.getVersion(subject, sourceInfo, false, true);
let source = {
id: 0,
version: version,
schema: sourceInfo.schema,
references: sourceInfo.references,
metadata: sourceInfo.metadata,
ruleSet: sourceInfo.ruleSet,
};
let migrationMode;
let migrations = [];
let first;
let last;
if (source.version < target.version) {
migrationMode = schemaregistry_client_1.RuleMode.UPGRADE;
first = source;
last = target;
}
else if (source.version > target.version) {
migrationMode = schemaregistry_client_1.RuleMode.DOWNGRADE;
first = target;
last = source;
}
else {
return migrations;
}
let previous = null;
let versions = await this.getSchemasBetween(subject, first, last, format);
for (let i = 0; i < versions.length; i++) {
let version = versions[i];
if (i === 0) {
previous = version;
continue;
}
if (version.ruleSet != null && this.hasRules(version.ruleSet, migrationMode)) {
let m;
if (migrationMode === schemaregistry_client_1.RuleMode.UPGRADE) {
m = {
ruleMode: migrationMode,
source: previous,
target: version,
};
}
else {
m = {
ruleMode: migrationMode,
source: version,
target: previous,
};
}
migrations.push(m);
}
previous = version;
}
if (migrationMode === schemaregistry_client_1.RuleMode.DOWNGRADE) {
migrations = migrations.reverse();
}
return migrations;
}
async getSchemasBetween(subject, first, last, format) {
if (last.version - first.version <= 1) {
return [first, last];
}
let version1 = first.version;
let version2 = last.version;
let result = [first];
for (let i = version1 + 1; i < version2; i++) {
let meta = await this.client.getSchemaMetadata(subject, i, true, format);
result.push(meta);
}
result.push(last);
return result;
}
async executeMigrations(migrations, subject, topic, msg) {
for (let migration of migrations) {
// TODO fix source, target?
msg = await this.executeRules(subject, topic, migration.ruleMode, migration.source, migration.target, msg, null);
}
return msg;
}
}
exports.Deserializer = Deserializer;
/**
* TopicNameStrategy creates a subject name by appending -[key|value] to the topic name.
* @param topic - the topic name
* @param serdeType - the serde type
*/
const TopicNameStrategy = (topic, serdeType) => {
let suffix = '-value';
if (serdeType === SerdeType.KEY) {
suffix = '-key';
}
return topic + suffix;
};
exports.TopicNameStrategy = TopicNameStrategy;
/**
* RuleContext represents a rule context
*/
class RuleContext {
constructor(source, target, subject, topic, isKey, ruleMode, rule, index, rules, inlineTags, fieldTransformer) {
this.source = source;
this.target = target;
this.subject = subject;
this.topic = topic;
this.isKey = isKey;
this.ruleMode = ruleMode;
this.rule = rule;
this.index = index;
this.rules = rules;
this.inlineTags = inlineTags;
this.fieldTransformer = fieldTransformer;
this.fieldContexts = [];
}
getParameter(name) {
const params = this.rule.params;
if (params != null) {
let value = params[name];
if (value != null) {
return value;
}
}
let metadata = this.target.metadata;
if (metadata != null && metadata.properties != null) {
let value = metadata.properties[name];
if (value != null) {
return value;
}
}
return null;
}
getInlineTags(name) {
let tags = this.inlineTags?.get(name);
if (tags != null) {
return tags;
}
return new Set();
}
currentField() {
let size = this.fieldContexts.length;
if (size === 0) {
return null;
}
return this.fieldContexts[size - 1];
}
enterField(containingMessage, fullName, name, fieldType, tags) {
let allTags = new Set(tags ?? this.getInlineTags(fullName));
for (let v of this.getTags(fullName)) {
allTags.add(v);
}
let fieldContext = new FieldContext(containingMessage, fullName, name, fieldType, allTags);
this.fieldContexts.push(fieldContext);
return fieldContext;
}
getTags(fullName) {
let tags = new Set();
let metadata = this.target.metadata;
if (metadata?.tags != null) {
for (let [k, v] of Object.entries(metadata.tags)) {
if ((0, wildcard_matcher_1.match)(fullName, k)) {
for (let tag of v) {
tags.add(tag);
}
}
}
}
return tags;
}
leaveField() {
let size = this.fieldContexts.length - 1;
this.fieldContexts = this.fieldContexts.slice(0, size);
}
}
exports.RuleContext = RuleContext;
/**
* FieldRuleExecutor represents a field rule executor
*/
class FieldRuleExecutor {
constructor() {
this.config = null;
}
async transform(ctx, msg) {
// TODO preserve source
switch (ctx.ruleMode) {
case schemaregistry_client_1.RuleMode.WRITE:
case schemaregistry_client_1.RuleMode.UPGRADE:
for (let i = 0; i < ctx.index; i++) {
let otherRule = ctx.rules[i];
if (areTransformsWithSameTag(ctx.rule, otherRule)) {
// ignore this transform if an earlier one has the same tag
return msg;
}
}
break;
case schemaregistry_client_1.RuleMode.READ:
case schemaregistry_client_1.RuleMode.DOWNGRADE:
for (let i = ctx.index + 1; i < ctx.rules.length; i++) {
let otherRule = ctx.rules[i];
if (areTransformsWithSameTag(ctx.rule, otherRule)) {
// ignore this transform if a later one has the same tag
return msg;
}
}
break;
}
let fieldTransform = this.newTransform(ctx);
return ctx.fieldTransformer(ctx, fieldTransform, msg);
}
}
exports.FieldRuleExecutor = FieldRuleExecutor;
function areTransformsWithSameTag(rule1, rule2) {
return rule1.tags != null && rule1.tags.length > 0
&& rule1.kind === 'TRANSFORM'
&& rule1.kind === rule2.kind
&& rule1.mode === rule2.mode
&& rule1.type === rule2.type
&& rule1.tags === rule2.tags;
}
/**
* FieldContext represents a field context
*/
class FieldContext {
constructor(containingMessage, fullName, name, fieldType, tags) {
this.containingMessage = containingMessage;
this.fullName = fullName;
this.name = name;
this.type = fieldType;
this.tags = new Set(tags);
}
isPrimitive() {
let t = this.type;
return t === FieldType.STRING || t === FieldType.BYTES || t === FieldType.INT
|| t === FieldType.LONG || t === FieldType.FLOAT || t === FieldType.DOUBLE
|| t === FieldType.BOOLEAN || t === FieldType.NULL;
}
typeName() {
return this.type.toString();
}
}
exports.FieldContext = FieldContext;
var FieldType;
(function (FieldType) {
FieldType["RECORD"] = "RECORD";
FieldType["ENUM"] = "ENUM";
FieldType["ARRAY"] = "ARRAY";
FieldType["MAP"] = "MAP";
FieldType["COMBINED"] = "COMBINED";
FieldType["FIXED"] = "FIXED";
FieldType["STRING"] = "STRING";
FieldType["BYTES"] = "BYTES";
FieldType["INT"] = "INT";
FieldType["LONG"] = "LONG";
FieldType["FLOAT"] = "FLOAT";
FieldType["DOUBLE"] = "DOUBLE";
FieldType["BOOLEAN"] = "BOOLEAN";
FieldType["NULL"] = "NULL";
})(FieldType || (exports.FieldType = FieldType = {}));
/**
* ErrorAction represents an error action
*/
class ErrorAction {
configure(clientConfig, config) {
}
type() {
return 'ERROR';
}
async run(ctx, msg, err) {
throw new SerializationError(err.message);
}
close() {
}
}
exports.ErrorAction = ErrorAction;
/**
* NoneAction represents a no-op action
*/
class NoneAction {
configure(clientConfig, config) {
}
type() {
return 'NONE';
}
async run(ctx, msg, err) {
return;
}
close() {
}
}
exports.NoneAction = NoneAction;
/**
* RuleError represents a rule error
*/
class RuleError extends Error {
/**
* Creates a new rule error.
* @param message - The error message.
*/
constructor(message) {
super(message);
}
}
exports.RuleError = RuleError;
/**
* RuleConditionError represents a rule condition error
*/
class RuleConditionError extends RuleError {
/**
* Creates a new rule condition error.
* @param rule - The rule.
*/
constructor(rule) {
super(RuleConditionError.error(rule));
this.rule = rule;
}
static error(rule) {
let errMsg = rule.doc;
if (!errMsg) {
if (rule.expr !== '') {
return `Expr failed: '${rule.expr}'`;
}
return `Condition failed: '${rule.name}'`;
}
return errMsg;
}
}
exports.RuleConditionError = RuleConditionError;