@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
454 lines (453 loc) • 17.6 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.JsonDeserializer = exports.JsonSerializer = exports.JSON_TYPE = void 0;
const serde_1 = require("./serde");
const schemaregistry_client_1 = require("../schemaregistry-client");
const _2019_1 = __importDefault(require("ajv/dist/2019"));
const _2020_1 = __importDefault(require("ajv/dist/2020"));
const draft6MetaSchema = __importStar(require("ajv/dist/refs/json-schema-draft-06.json"));
const draft7MetaSchema = __importStar(require("ajv/dist/refs/json-schema-draft-07.json"));
const draft_2020_12_1 = require("@criteria/json-schema/draft-2020-12");
const draft_07_1 = require("@criteria/json-schema/draft-07");
const json_schema_validation_1 = require("@criteria/json-schema-validation");
const lru_cache_1 = require("lru-cache");
const json_util_1 = require("./json-util");
const json_stringify_deterministic_1 = __importDefault(require("json-stringify-deterministic"));
exports.JSON_TYPE = "JSON";
/**
* JsonSerializer is a serializer for JSON messages.
*/
class JsonSerializer extends serde_1.Serializer {
/**
* Creates a new JsonSerializer.
* @param client - the schema registry client
* @param serdeType - the serializer type
* @param conf - the serializer configuration
* @param ruleRegistry - the rule registry
*/
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
this.schemaToTypeCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.schemaToValidateCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.fieldTransformer = async (ctx, fieldTransform, msg) => {
return await this.fieldTransform(ctx, fieldTransform, msg);
};
for (const rule of this.ruleRegistry.getExecutors()) {
rule.configure(client.config(), new Map(Object.entries(conf.ruleConfig ?? {})));
}
}
/**
* Serializes a message.
* @param topic - the topic
* @param msg - the message
* @param headers - optional headers
*/
async serialize(topic, msg, headers) {
if (this.client == null) {
throw new Error('client is not initialized');
}
if (msg == null) {
throw new Error('message is empty');
}
let schema = undefined;
// Don't derive the schema if it is being looked up in the following ways
if (this.config().useSchemaId == null &&
!this.config().useLatestVersion &&
this.config().useLatestWithMetadata == null) {
const jsonSchema = JsonSerializer.messageToSchema(msg);
schema = {
schemaType: 'JSON',
schema: JSON.stringify(jsonSchema),
};
}
const [schemaId, info] = await this.getSchemaId(exports.JSON_TYPE, topic, msg, schema);
const subject = this.subjectName(topic, info);
msg = await this.executeRules(subject, topic, schemaregistry_client_1.RuleMode.WRITE, null, info, msg, null);
if (this.conf.validate) {
const validate = await this.toValidateFunction(info);
if (validate != null && !validate(msg)) {
throw new serde_1.SerializationError('Invalid message');
}
}
let msgBytes = Buffer.from(JSON.stringify(msg));
msgBytes = await this.executeRulesWithPhase(subject, topic, schemaregistry_client_1.RulePhase.ENCODING, schemaregistry_client_1.RuleMode.WRITE, null, info, msgBytes, null);
return this.serializeSchemaId(topic, msgBytes, schemaId, headers);
}
async fieldTransform(ctx, fieldTransform, msg) {
const schema = await this.toType(ctx.target);
if (typeof schema === 'boolean') {
return msg;
}
return await transform(ctx, schema, '$', msg, fieldTransform);
}
async toType(info) {
return toType(this.client, this.conf, this, info, async (client, info) => {
const deps = new Map();
await this.resolveReferences(client, info, deps);
return deps;
});
}
async toValidateFunction(info) {
return await toValidateFunction(this.client, this.conf, this, info, async (client, info) => {
const deps = new Map();
await this.resolveReferences(client, info, deps);
return deps;
});
}
static messageToSchema(msg) {
return (0, json_util_1.generateSchema)(msg);
}
}
exports.JsonSerializer = JsonSerializer;
/**
* JsonDeserializer is a deserializer for JSON messages.
*/
class JsonDeserializer extends serde_1.Deserializer {
/**
* Creates a new JsonDeserializer.
* @param client - the schema registry client
* @param serdeType - the deserializer type
* @param conf - the deserializer configuration
* @param ruleRegistry - the rule registry
*/
constructor(client, serdeType, conf, ruleRegistry) {
super(client, serdeType, conf, ruleRegistry);
this.schemaToTypeCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.schemaToValidateCache = new lru_cache_1.LRUCache({ max: this.config().cacheCapacity ?? 1000 });
this.fieldTransformer = async (ctx, fieldTransform, msg) => {
return await this.fieldTransform(ctx, fieldTransform, msg);
};
for (const rule of this.ruleRegistry.getExecutors()) {
rule.configure(client.config(), new Map(Object.entries(conf.ruleConfig ?? {})));
}
}
/**
* Deserializes a message.
* @param topic - the topic
* @param payload - the message payload
* @param headers - optional headers
*/
async deserialize(topic, payload, headers) {
if (!Buffer.isBuffer(payload)) {
throw new Error('Invalid buffer');
}
if (payload.length === 0) {
return null;
}
const schemaId = new serde_1.SchemaId(exports.JSON_TYPE);
const [info, bytesRead] = await this.getWriterSchema(topic, payload, schemaId, headers);
payload = payload.subarray(bytesRead);
const subject = this.subjectName(topic, info);
payload = await this.executeRulesWithPhase(subject, topic, schemaregistry_client_1.RulePhase.ENCODING, schemaregistry_client_1.RuleMode.READ, null, info, payload, null);
const readerMeta = await this.getReaderSchema(subject);
let migrations = [];
if (readerMeta != null) {
migrations = await this.getMigrations(subject, info, readerMeta);
}
const msgBytes = payload;
let msg = JSON.parse(msgBytes.toString());
if (migrations.length > 0) {
msg = await this.executeMigrations(migrations, subject, topic, msg);
}
let target;
if (readerMeta != null) {
target = readerMeta;
}
else {
target = info;
}
msg = await this.executeRules(subject, topic, schemaregistry_client_1.RuleMode.READ, null, target, msg, null);
if (this.conf.validate) {
const validate = await this.toValidateFunction(info);
if (validate != null && !validate(JSON.parse(msg))) {
throw new serde_1.SerializationError('Invalid message');
}
}
return msg;
}
async fieldTransform(ctx, fieldTransform, msg) {
const schema = await this.toType(ctx.target);
return await transform(ctx, schema, '$', msg, fieldTransform);
}
toType(info) {
return toType(this.client, this.conf, this, info, async (client, info) => {
const deps = new Map();
await this.resolveReferences(client, info, deps);
return deps;
});
}
async toValidateFunction(info) {
return await toValidateFunction(this.client, this.conf, this, info, async (client, info) => {
const deps = new Map();
await this.resolveReferences(client, info, deps);
return deps;
});
}
}
exports.JsonDeserializer = JsonDeserializer;
async function toValidateFunction(client, conf, serde, info, refResolver) {
let fn = serde.schemaToValidateCache.get((0, json_stringify_deterministic_1.default)(info.schema));
if (fn != null) {
return fn;
}
const deps = await refResolver(client, info);
const json = JSON.parse(info.schema);
const spec = json.$schema;
if (spec === 'http://json-schema.org/draft/2020-12/schema'
|| spec === 'https://json-schema.org/draft/2020-12/schema') {
const ajv2020 = new _2020_1.default(conf);
ajv2020.addKeyword("confluent:tags");
deps.forEach((schema, name) => {
ajv2020.addSchema(JSON.parse(schema), name);
});
fn = ajv2020.compile(json);
}
else {
const ajv = new _2019_1.default(conf);
ajv.addKeyword("confluent:tags");
ajv.addMetaSchema(draft6MetaSchema);
ajv.addMetaSchema(draft7MetaSchema);
deps.forEach((schema, name) => {
ajv.addSchema(JSON.parse(schema), name);
});
fn = ajv.compile(json);
}
serde.schemaToValidateCache.set((0, json_stringify_deterministic_1.default)(info.schema), fn);
return fn;
}
async function toType(client, conf, serde, info, refResolver) {
let type = serde.schemaToTypeCache.get((0, json_stringify_deterministic_1.default)(info.schema));
if (type != null) {
return type;
}
const deps = await refResolver(client, info);
const retrieve = (uri) => {
const data = deps.get(uri);
if (data == null) {
throw new serde_1.SerializationError(`Schema not found: ${uri}`);
}
return JSON.parse(data);
};
const json = JSON.parse(info.schema);
const spec = json.$schema;
let schema;
if (spec === 'http://json-schema.org/draft/2020-12/schema'
|| spec === 'https://json-schema.org/draft/2020-12/schema') {
schema = await (0, draft_2020_12_1.dereferenceJSONSchema)(json, { retrieve });
}
else {
schema = await (0, draft_07_1.dereferenceJSONSchema)(json, { retrieve });
}
serde.schemaToTypeCache.set((0, json_stringify_deterministic_1.default)(info.schema), schema);
return schema;
}
async function transform(ctx, schema, path, msg, fieldTransform) {
if (msg == null || schema == null || typeof schema === 'boolean') {
return msg;
}
let fieldCtx = ctx.currentField();
if (fieldCtx != null) {
fieldCtx.type = getType(schema);
}
if (schema.type != null && Array.isArray(schema.type) && schema.type.length > 0) {
let originalType = schema.type;
let subschema = validateSubtypes(schema, msg);
try {
if (subschema != null) {
return await transform(ctx, subschema, path, msg, fieldTransform);
}
}
finally {
schema.type = originalType;
}
}
if (schema.allOf != null && schema.allOf.length > 0) {
let subschema = validateSubschemas(schema.allOf, msg);
if (subschema != null) {
return await transform(ctx, subschema, path, msg, fieldTransform);
}
}
if (schema.anyOf != null && schema.anyOf.length > 0) {
let subschema = validateSubschemas(schema.anyOf, msg);
if (subschema != null) {
return await transform(ctx, subschema, path, msg, fieldTransform);
}
}
if (schema.oneOf != null && schema.oneOf.length > 0) {
let subschema = validateSubschemas(schema.oneOf, msg);
if (subschema != null) {
return await transform(ctx, subschema, path, msg, fieldTransform);
}
}
if (schema.items != null) {
if (Array.isArray(msg)) {
for (let i = 0; i < msg.length; i++) {
msg[i] = await transform(ctx, schema.items, path, msg[i], fieldTransform);
}
return msg;
}
}
if (schema.$ref != null) {
return await transform(ctx, schema.$ref, path, msg, fieldTransform);
}
let type = getType(schema);
switch (type) {
case serde_1.FieldType.RECORD:
if (schema.properties != null) {
for (let [propName, propSchema] of Object.entries(schema.properties)) {
await transformField(ctx, path, propName, msg, propSchema, fieldTransform);
}
}
return msg;
case serde_1.FieldType.ENUM:
case serde_1.FieldType.STRING:
case serde_1.FieldType.INT:
case serde_1.FieldType.DOUBLE:
case serde_1.FieldType.BOOLEAN:
if (fieldCtx != null) {
const ruleTags = ctx.rule.tags;
if (ruleTags == null || ruleTags.length === 0 || !disjoint(new Set(ruleTags), fieldCtx.tags)) {
return await fieldTransform.transform(ctx, fieldCtx, msg);
}
}
}
return msg;
}
async function transformField(ctx, path, propName, msg, propSchema, fieldTransform) {
const fullName = path + '.' + propName;
try {
ctx.enterField(msg, fullName, propName, getType(propSchema), getInlineTags(propSchema));
let value = msg[propName];
if (value != null) {
const newVal = await transform(ctx, propSchema, fullName, value, fieldTransform);
if (ctx.rule.kind === 'CONDITION') {
if (newVal === false) {
throw new serde_1.RuleConditionError(ctx.rule);
}
}
else {
msg[propName] = newVal;
}
}
}
finally {
ctx.leaveField();
}
}
function validateSubtypes(schema, msg) {
if (typeof schema === 'boolean') {
return null;
}
if (schema.type == null || !Array.isArray(schema.type) || schema.type.length === 0) {
return null;
}
for (let typ of schema.type) {
schema.type = typ;
try {
(0, json_schema_validation_1.validateJSON)(msg, schema);
return schema;
}
catch (error) {
// ignore
}
}
return null;
}
function validateSubschemas(subschemas, msg) {
for (let subschema of subschemas) {
try {
(0, json_schema_validation_1.validateJSON)(msg, subschema);
return subschema;
}
catch (error) {
// ignore
}
}
return null;
}
function getType(schema) {
if (typeof schema === 'boolean') {
return serde_1.FieldType.NULL;
}
if (schema.type == null) {
if (schema.properties != null && Object.keys(schema.properties).length > 0) {
return serde_1.FieldType.RECORD;
}
return serde_1.FieldType.NULL;
}
if (Array.isArray(schema.type)) {
return serde_1.FieldType.COMBINED;
}
if (schema.const != null || schema.enum != null) {
return serde_1.FieldType.ENUM;
}
switch (schema.type) {
case 'object':
if (schema.properties == null || Object.keys(schema.properties).length === 0) {
return serde_1.FieldType.MAP;
}
return serde_1.FieldType.RECORD;
case 'array':
return serde_1.FieldType.ARRAY;
case 'string':
return serde_1.FieldType.STRING;
case 'integer':
return serde_1.FieldType.INT;
case 'number':
return serde_1.FieldType.DOUBLE;
case 'boolean':
return serde_1.FieldType.BOOLEAN;
case 'null':
return serde_1.FieldType.NULL;
default:
return serde_1.FieldType.NULL;
}
}
function getInlineTags(schema) {
let tagsKey = 'confluent:tags';
return new Set(schema[tagsKey]);
}
function disjoint(tags1, tags2) {
for (let tag of tags1) {
if (tags2.has(tag)) {
return false;
}
}
return true;
}