@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
1,361 lines (1,360 loc) • 51.8 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const globals_1 = require("@jest/globals");
const serde_1 = require("../../serde/serde");
const schemaregistry_client_1 = require("../../schemaregistry-client");
const local_driver_1 = require("../../rules/encryption/localkms/local-driver");
const encrypt_executor_1 = require("../../rules/encryption/encrypt-executor");
const json_1 = require("../../serde/json");
const rule_registry_1 = require("@confluentinc/schemaregistry/serde/rule-registry");
const json_stringify_deterministic_1 = __importDefault(require("json-stringify-deterministic"));
const jsonata_executor_1 = require("@confluentinc/schemaregistry/rules/jsonata/jsonata-executor");
const kms_registry_1 = require("@confluentinc/schemaregistry/rules/encryption/kms-registry");
const cel_executor_1 = require("../../rules/cel/cel-executor");
const cel_field_executor_1 = require("../../rules/cel/cel-field-executor");
const encryptionExecutor = encrypt_executor_1.EncryptionExecutor.register();
const fieldEncryptionExecutor = encrypt_executor_1.FieldEncryptionExecutor.register();
cel_executor_1.CelExecutor.register();
cel_field_executor_1.CelFieldExecutor.register();
jsonata_executor_1.JsonataExecutor.register();
local_driver_1.LocalKmsDriver.register();
//const baseURL = 'http://localhost:8081'
const baseURL = 'mock://';
const topic = 'topic1';
const subject = topic + '-value';
const rootSchema = `
{
"type": "object",
"properties": {
"otherField": { "$ref": "DemoSchema" }
}
}
`;
const rootSchema2020_12 = `
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"otherField": { "$ref": "DemoSchema" }
}
}
`;
const demoSchema = `
{
"type": "object",
"properties": {
"intField": { "type": "integer" },
"doubleField": { "type": "number" },
"stringField": {
"type": "string",
"confluent:tags": [ "PII" ]
},
"boolField": { "type": "boolean" },
"bytesField": {
"type": "string",
"contentEncoding": "base64",
"confluent:tags": [ "PII" ]
}
}
}
`;
const demoSchemaWithNullable = `
{
"type": "object",
"properties": {
"intField": { "type": "integer" },
"doubleField": { "type": "number" },
"stringField": {
"type": ["string", "null"],
"confluent:tags": [ "PII" ]
},
"boolField": { "type": "boolean" },
"bytesField": {
"type": "string",
"contentEncoding": "base64",
"confluent:tags": [ "PII" ]
}
}
}
`;
const demoSchemaWithUnion = `
{
"type": "object",
"properties": {
"intField": { "type": "integer" },
"doubleField": { "type": "number" },
"stringField": {
"oneOf": [
{
"type": "null"
},
{
"type": "string"
}
],
"confluent:tags": [ "PII" ]
},
"boolField": { "type": "boolean" },
"bytesField": {
"type": "string",
"contentEncoding": "base64",
"confluent:tags": [ "PII" ]
}
}
}
`;
const demoSchema2020_12 = `
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"intField": { "type": "integer" },
"doubleField": { "type": "number" },
"stringField": {
"type": "string",
"confluent:tags": [ "PII" ]
},
"boolField": { "type": "boolean" },
"bytesField": {
"type": "string",
"contentEncoding": "base64",
"confluent:tags": [ "PII" ]
}
}
}
`;
const complexSchema = `
{
"type": "object",
"properties": {
"arrayField": {
"type": "array",
"items": {
"type": "string"
},
"confluent:tags": [ "PII" ]
},
"objectField": {
"type": "object",
"properties": {
"stringField": { "type": "string" }
},
"confluent:tags": [ "PII" ]
},
"unionField": {
"oneOf": [
{
"type": "null"
},
{
"type": "string"
}
],
"confluent:tags": [ "PII" ]
}
}
}
`;
const defSchema = `
{
"$schema" : "http://json-schema.org/draft-07/schema#",
"additionalProperties" : false,
"definitions" : {
"Address" : {
"additionalProperties" : false,
"properties" : {
"doornumber" : {
"type" : "integer"
},
"doorpin" : {
"confluent:tags" : [ "PII" ],
"type" : "string"
}
},
"type" : "object"
}
},
"properties" : {
"address" : {
"$ref" : "#/definitions/Address"
},
"name" : {
"confluent:tags" : [ "PII" ],
"type" : "string"
}
},
"title" : "Sample Event",
"type" : "object"
}
`;
const messageSchema = `
{
"type": "object",
"properties": {
"messageType": {
"type": "string"
},
"version": {
"type": "string"
},
"payload": {
"type": "object",
"oneOf": [
{
"$ref": "#/$defs/authentication_request"
},
{
"$ref": "#/$defs/authentication_status"
}
]
}
},
"required": [
"payload",
"messageType",
"version"
],
"$defs": {
"authentication_request": {
"properties": {
"messageId": {
"type": "string",
"confluent:tags": ["PII"]
},
"timestamp": {
"type": "integer",
"minimum": 0
},
"requestId": {
"type": "string"
}
},
"required": [
"messageId",
"timestamp"
]
},
"authentication_status": {
"properties": {
"messageId": {
"type": "string",
"confluent:tags": ["PII"]
},
"authType": {
"type": [
"string",
"null"
]
}
},
"required": [
"messageId",
"authType"
]
}
}
}
`;
(0, globals_1.describe)('JsonSerializer', () => {
(0, globals_1.afterEach)(async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
await client.deleteSubject(subject, false);
await client.deleteSubject(subject, true);
});
(0, globals_1.it)('basic serialization', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
autoRegisterSchemas: true,
validate: true
});
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('basic serialization 2020-12', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
useLatestVersion: true,
validate: true
});
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let info = {
schemaType: 'JSON',
schema: demoSchema2020_12
};
await client.register(subject, info, false);
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('guid in header', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
autoRegisterSchemas: true,
validate: true,
schemaIdSerializer: serde_1.HeaderSchemaIdSerializer
});
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let headers = {};
let bytes = await ser.serialize(topic, obj, headers);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes, headers);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize nested', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
autoRegisterSchemas: true,
validate: true
});
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
useLatestVersion: true,
validate: true
});
let info = {
schemaType: 'JSON',
schema: demoSchema
};
await client.register('demo-value', info, false);
info = {
schemaType: 'JSON',
schema: rootSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize reference 2020_12', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
useLatestVersion: true,
validate: true
});
let info = {
schemaType: 'JSON',
schema: demoSchema2020_12
};
await client.register('demo-value', info, false);
info = {
schemaType: 'JSON',
schema: rootSchema2020_12,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('basic failing validation', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, {
useLatestVersion: true,
validate: true
});
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let jsonSchema = json_1.JsonSerializer.messageToSchema(obj);
let info = {
schemaType: 'JSON',
schema: JSON.stringify(jsonSchema)
};
await client.register(subject, info, false);
let bytes = await ser.serialize(topic, obj);
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
let diffObj = {
intField: '123',
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
await (0, globals_1.expect)(() => ser.serialize(topic, diffObj)).rejects.toThrow(serde_1.SerializationError);
});
(0, globals_1.it)('cel field transform', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value + '-suffix'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual('hi-suffix');
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field transform with nullable', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value + '-suffix'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchemaWithNullable,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual('hi-suffix');
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field transform with union of refs', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'messageId' ; value + '-suffix'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: messageSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
messageType: 'authentication_request',
version: '1.0',
payload: {
messageId: '12345',
timestamp: 123456789
}
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.messageType).toEqual(obj.messageType);
(0, globals_1.expect)(obj2.version).toEqual(obj.version);
(0, globals_1.expect)(obj2.payload.messageId).toEqual('12345-suffix');
(0, globals_1.expect)(obj2.payload.timestamp).toEqual(obj.payload.timestamp);
});
(0, globals_1.it)('basic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([0, 0, 0, 1]).toString('base64');
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
(0, kms_registry_1.clearKmsClients)();
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).not.toEqual(obj);
});
(0, globals_1.it)('payload encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = encryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT_PAYLOAD',
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
encodingRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('basic encryption 2020-12', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchema2020_12,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([0, 0, 0, 1]).toString('base64');
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
(0, kms_registry_1.clearKmsClients)();
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).not.toEqual(obj);
});
(0, globals_1.it)('encryption with def', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: defSchema,
ruleSet
};
await client.register(subject, info, false);
let addr = {
doornumber: 123,
doorpin: 'hi'
};
let obj = {
address: addr,
name: 'bob'
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.name = 'bob';
obj.address.doorpin = 'hi';
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
(0, kms_registry_1.clearKmsClients)();
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).not.toEqual(obj);
});
(0, globals_1.it)('encryption with union', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: demoSchemaWithUnion,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([0, 0, 0, 1]).toString('base64');
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('encryption with reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let info = {
schemaType: 'JSON',
schema: demoSchema,
};
await client.register('demo-value', info, false);
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
info = {
schemaType: 'JSON',
schema: rootSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}],
ruleSet
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
nested.stringField = 'hi';
nested.bytesField = Buffer.from([0, 0, 0, 1]).toString('base64');
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('complex encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
objectField: { 'stringField': 'world' },
unionField: 'bye',
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello']);
(0, globals_1.expect)(obj2.objectField.stringField).toEqual('world');
(0, globals_1.expect)(obj2.unionField).toEqual('bye');
});
(0, globals_1.it)('complex encryption with null', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'JSON',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
objectField: { 'stringField': 'world' },
unionField: null,
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello']);
(0, globals_1.expect)(obj2.objectField.stringField).toEqual('world');
(0, globals_1.expect)(obj2.unionField).toEqual(null);
});
(0, globals_1.it)('jsonata fully compatible', async () => {
let rule1To2 = "$merge([$sift($, function($v, $k) {$k != 'size'}), {'height': $.'size'}])";
let rule2To1 = "$merge([$sift($, function($v, $k) {$k != 'height'}), {'size': $.'height'}])";
let rule2To3 = "$merge([$sift($, function($v, $k) {$k != 'height'}), {'length': $.'height'}])";
let rule3To2 = "$merge([$sift($, function($v, $k) {$k != 'length'}), {'height': $.'length'}])";
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
client.updateConfig(subject, {
compatibilityGroup: 'application.version'
});
let widget = {
name: 'alice',
size: 123,
version: 1,
};
let jsonSchema = json_1.JsonSerializer.messageToSchema(widget);
let info = {
schemaType: 'JSON',
schema: JSON.stringify(jsonSchema),
metadata: {
properties: {
"application.version": "v1"
}
}
};
await client.register(subject, info, false);
let newWidget = {
name: 'alice',
height: 123,
version: 1,
};
jsonSchema = json_1.JsonSerializer.messageToSchema(newWidget);
info = {
schemaType: 'JSON',
schema: JSON.stringify(jsonSchema),
metadata: {
properties: {
"application.version": "v2"
}
},
ruleSet: {
migrationRules: [
{
name: 'myRule1',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.UPGRADE,
type: 'JSONATA',
expr: rule1To2,
},
{
name: 'myRule2',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.DOWNGRADE,
type: 'JSONATA',
expr: rule2To1,
},
]
}
};
await client.register(subject, info, false);
let newerWidget = {
name: 'alice',
length: 123,
version: 1,
};
jsonSchema = json_1.JsonSerializer.messageToSchema(newerWidget);
info = {
schemaType: 'JSON',
schema: JSON.stringify(jsonSchema),
metadata: {
properties: {
"application.version": "v3"
}
},
ruleSet: {
migrationRules: [
{
name: 'myRule1',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.UPGRADE,
type: 'JSONATA',
expr: rule2To3,
},
{
name: 'myRule2',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.DOWNGRADE,
type: 'JSONATA',
expr: rule3To2,
},
]
}
};
await client.register(subject, info, false);
let serConfig1 = {
useLatestWithMetadata: {
"application.version": "v1"
}
};
let ser1 = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig1);
let bytes = await ser1.serialize(topic, widget);
await deserializeWithAllVersions(client, ser1, bytes, widget, newWidget, newerWidget);
let serConfig2 = {
useLatestWithMetadata: {
"application.version": "v2"
}
};
let ser2 = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig2);
bytes = await ser2.serialize(topic, newWidget);
await deserializeWithAllVersions(client, ser2, bytes, widget, newWidget, newerWidget);
let serConfig3 = {
useLatestWithMetadata: {
"application.version": "v3"
}
};
let ser3 = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig3);
bytes = await ser3.serialize(topic, newerWidget);
await deserializeWithAllVersions(client, ser3, bytes, widget, newWidget, newerWidget);
});
async function deserializeWithAllVersions(client, ser, bytes, widget, newWidget, newerWidget) {
let deserConfig1 = {
useLatestWithMetadata: {
"application.version": "v1"
}
};
let deser1 = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig1);
deser1.client = ser.client;
let newobj = await deser1.deserialize(topic, bytes);
(0, globals_1.expect)((0, json_stringify_deterministic_1.default)(newobj)).toEqual((0, json_stringify_deterministic_1.default)(widget));
let deserConfig2 = {
useLatestWithMetadata: {
"application.version": "v2"
}
};
let deser2 = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig2);
newobj = await deser2.deserialize(topic, bytes);
(0, globals_1.expect)((0, json_stringify_deterministic_1.default)(newobj)).toEqual((0, json_stringify_deterministic_1.default)(newWidget));
let deserConfig3 = {
useLatestWithMetadata: {
"application.version": "v3"
}
};
let deser3 = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig3);
newobj = await deser3.deserialize(topic, bytes);
(0, globals_1.expect)((0, json_stringify_deterministic_1.default)(newobj)).toEqual((0, json_stringify_deterministic_1.default)(newerWidget));
}
});
(0, globals_1.describe)('JsonSerdeWithAssociatedNameStrategy', () => {
(0, globals_1.it)('serializes and deserializes with associated name strategy', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const info = { schemaType: 'JSON', schema: demoSchema };
const id = await client.register('my-custom-subject', info, false);
(0, globals_1.expect)(id).toBeGreaterThan(0);
const request = {
resourceName: 'topic1',
resourceNamespace: '-',
resourceId: 'lkc-123:topic1',
resourceType: 'topic',
associations: [{ subject: 'my-custom-subject', associationType: 'value', lifecycle: schemaregistry_client_1.LifecyclePolicy.STRONG }]
};
await client.createAssociation(request);
const serConfig = {
autoRegisterSchemas: false,
useLatestVersion: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
await client.deleteAssociations('lkc-123:topic1', 'topic', ['value'], true);
});
(0, globals_1.it)('falls back to topic name strategy when no association found', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const info = { schemaType: 'JSON', schema: demoSchema };
const id = await client.register('topic1-value', info, false);
(0, globals_1.expect)(id).toBeGreaterThan(0);
// No association created - should fall back to TopicNameStrategy
const serConfig = {
autoRegisterSchemas: false,
useLatestVersion: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
const bytes = await ser.serialize(topic, obj);
const deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, {});
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('throws error when no association found and fallback is NONE', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const info = { schemaType: 'JSON', schema: demoSchema };
const id = await client.register('topic1-value', info, false);
(0, globals_1.expect)(id).toBeGreaterThan(0);
// No association created, and fallback is NONE - should error
const serConfig = {
autoRegisterSchemas: false,
useLatestVersion: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.FALLBACK_TYPE]: 'NONE' }
};
const ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
await (0, globals_1.expect)(ser.serialize(topic, obj)).rejects.toThrow();
});
(0, globals_1.it)('uses kafka cluster id as namespace when configured', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const info = { schemaType: 'JSON', schema: demoSchema };
const id = await client.register('my-custom-subject', info, false);
(0, globals_1.expect)(id).toBeGreaterThan(0);
const request = {
resourceName: 'topic1',
resourceNamespace: 'lkc-my-cluster',
resourceId: 'lkc-my-cluster:topic1',
resourceType: 'topic',
associations: [{ subject: 'my-custom-subject', associationType: 'value', lifecycle: schemaregistry_client_1.LifecyclePolicy.STRONG }]
};
await client.createAssociation(request);
const serConfig = {
autoRegisterSchemas: false,
useLatestVersion: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.KAFKA_CLUSTER_ID]: 'lkc-my-cluster' }
};
const ser = new json_1.JsonSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([0, 0, 0, 1]).toString('base64')
};
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.KAFKA_CLUSTER_ID]: 'lkc-my-cluster' }
};
const deser = new json_1.JsonDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
await client.deleteAssociations('lkc-my-cluster:topic1', 'topic', ['value'], true);
});
(0, globals_1.it)('serializes and deserializes correctly across multiple calls with caching', async () => {
const conf = { baseURLs: [baseURL], cacheCapa