@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
529 lines (528 loc) • 25.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const globals_1 = require("@jest/globals");
const protobuf_1 = require("../../serde/protobuf");
const serde_1 = require("../../serde/serde");
const schemaregistry_client_1 = require("../../schemaregistry-client");
const local_driver_1 = require("../../rules/encryption/localkms/local-driver");
const encrypt_executor_1 = require("../../rules/encryption/encrypt-executor");
const example_pb_1 = require("./test/example_pb");
const protobuf_2 = require("@bufbuild/protobuf");
const wkt_1 = require("@bufbuild/protobuf/wkt");
const nested_pb_1 = require("./test/nested_pb");
const test_pb_1 = require("./test/test_pb");
const dep_pb_1 = require("./test/dep_pb");
const rule_registry_1 = require("@confluentinc/schemaregistry/serde/rule-registry");
const cycle_pb_1 = require("./test/cycle_pb");
const kms_registry_1 = require("@confluentinc/schemaregistry/rules/encryption/kms-registry");
const encryptionExecutor = encrypt_executor_1.EncryptionExecutor.register();
const fieldEncryptionExecutor = encrypt_executor_1.FieldEncryptionExecutor.register();
local_driver_1.LocalKmsDriver.register();
//const baseURL = 'http://localhost:8081'
const baseURL = 'mock://';
const topic = 'topic1';
const subject = topic + '-value';
(0, globals_1.describe)('ProtobufSerializer', () => {
(0, globals_1.afterEach)(async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
await client.deleteSubject(subject, false);
await client.deleteSubject(subject, true);
});
(0, globals_1.it)('basic serialization', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
ser.registry.add(example_pb_1.AuthorSchema);
let obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
let bytes = await ser.serialize(topic, obj);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('guid in header', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true, schemaIdSerializer: serde_1.HeaderSchemaIdSerializer });
ser.registry.add(example_pb_1.AuthorSchema);
let obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
let headers = {};
let bytes = await ser.serialize(topic, obj, headers);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes, headers);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize second messsage', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
ser.registry.add(example_pb_1.PizzaSchema);
let obj = (0, protobuf_2.create)(example_pb_1.PizzaSchema, {
size: 'Extra extra large',
toppings: ['anchovies', 'mushrooms']
});
let bytes = await ser.serialize(topic, obj);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('deserialize with relative type names in FileDescriptorProto', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
// Build a FileDescriptorProto with various relative type name forms,
// simulating what the schema registry returns for format=serialized
const fileDescProto = (0, protobuf_2.create)(wkt_1.FileDescriptorProtoSchema, {
name: 'top.proto',
package: 'test',
syntax: 'proto3',
dependency: ['google/protobuf/timestamp.proto'],
messageType: [{
name: 'MyMessage',
field: [{
name: 'my_field',
number: 1,
type: wkt_1.FieldDescriptorProto_Type.ENUM,
label: wkt_1.FieldDescriptorProto_Label.OPTIONAL,
typeName: 'MyEnum', // case 1: unqualified relative name
proto3Optional: true,
}, {
name: 'my_field2',
number: 2,
type: wkt_1.FieldDescriptorProto_Type.ENUM,
label: wkt_1.FieldDescriptorProto_Label.OPTIONAL,
typeName: 'test.MyEnum', // case 2: package-qualified, missing leading dot
proto3Optional: true,
}, {
name: 'my_nested',
number: 3,
type: wkt_1.FieldDescriptorProto_Type.MESSAGE,
label: wkt_1.FieldDescriptorProto_Label.OPTIONAL,
typeName: 'Outer.Inner', // case 3: nested type reference
proto3Optional: true,
}, {
name: 'my_ts',
number: 4,
type: wkt_1.FieldDescriptorProto_Type.MESSAGE,
label: wkt_1.FieldDescriptorProto_Label.OPTIONAL,
typeName: 'google.protobuf.Timestamp', // case 4: cross-file, missing leading dot
proto3Optional: true,
}],
oneofDecl: [
{ name: '_my_field' },
{ name: '_my_field2' },
{ name: '_my_nested' },
{ name: '_my_ts' },
],
}, {
name: 'Outer',
nestedType: [{
name: 'Inner',
field: [{
name: 'value',
number: 1,
type: wkt_1.FieldDescriptorProto_Type.INT32,
label: wkt_1.FieldDescriptorProto_Label.OPTIONAL,
proto3Optional: true,
}],
oneofDecl: [{ name: '_value' }],
}],
}],
enumType: [{
name: 'MyEnum',
value: [
{ name: 'MY_ENUM_UNSPECIFIED', number: 0 },
{ name: 'MY_ENUM_FOO', number: 1 },
]
}]
});
const schema = Buffer.from((0, protobuf_2.toBinary)(wkt_1.FileDescriptorProtoSchema, fileDescProto)).toString('base64');
const info = { schemaType: 'PROTOBUF', schema };
const schemaId = await client.register(subject, info, false);
// MyMessage { my_field = 1, my_field2 = 1, my_nested = { value = 42 }, my_ts = { seconds = 1000 } }
// field 1 varint 1: 0x08, 0x01
// field 2 varint 1: 0x10, 0x01
// field 3 LEN {08 2A}: 0x1a, 0x02, 0x08, 0x2a (Inner { value = 42 })
// field 4 LEN {08 e807}: 0x22, 0x03, 0x08, 0xe8, 0x07 (Timestamp { seconds = 1000 })
const msgBytes = Buffer.from([
0x08, 0x01,
0x10, 0x01,
0x1a, 0x02, 0x08, 0x2a,
0x22, 0x03, 0x08, 0xe8, 0x07,
]);
const sid = new serde_1.SchemaId('PROTOBUF', schemaId, undefined, [0]);
const buf = Buffer.concat([sid.idToBytes(), msgBytes]);
const deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
const result = await deser.deserialize(topic, buf);
(0, globals_1.expect)(result.myField).toEqual(1); // MY_ENUM_FOO = 1
(0, globals_1.expect)(result.myField2).toEqual(1); // MY_ENUM_FOO = 1 (package-qualified)
(0, globals_1.expect)(result.myNested.value).toEqual(42); // nested type
(0, globals_1.expect)(result.myTs.seconds).toEqual(BigInt(1000)); // cross-file Timestamp
});
(0, globals_1.it)('serialize nested messsage', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
ser.registry.add(nested_pb_1.NestedMessage_InnerMessageSchema);
let obj = (0, protobuf_2.create)(nested_pb_1.NestedMessage_InnerMessageSchema, {
id: "inner"
});
let bytes = await ser.serialize(topic, obj);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
ser.registry.add(test_pb_1.TestMessageSchema);
ser.registry.add(dep_pb_1.DependencyMessageSchema);
let msg = (0, protobuf_2.create)(test_pb_1.TestMessageSchema, {
testString: "hi",
testBool: true,
testBytes: Buffer.from([1, 2]),
testDouble: 1.23,
testFloat: 3.45,
testFixed32: 67,
testFixed64: 89n,
testInt32: 100,
testInt64: 200n,
testSfixed32: 300,
testSfixed64: 400n,
testSint32: 500,
testSint64: 600n,
testUint32: 700,
testUint64: 800n,
});
let obj = (0, protobuf_2.create)(dep_pb_1.DependencyMessageSchema, {
isActive: true,
testMesssage: msg
});
let bytes = await ser.serialize(topic, obj);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.testMesssage.testString).toEqual(msg.testString);
(0, globals_1.expect)(obj2.testMesssage.testBool).toEqual(msg.testBool);
(0, globals_1.expect)(obj2.testMesssage.testBytes).toEqual(msg.testBytes);
(0, globals_1.expect)(obj2.testMesssage.testDouble).toBeCloseTo(msg.testDouble, 0.001);
(0, globals_1.expect)(obj2.testMesssage.testFloat).toBeCloseTo(msg.testFloat, 0.001);
(0, globals_1.expect)(obj2.testMesssage.testFixed32).toEqual(msg.testFixed32);
(0, globals_1.expect)(obj2.testMesssage.testFixed64).toEqual(msg.testFixed64);
});
(0, globals_1.it)('serialize cycle', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
ser.registry.add(cycle_pb_1.LinkedListSchema);
let inner = (0, protobuf_2.create)(cycle_pb_1.LinkedListSchema, {
value: 100,
});
let obj = (0, protobuf_2.create)(cycle_pb_1.LinkedListSchema, {
value: 1,
next: inner
});
let bytes = await ser.serialize(topic, obj);
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('basic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'PROTOBUF',
schema: Buffer.from((0, protobuf_2.toBinary)(wkt_1.FileDescriptorProtoSchema, example_pb_1.file_test_schemaregistry_serde_example.proto)).toString('base64'),
ruleSet
};
await client.register(subject, info, false);
let obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial'],
piiOneof: {
case: 'oneofString',
value: 'oneof'
}
});
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.name = 'Kafka';
obj.picture = Buffer.from([1, 2]);
obj.piiOneof = { case: 'oneofString', value: 'oneof' };
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
(0, kms_registry_1.clearKmsClients)();
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).not.toEqual(obj);
});
(0, globals_1.it)('payload encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
let dekClient = encryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT_PAYLOAD',
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
encodingRules: [encRule]
};
let info = {
schemaType: 'PROTOBUF',
schema: Buffer.from((0, protobuf_2.toBinary)(wkt_1.FileDescriptorProtoSchema, example_pb_1.file_test_schemaregistry_serde_example.proto)).toString('base64'),
ruleSet
};
await client.register(subject, info, false);
let obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial'],
piiOneof: {
case: 'oneofString',
value: 'oneof'
}
});
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
});
(0, globals_1.describe)('ProtobufSerdeWithAssociatedNameStrategy', () => {
(0, globals_1.it)('serializes and deserializes with associated name strategy', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const request = {
resourceName: 'topic1',
resourceNamespace: '-',
resourceId: 'lkc-123:topic1',
resourceType: 'topic',
associations: [{ subject: 'my-custom-subject', associationType: 'value', lifecycle: schemaregistry_client_1.LifecyclePolicy.STRONG }]
};
await client.createAssociation(request);
const serConfig = {
autoRegisterSchemas: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
const obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
await client.deleteAssociations('lkc-123:topic1', 'topic', ['value'], true);
});
(0, globals_1.it)('falls back to topic name strategy when no association found', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
// No association created - should fall back to TopicNameStrategy
const serConfig = {
autoRegisterSchemas: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
const obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
const bytes = await ser.serialize(topic, obj);
const deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, {});
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('throws error when no association found and fallback is NONE', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
// No association created, and fallback is NONE - should error
const serConfig = {
autoRegisterSchemas: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.FALLBACK_TYPE]: 'NONE' }
};
const ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
const obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
await (0, globals_1.expect)(ser.serialize(topic, obj)).rejects.toThrow();
});
(0, globals_1.it)('uses kafka cluster id as namespace when configured', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const request = {
resourceName: 'topic1',
resourceNamespace: 'lkc-my-cluster',
resourceId: 'lkc-my-cluster:topic1',
resourceType: 'topic',
associations: [{ subject: 'my-custom-subject', associationType: 'value', lifecycle: schemaregistry_client_1.LifecyclePolicy.STRONG }]
};
await client.createAssociation(request);
const serConfig = {
autoRegisterSchemas: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.KAFKA_CLUSTER_ID]: 'lkc-my-cluster' }
};
const ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
const obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED,
subjectNameStrategyConfig: { [serde_1.KAFKA_CLUSTER_ID]: 'lkc-my-cluster' }
};
const deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
await client.deleteAssociations('lkc-my-cluster:topic1', 'topic', ['value'], true);
});
(0, globals_1.it)('serializes and deserializes correctly across multiple calls with caching', async () => {
const conf = { baseURLs: [baseURL], cacheCapacity: 1000 };
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const request = {
resourceName: 'topic1',
resourceNamespace: '-',
resourceId: 'lkc-123:topic1',
resourceType: 'topic',
associations: [{ subject: 'my-cached-subject', associationType: 'value', lifecycle: schemaregistry_client_1.LifecyclePolicy.STRONG }]
};
await client.createAssociation(request);
const serConfig = {
autoRegisterSchemas: true,
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const ser = new protobuf_1.ProtobufSerializer(client, serde_1.SerdeType.VALUE, serConfig);
ser.registry.add(example_pb_1.AuthorSchema);
const obj = (0, protobuf_2.create)(example_pb_1.AuthorSchema, {
name: 'Kafka',
id: 123,
picture: Buffer.from([1, 2]),
works: ['The Castle', 'The Trial']
});
const deserConfig = {
subjectNameStrategyType: serde_1.SubjectNameStrategyType.ASSOCIATED
};
const deser = new protobuf_1.ProtobufDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
// Serialize multiple times - should use cache after first call
for (let i = 0; i < 5; i++) {
const bytes = await ser.serialize(topic, obj);
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
}
await client.deleteAssociations('lkc-123:topic1', 'topic', ['value'], true);
});
});