@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
1,469 lines (1,468 loc) • 79.8 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const globals_1 = require("@jest/globals");
const avro_1 = require("../../serde/avro");
const serde_1 = require("../../serde/serde");
const schemaregistry_client_1 = require("../../schemaregistry-client");
const local_driver_1 = require("../../rules/encryption/localkms/local-driver");
const encrypt_executor_1 = require("../../rules/encryption/encrypt-executor");
const gcp_driver_1 = require("../../rules/encryption/gcpkms/gcp-driver");
const aws_driver_1 = require("../../rules/encryption/awskms/aws-driver");
const azure_driver_1 = require("../../rules/encryption/azurekms/azure-driver");
const hcvault_driver_1 = require("../../rules/encryption/hcvault/hcvault-driver");
const jsonata_executor_1 = require("@confluentinc/schemaregistry/rules/jsonata/jsonata-executor");
const json_stringify_deterministic_1 = __importDefault(require("json-stringify-deterministic"));
const rule_registry_1 = require("@confluentinc/schemaregistry/serde/rule-registry");
const kms_registry_1 = require("@confluentinc/schemaregistry/rules/encryption/kms-registry");
const cel_executor_1 = require("../../rules/cel/cel-executor");
const cel_field_executor_1 = require("../../rules/cel/cel-field-executor");
const rootSchema = `
{
"name": "NestedTestRecord",
"type": "record",
"fields": [
{
"name": "otherField",
"type": "DemoSchema"
}
]
}
`;
const demoSchema = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": "string",
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes",
"confluent:tags": [ "PII" ]
}
]
}
`;
const demoSchemaWithMissing = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": "string",
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes",
"confluent:tags": [ "PII" ]
},
{
"name": "missing",
"type": ["null", "string"],
"default": null
}
]
}
`;
const nameSchema = `
{
"type": "record",
"namespace": "examples",
"name": "NameSchema",
"fields": [
{ "name": "fullName", "type": "string" },
{ "name": "lastName", "type": "string" }
],
"version": "1"
}
`;
const demoSchemaSingleTag = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": "string",
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes"
}
]
}
`;
const demoSchemaWithLogicalType = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": {
"type": "string",
"logicalType": "uuid"
},
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes",
"confluent:tags": [ "PII" ]
}
]
}
`;
const rootPointerSchema = `
{
"name": "NestedTestPointerRecord",
"type": "record",
"fields": [
{
"name": "otherField",
"type": ["null", "DemoSchema"]
}
]
}
`;
const f1Schema = `
{
"name": "F1Schema",
"type": "record",
"fields": [
{
"name": "f1",
"type": "string",
"confluent:tags": [ "PII" ]
}
]
}
`;
const demoSchemaWithUnion = `
{
"name": "DemoSchemaWithUnion",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": ["null", "string"],
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": ["null", "bytes"],
"confluent:tags": [ "PII" ]
}
]
}
`;
const schemaEvolution1 = `
{
"name": "SchemaEvolution",
"type": "record",
"fields": [
{
"name": "fieldToDelete",
"type": "string"
}
]
}
`;
const schemaEvolution2 = `
{
"name": "SchemaEvolution",
"type": "record",
"fields": [
{
"name": "newOptionalField",
"type": ["string", "null"],
"default": "optional"
}
]
}
`;
const complexSchema = `
{
"name": "ComplexSchema",
"type": "record",
"fields": [
{
"name": "arrayField",
"type": {
"type": "array",
"items": "string"
},
"confluent:tags": [ "PII" ]
},
{
"name": "mapField",
"type": {
"type": "map",
"values": "string"
},
"confluent:tags": [ "PII" ]
},
{
"name": "unionField",
"type": ["null", "string"],
"confluent:tags": [ "PII" ]
}
]
}
`;
const unionFieldSchema = `
{
"type": "record",
"name": "UnionTest",
"namespace": "test",
"fields": [
{
"name": "color",
"type": [
"string",
{
"type": "enum",
"name": "Color",
"symbols": [
"RED",
"BLUE"
]
}
],
"default": "BLUE"
}
],
"version": "1"
}`;
const complexNestedSchema = `
{
"type": "record",
"name": "UnionTest",
"namespace": "test",
"fields": [
{
"name": "emails",
"type": [
"null",
{
"type": "array",
"items": {
"type": "record",
"name": "Email",
"fields": [
{
"name": "email",
"type": [
"null",
"string"
],
"doc": "Email address",
"default": null,
"confluent:tags": [
"PII"
]
}
]
}
}
],
"doc": "Communication Email",
"default": null
}
]
}`;
const wrappedUnionSchema = `{
"fields": [
{
"name": "id",
"type": "int"
},
{
"name": "result",
"type": [
"null",
{
"fields": [
{
"name": "code",
"type": "int"
},
{
"confluent:tags": [
"PII"
],
"name": "secret",
"type": [
"null",
"string"
]
}
],
"name": "Data",
"type": "record"
},
{
"fields": [
{
"name": "code",
"type": "int"
},
{
"name": "reason",
"type": [
"null",
"string"
]
}
],
"name": "Error",
"type": "record"
}
]
}
],
"name": "Result",
"namespace": "com.acme",
"type": "record"
}`;
class FakeClock extends encrypt_executor_1.Clock {
constructor() {
super(...arguments);
this.fixedNow = 0;
}
now() {
return this.fixedNow;
}
}
const encryptionExecutor = encrypt_executor_1.EncryptionExecutor.registerWithClock(new FakeClock());
const fieldEncryptionExecutor = encrypt_executor_1.FieldEncryptionExecutor.registerWithClock(new FakeClock());
cel_executor_1.CelExecutor.register();
cel_field_executor_1.CelFieldExecutor.register();
jsonata_executor_1.JsonataExecutor.register();
aws_driver_1.AwsKmsDriver.register();
azure_driver_1.AzureKmsDriver.register();
gcp_driver_1.GcpKmsDriver.register();
hcvault_driver_1.HcVaultDriver.register();
local_driver_1.LocalKmsDriver.register();
//const baseURL = 'http://localhost:8081'
const baseURL = 'mock://';
const topic = 'topic1';
const subject = topic + '-value';
(0, globals_1.describe)('AvroSerializer', () => {
(0, globals_1.afterEach)(async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
await client.deleteSubject(subject, false);
await client.deleteSubject(subject, true);
});
(0, globals_1.it)('basic serialization', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('bad serialization', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: nameSchema
};
await client.register(subject, info, false);
try {
await ser.serialize(topic, { lastName: "lastName" });
}
catch (err) {
(0, globals_1.expect)(err).toBeInstanceOf(serde_1.SerializationError);
}
});
(0, globals_1.it)('guid in header', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true, schemaIdSerializer: serde_1.HeaderSchemaIdSerializer });
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let headers = {};
let bytes = await ser.serialize(topic, obj, headers);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes, headers);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('serialize bytes', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
let obj = Buffer.from([0x02, 0x03, 0x04]);
let bytes = await ser.serialize(topic, obj);
(0, globals_1.expect)(bytes).toEqual(Buffer.from([0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04]));
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2).toEqual(obj);
});
(0, globals_1.it)('serialize nested', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: rootPointerSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('field with union with non-applicable rule', async () => {
const conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
const ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const dekClient = fieldEncryptionExecutor.executor.client;
const encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
const ruleSet = {
domainRules: [encRule]
};
const info = {
schemaType: 'AVRO',
schema: unionFieldSchema,
ruleSet
};
await client.register(subject, info, false);
const obj = {
color: { "test.Color": "BLUE" }
};
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
const deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.color).toEqual(obj.color);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: rootPointerSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('serialize union with references', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: complexSchema,
};
await client.register('complex-value', info, false);
info = {
schemaType: 'AVRO',
schema: '[ "DemoSchema", "ComplexSchema" ]',
references: [
{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
},
{
name: 'ComplexSchema',
subject: 'complex-value',
version: 1
}
]
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
// need to wrap union
let union = { DemoSchema: obj };
let bytes = await ser.serialize(topic, union);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let union2 = await deser.deserialize(topic, bytes);
// need to unwrap union
let obj2 = union2.DemoSchema;
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('schema evolution', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let obj = {
fieldToDelete: "bye",
};
let info = {
schemaType: 'AVRO',
schema: schemaEvolution1,
};
await client.register(subject, info, false);
let bytes = await ser.serialize(topic, obj);
info = {
schemaType: 'AVRO',
schema: schemaEvolution2,
};
await client.register(subject, info, false);
client.clearLatestCaches();
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.fieldToDelete).toEqual(undefined);
(0, globals_1.expect)(obj2.newOptionalField).toEqual("optional");
});
(0, globals_1.it)('cel condition', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'CONDITION',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL',
expr: "message.stringField == 'hi'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel condition fail', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'CONDITION',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL',
expr: "message.stringField != 'hi'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
try {
await ser.serialize(topic, obj);
(0, globals_1.expect)(true).toBe(false);
}
catch (err) {
(0, globals_1.expect)(err).toBeInstanceOf(serde_1.SerializationError);
}
});
(0, globals_1.it)('cel condition ignore fail', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'CONDITION',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL',
expr: "message.stringField != 'hi'",
onFailure: 'NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field transform', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value + '-suffix'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual('hi-suffix');
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field transform missing prop', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value + '-suffix'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchemaWithMissing,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual('hi-suffix-suffix');
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field complex transform', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "typeName == 'STRING' ; value + '-suffix'",
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
mapField: { 'key': 'world' },
unionField: 'bye',
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello-suffix']);
(0, globals_1.expect)(obj2.mapField).toEqual({ 'key': 'world-suffix' });
(0, globals_1.expect)(obj2.unionField).toEqual('bye-suffix');
});
(0, globals_1.it)('cel field complex transform with null', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "typeName == 'STRING' ; value + '-suffix'",
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
mapField: { 'key': 'world' },
unionField: null,
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello-suffix']);
(0, globals_1.expect)(obj2.mapField).toEqual({ 'key': 'world-suffix' });
(0, globals_1.expect)(obj2.unionField).toEqual(null);
});
(0, globals_1.it)('cel field condition', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'CONDITION',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value == 'hi'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('cel field condition fail', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let encRule = {
name: 'test-cel',
kind: 'CONDITION',
mode: schemaregistry_client_1.RuleMode.WRITE,
type: 'CEL_FIELD',
expr: "name == 'stringField' ; value != 'hi'"
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
try {
await ser.serialize(topic, obj);
(0, globals_1.expect)(true).toBe(false);
}
catch (err) {
(0, globals_1.expect)(err).toBeInstanceOf(serde_1.SerializationError);
}
});
(0, globals_1.it)('basic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
registry.registerOverride({ type: 'ENCRYPT', disabled: true });
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
(0, kms_registry_1.clearKmsClients)();
registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
});
(0, globals_1.it)('payload encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = encryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT_PAYLOAD',
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
encodingRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
encryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('encryption with alternate keks', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret',
'encrypt.alternate.kms.key.ids': 'mykey2,mykey3'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = encryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT_PAYLOAD',
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
encodingRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
encryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('deterministic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
'encrypt.dek.algorithm': 'AES256_SIV',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
registry.registerOverride({ type: 'ENCRYPT', disabled: true });
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
(0, kms_registry_1.clearKmsClients)();
registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
});
(0, globals_1.it)('basic encryption with wrapped union', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.executor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: wrappedUnionSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
id: 123,
result: {
'com.acme.Data': {
code: 456,
secret: 'mypii'
}
}
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.result["com.acme.Data"].secret = 'mypii';
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.executor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.result["com.acme.Data"].code).toEqual(obj.result["com.acme.Data"].code);
(0, globals_1.expect)(obj