@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
1,401 lines (1,399 loc) • 53.2 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const globals_1 = require("@jest/globals");
const avro_1 = require("../../serde/avro");
const serde_1 = require("../../serde/serde");
const schemaregistry_client_1 = require("../../schemaregistry-client");
const local_driver_1 = require("../../rules/encryption/localkms/local-driver");
const encrypt_executor_1 = require("../../rules/encryption/encrypt-executor");
const gcp_driver_1 = require("../../rules/encryption/gcpkms/gcp-driver");
const aws_driver_1 = require("../../rules/encryption/awskms/aws-driver");
const azure_driver_1 = require("../../rules/encryption/azurekms/azure-driver");
const hcvault_driver_1 = require("../../rules/encryption/hcvault/hcvault-driver");
const jsonata_executor_1 = require("@confluentinc/schemaregistry/rules/jsonata/jsonata-executor");
const json_stringify_deterministic_1 = __importDefault(require("json-stringify-deterministic"));
const rule_registry_1 = require("@confluentinc/schemaregistry/serde/rule-registry");
const kms_registry_1 = require("@confluentinc/schemaregistry/rules/encryption/kms-registry");
const rootSchema = `
{
"name": "NestedTestRecord",
"type": "record",
"fields": [
{
"name": "otherField",
"type": "DemoSchema"
}
]
}
`;
const demoSchema = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": "string",
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes",
"confluent:tags": [ "PII" ]
}
]
}
`;
const demoSchemaSingleTag = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": "string",
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes"
}
]
}
`;
const demoSchemaWithLogicalType = `
{
"name": "DemoSchema",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": {
"type": "string",
"logicalType": "uuid"
},
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": "bytes",
"confluent:tags": [ "PII" ]
}
]
}
`;
const rootPointerSchema = `
{
"name": "NestedTestPointerRecord",
"type": "record",
"fields": [
{
"name": "otherField",
"type": ["null", "DemoSchema"]
}
]
}
`;
const f1Schema = `
{
"name": "F1Schema",
"type": "record",
"fields": [
{
"name": "f1",
"type": "string",
"confluent:tags": [ "PII" ]
}
]
}
`;
const demoSchemaWithUnion = `
{
"name": "DemoSchemaWithUnion",
"type": "record",
"fields": [
{
"name": "intField",
"type": "int"
},
{
"name": "doubleField",
"type": "double"
},
{
"name": "stringField",
"type": ["null", "string"],
"confluent:tags": [ "PII" ]
},
{
"name": "boolField",
"type": "boolean"
},
{
"name": "bytesField",
"type": ["null", "bytes"],
"confluent:tags": [ "PII" ]
}
]
}
`;
const schemaEvolution1 = `
{
"name": "SchemaEvolution",
"type": "record",
"fields": [
{
"name": "fieldToDelete",
"type": "string"
}
]
}
`;
const schemaEvolution2 = `
{
"name": "SchemaEvolution",
"type": "record",
"fields": [
{
"name": "newOptionalField",
"type": ["string", "null"],
"default": "optional"
}
]
}
`;
const complexSchema = `
{
"name": "ComplexSchema",
"type": "record",
"fields": [
{
"name": "arrayField",
"type": {
"type": "array",
"items": "string"
},
"confluent:tags": [ "PII" ]
},
{
"name": "mapField",
"type": {
"type": "map",
"values": "string"
},
"confluent:tags": [ "PII" ]
},
{
"name": "unionField",
"type": ["null", "string"],
"confluent:tags": [ "PII" ]
}
]
}
`;
const unionFieldSchema = `
{
"type": "record",
"name": "UnionTest",
"namespace": "test",
"fields": [
{
"name": "color",
"type": [
"string",
{
"type": "enum",
"name": "Color",
"symbols": [
"RED",
"BLUE"
]
}
],
"default": "BLUE"
}
],
"version": "1"
}`;
const complexNestedSchema = `
{
"type": "record",
"name": "UnionTest",
"namespace": "test",
"fields": [
{
"name": "emails",
"type": [
"null",
{
"type": "array",
"items": {
"type": "record",
"name": "Email",
"fields": [
{
"name": "email",
"type": [
"null",
"string"
],
"doc": "Email address",
"default": null,
"confluent:tags": [
"PII"
]
}
]
}
}
],
"doc": "Communication Email",
"default": null
}
]
}`;
class FakeClock extends encrypt_executor_1.Clock {
constructor() {
super(...arguments);
this.fixedNow = 0;
}
now() {
return this.fixedNow;
}
}
const fieldEncryptionExecutor = encrypt_executor_1.FieldEncryptionExecutor.registerWithClock(new FakeClock());
jsonata_executor_1.JsonataExecutor.register();
aws_driver_1.AwsKmsDriver.register();
azure_driver_1.AzureKmsDriver.register();
gcp_driver_1.GcpKmsDriver.register();
hcvault_driver_1.HcVaultDriver.register();
local_driver_1.LocalKmsDriver.register();
//const baseURL = 'http://localhost:8081'
const baseURL = 'mock://';
const topic = 'topic1';
const subject = topic + '-value';
(0, globals_1.describe)('AvroSerializer', () => {
(0, globals_1.afterEach)(async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
await client.deleteSubject(subject, false);
await client.deleteSubject(subject, true);
});
(0, globals_1.it)('basic serialization', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('serialize nested', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { autoRegisterSchemas: true });
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: rootPointerSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('field with union with non-applicable rule', async () => {
const conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
const client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
const serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
const ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
const dekClient = fieldEncryptionExecutor.client;
const encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
const ruleSet = {
domainRules: [encRule]
};
const info = {
schemaType: 'AVRO',
schema: unionFieldSchema,
ruleSet
};
await client.register(subject, info, false);
const obj = {
color: { "test.Color": "BLUE" }
};
const bytes = await ser.serialize(topic, obj);
const deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
const deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
const obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.color).toEqual(obj.color);
});
(0, globals_1.it)('serialize reference', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: rootPointerSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}]
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('serialize union with references', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
info = {
schemaType: 'AVRO',
schema: complexSchema,
};
await client.register('complex-value', info, false);
info = {
schemaType: 'AVRO',
schema: '[ "DemoSchema", "ComplexSchema" ]',
references: [
{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
},
{
name: 'ComplexSchema',
subject: 'complex-value',
version: 1
}
]
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
// need to wrap union
let union = { DemoSchema: obj };
let bytes = await ser.serialize(topic, union);
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {});
let union2 = await deser.deserialize(topic, bytes);
// need to unwrap union
let obj2 = union2.DemoSchema;
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('schema evolution', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let obj = {
fieldToDelete: "bye",
};
let info = {
schemaType: 'AVRO',
schema: schemaEvolution1,
};
await client.register(subject, info, false);
let bytes = await ser.serialize(topic, obj);
info = {
schemaType: 'AVRO',
schema: schemaEvolution2,
};
await client.register(subject, info, false);
client.clearLatestCaches();
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, { useLatestVersion: true });
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.fieldToDelete).toEqual(undefined);
(0, globals_1.expect)(obj2.newOptionalField).toEqual("optional");
});
(0, globals_1.it)('basic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
registry.registerOverride({ type: 'ENCRYPT', disabled: true });
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
(0, kms_registry_1.clearKmsClients)();
registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
});
(0, globals_1.it)('deterministic encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
'encrypt.dek.algorithm': 'AES256_SIV',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
let registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
registry.registerOverride({ type: 'ENCRYPT', disabled: true });
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
(0, kms_registry_1.clearKmsClients)();
registry = new rule_registry_1.RuleRegistry();
registry.registerExecutor(new encrypt_executor_1.FieldEncryptionExecutor());
deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, {}, registry);
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.stringField).not.toEqual("hi");
(0, globals_1.expect)(obj2.bytesField).not.toEqual(Buffer.from([1, 2]));
});
(0, globals_1.it)('basic encryption with logical type', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchemaWithLogicalType,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('basic encryption with dek rotation', async () => {
fieldEncryptionExecutor.clock.fixedNow = Date.now();
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
'encrypt.dek.expiry.days': '1',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchemaSingleTag,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
let dek = await dekClient.getDek("kek1", subject, 'AES256_GCM', -1, false);
(0, globals_1.expect)(1).toEqual(dek.version);
// advance time by 2 days
fieldEncryptionExecutor.clock.fixedNow += 2 * 24 * 60 * 60 * 1000;
bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
dek = await dekClient.getDek("kek1", subject, 'AES256_GCM', -1, false);
(0, globals_1.expect)(2).toEqual(dek.version);
// advance time by 2 days
fieldEncryptionExecutor.clock.fixedNow += 2 * 24 * 60 * 60 * 1000;
bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
dek = await dekClient.getDek("kek1", subject, 'AES256_GCM', -1, false);
(0, globals_1.expect)(3).toEqual(dek.version);
});
(0, globals_1.it)('basic encryption with preserialized data', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: f1Schema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
f1: 'hello world'
};
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let dekClient = fieldEncryptionExecutor.client;
await dekClient.registerKek("kek1", "local-kms", "mykey", false);
const encryptedDek = "07V2ndh02DA73p+dTybwZFm7DKQSZN1tEwQh+FoX1DZLk4Yj2LLu4omYjp/84tAg3BYlkfGSz+zZacJHIE4=";
await dekClient.registerDek("kek1", subject, "AES256_GCM", 1, encryptedDek);
const bytes = Buffer.from([0, 0, 0, 0, 1, 104, 122, 103, 121, 47, 106, 70, 78, 77, 86, 47, 101, 70, 105, 108, 97, 72, 114, 77, 121, 101, 66, 103, 100, 97, 86, 122, 114, 82, 48, 117, 100, 71, 101, 111, 116, 87, 56, 99, 65, 47, 74, 97, 108, 55, 117, 107, 114, 43, 77, 47, 121, 122]);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.f1).toEqual(obj.f1);
});
(0, globals_1.it)('deterministic encryption with preserialized data', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
'encrypt.dek.algorithm': 'AES256_SIV',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: f1Schema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
f1: 'hello world'
};
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let dekClient = fieldEncryptionExecutor.client;
await dekClient.registerKek("kek1", "local-kms", "mykey", false);
const encryptedDek = "YSx3DTlAHrmpoDChquJMifmPntBzxgRVdMzgYL82rgWBKn7aUSnG+WIu9ozBNS3y2vXd++mBtK07w4/W/G6w0da39X9hfOVZsGnkSvry/QRht84V8yz3dqKxGMOK5A==";
await dekClient.registerDek("kek1", subject, "AES256_SIV", 1, encryptedDek);
const bytes = Buffer.from([0, 0, 0, 0, 1, 72, 68, 54, 89, 116, 120, 114, 108, 66, 110, 107, 84, 87, 87, 57, 78, 54, 86, 98, 107, 51, 73, 73, 110, 106, 87, 72, 56, 49, 120, 109, 89, 104, 51, 107, 52, 100]);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.f1).toEqual(obj.f1);
});
(0, globals_1.it)('dek rotation encryption with preserialized data', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
'encrypt.dek.expiry.days': '1',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: f1Schema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
f1: 'hello world'
};
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
let dekClient = fieldEncryptionExecutor.client;
await dekClient.registerKek("kek1", "local-kms", "mykey", false);
const encryptedDek = "W/v6hOQYq1idVAcs1pPWz9UUONMVZW4IrglTnG88TsWjeCjxmtRQ4VaNe/I5dCfm2zyY9Cu0nqdvqImtUk4=";
await dekClient.registerDek("kek1", subject, "AES256_GCM", 1, encryptedDek);
const bytes = Buffer.from([0, 0, 0, 0, 1, 120, 65, 65, 65, 65, 65, 65, 71, 52, 72, 73, 54, 98, 49, 110, 88, 80, 88, 113, 76, 121, 71, 56, 99, 73, 73, 51, 53, 78, 72, 81, 115, 101, 113, 113, 85, 67, 100, 43, 73, 101, 76, 101, 70, 86, 65, 101, 78, 112, 83, 83, 51, 102, 120, 80, 110, 74, 51, 50, 65, 61]);
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.f1).toEqual(obj.f1);
});
(0, globals_1.it)('encryption with references', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let info = {
schemaType: 'AVRO',
schema: demoSchema,
};
await client.register('demo-value', info, false);
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
info = {
schemaType: 'AVRO',
schema: rootSchema,
references: [{
name: 'DemoSchema',
subject: 'demo-value',
version: 1
}],
ruleSet
};
await client.register(subject, info, false);
let nested = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let obj = {
otherField: nested
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
nested.stringField = 'hi';
nested.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.otherField.intField).toEqual(nested.intField);
(0, globals_1.expect)(obj2.otherField.doubleField).toBeCloseTo(nested.doubleField, 0.001);
(0, globals_1.expect)(obj2.otherField.stringField).toEqual(nested.stringField);
(0, globals_1.expect)(obj2.otherField.boolField).toEqual(nested.boolField);
(0, globals_1.expect)(obj2.otherField.bytesField).toEqual(nested.bytesField);
});
(0, globals_1.it)('encryption with union', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,ERROR'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: demoSchemaWithUnion,
ruleSet
};
await client.register(subject, info, false);
let obj = {
intField: 123,
doubleField: 45.67,
stringField: 'hi',
boolField: true,
bytesField: Buffer.from([1, 2]),
};
let bytes = await ser.serialize(topic, obj);
// reset encrypted field
obj.stringField = 'hi';
obj.bytesField = Buffer.from([1, 2]);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.intField).toEqual(obj.intField);
(0, globals_1.expect)(obj2.doubleField).toBeCloseTo(obj.doubleField, 0.001);
(0, globals_1.expect)(obj2.stringField).toEqual(obj.stringField);
(0, globals_1.expect)(obj2.boolField).toEqual(obj.boolField);
(0, globals_1.expect)(obj2.bytesField).toEqual(obj.bytesField);
});
(0, globals_1.it)('complex encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
mapField: { 'key': 'world' },
unionField: 'bye',
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello']);
(0, globals_1.expect)(obj2.mapField).toEqual({ 'key': 'world' });
(0, globals_1.expect)(obj2.unionField).toEqual('bye');
});
(0, globals_1.it)('complex encryption with null', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: complexSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
arrayField: ['hello'],
mapField: { 'key': 'world' },
unionField: null
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.arrayField).toEqual(['hello']);
(0, globals_1.expect)(obj2.mapField).toEqual({ 'key': 'world' });
(0, globals_1.expect)(obj2.unionField).toEqual(null);
});
(0, globals_1.it)('complex nested encryption', async () => {
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
let serConfig = {
useLatestVersion: true,
ruleConfig: {
secret: 'mysecret'
}
};
let ser = new avro_1.AvroSerializer(client, serde_1.SerdeType.VALUE, serConfig);
let dekClient = fieldEncryptionExecutor.client;
let encRule = {
name: 'test-encrypt',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.WRITEREAD,
type: 'ENCRYPT',
tags: ['PII'],
params: {
'encrypt.kek.name': 'kek1',
'encrypt.kms.type': 'local-kms',
'encrypt.kms.key.id': 'mykey',
},
onFailure: 'ERROR,NONE'
};
let ruleSet = {
domainRules: [encRule]
};
let info = {
schemaType: 'AVRO',
schema: complexNestedSchema,
ruleSet
};
await client.register(subject, info, false);
let obj = {
emails: [{
email: "john@acme.com",
}],
};
let bytes = await ser.serialize(topic, obj);
let deserConfig = {
ruleConfig: {
secret: 'mysecret'
}
};
let deser = new avro_1.AvroDeserializer(client, serde_1.SerdeType.VALUE, deserConfig);
fieldEncryptionExecutor.client = dekClient;
let obj2 = await deser.deserialize(topic, bytes);
(0, globals_1.expect)(obj2.emails[0].email).toEqual('john@acme.com');
});
(0, globals_1.it)('jsonata fully compatible', async () => {
let rule1To2 = "$merge([$sift($, function($v, $k) {$k != 'size'}), {'height': $.'size'}])";
let rule2To1 = "$merge([$sift($, function($v, $k) {$k != 'height'}), {'size': $.'height'}])";
let rule2To3 = "$merge([$sift($, function($v, $k) {$k != 'height'}), {'length': $.'height'}])";
let rule3To2 = "$merge([$sift($, function($v, $k) {$k != 'length'}), {'height': $.'length'}])";
let conf = {
baseURLs: [baseURL],
cacheCapacity: 1000
};
let client = schemaregistry_client_1.SchemaRegistryClient.newClient(conf);
client.updateConfig(subject, {
compatibilityGroup: 'application.version'
});
let widget = {
name: 'alice',
size: 123,
version: 1,
};
let avroSchema = avro_1.AvroSerializer.messageToSchema(widget);
let info = {
schemaType: 'AVRO',
schema: JSON.stringify(avroSchema),
metadata: {
properties: {
"application.version": "v1"
}
}
};
await client.register(subject, info, false);
let newWidget = {
name: 'alice',
height: 123,
version: 1,
};
avroSchema = avro_1.AvroSerializer.messageToSchema(newWidget);
info = {
schemaType: 'AVRO',
schema: JSON.stringify(avroSchema),
metadata: {
properties: {
"application.version": "v2"
}
},
ruleSet: {
migrationRules: [
{
name: 'myRule1',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.UPGRADE,
type: 'JSONATA',
expr: rule1To2,
},
{
name: 'myRule2',
kind: 'TRANSFORM',
mode: schemaregistry_client_1.RuleMode.DOWNGRADE,
type: 'JSONATA',
expr: rule2To1,
},
]
}
};
await client.register(subject, info, false);
let newerWidget = {
name: 'alice',
length: 123,
version: 1,
};
avroSchema = avro_1.AvroSerializer.messageToSchema(newerWidget);
info = {
schemaType: 'AVRO',
schema: JSON.stringify(avroSchema),
metadata: {
properties: {
"application.version": "v3"
}
},
ruleSet: {
migrationRules: [
{