@confluentinc/schemaregistry
Version:
Node.js client for Confluent Schema Registry
645 lines (644 loc) • 25.7 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.KmsClientWrapper = exports.FieldEncryptionExecutorTransform = exports.FieldEncryptionExecutor = exports.EncryptionExecutorTransform = exports.Cryptor = exports.EncryptionExecutor = exports.Clock = exports.DekFormat = void 0;
const serde_1 = require("../../serde/serde");
const schemaregistry_client_1 = require("../../schemaregistry-client");
const dekregistry_client_1 = require("./dekregistry/dekregistry-client");
const rule_registry_1 = require("../../serde/rule-registry");
const rest_error_1 = require("../../rest-error");
const Random = __importStar(require("./tink/random"));
const Registry = __importStar(require("./kms-registry"));
const aes_gcm_pb_1 = require("./tink/proto/aes_gcm_pb");
const aes_siv_pb_1 = require("./tink/proto/aes_siv_pb");
const protobuf_1 = require("@bufbuild/protobuf");
const aes_gcm_1 = require("./tink/aes_gcm");
const aes_siv_1 = require("./tink/aes_siv");
const json_util_1 = require("../../serde/json-util");
// EncryptKekName represents a kek name
const ENCRYPT_KEK_NAME = 'encrypt.kek.name';
// EncryptKmsKeyId represents a kms key ID
const ENCRYPT_KMS_KEY_ID = 'encrypt.kms.key.id';
// EncryptKmsType represents a kms type
const ENCRYPT_KMS_TYPE = 'encrypt.kms.type';
// EncryptDekAlgorithm represents a dek algorithm
const ENCRYPT_DEK_ALGORITHM = 'encrypt.dek.algorithm';
// EncryptDekExpiryDays represents dek expiry days
const ENCRYPT_DEK_EXPIRY_DAYS = 'encrypt.dek.expiry.days';
// EncryptAlternateKmsKeyIds represents alternate kms key IDs
const ENCRYPT_ALTERNATE_KMS_KEY_IDS = 'encrypt.alternate.kms.key.ids';
// MillisInDay represents number of milliseconds in a day
const MILLIS_IN_DAY = 24 * 60 * 60 * 1000;
var DekFormat;
(function (DekFormat) {
DekFormat["AES128_GCM"] = "AES128_GCM";
DekFormat["AES256_GCM"] = "AES256_GCM";
DekFormat["AES256_SIV"] = "AES256_SIV";
})(DekFormat || (exports.DekFormat = DekFormat = {}));
class Clock {
now() {
return Date.now();
}
}
exports.Clock = Clock;
class EncryptionExecutor {
/**
* Register the field encryption executor with the rule registry.
*/
static register() {
return this.registerWithClock(new Clock());
}
static registerWithClock(clock) {
const executor = new EncryptionExecutor(clock);
rule_registry_1.RuleRegistry.registerRuleExecutor(executor);
return executor;
}
constructor(clock = new Clock()) {
this.config = null;
this.client = null;
this.clock = clock;
}
configure(clientConfig, config) {
if (this.client != null) {
if (!(0, json_util_1.deepEqual)(this.client.config(), clientConfig)) {
throw new serde_1.RuleError('executor already configured');
}
}
else {
this.client = dekregistry_client_1.DekRegistryClient.newClient(clientConfig);
}
if (this.config != null) {
if (config != null) {
for (let [key, value] of config) {
let v = this.config.get(key);
if (v != null) {
if (v !== value) {
throw new serde_1.RuleError('rule config key already set: {key}');
}
}
else {
this.config.set(key, value);
}
}
}
}
else {
this.config = config != null ? config : new Map();
}
}
type() {
return 'ENCRYPT_PAYLOAD';
}
async transform(ctx, msg) {
const transform = this.newTransform(ctx);
return await transform.transform(ctx, serde_1.FieldType.BYTES, msg);
}
newTransform(ctx) {
const cryptor = this.getCryptor(ctx);
const kekName = this.getKekName(ctx);
const dekExpiryDays = this.getDekExpiryDays(ctx);
return new EncryptionExecutorTransform(this, cryptor, kekName, dekExpiryDays);
}
async close() {
if (this.client != null) {
await this.client.close();
}
}
getCryptor(ctx) {
let dekAlgorithm = DekFormat.AES256_GCM;
const dekAlgorithmStr = ctx.getParameter(ENCRYPT_DEK_ALGORITHM);
if (dekAlgorithmStr != null) {
dekAlgorithm = DekFormat[dekAlgorithmStr];
}
return new Cryptor(dekAlgorithm);
}
getKekName(ctx) {
const kekName = ctx.getParameter(ENCRYPT_KEK_NAME);
if (kekName == null) {
throw new serde_1.RuleError('no kek name found');
}
if (kekName === '') {
throw new serde_1.RuleError('empty kek name');
}
return kekName;
}
getDekExpiryDays(ctx) {
const expiryDaysStr = ctx.getParameter(ENCRYPT_DEK_EXPIRY_DAYS);
if (expiryDaysStr == null) {
return 0;
}
const expiryDays = Number(expiryDaysStr);
if (isNaN(expiryDays)) {
throw new serde_1.RuleError('invalid expiry days');
}
if (expiryDays < 0) {
throw new serde_1.RuleError('negative expiry days');
}
return expiryDays;
}
}
exports.EncryptionExecutor = EncryptionExecutor;
class Cryptor {
constructor(dekFormat) {
this.dekFormat = dekFormat;
this.isDeterministic = dekFormat === DekFormat.AES256_SIV;
}
keySize() {
switch (this.dekFormat) {
case DekFormat.AES256_SIV:
// Generate 2 256-bit keys
return 64;
case DekFormat.AES128_GCM:
// Generate 128-bit key
return 16;
case DekFormat.AES256_GCM:
// Generate 256-bit key
return 32;
default:
throw new serde_1.RuleError('unsupported dek format');
}
}
generateKey() {
let rawKey = Random.randBytes(this.keySize());
switch (this.dekFormat) {
case DekFormat.AES256_SIV:
const aesSivKey = (0, protobuf_1.create)(aes_siv_pb_1.AesSivKeySchema, {
version: 0,
keyValue: rawKey
});
return Buffer.from((0, protobuf_1.toBinary)(aes_siv_pb_1.AesSivKeySchema, aesSivKey));
case DekFormat.AES128_GCM:
case DekFormat.AES256_GCM:
const aesGcmKey = (0, protobuf_1.create)(aes_gcm_pb_1.AesGcmKeySchema, {
version: 0,
keyValue: rawKey
});
return Buffer.from((0, protobuf_1.toBinary)(aes_gcm_pb_1.AesGcmKeySchema, aesGcmKey));
default:
throw new serde_1.RuleError('unsupported dek format');
}
}
async encrypt(dek, plaintext) {
let rawKey;
switch (this.dekFormat) {
case DekFormat.AES256_SIV:
const aesSivKey = (0, protobuf_1.fromBinary)(aes_siv_pb_1.AesSivKeySchema, dek);
rawKey = new Uint8Array(aesSivKey.keyValue);
return Buffer.from(await this.encryptWithAesSiv(rawKey, new Uint8Array(plaintext)));
case DekFormat.AES128_GCM:
case DekFormat.AES256_GCM:
const aesGcmKey = (0, protobuf_1.fromBinary)(aes_gcm_pb_1.AesGcmKeySchema, dek);
rawKey = new Uint8Array(aesGcmKey.keyValue);
return Buffer.from(await this.encryptWithAesGcm(rawKey, new Uint8Array(plaintext)));
default:
throw new serde_1.RuleError('unsupported dek format');
}
}
async decrypt(dek, ciphertext) {
let rawKey;
switch (this.dekFormat) {
case DekFormat.AES256_SIV:
const aesSivKey = (0, protobuf_1.fromBinary)(aes_siv_pb_1.AesSivKeySchema, dek);
rawKey = new Uint8Array(aesSivKey.keyValue);
return Buffer.from(await this.decryptWithAesSiv(rawKey, new Uint8Array(ciphertext)));
case DekFormat.AES128_GCM:
case DekFormat.AES256_GCM:
const aesGcmKey = (0, protobuf_1.fromBinary)(aes_gcm_pb_1.AesGcmKeySchema, dek);
rawKey = new Uint8Array(aesGcmKey.keyValue);
return Buffer.from(await this.decryptWithAesGcm(rawKey, new Uint8Array(ciphertext)));
default:
throw new serde_1.RuleError('unsupported dek format');
}
}
async encryptWithAesSiv(key, plaintext) {
const aead = await (0, aes_siv_1.fromRawKey)(key);
return aead.encrypt(plaintext, Cryptor.EMPTY_AAD);
}
async decryptWithAesSiv(key, ciphertext) {
const aead = await (0, aes_siv_1.fromRawKey)(key);
return aead.decrypt(ciphertext, Cryptor.EMPTY_AAD);
}
async encryptWithAesGcm(key, plaintext) {
const aead = await (0, aes_gcm_1.fromRawKey)(key);
return aead.encrypt(plaintext, Cryptor.EMPTY_AAD);
}
async decryptWithAesGcm(key, ciphertext) {
const aead = await (0, aes_gcm_1.fromRawKey)(key);
return aead.decrypt(ciphertext, Cryptor.EMPTY_AAD);
}
}
exports.Cryptor = Cryptor;
Cryptor.EMPTY_AAD = new Uint8Array(0);
class EncryptionExecutorTransform {
constructor(executor, cryptor, kekName, dekExpiryDays) {
this.kek = null;
this.executor = executor;
this.cryptor = cryptor;
this.kekName = kekName;
this.dekExpiryDays = dekExpiryDays;
}
isDekRotated() {
return this.dekExpiryDays > 0;
}
async getKek(ctx) {
if (this.kek == null) {
this.kek = await this.getOrCreateKek(ctx);
}
return this.kek;
}
async getOrCreateKek(ctx) {
const isRead = ctx.ruleMode === schemaregistry_client_1.RuleMode.READ;
const kmsType = ctx.getParameter(ENCRYPT_KMS_TYPE);
const kmsKeyId = ctx.getParameter(ENCRYPT_KMS_KEY_ID);
const kekId = {
name: this.kekName,
deleted: false,
};
let kek = await this.retrieveKekFromRegistry(kekId);
if (kek == null) {
if (isRead) {
throw new serde_1.RuleError(`no kek found for ${this.kekName} during consume`);
}
if (kmsType == null || kmsType.length === 0) {
throw new serde_1.RuleError(`no kms type found for ${this.kekName} during produce`);
}
if (kmsKeyId == null || kmsKeyId.length === 0) {
throw new serde_1.RuleError(`no kms key id found for ${this.kekName} during produce`);
}
kek = await this.storeKekToRegistry(kekId, kmsType, kmsKeyId, false);
if (kek == null) {
// handle conflicts (409)
kek = await this.retrieveKekFromRegistry(kekId);
}
if (kek == null) {
throw new serde_1.RuleError(`no kek found for ${this.kekName} during produce`);
}
}
if (kmsType != null && kmsType.length !== 0 && kmsType !== kek.kmsType) {
throw new serde_1.RuleError(`found ${this.kekName} with kms type ${kek.kmsType} which differs from rule kms type ${kmsType}`);
}
if (kmsKeyId != null && kmsKeyId.length !== 0 && kmsKeyId !== kek.kmsKeyId) {
throw new serde_1.RuleError(`found ${this.kekName} with kms key id ${kek.kmsKeyId} which differs from rule kms keyId ${kmsKeyId}`);
}
return kek;
}
async retrieveKekFromRegistry(key) {
try {
return await this.executor.client.getKek(key.name, key.deleted);
}
catch (err) {
if (err instanceof rest_error_1.RestError && err.status === 404) {
return null;
}
throw new serde_1.RuleError(`could not get kek ${key.name}: ${err}`);
}
}
async storeKekToRegistry(key, kmsType, kmsKeyId, shared) {
try {
return await this.executor.client.registerKek(key.name, kmsType, kmsKeyId, shared);
}
catch (err) {
if (err instanceof rest_error_1.RestError && err.status === 409) {
return null;
}
throw new serde_1.RuleError(`could not register kek ${key.name}: ${err}`);
}
}
async getOrCreateDek(ctx, version) {
const kek = await this.getKek(ctx);
const isRead = ctx.ruleMode === schemaregistry_client_1.RuleMode.READ;
if (version == null || version === 0) {
version = 1;
}
const dekId = {
kekName: this.kekName,
subject: ctx.subject,
version,
algorithm: this.cryptor.dekFormat,
deleted: isRead
};
let dek = await this.retrieveDekFromRegistry(dekId);
const isExpired = this.isExpired(ctx, dek);
let kmsClient = null;
if (dek == null || isExpired) {
if (isRead) {
throw new serde_1.RuleError(`no dek found for ${this.kekName} during consume`);
}
let encryptedDek = null;
if (!kek.shared) {
kmsClient = new KmsClientWrapper(this.executor.config, kek);
// Generate new dek
const rawDek = this.cryptor.generateKey();
encryptedDek = await kmsClient.encrypt(rawDek);
}
const newVersion = isExpired ? dek.version + 1 : null;
try {
dek = await this.createDek(dekId, newVersion, encryptedDek);
}
catch (err) {
if (dek == null) {
throw err;
}
console.warn("failed to create dek for %s, subject %s, version %d, using existing dek", this.kekName, ctx.subject, newVersion);
}
}
const keyMaterialBytes = await this.executor.client.getDekKeyMaterialBytes(dek);
if (keyMaterialBytes == null) {
if (kmsClient == null) {
kmsClient = new KmsClientWrapper(this.executor.config, kek);
}
const encryptedKeyMaterialBytes = await this.executor.client.getDekEncryptedKeyMaterialBytes(dek);
const rawDek = await kmsClient.decrypt(encryptedKeyMaterialBytes);
await this.executor.client.setDekKeyMaterial(dek, rawDek);
}
return dek;
}
async createDek(dekId, newVersion, encryptedDek) {
const newDekId = {
kekName: dekId.kekName,
subject: dekId.subject,
version: newVersion,
algorithm: dekId.algorithm,
deleted: dekId.deleted,
};
// encryptedDek may be passed as null if kek is shared
let dek = await this.storeDekToRegistry(newDekId, encryptedDek);
if (dek == null) {
// handle conflicts (409)
dek = await this.retrieveDekFromRegistry(dekId);
}
if (dek == null) {
throw new serde_1.RuleError(`no dek found for ${dekId.kekName} during produce`);
}
return dek;
}
async retrieveDekFromRegistry(key) {
try {
let dek;
let version = key.version;
if (version == null || version === 0) {
version = 1;
}
dek = await this.executor.client.getDek(key.kekName, key.subject, key.algorithm, version, key.deleted);
return dek != null && dek.encryptedKeyMaterial != null ? dek : null;
}
catch (err) {
if (err instanceof rest_error_1.RestError && err.status === 404) {
return null;
}
throw new serde_1.RuleError(`could not get dek for kek ${key.kekName}, subject ${key.subject}: ${err}`);
}
}
async storeDekToRegistry(key, encryptedDek) {
try {
let dek;
let encryptedDekStr = undefined;
if (encryptedDek != null) {
encryptedDekStr = encryptedDek.toString('base64');
}
let version = key.version;
if (version == null || version === 0) {
version = 1;
}
dek = await this.executor.client.registerDek(key.kekName, key.subject, key.algorithm, version, encryptedDekStr);
return dek;
}
catch (err) {
if (err instanceof rest_error_1.RestError && err.status === 409) {
return null;
}
throw new serde_1.RuleError(`could not register dek for kek ${key.kekName}, subject ${key.subject}: ${err}`);
}
}
isExpired(ctx, dek) {
const now = this.executor.clock.now();
return ctx.ruleMode !== schemaregistry_client_1.RuleMode.READ &&
this.dekExpiryDays > 0 &&
dek != null &&
(now - dek.ts) / MILLIS_IN_DAY >= this.dekExpiryDays;
}
async transform(ctx, fieldType, fieldValue) {
if (fieldValue == null) {
return null;
}
switch (ctx.ruleMode) {
case schemaregistry_client_1.RuleMode.WRITE: {
let plaintext = this.toBytes(fieldType, fieldValue);
if (plaintext == null) {
throw new serde_1.RuleError(`type ${fieldType} not supported for encryption`);
}
let version = null;
if (this.isDekRotated()) {
version = -1;
}
let dek = await this.getOrCreateDek(ctx, version);
let keyMaterialBytes = await this.executor.client.getDekKeyMaterialBytes(dek);
let ciphertext = await this.cryptor.encrypt(keyMaterialBytes, plaintext);
if (this.isDekRotated()) {
ciphertext = this.prefixVersion(dek.version, ciphertext);
}
if (fieldType === serde_1.FieldType.STRING) {
return ciphertext.toString('base64');
}
else {
return this.toObject(fieldType, ciphertext);
}
}
case schemaregistry_client_1.RuleMode.READ: {
let ciphertext;
if (fieldType === serde_1.FieldType.STRING) {
ciphertext = Buffer.from(fieldValue, 'base64');
}
else {
ciphertext = this.toBytes(fieldType, fieldValue);
}
if (ciphertext == null) {
return fieldValue;
}
let version = null;
if (this.isDekRotated()) {
version = this.extractVersion(ciphertext);
if (version == null) {
throw new serde_1.RuleError('no version found in ciphertext');
}
ciphertext = ciphertext.subarray(5);
}
let dek = await this.getOrCreateDek(ctx, version);
let keyMaterialBytes = await this.executor.client.getDekKeyMaterialBytes(dek);
let plaintext = await this.cryptor.decrypt(keyMaterialBytes, ciphertext);
return this.toObject(fieldType, plaintext);
}
default:
throw new serde_1.RuleError(`unsupported rule mode ${ctx.ruleMode}`);
}
}
prefixVersion(version, ciphertext) {
const versionBuf = Buffer.alloc(4);
versionBuf.writeInt32BE(version);
return Buffer.concat([serde_1.MAGIC_BYTE_V0, versionBuf, ciphertext]);
}
extractVersion(ciphertext) {
let magicByte = ciphertext.subarray(0, 1);
if (!magicByte.equals(serde_1.MAGIC_BYTE_V0)) {
throw new serde_1.RuleError(`Message encoded with magic byte ${JSON.stringify(magicByte)}, expected ${JSON.stringify(serde_1.MAGIC_BYTE_V0)}`);
}
return ciphertext.subarray(1, 5).readInt32BE(0);
}
toBytes(type, value) {
switch (type) {
case serde_1.FieldType.BYTES:
return value;
case serde_1.FieldType.STRING:
return Buffer.from(value);
default:
return null;
}
}
toObject(type, value) {
switch (type) {
case serde_1.FieldType.BYTES:
return value;
case serde_1.FieldType.STRING:
return value.toString();
default:
return null;
}
}
}
exports.EncryptionExecutorTransform = EncryptionExecutorTransform;
function getKmsClient(config, kmsType, kmsKeyId) {
let keyUrl = kmsType + '://' + kmsKeyId;
let kmsClient = Registry.getKmsClient(keyUrl);
if (kmsClient == null) {
let kmsDriver = Registry.getKmsDriver(keyUrl);
kmsClient = kmsDriver.newKmsClient(config, keyUrl);
Registry.registerKmsClient(kmsClient);
}
return kmsClient;
}
class FieldEncryptionExecutor extends serde_1.FieldRuleExecutor {
/**
* Register the field encryption executor with the rule registry.
*/
static register() {
return this.registerWithClock(new Clock());
}
static registerWithClock(clock) {
const executor = new FieldEncryptionExecutor(clock);
rule_registry_1.RuleRegistry.registerRuleExecutor(executor);
return executor;
}
constructor(clock = new Clock()) {
super();
this.executor = new EncryptionExecutor(clock);
}
configure(clientConfig, config) {
this.executor.configure(clientConfig, config);
}
type() {
return 'ENCRYPT';
}
newTransform(ctx) {
const executorTransform = this.executor.newTransform(ctx);
return new FieldEncryptionExecutorTransform(executorTransform);
}
async close() {
return this.executor.close();
}
}
exports.FieldEncryptionExecutor = FieldEncryptionExecutor;
class FieldEncryptionExecutorTransform {
constructor(executorTransform) {
this.executorTransform = executorTransform;
}
async transform(ctx, fieldCtx, fieldValue) {
return await this.executorTransform.transform(ctx, fieldCtx.type, fieldValue);
}
}
exports.FieldEncryptionExecutorTransform = FieldEncryptionExecutorTransform;
class KmsClientWrapper {
constructor(config, kek) {
this.config = config;
this.kek = kek;
this.kekId = kek.kmsType + '://' + kek.kmsKeyId;
this.kmsKeyIds = this.getKmsKeyIds();
}
getKmsKeyIds() {
let kmsKeyIds = [this.kek.kmsKeyId];
let alternateKmsKeyIds;
if (this.kek.kmsProps != null) {
alternateKmsKeyIds = this.kek.kmsProps[ENCRYPT_ALTERNATE_KMS_KEY_IDS];
}
if (alternateKmsKeyIds == null) {
alternateKmsKeyIds = this.config.get(ENCRYPT_ALTERNATE_KMS_KEY_IDS);
}
if (alternateKmsKeyIds != null) {
kmsKeyIds = kmsKeyIds.concat(alternateKmsKeyIds.split(',').map(id => id.trim()));
}
return kmsKeyIds;
}
supported(keyUri) {
return this.kekId === keyUri;
}
async encrypt(rawKey) {
for (let i = 0; i < this.kmsKeyIds.length; i++) {
try {
let kmsClient = getKmsClient(this.config, this.kek.kmsType, this.kmsKeyIds[i]);
return await kmsClient.encrypt(rawKey);
}
catch (e) {
if (i === this.kmsKeyIds.length - 1) {
throw new serde_1.RuleError(`failed to encrypt key with all KMS keys: ${e}`);
}
}
}
throw new serde_1.RuleError('no KEK found for encryption');
}
async decrypt(encryptedKey) {
for (let i = 0; i < this.kmsKeyIds.length; i++) {
try {
let kmsClient = getKmsClient(this.config, this.kek.kmsType, this.kmsKeyIds[i]);
return await kmsClient.decrypt(encryptedKey);
}
catch (e) {
if (i === this.kmsKeyIds.length - 1) {
throw new serde_1.RuleError(`failed to decrypt key with all KMS keys: ${e}`);
}
}
}
throw new serde_1.RuleError('no KEK found for decryption');
}
}
exports.KmsClientWrapper = KmsClientWrapper;