target-clickhouse
Version:
A Singer target for Clickhouse
314 lines • 14.7 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
if (ar || !(i in from)) {
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
ar[i] = from[i];
}
}
return to.concat(ar || Array.prototype.slice.call(from));
};
exports.__esModule = true;
exports.escapeIdentifier = exports.getSimpleColumnSqlType = exports.buildMeta = exports.formatParentPKColumn = exports.formatRootPKColumn = exports.formatLevelIndexColumn = exports.PKType = exports.JsonSchemaInspectorContext = exports.nestedSubObjectSeparator = void 0;
var singer_node_1 = require("singer-node");
var utils_1 = require("./utils");
var SchemaTranslator_1 = require("./SchemaTranslator");
var get = require("lodash.get");
var sha1 = require('sha1');
exports.nestedSubObjectSeparator = "$%€£";
var JsonSchemaInspectorContext = (function () {
function JsonSchemaInspectorContext(alias, schema, keyProperties, subtableSeparator, parentCtx, level, tableName, cleaningColumn, allKeyProperties) {
if (subtableSeparator === void 0) { subtableSeparator = "__"; }
if (level === void 0) { level = 0; }
if (tableName === void 0) { tableName = JsonSchemaInspectorContext.defaultTableName(alias, subtableSeparator, parentCtx); }
if (allKeyProperties === void 0) { allKeyProperties = { props: [], children: {} }; }
this.alias = alias;
this.schema = schema;
this.keyProperties = keyProperties;
this.subtableSeparator = subtableSeparator;
this.parentCtx = parentCtx;
this.level = level;
this.tableName = tableName;
this.cleaningColumn = cleaningColumn;
this.allKeyProperties = allKeyProperties;
}
JsonSchemaInspectorContext.defaultTableName = function (alias, subtableSeparator, parentCtx) {
return "".concat(parentCtx ? "".concat(parentCtx.tableName).concat(subtableSeparator) : "").concat(alias);
};
JsonSchemaInspectorContext.prototype.isTypeObject = function () {
var _a, _b;
return (_b = (_a = this.schema.type) === null || _a === void 0 ? void 0 : _a.includes("object")) !== null && _b !== void 0 ? _b : false;
};
JsonSchemaInspectorContext.prototype.isRoot = function () {
return this.parentCtx === undefined;
};
JsonSchemaInspectorContext.prototype.getRootContext = function () {
return this.isRoot() ? this : this.parentCtx.getRootContext();
};
return JsonSchemaInspectorContext;
}());
exports.JsonSchemaInspectorContext = JsonSchemaInspectorContext;
var PKType;
(function (PKType) {
PKType["ROOT"] = "ROOT";
PKType["PARENT"] = "PARENT";
PKType["CURRENT"] = "CURRENT";
PKType["LEVEL"] = "LEVEL";
})(PKType = exports.PKType || (exports.PKType = {}));
var formatLevelIndexColumn = function (level) { return "_level_".concat(level, "_index"); };
exports.formatLevelIndexColumn = formatLevelIndexColumn;
var formatRootPKColumn = function (prop) { return "_root_".concat(prop); };
exports.formatRootPKColumn = formatRootPKColumn;
var formatParentPKColumn = function (prop) { return "_parent_".concat(prop); };
exports.formatParentPKColumn = formatParentPKColumn;
var buildMetaPkProp = function (prop, ctx, pkType, fieldFormatter) {
var _a;
return (__assign(__assign({ prop: prop, valueExtractor: buildValueExtractor(prop), sqlIdentifier: escapeIdentifier((_a = fieldFormatter === null || fieldFormatter === void 0 ? void 0 : fieldFormatter(prop)) !== null && _a !== void 0 ? _a : prop, ctx.subtableSeparator) }, getSimpleColumnType(ctx, prop)), { nullable: false, lowCardinality: false, nestedArray: false, pkType: pkType }));
};
var buildValueExtractor = function (prop) {
if (prop) {
var propParts_1 = prop.split(exports.nestedSubObjectSeparator);
if (propParts_1.length == 1) {
var uniqPart_1 = propParts_1[0];
return function (data) { return data[uniqPart_1]; };
}
else {
return function (data) { return get(data, propParts_1); };
}
}
else {
return function (data) { return data; };
}
};
var buildMetaPkProps = function (ctx) {
var _a, _b, _c;
return []
.concat(ctx.isRoot() ? [] : ctx.getRootContext().keyProperties.map((function (prop) { return buildMetaPkProp(prop, ctx.getRootContext(), PKType.ROOT, exports.formatRootPKColumn); })))
.concat((!(((_b = (_a = ctx.parentCtx) === null || _a === void 0 ? void 0 : _a.allKeyProperties) === null || _b === void 0 ? void 0 : _b.props.length) === 0) && ((_c = ctx.parentCtx) === null || _c === void 0 ? void 0 : _c.keyProperties.map((function (prop) { return buildMetaPkProp(prop, ctx.parentCtx, PKType.PARENT, exports.formatParentPKColumn); })))) || [])
.concat(ctx.keyProperties.map((function (prop) { return buildMetaPkProp(prop, ctx, PKType.CURRENT); })))
.concat(Array.from(Array(ctx.level).keys()).map(function (value) {
var prop = (0, exports.formatLevelIndexColumn)(value);
return {
prop: prop,
sqlIdentifier: escapeIdentifier(prop, ctx.subtableSeparator),
chType: "Int32",
nullable: false,
lowCardinality: false,
nestedArray: false,
pkType: PKType.LEVEL
};
}));
};
var buildMeta = function (ctx) { return (__assign({ prop: ctx.alias, sqlTableName: escapeIdentifier(ctx.tableName, ctx.subtableSeparator), pkMappings: buildMetaPkProps(ctx), cleaningColumn: ctx.cleaningColumn }, buildMetaProps(ctx))); };
exports.buildMeta = buildMeta;
function makeNullable(type) {
if (!type) {
return [];
}
return (0, utils_1.asArray)(type)
.concat(!type.includes("null") ? ["null"] : []);
}
function flattenNestedObject(propDef, key, ctx) {
var _a;
var nullable = getNullable(propDef);
var nestedSchema = Object.entries((_a = propDef.properties) !== null && _a !== void 0 ? _a : {})
.reduce(function (acc, _a) {
var _b;
var nestedKey = _a[0], nestedPropDef = _a[1];
if (typeof nestedPropDef === "boolean") {
throw new Error("unhandled boolean propdef");
}
return __assign(__assign({}, acc), { properties: __assign(__assign({}, acc.properties), (_b = {}, _b["".concat(key).concat(exports.nestedSubObjectSeparator).concat(nestedKey)] = __assign(__assign({}, nestedPropDef), { type: nullable ? makeNullable(nestedPropDef.type) : nestedPropDef.type }), _b)) });
}, { type: "object", properties: {} });
return buildMetaProps(new JsonSchemaInspectorContext(ctx.alias, nestedSchema, [], ctx.subtableSeparator, ctx, ctx.level, ctx.tableName));
}
var createSubTable = function (propDef, key, ctx) {
var _a, _b;
return (0, exports.buildMeta)(new JsonSchemaInspectorContext(key, (propDef.items || { type: "string" }), (_b = (_a = ctx.allKeyProperties.children[key]) === null || _a === void 0 ? void 0 : _a.props) !== null && _b !== void 0 ? _b : [], ctx.subtableSeparator, ctx, ctx.level + 1, undefined, undefined, ctx.allKeyProperties.children[key]));
};
function buildMetaProps(ctx) {
var _a;
if (ctx.isTypeObject()) {
return Object.entries((_a = ctx.schema.properties) !== null && _a !== void 0 ? _a : {})
.filter(function (_a) {
var key = _a[0];
return !ctx.keyProperties.includes(key);
})
.reduce(function (acc, _a) {
var key = _a[0], propDef = _a[1];
if (typeof propDef === "boolean") {
throwError(ctx, "propDef as boolean not supported");
return acc;
}
var propDefTypes = (0, utils_1.asArray)(propDef.type);
if (propDefTypes.includes("object")) {
var _b = flattenNestedObject(propDef, key, ctx), nestedSimpleColumnMappings = _b.simpleColumnMappings, nestedChildren = _b.children;
return __assign(__assign({}, acc), { simpleColumnMappings: acc.simpleColumnMappings.concat(nestedSimpleColumnMappings), children: acc.children.concat(nestedChildren) });
}
else if (propDefTypes.includes("array") && propDef.format !== "nested") {
if (ctx.getRootContext().keyProperties.length === 0 && ctx.getRootContext().allKeyProperties.props.length === 0) {
throwError(ctx, "".concat(key, " refused: array child with no root key properties"));
}
return __assign(__assign({}, acc), { children: __spreadArray(__spreadArray([], acc.children, true), [createSubTable(propDef, key, ctx)], false) });
}
else {
var colType = getSimpleColumnType(ctx, key);
if (colType) {
return __assign(__assign({}, acc), { simpleColumnMappings: __spreadArray(__spreadArray([], acc.simpleColumnMappings, true), [__assign({ prop: key, valueExtractor: buildValueExtractor(key), sqlIdentifier: escapeIdentifier(key, ctx.subtableSeparator) }, colType)], false) });
}
else {
(0, singer_node_1.log_warning)("'".concat(ctx.alias, "': '").concat(key, "': could not be registered (type '").concat(propDef.type, "' unrecognized)"));
return acc;
}
}
}, { simpleColumnMappings: [], children: [] });
}
else {
if (!ctx.schema.type) {
return {
simpleColumnMappings: [],
children: []
};
}
return {
simpleColumnMappings: [__assign(__assign({ valueExtractor: buildValueExtractor(undefined), sqlIdentifier: escapeIdentifier("value", ctx.subtableSeparator) }, getSimpleColumnType(ctx, undefined)), { nullable: getNullable(ctx.schema), lowCardinality: false, nestedArray: false })],
children: []
};
}
}
function excludeNullFromArray(array) {
return (0, utils_1.asArray)(array).filter(function (type) { return type !== "null"; });
}
function getNullable(propDef) {
var _a;
if (typeof propDef === "boolean") {
throw new Error("boolean propDef not handled");
}
return (_a = (0, utils_1.asArray)(propDef.type).includes("null")) !== null && _a !== void 0 ? _a : false;
}
var getLowCardinality = function (propDef) { return propDef.lowCardinality !== null && propDef.lowCardinality === true; };
function getSimpleColumnType(ctx, key) {
var _a;
var propDef = key ? (_a = ctx.schema.properties) === null || _a === void 0 ? void 0 : _a[key] : ctx.schema;
var nestedArray = false;
if (!propDef || typeof propDef === "boolean") {
throwError(ctx, "Key '".concat(key, "' does not match any usable prop in schema props '").concat(ctx.schema.properties, "'"));
return;
}
if (propDef.format === "nested" && propDef.type === "array") {
propDef = propDef.items;
nestedArray = true;
}
var type = excludeNullFromArray(propDef.type)[0];
var chType = getSimpleColumnSqlType(ctx, propDef, key);
return chType ? {
valueTranslator: SchemaTranslator_1["default"].buildTranslator(type),
typeFormat: propDef.format,
chType: chType,
nullable: getNullable(propDef),
lowCardinality: getLowCardinality(propDef),
nestedArray: nestedArray
} : undefined;
}
function getSimpleColumnSqlType(ctx, propDef, key) {
var type = excludeNullFromArray(propDef.type)[0];
var format = propDef.format;
if (type === "string") {
if (format === "date" || format === "x-excel-date") {
return "Date";
}
else if (format === "date-time") {
return "DateTime";
}
else if (format === "date-time64") {
return "DateTime64";
}
else if (format === "uuid") {
return "UUID";
}
else {
return "String";
}
}
else if (type === "integer") {
if (!format) {
return "Int64";
}
else if (format === "int128") {
return "Int128";
}
else if (format === "int64") {
return "Int64";
}
else if (format === "int32") {
return "Int32";
}
else if (format === "int16") {
return "Int16";
}
else if (format === "int8") {
return "Int8";
}
else {
throwError(ctx, "".concat(key, ": unsupported integer format [").concat(format, "]"));
}
}
else if (type === "number") {
if (!format) {
return "Decimal(".concat(propDef.precision || 16, ", ").concat(propDef.decimals || 2, ")");
}
else if (format === "float64") {
return "Float64";
}
else if (format === "float32") {
return "Float32";
}
else {
throwError(ctx, "".concat(key, ": unsupported number format [").concat(format, "]"));
}
}
else if (type === "boolean") {
if (!format) {
return "UInt8";
}
else {
throwError(ctx, "".concat(key, ": unsupported number format [").concat(format, "]"));
}
}
else {
return undefined;
}
}
exports.getSimpleColumnSqlType = getSimpleColumnSqlType;
function escapeIdentifier(id, subtableSeparator) {
if (subtableSeparator === void 0) { subtableSeparator = "__"; }
id = id.replaceAll(exports.nestedSubObjectSeparator, subtableSeparator);
if (id.length > 64) {
var uid = sha1(id).substring(0, 10);
id = id.substring(0, 64 - uid.length - 27) + uid + id.substring(id.length - 27);
}
return "`".concat(id, "`");
}
exports.escapeIdentifier = escapeIdentifier;
function throwError(ctx, msg, childAlias) {
var alias = "".concat(ctx.alias).concat(childAlias ? (".".concat(childAlias)) : "");
if (ctx.parentCtx) {
throwError(ctx.parentCtx, msg, alias);
}
else {
(0, singer_node_1.log_error)("".concat(alias, ": ").concat(msg));
throw new Error("".concat(alias, ": ").concat(msg));
}
}
//# sourceMappingURL=jsonSchemaInspector.js.map