target-clickhouse
Version:
A Singer target for Clickhouse
252 lines • 14.3 kB
JavaScript
;
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
if (ar || !(i in from)) {
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
ar[i] = from[i];
}
}
return to.concat(ar || Array.prototype.slice.call(from));
};
exports.__esModule = true;
exports.updateSchema = exports.getColumnsIntersections = exports.toQualifiedType = exports.dropStreamTablesQueries = exports.listTableNames = exports.translateCH = exports.extractValue = void 0;
var jsonSchemaInspector_1 = require("./jsonSchemaInspector");
var singer_node_1 = require("singer-node");
var Either_1 = require("./Either");
function extractValue(data, mapping, translateValue) {
var v = mapping.valueExtractor(data);
if (v === undefined) {
v = null;
}
if (!translateValue) {
return v;
}
return mapping.valueTranslator ? mapping.valueTranslator(v) : v;
}
exports.extractValue = extractValue;
function resolveVersionColumn(isRoot, hasPkMappings, withType) {
if (withType === void 0) { withType = true; }
var type = withType ? ' UInt64' : '';
if (isRoot) {
if (hasPkMappings) {
return "`_ver`".concat(type);
}
return "";
}
else {
return "`_root_ver`".concat(type);
}
}
var resolveEngine = function (isRoot, hasPkMappings) { return isRoot && hasPkMappings ? "ReplacingMergeTree(_ver)" : "MergeTree"; };
var buildOrderByContent = function (sqlIdentifiers) { return "".concat(sqlIdentifiers.length > 1 ? '(' : '').concat(sqlIdentifiers.length > 0 ? sqlIdentifiers.map(function (sqlIdentifier) { return sqlIdentifier; }).join(", ") : "tuple()").concat(sqlIdentifiers.length > 1 ? ')' : ''); };
var resolveOrderBy = function (meta, isRoot) {
if (isRoot) {
return buildOrderByContent(meta.pkMappings
.filter(function (pkMap) { return pkMap.pkType === jsonSchemaInspector_1.PKType.CURRENT; })
.map(function (pkMap) { return pkMap.sqlIdentifier; }));
}
else {
return buildOrderByContent(meta.pkMappings
.filter(function (pkMap) { return pkMap.pkType === jsonSchemaInspector_1.PKType.ROOT || pkMap.pkType === jsonSchemaInspector_1.PKType.LEVEL; })
.map(function (pkMap) { return pkMap.sqlIdentifier; }));
}
};
function translateCH(database, meta, recursive, parentMeta, rootMeta) {
if (meta.simpleColumnMappings.length == 0 && meta.pkMappings.length == 0) {
throw new Error("Attempting to create table without columns");
}
var isNodeRoot = rootMeta === undefined;
var createDefs = meta.pkMappings
.map(function (fkMapping) { return "".concat(fkMapping.sqlIdentifier, " ").concat(fkMapping.chType); })
.concat(meta.simpleColumnMappings.map(function (mapping) { return "".concat(mapping.sqlIdentifier, " ").concat((0, exports.toQualifiedType)(mapping)); }))
.concat(resolveVersionColumn(isNodeRoot, meta.pkMappings.length > 0));
var ret = [
"CREATE TABLE ".concat(database, ".").concat(meta.sqlTableName, " ( ").concat(createDefs.filter(Boolean).join(", "), " ) ENGINE = ").concat(resolveEngine(isNodeRoot, meta.pkMappings.length > 0), " ORDER BY ").concat(resolveOrderBy(meta, isNodeRoot))
];
if (recursive) {
ret.push.apply(ret, meta.children.flatMap(function (child) { return translateCH(database, child, recursive, meta, rootMeta || meta); }));
}
return ret;
}
exports.translateCH = translateCH;
var listTableNames = function (meta) { return __spreadArray([
meta.sqlTableName
], meta.children.flatMap(exports.listTableNames), true); };
exports.listTableNames = listTableNames;
var dropStreamTablesQueries = function (meta) { return __spreadArray([
"DROP TABLE if exists ".concat(meta.sqlTableName)
], meta.children.flatMap(exports.dropStreamTablesQueries), true); };
exports.dropStreamTablesQueries = dropStreamTablesQueries;
var toQualifiedType = function (mapping) {
var _a;
var modifiers = [
mapping.nullable ? "Nullable" : null,
mapping.lowCardinality ? "LowCardinality" : null,
mapping.nestedArray ? "Array" : null,
].filter(Boolean);
return (_a = modifiers.reduce(function (acc, modifier) { return "".concat(modifier, "(").concat(acc, ")"); }, mapping.chType)) !== null && _a !== void 0 ? _a : "undefined type";
};
exports.toQualifiedType = toQualifiedType;
var mapToColumn = function (col) {
var _a;
return ({
name: (_a = unescape(col.sqlIdentifier)) !== null && _a !== void 0 ? _a : "",
type: (0, exports.toQualifiedType)(col),
is_in_sorting_key: false
});
};
var pkMapToColumn = function (col) { return (__assign(__assign({}, mapToColumn(col)), { is_in_sorting_key: true })); };
var unescape = function (v) { return v.replace(/`/g, ""); };
function getColumnsIntersections(existingCols, requiredCols) {
var missing = requiredCols.filter(function (required) {
return existingCols.find(function (existing) { return existing.name === required.name; }) === undefined;
});
var modified = existingCols.reduce(function (acc, existing) {
var matching = requiredCols.find(function (required) { return required.name === existing.name && required.type !== existing.type; });
if (matching) {
return __spreadArray(__spreadArray([], acc, true), [{
existing: existing,
"new": matching
}], false);
}
else {
return acc;
}
}, []);
var obsolete = existingCols.filter(function (existing) {
return (requiredCols.find(function (required) { return required.name === existing.name; })) === undefined;
});
return {
missing: missing,
modified: modified,
obsolete: obsolete
};
}
exports.getColumnsIntersections = getColumnsIntersections;
function checkPrimaryKeysConsistency(existingColumns, meta) {
var tablePks = existingColumns.filter(function (c) { return c.is_in_sorting_key; }).map(function (c) { return c.name; });
var schemaPks = meta.pkMappings.map(function (p) { return p.prop; });
var newPks = schemaPks.filter(function (pk) { return !tablePks.includes(pk); }).map(function (pk) { return "Could not add new PK property to ".concat(pk, " in the table"); });
var removedPks = tablePks.filter(function (pk) { return !schemaPks.includes(pk); }).map(function (pk) { return "Could not remove the PK property of ".concat(pk, " in the table"); });
var errors = __spreadArray(__spreadArray([], newPks, true), removedPks, true);
errors.forEach(function (it) { return (0, singer_node_1.log_error)(it); });
if (errors.length > 0) {
throw new Error("Could not update table because of key properties");
}
}
function updateSchema(meta, ch, existingTables) {
return __awaiter(this, void 0, void 0, function () {
var isRoot, existingColumns, expectedColumns, intersections, added, updated, removed, errors;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4, Promise.all(meta.children.map(function (child) { return updateSchema(child, ch, existingTables); }))];
case 1:
_a.sent();
isRoot = meta.pkMappings.filter(function (pkMap) { return pkMap.pkType === jsonSchemaInspector_1.PKType.ROOT; }).length == 0;
if (!!existingTables.includes(unescape(meta.sqlTableName))) return [3, 3];
return [4, Promise.all(translateCH(ch.getDatabase(), meta, false, undefined, meta).map(ch.runQuery.bind(ch)))];
case 2:
_a.sent();
_a.label = 3;
case 3: return [4, ch.listColumns(unescape(meta.sqlTableName))];
case 4:
existingColumns = _a.sent();
expectedColumns = meta.pkMappings
.filter(function (pkMap) {
if (isRoot) {
return pkMap.pkType === jsonSchemaInspector_1.PKType.CURRENT;
}
else {
return pkMap.pkType === jsonSchemaInspector_1.PKType.ROOT || pkMap.pkType === jsonSchemaInspector_1.PKType.LEVEL;
}
})
.map(pkMapToColumn)
.concat(meta.pkMappings
.filter(function (pkMap) { return !isRoot && (pkMap.pkType === jsonSchemaInspector_1.PKType.CURRENT || pkMap.pkType === jsonSchemaInspector_1.PKType.PARENT); }).map(mapToColumn))
.concat(meta.simpleColumnMappings.map(mapToColumn))
.concat(!isRoot || (isRoot && meta.pkMappings.find(function (pk) { return pk.pkType === jsonSchemaInspector_1.PKType.CURRENT; }) !== undefined) ? [{
name: isRoot ? "_ver" : "_root_ver",
type: "UInt64",
is_in_sorting_key: false
}] : []);
intersections = getColumnsIntersections(existingColumns, expectedColumns);
if (isRoot) {
checkPrimaryKeysConsistency(existingColumns, meta);
}
return [4, Promise.all(intersections.missing.map(function (elem) { return ch.addColumn(meta.sqlTableName, elem); }))];
case 5:
added = (_a.sent())
.map(function (res) { return (0, Either_1.mapLeft)(res, function (ctx) {
return "Could not create column ".concat(ctx["new"].name, " ").concat(ctx["new"].type);
}); });
return [4, Promise.all(intersections.modified.map(function (elem) { return ch.updateColumn(meta.sqlTableName, elem.existing, elem["new"]); }))];
case 6:
updated = (_a.sent())
.map(function (res) { return (0, Either_1.mapLeft)(res, function (ctx) {
return "Could not update column ".concat(ctx["new"].name, " from ").concat(ctx.existing.type, " to ").concat(ctx["new"].type);
}); });
return [4, Promise.all(intersections.obsolete.map(function (elem) { return ch.removeColumn(meta.sqlTableName, elem); }))];
case 7:
removed = (_a.sent())
.map(function (res) { return (0, Either_1.mapLeft)(res, function (ctx) {
return "Could not drop column ".concat(ctx.existing.name, " ").concat(ctx.existing.type);
}); });
errors = __spreadArray(__spreadArray(__spreadArray([], (0, Either_1.listLeft)(added), true), (0, Either_1.listLeft)(updated), true), (0, Either_1.listLeft)(removed), true);
errors.forEach(function (it) { return (0, singer_node_1.log_error)(it); });
if (errors.length > 0) {
throw new Error("Could not update table");
}
return [2];
}
});
});
}
exports.updateSchema = updateSchema;
//# sourceMappingURL=jsonSchemaTranslator.js.map