target-clickhouse
Version:
A Singer target for Clickhouse
218 lines • 11.8 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
if (ar || !(i in from)) {
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
ar[i] = from[i];
}
}
return to.concat(ar || Array.prototype.slice.call(from));
};
exports.__esModule = true;
var jsonSchemaInspector_1 = require("./jsonSchemaInspector");
var jsonSchemaTranslator_1 = require("./jsonSchemaTranslator");
var singer_node_1 = require("singer-node");
var get = require("lodash.get");
var RecordProcessor = (function () {
function RecordProcessor(meta, clickhouse, config, level) {
var _this = this;
if (level === void 0) { level = 0; }
this.meta = meta;
this.clickhouse = clickhouse;
this.config = config;
this.level = level;
this.bufferedDatasToStream = [];
this.buildSQLInsertValues = function (data, pkValues, version) {
if (pkValues === void 0) { pkValues = []; }
var noPk = pkValues.length;
var noSimpleColumn = _this.meta.simpleColumnMappings.length;
var result = new Array(noPk + noSimpleColumn + (version !== undefined ? 1 : 0));
for (var i = 0; i < noPk; i++) {
result[i] = pkValues[i];
}
for (var i = 0; i < noSimpleColumn; i++) {
result[i + noPk] = (0, jsonSchemaTranslator_1.extractValue)(data, _this.meta.simpleColumnMappings[i], _this.config.translateValues);
}
if (version !== undefined)
result[noPk + noSimpleColumn] = version;
return result;
};
this.meta = meta;
this.isRoot = level === 0;
this.isWithParentPK = !this.isRoot && this.meta.pkMappings.find(function (pk) { return pk.pkType === jsonSchemaInspector_1.PKType.PARENT; }) !== undefined;
this.hasChildren = meta.children.length > 0;
this.children = meta.children.reduce(function (acc, child) {
var _a;
var processor = new RecordProcessor(child, _this.clickhouse, _this.config, _this.level + 1);
return __assign(__assign({}, acc), (_a = {}, _a[child.sqlTableName] = processor, _a));
}, {});
this.currentPkMappings = this.meta.pkMappings.filter(function (pkMap) { return pkMap.pkType === jsonSchemaInspector_1.PKType.CURRENT; });
}
RecordProcessor.prototype.pushRecord = function (data, abort, maxVer, parentMeta, rootVer, indexInParent, messageCount) {
var _a;
if (indexInParent === void 0) { indexInParent = -1; }
if (messageCount === void 0) { messageCount = 0; }
if (!this.isInitialized()) {
this.startIngestion(messageCount, abort);
}
var resolvedRootVer = (this.isRoot && this.meta.pkMappings.length > 0) ? maxVer + 1 : rootVer;
var currentPkValues = new Array(this.currentPkMappings.length);
for (var i = 0; i < this.currentPkMappings.length; i++) {
currentPkValues[i] = (0, jsonSchemaTranslator_1.extractValue)(data, this.currentPkMappings[i], this.config.translateValues);
}
var sourceMetaPK = {
values: currentPkValues,
rootValues: this.isRoot ? undefined : ((_a = parentMeta === null || parentMeta === void 0 ? void 0 : parentMeta.rootValues) !== null && _a !== void 0 ? _a : parentMeta === null || parentMeta === void 0 ? void 0 : parentMeta.values),
parentValues: this.isRoot ? undefined : parentMeta === null || parentMeta === void 0 ? void 0 : parentMeta.values,
levelValues: this.isRoot ? undefined : __spreadArray(__spreadArray([], ((parentMeta === null || parentMeta === void 0 ? void 0 : parentMeta.levelValues) || []), true), [indexInParent], false)
};
var pkValues = currentPkValues;
if (!this.isRoot) {
pkValues = sourceMetaPK.rootValues
.concat(this.isWithParentPK ? sourceMetaPK.parentValues : [])
.concat(pkValues)
.concat(sourceMetaPK.levelValues);
}
var dataToStream = JSON.stringify(this.buildSQLInsertValues(data, pkValues, resolvedRootVer));
this.bufferedDatasToStream.push(dataToStream);
if (this.bufferedDatasToStream.length == this.config.batchSize) {
this.ingestionCtx.autoEndTimeout.refresh();
this.sendBufferedDatasToStream();
}
if (this.hasChildren) {
for (var _i = 0, _b = this.meta.children; _i < _b.length; _i++) {
var child = _b[_i];
var childProcessor = this.children[child.sqlTableName];
var childDataRaw = get(data, child.prop.split(jsonSchemaInspector_1.nestedSubObjectSeparator));
var childDataAsArray = Array.isArray(childDataRaw) ? childDataRaw : (childDataRaw ? [childDataRaw] : []);
for (var idx = 0; idx < childDataAsArray.length; idx++) {
childProcessor.pushRecord(childDataAsArray[idx], abort, maxVer, sourceMetaPK, resolvedRootVer, idx, messageCount);
}
}
}
};
RecordProcessor.prototype.endIngestion = function () {
return __awaiter(this, void 0, void 0, function () {
var _a, promise, stream, autoEndTimeout;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!this.isInitialized()) return [3, 2];
(0, singer_node_1.log_debug)("[".concat(this.meta.prop, "] closing stream to insert data"));
_a = this.ingestionCtx, promise = _a.promise, stream = _a.stream, autoEndTimeout = _a.autoEndTimeout;
clearTimeout(autoEndTimeout);
this.sendBufferedDatasToStream();
stream.end();
this.ingestionCtx = undefined;
return [4, Promise.all([
promise,
Promise.all(Object.values(this.children).map(function (child) { return child.endIngestion(); })),
])];
case 1:
_b.sent();
_b.label = 2;
case 2: return [2];
}
});
});
};
RecordProcessor.prototype.buildSQLInsertField = function () {
var isRoot = this.meta.pkMappings.find(function (pkMap) { return pkMap.pkType === jsonSchemaInspector_1.PKType.ROOT; }) === undefined;
return this.meta.pkMappings
.map(function (pkMap) { return pkMap.sqlIdentifier; })
.concat(this.meta.simpleColumnMappings.map(function (cMap) { return cMap.sqlIdentifier; }))
.concat(isRoot ? (this.meta.pkMappings.length > 0 ? ["`_ver`"] : []) : ["`_root_ver`"]);
};
RecordProcessor.prototype.isInitialized = function () {
return this.ingestionCtx !== undefined;
};
RecordProcessor.prototype.sendBufferedDatasToStream = function () {
if (this.bufferedDatasToStream.length > 0) {
this.bufferedDatasToStream.push("");
if (!this.ingestionCtx) {
throw new Error("ingestionCtx is undefined but there is still bufferedData");
}
this.ingestionCtx.stream.write(Buffer.from(this.bufferedDatasToStream.join('\n')));
this.bufferedDatasToStream = [];
}
};
RecordProcessor.prototype.startIngestion = function (messageCount, abort) {
var _this = this;
var insertQuery = "INSERT INTO ".concat(this.meta.sqlTableName, " (").concat(this.buildSQLInsertField().join(","), ") FORMAT JSONCompactEachRow");
if (this.isRoot) {
(0, singer_node_1.log_info)("[".concat(this.meta.prop, "] handling lines starting at ").concat(messageCount));
}
var promiseResolve;
var promiseReject;
var promise = new Promise(function (resolve, reject) {
promiseResolve = resolve;
promiseReject = reject;
});
this.ingestionCtx = {
stream: this.clickhouse.createWriteStream(insertQuery, function (err) {
if (err) {
abort(err);
promiseReject(err);
}
else {
promiseResolve();
}
}),
promise: promise,
autoEndTimeout: setTimeout(function () {
(0, singer_node_1.log_debug)("[".concat(_this.meta.prop, "] auto closing stream to insert data due to inactivity"));
_this.endIngestion();
}, this.config.autoEndTimeoutMs)
};
};
return RecordProcessor;
}());
exports["default"] = RecordProcessor;
//# sourceMappingURL=RecordProcessor.js.map