target-clickhouse
Version:
A Singer target for Clickhouse
272 lines • 14.2 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
exports.__esModule = true;
var ono_1 = require("ono");
var retry = require("retry");
var singer_node_1 = require("singer-node");
var utils_1 = require("./utils");
var Either_1 = require("./Either");
var ClickHouse = require("@apla/clickhouse");
var ClickhouseConnection = (function () {
function ClickhouseConnection(connInfo) {
this.connInfo = connInfo;
}
ClickhouseConnection.prototype.getDatabase = function () {
return this.connInfo.database;
};
ClickhouseConnection.prototype.listTables = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4, this.runQuery("SHOW TABLES")];
case 1: return [2, (_a.sent()).data.map(function (_a) {
var tableName = _a[0];
return tableName;
})];
}
});
});
};
ClickhouseConnection.prototype.addColumn = function (table, newCol) {
return __awaiter(this, void 0, void 0, function () {
var e_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
(0, singer_node_1.log_info)("[".concat(table, "] Adding column ").concat(table, ".").concat(newCol.name, " ").concat(newCol.type));
return [4, this.runQuery("ALTER TABLE ".concat(table, "\n ADD COLUMN `").concat(newCol.name, "` ").concat(newCol.type), 2)];
case 1:
_a.sent();
return [2, (0, Either_1.makeRight)(true)];
case 2:
e_1 = _a.sent();
return [2, (0, Either_1.makeLeft)({
"new": newCol,
err: e_1
})];
case 3: return [2];
}
});
});
};
ClickhouseConnection.prototype.renameObsoleteColumn = function (table) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
(0, singer_node_1.log_info)("[".concat(table, "] Renaming table ").concat(table));
return [2, this.runQuery("RENAME TABLE `".concat(table, "` TO `").concat(ClickhouseConnection.droppedTablePrefix).concat(table, "`"))];
});
});
};
ClickhouseConnection.prototype.removeColumn = function (table, existing) {
return __awaiter(this, void 0, void 0, function () {
var e_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
(0, singer_node_1.log_info)("[".concat(table, "] Removing column ").concat(table, ".").concat(existing.name, " "));
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
return [4, this.runQuery("ALTER TABLE ".concat(table, "\n DROP COLUMN `").concat(existing.name, "`"))];
case 2:
_a.sent();
return [2, (0, Either_1.makeRight)(true)];
case 3:
e_2 = _a.sent();
return [2, (0, Either_1.makeLeft)({
existing: existing,
err: e_2
})];
case 4: return [2];
}
});
});
};
ClickhouseConnection.prototype.updateColumn = function (table, existing, newCol) {
return __awaiter(this, void 0, void 0, function () {
var e_3, revertError_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 7]);
(0, singer_node_1.log_info)("[".concat(table, "] Updating column ").concat(table, ".").concat(existing.name, " from ").concat(existing.type, " to ").concat(newCol.type));
return [4, this.runQuery("ALTER TABLE ".concat(table, "\n MODIFY COLUMN `").concat(newCol.name, "` ").concat(newCol.type), 0)];
case 1:
_a.sent();
return [2, (0, Either_1.makeRight)(true)];
case 2:
e_3 = _a.sent();
_a.label = 3;
case 3:
_a.trys.push([3, 5, , 6]);
return [4, this.runQuery("ALTER TABLE ".concat(table, "\n MODIFY COLUMN `").concat(existing.name, "` ").concat(existing.type))];
case 4:
_a.sent();
return [3, 6];
case 5:
revertError_1 = _a.sent();
(0, singer_node_1.log_error)("could not revert update");
return [3, 6];
case 6: return [2, (0, Either_1.makeLeft)({
existing: existing,
"new": newCol,
err: e_3
})];
case 7: return [2];
}
});
});
};
ClickhouseConnection.prototype.listColumns = function (table) {
return __awaiter(this, void 0, void 0, function () {
var res;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4, this.runQuery("SELECT name, type, is_in_sorting_key\n FROM system.columns\n WHERE database = '".concat((0, utils_1.escapeValue)(this.connInfo.database), "'\n AND table = '").concat((0, utils_1.escapeValue)(table), "'"))];
case 1:
res = _a.sent();
return [2, res.data.map(function (row) { return ({
name: row[0],
type: row[1],
is_in_sorting_key: Boolean(row[2])
}); })];
}
});
});
};
ClickhouseConnection.prototype.createWriteStream = function (query, callback) {
(0, singer_node_1.log_debug)("building stream to query sql ".concat(query));
if (!this.connection) {
throw new Error("Clickhouse connection was not initialized");
}
return this.connection.query(query, { omitFormat: true }, callback)
.on('error', function () {
(0, singer_node_1.log_error)("rejecting query ".concat(query));
});
};
ClickhouseConnection.prototype.runQuery = function (query, retries) {
if (retries === void 0) { retries = 2; }
return __awaiter(this, void 0, void 0, function () {
var conn;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4, this.getConnectionPooled()];
case 1:
conn = _a.sent();
return [2, new Promise(function (resolve, reject) {
var op = retry.operation({
retries: retries,
factor: 4
});
op.attempt(function () {
return __awaiter(this, void 0, void 0, function () {
var res, err_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
(0, singer_node_1.log_debug)("query sql [".concat(query, "]"));
return [4, conn.querying(query)];
case 1:
res = _a.sent();
resolve(res);
return [3, 3];
case 2:
err_1 = _a.sent();
(0, singer_node_1.log_warning)("query sql failed [".concat(query, "]: ").concat(err_1.message));
if (op.retry(err_1)) {
return [2];
}
reject((0, ono_1.ono)(err_1, "even after retries, ch query failed", err_1));
return [3, 3];
case 3: return [2];
}
});
});
});
})];
}
});
});
};
ClickhouseConnection.prototype.getConnectionPooled = function () {
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
if (!this.connection) {
this.connection = new ClickHouse({
host: this.connInfo.host,
user: this.connInfo.username,
port: this.connInfo.port,
password: this.connInfo.password,
queryOptions: {
mutations_sync: 2,
database: this.connInfo.database,
date_time_input_format: "best_effort",
insert_null_as_default: 0,
input_format_null_as_default: 0,
input_format_defaults_for_omitted_fields: 0,
http_receive_timeout: this.connInfo.insert_stream_timeout_sec
}
});
return [2, new Promise(function (resolve, reject) {
_this.connection.query("SELECT 1", function (err) {
if (err) {
_this.connection = undefined;
reject(err);
}
else {
resolve(_this.connection);
}
});
})];
}
else {
return [2, this.connection];
}
return [2];
});
});
};
ClickhouseConnection.droppedTablePrefix = "_dropped_";
ClickhouseConnection.archivedTablePrefix = "_archived_";
return ClickhouseConnection;
}());
exports["default"] = ClickhouseConnection;
//# sourceMappingURL=ClickhouseConnection.js.map