@voicenter-team/mysql-dynamic-cluster
Version:
Galera cluster with implementation of dynamic choose mysql server for queries, caching, hashing it and metrics
319 lines • 13.6 kB
JavaScript
"use strict";
/**
* Created by Bohdan on Sep, 2021
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GaleraCluster = void 0;
const Logger_1 = __importDefault(require("../utils/Logger"));
const mysql2_1 = require("mysql2");
const Pool_1 = require("../pool/Pool");
const ClusterHashing_1 = require("./ClusterHashing");
const MetricNames_1 = __importDefault(require("../metrics/MetricNames"));
const Metrics_1 = __importDefault(require("../metrics/Metrics"));
const Events_1 = __importDefault(require("../utils/Events"));
const Redis_1 = __importDefault(require("../Redis/Redis"));
const QueryTimer_1 = require("../utils/QueryTimer");
const ServiceNames_1 = __importDefault(require("../utils/ServiceNames"));
const configs_1 = __importDefault(require("../configs"));
class GaleraCluster {
/**
* @param userSettings global user settings
*/
constructor() {
this.connected = false;
this._pools = [];
this._nullServiceName = "mdc";
Logger_1.default.debug("Configuring cluster...");
this._useClusterHashing = configs_1.default.get('useClusterHashing');
this._clusterName = configs_1.default.get('clusterName');
this._errorRetryCount = configs_1.default.get('errorRetryCount');
this._useRedis = configs_1.default.get('redis.enabled');
const poolIds = this._sortPoolIds(configs_1.default.get('hosts'));
configs_1.default.get('hosts').forEach(poolSettings => {
if (!poolSettings.id) {
poolSettings.id = poolIds.length <= 0 ? 0 : poolIds[poolIds.length - 1] + 1;
poolIds.push(poolSettings.id);
}
if (poolSettings.host)
this._pools.push(new Pool_1.Pool(poolSettings, this._clusterName));
else
Logger_1.default.error(`Pool not valid host ${poolSettings.host}`, poolSettings);
});
this._serviceNames = new ServiceNames_1.default(this, configs_1.default.get('serviceMetrics'));
this._clusterHashing = new ClusterHashing_1.ClusterHashing(this, this._clusterName, configs_1.default.get('clusterHashing'));
Logger_1.default.info("Cluster configuration finished");
}
/**
* Get all pools
* @internal
*/
get pools() {
return this._pools;
}
/**
* Sort pool ids for mix userSettings and autogenerated ids
* @param pools pools passed in userSettings
* @private
*/
_sortPoolIds(pools) {
const poolIds = [];
pools.forEach(pool => {
if (pool.id)
poolIds.push(pool.id);
});
poolIds.sort((a, b) => a - b);
return poolIds;
}
/**
* Connect all cluster pools what created from host information in config
*/
connect() {
return __awaiter(this, void 0, void 0, function* () {
return new Promise((resolve, reject) => {
Logger_1.default.debug("Connecting all pools");
this._pools.forEach((pool) => {
pool.connect().then(() => __awaiter(this, void 0, void 0, function* () {
if (this.connected)
return;
this.connected = true;
if (this._useClusterHashing)
yield this._enableHashing();
Events_1.default.emit('connected');
Logger_1.default.info('Cluster connected');
resolve();
})).catch(err => {
Logger_1.default.error(err.message);
reject(err.message);
});
});
});
});
}
/**
* Enable hashing for cluster
*/
_enableHashing() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield this._clusterHashing.connect();
Events_1.default.emit('hashing_created');
Logger_1.default.info("Cluster hashing enabled");
}
catch (e) {
Logger_1.default.error(e.message);
}
});
}
/**
* Disconnect all cluster pools
*/
disconnect() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
Logger_1.default.debug("disconnecting all pools");
this.connected = false;
(_a = this._clusterHashing) === null || _a === void 0 ? void 0 : _a.stop();
Redis_1.default.disconnect();
this._pools.forEach((pool) => {
pool.disconnect();
});
Events_1.default.emit('disconnected');
});
}
/**
* Connect to the event
* @param event event name
* @param callback function what will be called after emit
*/
on(event, callback) {
Events_1.default.on(event, callback);
}
/**
* Cluster mysql query
* @param sql MySQL query
* @param values values what passed in sql string
* @param queryOptions params for configure query
*/
query(sql, values, queryOptions) {
return __awaiter(this, void 0, void 0, function* () {
queryOptions = Object.assign({ redis: this._useRedis, maxRetry: this._errorRetryCount, redisRefreshCache: false }, queryOptions);
let activePools; // available pools for query what passed the validator
let retryCount; // max retry query count after error
let serviceId = queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.serviceId;
Metrics_1.default.mark(MetricNames_1.default.cluster.queryPerMinute);
Metrics_1.default.inc(MetricNames_1.default.cluster.allQueries);
try {
if (!serviceId && (queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.serviceName)) {
serviceId = yield this._serviceNames.getID(queryOptions.serviceName);
queryOptions.serviceId = serviceId;
}
if (!serviceId) {
queryOptions.serviceId = 0;
queryOptions.serviceName = `${this._clusterName}_${this._nullServiceName}`;
}
activePools = yield this._getActivePools(serviceId);
retryCount = this._maxRetryCount(queryOptions.maxRetry, activePools.length);
}
catch (e) {
Metrics_1.default.inc(MetricNames_1.default.cluster.errorQueries);
throw new Error(e);
}
sql = this._formatSQL(sql, values);
Logger_1.default.debug("formatSQL", { sql, values });
let redisData = null;
if (queryOptions.redis && !queryOptions.redisRefreshCache) {
const redisLatency = new QueryTimer_1.QueryTimer(MetricNames_1.default.redis.latency);
redisLatency.start();
Metrics_1.default.inc(MetricNames_1.default.redis.uses);
const redisResult = yield Redis_1.default.get(sql);
redisLatency.end();
redisLatency.save();
if (redisResult) {
Logger_1.default.debug("Get result of query from redis");
redisData = JSON.parse(redisResult);
if (redisData.expired > Date.now()) {
Metrics_1.default.inc(MetricNames_1.default.cluster.successfulQueries);
return redisData.data;
}
Metrics_1.default.inc(MetricNames_1.default.redis.expired);
}
}
const errorList = [];
for (let i = 0; i < retryCount; i++) {
try {
Logger_1.default.debug("Query use host: " + activePools[i].host);
const res = yield this._queryRequest(sql, activePools[i], queryOptions);
Metrics_1.default.inc(MetricNames_1.default.cluster.successfulQueries);
return res;
}
catch (e) {
errorList.push({
error: e,
pool: activePools[i],
});
Logger_1.default.error(e.message);
if (i + 1 < retryCount) {
Logger_1.default.debug("Retrying query...");
}
}
}
let errorMessage = "";
errorList.forEach(err => {
errorMessage += `Pool: ${err.pool.name}; Error: ${err.error.message}\n`;
});
Logger_1.default.error("All pools have error. Error messages: \n" + errorMessage);
if (redisData) {
Logger_1.default.warn("Use old data from Redis");
Metrics_1.default.inc(MetricNames_1.default.cluster.successfulQueries);
return redisData.data;
}
Metrics_1.default.inc(MetricNames_1.default.cluster.errorQueries);
throw new Error(errorMessage);
});
}
/**
* Query request to pool
* @param sql MySQL query string
* @param pool active pool
* @param queryOptions options to configure query
* @private
*/
_queryRequest(sql, pool, queryOptions) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const queryTimer = new QueryTimer_1.QueryTimer(MetricNames_1.default.cluster.queryTime);
try {
queryTimer.start();
const result = yield pool.query(sql, queryOptions);
queryTimer.end();
queryTimer.save();
if ((queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.serviceId) && this._clusterHashing.connected) {
yield ((_a = this._clusterHashing) === null || _a === void 0 ? void 0 : _a.updateNodeForService(queryOptions === null || queryOptions === void 0 ? void 0 : queryOptions.serviceId, pool.id));
}
return result;
}
catch (e) {
queryTimer.end();
queryTimer.save();
throw new Error("Query error: " + e.message);
}
});
}
/**
* Count max retry after error
* @param maxRetry max retry count after error
* @param activePoolsLength length active pools what passed the validator
* @private
*/
_maxRetryCount(maxRetry, activePoolsLength) {
let retryCount = maxRetry && maxRetry > 0 ? maxRetry : this._errorRetryCount;
if (retryCount > activePoolsLength) {
Logger_1.default.warn("Active pools less than error retry count");
}
retryCount = Math.min(retryCount, activePoolsLength);
return retryCount;
}
/**
* Mix together values and sql string
* @param sql MySQL query
* @param values values to mix with sql
* @private
*/
_formatSQL(sql, values) {
if (values) {
if (Array.isArray(values)) {
return (0, mysql2_1.format)(sql, values);
}
else if (typeof values === 'string') {
return (0, mysql2_1.format)(sql, values);
}
else {
return sql.replace(/:(\w+)/g, (txt, key) => {
return values.hasOwnProperty(key) ? values[key] : txt;
});
}
}
return sql;
}
/**
* Get all active pools what passed the validator
* @param serviceId serviceId of what need to hashing in the cluster
* @private
*/
_getActivePools(serviceId) {
return __awaiter(this, void 0, void 0, function* () {
let activePools;
let poolIdService = -1;
if (serviceId && this._clusterHashing.connected) {
poolIdService = this._clusterHashing.getNodeByService(serviceId);
}
activePools = this._pools.filter(pool => {
return pool.status.isValid && pool.id !== poolIdService;
});
activePools.sort((a, b) => a.status.loadScore - b.status.loadScore);
if (poolIdService >= 0) {
const servicePool = this._pools.find(pool => pool.id === poolIdService);
if (servicePool)
activePools.unshift(servicePool);
}
if (activePools.length < 1) {
throw new Error("There is no pool that satisfies the parameters");
}
return activePools;
});
}
}
exports.GaleraCluster = GaleraCluster;
//# sourceMappingURL=GaleraCluster.js.map