@naturalcycles/mysql-lib
Version:
MySQL client implementing CommonDB interface
298 lines (297 loc) • 11.2 kB
JavaScript
import { __decorate } from "tslib";
import { Readable, Transform } from 'node:stream';
import { promisify } from 'node:util';
import { BaseCommonDB, commonDBFullSupport, CommonDBType, DBQuery } from '@naturalcycles/db-lib';
import { _Memo } from '@naturalcycles/js-lib/decorators/memo.decorator.js';
import { _assert } from '@naturalcycles/js-lib/error/assert.js';
import { commonLoggerPrefix } from '@naturalcycles/js-lib/log';
import { _filterUndefinedValues, _mapKeys, _mapValues, _omit } from '@naturalcycles/js-lib/object';
import { white } from '@naturalcycles/nodejs-lib/colors';
import * as mysql from 'mysql';
import { dbQueryToSQLDelete, dbQueryToSQLSelect, dbQueryToSQLUpdate, insertSQL, } from './query.util.js';
import { jsonSchemaToMySQLDDL, mapNameFromMySQL, mysqlTableStatsToJsonSchemaField, } from './schema/mysql.schema.util.js';
const BOOLEAN_TYPES = new Set(['TINY', 'TINYINT', 'INT']);
const BOOLEAN_BIT_TYPES = new Set(['BIT']);
const typeCast = (field, next) => {
// cast to boolean
if (field.length === 1) {
if (BOOLEAN_BIT_TYPES.has(field.type)) {
const b = field.buffer();
if (b === null || b === undefined)
return undefined; // null = undefined
return b[0] === 1; // 1 = true, 0 = false
}
if (BOOLEAN_TYPES.has(field.type)) {
const s = field.string();
// console.log(field.name, field.type, s, s?.charCodeAt(0))
if (s === null || s === undefined)
return undefined; // null = undefined
return s === '1'; // 1 = true, 0 = false
}
}
return next();
};
export class MysqlDB extends BaseCommonDB {
dbType = CommonDBType.relational;
support = {
...commonDBFullSupport,
updateSaveMethod: false, // todo: can be implemented
transactions: false, // todo: can be implemented
increment: false, // todo: can be implemented
};
constructor(cfg = {}) {
super();
this.cfg = {
typeCast,
// charset: 'utf8mb4', // for emoji support
// host: 'localhost',
// user: MYSQL_USER,
// password: MYSQL_PW,
// database: MYSQL_DB,
...cfg,
logger: commonLoggerPrefix(cfg.logger || console, '[mysql]'),
};
}
cfg;
async ping() {
const con = await this.getConnection();
con.release();
}
pool() {
const pool = mysql.createPool(this.cfg);
const { host, database, logger } = this.cfg;
logger.log(`connected to ${white(host + '/' + database)}`);
if (this.cfg.debugConnections) {
pool.on('acquire', con => {
logger.log(`acquire(${con.threadId})`);
});
pool.on('connection', con => {
logger.log(`connection(${con.threadId})`);
});
pool.on('enqueue', () => {
logger.log(`enqueue`);
});
pool.on('release', con => {
logger.log(`release(${con.threadId})`);
});
}
pool.on('error', err => {
logger.error(err);
});
return pool;
}
async close() {
const pool = this.pool();
await promisify(pool.end.bind(pool))();
this.cfg.logger.log('closed');
}
/**
* Be careful to always call `con.release()` when you get connection with this method.
*/
async getConnection() {
const pool = this.pool();
return await promisify(pool.getConnection.bind(pool))();
}
/**
* Manually create a single (not pool) connection.
* Be careful to manage this connection yourself (open, release, etc).
*/
async createSingleConnection() {
const con = mysql.createConnection(this.cfg);
await promisify(con.connect.bind(con))();
const { threadId } = con;
if (this.cfg.debugConnections) {
this.cfg.logger.log(`createSingleConnection(${threadId})`);
con.on('connect', () => this.cfg.logger.log(`createSingleConnection(${threadId}).connect`));
con.on('drain', () => this.cfg.logger.log(`createSingleConnection(${threadId}).drain`));
con.on('enqueue', () => this.cfg.logger.log(`createSingleConnection(${threadId}).enqueue`));
con.on('end', () => this.cfg.logger.log(`createSingleConnection(${threadId}).end`));
}
con.on('error', err => {
this.cfg.logger.error(`createSingleConnection(${threadId}).error`, err);
});
return con;
}
// GET
async getByIds(table, ids, opt = {}) {
if (!ids.length)
return [];
const q = new DBQuery(table).filterEq('id', ids);
const { rows } = await this.runQuery(q, opt);
return rows.map(r => _mapKeys(r, k => mapNameFromMySQL(k)));
}
// QUERY
async runQuery(q, _opt = {}) {
const sql = dbQueryToSQLSelect(q);
if (!sql) {
return {
rows: [],
};
}
const rows = (await this.runSQL({ sql })).map(row => _mapKeys(_filterUndefinedValues(row, { mutate: true }), k => mapNameFromMySQL(k)));
// edge case where 0 fields are selected
if (q._selectedFieldNames?.length === 0) {
return {
rows: rows.map(_ => ({})),
};
}
return {
rows,
};
}
async runSQL(q) {
if (this.cfg.logSQL)
this.cfg.logger.log(...[q.sql, q.values].filter(Boolean));
return await new Promise((resolve, reject) => {
this.pool().query(q, (err, res) => {
if (err)
return reject(err);
resolve(res);
});
});
}
/**
* Allows to run semicolon-separated "SQL file".
* E.g "ddl reset script".
*/
async runSQLString(s) {
const queries = s
.split(';')
.map(s => s.trim())
.filter(Boolean);
for (const sql of queries) {
await this.runSQL({ sql });
}
}
async runQueryCount(q, _opt) {
const { rows } = await this.runQuery(q.select(['count(*) as _count']));
return rows[0]._count;
}
streamQuery(q, _opt = {}) {
const sql = dbQueryToSQLSelect(q);
if (!sql) {
return Readable.from([]);
}
if (this.cfg.logSQL)
this.cfg.logger.log(`stream: ${sql}`);
// todo: this is nice, but `mysql` package uses `readable-stream@2` which is not compatible with `node:stream` iterable helpers
// return (this.pool().query(sql).stream() as ReadableTyped<ROW>).map(row =>
// _filterUndefinedValues(row, true),
// )
return this.pool()
.query(sql)
.stream()
.pipe(new Transform({
objectMode: true,
transform(row, _encoding, cb) {
cb(null, _filterUndefinedValues(row, { mutate: true }));
},
}));
}
// SAVE
async saveBatch(table, rowsInput, opt = {}) {
if (!rowsInput.length)
return;
// Stringify object values
const rows = rowsInput.map(row => _mapValues(row, (_k, v) => {
if (v && typeof v === 'object' && !Buffer.isBuffer(v)) {
// This is to avoid implicit Date stringification and mismatch: it gets saved as Date, but loaded as String
_assert(!(v instanceof Date), 'mysql-lib does not support Date values, please stringify them before passing');
return JSON.stringify(v);
}
return v;
}));
if (opt.assignGeneratedIds) {
// Insert rows one-by-one, to get their auto-generated id
let i = -1;
for (const row of rows) {
i++;
if (row.id) {
// Update already existing
const query = new DBQuery(table).filterEq('id', row.id);
await this.patchByQuery(query, _omit(row, ['id']));
}
else {
// Create new
const sql = insertSQL(table, [row], 'INSERT', this.cfg.logger)[0];
const { insertId } = await this.runSQL({ sql });
// Mutate the input row with insertIt
rowsInput[i].id = insertId; // this is because we no longer support number ids in CommonDB
}
}
return;
}
if (opt.saveMethod === 'update') {
// TODO: This fails if a combination of entities with id and without id are parsed
for (const row of rows) {
// Update already existing
_assert(row.id, 'id is required for updating');
const query = new DBQuery(table).filterEq('id', row.id);
await this.patchByQuery(query, _omit(row, ['id']));
}
return;
}
const verb = opt.saveMethod === 'insert' ? 'INSERT' : 'REPLACE';
// inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
const sqls = insertSQL(table, rows, verb, this.cfg.logger);
for (const sql of sqls) {
await this.runSQL({ sql });
}
}
// DELETE
/**
* Limitation: always returns [], regardless of which rows are actually deleted
*/
async deleteByIds(table, ids, _opt) {
if (!ids.length)
return 0;
const sql = dbQueryToSQLDelete(new DBQuery(table).filterEq('id', ids));
if (!sql)
return 0;
const { affectedRows } = await this.runSQL({ sql });
return affectedRows;
}
async deleteByQuery(q, _opt) {
const sql = dbQueryToSQLDelete(q);
if (!sql)
return 0;
const { affectedRows } = await this.runSQL({ sql });
return affectedRows;
}
/**
* Use with caution!
*/
async dropTable(table) {
await this.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` });
}
/**
* dropIfExists=true needed as a safety check
*/
async createTable(table, schema, opt = {}) {
if (opt.dropIfExists)
await this.dropTable(table);
const sql = jsonSchemaToMySQLDDL(table, schema);
await this.runSQL({ sql });
}
async getTables() {
return (await this.runSQL({ sql: `show tables` }))
.map(r => Object.values(r)[0])
.filter(Boolean);
}
async getTableSchema(table) {
const stats = await this.runSQL({
sql: `describe ${mysql.escapeId(table)}`,
});
return mysqlTableStatsToJsonSchemaField(table, stats, this.cfg.logger);
}
async patchByQuery(q, patch) {
const sql = dbQueryToSQLUpdate(q, patch);
if (!sql)
return 0;
const { affectedRows } = await this.runSQL({ sql });
return affectedRows;
}
}
__decorate([
_Memo()
], MysqlDB.prototype, "pool", null);