knex/lib/client.js

A batteries-included SQL query & schema builder for PostgresSQL, MySQL, CockroachDB, MSSQL and SQLite3

const { TimeoutError } = require('tarn');
const { KnexPool, isTarnPool } = require('./pool');
const { EventEmitter } = require('events');
const { promisify } = require('util');
const { makeEscape } = require('./util/string');
const cloneDeepWith = require('lodash/cloneDeepWith');
const defaults = require('lodash/defaults');
const uniqueId = require('lodash/uniqueId');

const Runner = require('./execution/runner');
const Transaction = require('./execution/transaction');
const {
  executeQuery,
  enrichQueryObject,
} = require('./execution/internal/query-executioner');
const QueryBuilder = require('./query/querybuilder');
const QueryCompiler = require('./query/querycompiler');
const SchemaBuilder = require('./schema/builder');
const SchemaCompiler = require('./schema/compiler');
const TableBuilder = require('./schema/tablebuilder');
const TableCompiler = require('./schema/tablecompiler');
const ColumnBuilder = require('./schema/columnbuilder');
const ColumnCompiler = require('./schema/columncompiler');
const { KnexTimeoutError } = require('./util/timeout');
const { outputQuery, unwrapRaw } = require('./formatter/wrappingFormatter');
const { compileCallback } = require('./formatter/formatterUtils');
const Raw = require('./raw');
const Ref = require('./ref');
const Formatter = require('./formatter');
const Logger = require('./logger');
const { POOL_CONFIG_OPTIONS } = require('./constants');
const ViewBuilder = require('./schema/viewbuilder.js');
const ViewCompiler = require('./schema/viewcompiler.js');
const isPlainObject = require('lodash/isPlainObject');
const { setHiddenProperty } = require('./util/security.js');

const debug = require('debug')('knex:client');

function preserveClassInstances(value) {
  if (
    value !== null &&
    typeof value === 'object' &&
    !Array.isArray(value) &&
    !isPlainObject(value)
  ) {
    return value;
  }
}

// The base client provides the general structure
// for a dialect specific client object.

class Client extends EventEmitter {
  constructor(config = {}) {
    super();
    this.config = { ...config };
    if (config.connection && typeof config.connection === 'object') {
      this.config.connection = { ...config.connection };
    }
    this.logger = new Logger(this.config);

    if (this.config.connection && config.connection.password) {
      setHiddenProperty(this.config.connection, config.connection);
    }

    //Client is a required field, so throw error if it's not supplied.
    //If 'this.dialect' is set, then this is a 'super()' call, in which case
    //'client' does not have to be set as it's already assigned on the client prototype.

    if (this.dialect && !this.config.client) {
      this.logger.warn(
        `Using 'this.dialect' to identify the client is deprecated and support for it will be removed in the future. Please use configuration option 'client' instead.`
      );
    }

    const dbClient = this.config.client || this.dialect;
    if (!dbClient) {
      throw new Error(
        `knex: Required configuration option 'client' is missing.`
      );
    }

    if (config.version) {
      this.version = config.version;
    }

    if (this.config.connection && this.config.connection instanceof Function) {
      this.connectionConfigProvider = this.config.connection;
      this.connectionConfigExpirationChecker = () => true; // causes the provider to be called on first use
    } else {
      this.connectionSettings = cloneDeepWith(
        this.config.connection || {},
        preserveClassInstances
      );
      if (config.connection && config.connection.password) {
        setHiddenProperty(this.connectionSettings, this.config.connection);
      }
      this.connectionConfigExpirationChecker = null;
    }
    if (this.driverName && config.connection) {
      this.initializeDriver();
    }
    const shouldInitPool =
      this.driverName && (config.connection || config.connectionPool);
    if (shouldInitPool) {
      if (!config.pool || (config.pool && config.pool.max !== 0)) {
        this.initializePool(this.config);
      }
    }
    this.valueForUndefined = this.raw('DEFAULT');
    if (config.useNullAsDefault) {
      this.valueForUndefined = null;
    }
  }
  formatter(builder) {
    return new Formatter(this, builder);
  }

  queryBuilder() {
    return new QueryBuilder(this);
  }

  queryCompiler(builder, formatter) {
    return new QueryCompiler(this, builder, formatter);
  }

  schemaBuilder() {
    return new SchemaBuilder(this);
  }

  schemaCompiler(builder) {
    return new SchemaCompiler(this, builder);
  }

  tableBuilder(type, tableName, tableNameLike, fn) {
    return new TableBuilder(this, type, tableName, tableNameLike, fn);
  }

  viewBuilder(type, viewBuilder, fn) {
    return new ViewBuilder(this, type, viewBuilder, fn);
  }

  tableCompiler(tableBuilder) {
    return new TableCompiler(this, tableBuilder);
  }

  viewCompiler(viewCompiler) {
    return new ViewCompiler(this, viewCompiler);
  }

  columnBuilder(tableBuilder, type, args) {
    return new ColumnBuilder(this, tableBuilder, type, args);
  }

  columnCompiler(tableBuilder, columnBuilder) {
    return new ColumnCompiler(this, tableBuilder, columnBuilder);
  }

  runner(builder) {
    return new Runner(this, builder);
  }

  transaction(container, config, outerTx) {
    return new Transaction(this, container, config, outerTx);
  }

  raw() {
    return new Raw(this).set(...arguments);
  }

  ref() {
    return new Ref(this, ...arguments);
  }
  query(connection, queryParam) {
    const queryObject = enrichQueryObject(connection, queryParam, this);
    return executeQuery(connection, queryObject, this);
  }

  stream(connection, queryParam, stream, options) {
    const queryObject = enrichQueryObject(connection, queryParam, this);
    return this._stream(connection, queryObject, stream, options);
  }

  prepBindings(bindings) {
    return bindings;
  }

  positionBindings(sql) {
    return sql;
  }

  postProcessResponse(resp, queryContext) {
    if (this.config.postProcessResponse) {
      return this.config.postProcessResponse(resp, queryContext);
    }
    return resp;
  }

  wrapIdentifier(value, queryContext) {
    return this.customWrapIdentifier(
      value,
      this.wrapIdentifierImpl,
      queryContext
    );
  }

  customWrapIdentifier(value, origImpl, queryContext) {
    if (this.config.wrapIdentifier) {
      return this.config.wrapIdentifier(value, origImpl, queryContext);
    }
    return origImpl(value);
  }

  wrapIdentifierImpl(value) {
    return value !== '*' ? `"${value.replace(/"/g, '""')}"` : '*';
  }

  initializeDriver() {
    try {
      this.driver = this._driver();
    } catch (e) {
      const packageName = this._packageName || this.driverName;
      const message = `Knex: run\n$ npm install ${packageName} --save`;
      this.logger.error(`${message}\n${e.message}\n${e.stack}`);
      throw new Error(`${message}\n${e.message}`);
    }
  }

  poolDefaults() {
    return { min: 2, max: 10, propagateCreateError: true };
  }

  getPoolSettings(poolConfig) {
    const {
      afterCreate: userAfterCreate,
      validate: userValidate,
      maxConnectionLifetimeMillis,
      maxConnectionLifetimeJitterMillis,
      ...tarnOptions
    } = defaults({}, poolConfig, this.poolDefaults());

    POOL_CONFIG_OPTIONS.forEach((option) => {
      if (option in tarnOptions) {
        this.logger.warn(
          [
            `Pool config option "${option}" is no longer supported.`,
            `See https://github.com/Vincit/tarn.js for possible pool config options.`,
          ].join(' ')
        );
      }
    });

    const DEFAULT_ACQUIRE_TIMEOUT = 60000;
    const timeouts = [
      this.config.acquireConnectionTimeout,
      tarnOptions.acquireTimeoutMillis,
    ].filter((timeout) => timeout !== undefined);

    if (!timeouts.length) {
      timeouts.push(DEFAULT_ACQUIRE_TIMEOUT);
    }

    // acquire connection timeout can be set on config or config.pool
    // choose the smallest, positive timeout setting and set on poolConfig
    const acquireTimeoutMillis = Math.min(...timeouts);

    const hasMaxConnectionLifetime =
      typeof maxConnectionLifetimeMillis === 'number' &&
      Number.isFinite(maxConnectionLifetimeMillis) &&
      maxConnectionLifetimeMillis > 0;

    const generateLifetimeLimit = () => {
      if (!hasMaxConnectionLifetime) {
        return null;
      }
      const jitter =
        typeof maxConnectionLifetimeJitterMillis === 'number' &&
        maxConnectionLifetimeJitterMillis > 0
          ? Math.floor(Math.random() * maxConnectionLifetimeJitterMillis)
          : 0;
      return performance.now() + maxConnectionLifetimeMillis + jitter;
    };

    const updatePoolConnectionSettingsFromProvider = async (
      forceRefresh = false
    ) => {
      if (!this.connectionConfigProvider) {
        return; // static configuration, nothing to update
      }
      if (
        !forceRefresh &&
        (!this.connectionConfigExpirationChecker ||
          !this.connectionConfigExpirationChecker())
      ) {
        return; // not expired, reuse existing connection
      }
      const providerResult = await this.connectionConfigProvider();
      if (providerResult.expirationChecker) {
        this.connectionConfigExpirationChecker =
          providerResult.expirationChecker;
        delete providerResult.expirationChecker; // MySQL2 driver warns on receiving extra properties
      } else {
        this.connectionConfigExpirationChecker = null;
      }
      this.connectionSettings = providerResult;
    };

    const connectionIsDisposed = (connection) => {
      if (connection.__knex__disposed) {
        this.logger.warn(`Connection Error: ${connection.__knex__disposed}`);
        return true;
      }
      return false;
    };

    const connectionIsOverLifetime = (connection) => {
      if (!hasMaxConnectionLifetime) {
        return false;
      }
      if (!connection.__knexLifetimeLimit) {
        connection.__knexLifetimeLimit = generateLifetimeLimit();
      }
      return (
        connection.__knexLifetimeLimit &&
        performance.now() > connection.__knexLifetimeLimit
      );
    };

    const configWasRefreshed = async () => {
      if (!this.connectionConfigExpirationChecker) {
        return false;
      }
      if (!this.connectionConfigExpirationChecker()) {
        return false;
      }
      await updatePoolConnectionSettingsFromProvider(true);
      if (
        this.connectionConfigExpirationChecker &&
        this.connectionConfigExpirationChecker()
      ) {
        throw new Error(
          'Connection configuration still reported expired after refresh'
        );
      }
      return true;
    };

    const tarnPoolConfig = {
      ...tarnOptions,
      acquireTimeoutMillis,
      create: async () => {
        await updatePoolConnectionSettingsFromProvider();
        const connection = await this.acquireRawConnection();
        connection.__knexUid = uniqueId('__knexUid');
        if (hasMaxConnectionLifetime) {
          connection.__knexLifetimeLimit = generateLifetimeLimit();
        }
        if (userAfterCreate) {
          await promisify(userAfterCreate)(connection);
        }
        return connection;
      },

      destroy: (connection) => {
        if (connection !== void 0) {
          return this.destroyRawConnection(connection);
        }
      },

      validate: async (connection) => {
        if (connectionIsDisposed(connection)) {
          return false;
        }

        if (connectionIsOverLifetime(connection)) {
          return false;
        }

        if (await configWasRefreshed()) {
          return false;
        }

        if (userValidate) {
          try {
            const validationResult = await userValidate(connection);
            if (!validationResult) {
              return false;
            }
          } catch (validationError) {
            this.logger.warn(
              `Pool validate threw an error: ${validationError.message}`
            );
            return false;
          }
        }

        return this.validateConnection(connection);
      },
    };

    if (tarnPoolConfig.afterCreate) {
      delete tarnPoolConfig.afterCreate;
    }

    return tarnPoolConfig;
  }

  initializePool(config = this.config) {
    if (this.pool) {
      this.logger.warn('The pool has already been initialized');
      return;
    }
    if (config.connectionPool) {
      // External pool mode — delegate to a tarn-compatible or native pool
      this.connectionSettings = {};
      this.connectionConfigExpirationChecker = null;
      if (this.driverName) {
        this.initializeDriver();
      }
      this.initializeExternalPool(config.connectionPool);
    } else {
      this.pool = new KnexPool(this.getPoolSettings(config.pool));
      this._ownsPool = true;
    }
  }

  // Accept an external pool — either a tarn-compatible pool (KnexPool, tarn Pool)
  // or a native driver pool (pg.Pool, mysql.createPool(), etc.)
  initializeExternalPool(externalPool) {
    if (this.pool) {
      this.logger.warn('The pool has already been initialized');
      return;
    }
    if (isTarnPool(externalPool)) {
      // Tarn-compatible pool — use directly
      this.pool = externalPool;
    } else {
      // Native driver pool — wrap via dialect-specific adapter
      this.pool = this.wrapNativePool(externalPool);
    }
    this._ownsPool = false;
  }

  // Override in dialects that support native driver pools.
  // Must return an object with { acquire(), release(), destroy() } matching tarn's interface.
  wrapNativePool(_nativePool) {
    throw new Error(
      `The '${
        this.driverName || 'base'
      }' dialect does not support native driver pools in 'connectionPool'. ` +
        `Pass a tarn Pool or KnexPool instance instead.`
    );
  }

  validateConnection(connection) {
    return true;
  }

  // Acquire a connection from the pool.
  async acquireConnection() {
    if (!this.pool) {
      throw new Error('Unable to acquire a connection');
    }
    try {
      const connection = await this.pool.acquire().promise;
      debug('acquired connection from pool: %s', connection.__knexUid);
      if (connection.config) {
        if (connection.config.password) {
          setHiddenProperty(connection.config);
        }
        if (
          connection.config.authentication &&
          connection.config.authentication.options &&
          connection.config.authentication.options.password
        ) {
          setHiddenProperty(connection.config.authentication.options);
        }
      }
      return connection;
    } catch (error) {
      // Tarn only gives us the propagated error's message, not its stack. However, for the "original"
      // promise that rejected, the stack will be present when Tarn passes us the error directly (not
      // as a timeout). The stack is likely of little use, but let's give all the information we can.
      const withStack =
        error instanceof Error ? error.stack ?? error.message : String(error);

      // Errors like this can be noisy; use logger.warn to make it easier for operators to opt out of
      // noise but still surface more critical problems
      this.logger.warn(`Acquire connection error: ${withStack}`);

      let convertedError = error;
      if (error instanceof TimeoutError) {
        convertedError = new KnexTimeoutError(
          'Knex: Timeout acquiring a connection. The pool is probably full. ' +
            'Are you missing a .transacting(trx) call?'
        );
        convertedError.cause = error;
      }
      throw convertedError;
    }
  }

  // Releases a connection back to the connection pool,
  // returning a promise resolved when the connection is released.
  releaseConnection(connection) {
    debug('releasing connection to pool: %s', connection.__knexUid);
    const didRelease = this.pool.release(connection);

    if (!didRelease) {
      debug('pool refused connection: %s', connection.__knexUid);
    }

    return Promise.resolve();
  }

  // Destroy the current connection pool for the client.
  // When the pool is externally provided (_ownsPool === false), we only drop
  // our reference — the caller is responsible for tearing it down.
  async destroy(callback) {
    try {
      if (this.pool && this.pool.destroy && this._ownsPool !== false) {
        await this.pool.destroy();
      }
      this.pool = undefined;

      if (typeof callback === 'function') {
        callback();
      }
    } catch (err) {
      if (typeof callback === 'function') {
        return callback(err);
      }
      throw err;
    }
  }

  // Return the database being used by this client.
  database() {
    return this.connectionSettings.database;
  }

  toString() {
    return '[object KnexClient]';
  }

  assertCanCancelQuery() {
    if (!this.canCancelQuery) {
      throw new Error('Query cancelling not supported for this dialect');
    }
  }

  cancelQuery() {
    throw new Error('Query cancelling not supported for this dialect');
  }

  // Formatter part

  alias(first, second) {
    return first + ' as ' + second;
  }

  // Checks whether a value is a function... if it is, we compile it
  // otherwise we check whether it's a raw
  parameter(value, builder, bindingsHolder) {
    if (typeof value === 'function') {
      return outputQuery(
        compileCallback(value, undefined, this, bindingsHolder),
        true,
        builder,
        this
      );
    }
    return unwrapRaw(value, true, builder, this, bindingsHolder) || '?';
  }

  // Turns a list of values into a list of ?'s, joining them with commas unless
  // a "joining" value is specified (e.g. ' and ')
  parameterize(values, notSetValue, builder, bindingsHolder) {
    if (typeof values === 'function')
      return this.parameter(values, builder, bindingsHolder);
    values = Array.isArray(values) ? values : [values];
    let str = '',
      i = -1;
    while (++i < values.length) {
      if (i > 0) str += ', ';
      let value = values[i];
      // json columns can have object in values.
      if (isPlainObject(value)) {
        value = JSON.stringify(value);
      }
      str += this.parameter(
        value === undefined ? notSetValue : value,
        builder,
        bindingsHolder
      );
    }
    return str;
  }

  // Formats `values` into a parenthesized list of parameters for a `VALUES`
  // clause.
  //
  // [1, 2]                  -> '(?, ?)'
  // [[1, 2], [3, 4]]        -> '((?, ?), (?, ?))'
  // knex('table')           -> '(select * from "table")'
  // knex.raw('select ?', 1) -> '(select ?)'
  //
  values(values, builder, bindingsHolder) {
    if (Array.isArray(values)) {
      if (Array.isArray(values[0])) {
        return `(${values
          .map(
            (value) =>
              `(${this.parameterize(
                value,
                undefined,
                builder,
                bindingsHolder
              )})`
          )
          .join(', ')})`;
      }
      return `(${this.parameterize(
        values,
        undefined,
        builder,
        bindingsHolder
      )})`;
    }

    if (values && values.isRawInstance) {
      return `(${this.parameter(values, builder, bindingsHolder)})`;
    }

    return this.parameter(values, builder, bindingsHolder);
  }

  processPassedConnection(connection) {
    // Default implementation is noop
  }

  toPathForJson(jsonPath) {
    // By default, we want a json path, so if this function is not overriden,
    // we return the path.
    return jsonPath;
  }
}

Object.assign(Client.prototype, {
  _escapeBinding: makeEscape({
    escapeString(str) {
      return `'${str.replace(/'/g, "''")}'`;
    },
  }),

  canCancelQuery: false,
});

module.exports = Client;