mongodb
Version:
The official MongoDB driver for Node.js
830 lines (745 loc) • 26.2 kB
text/typescript
import { clearTimeout, setTimeout } from 'timers';
import type { ObjectId } from '../bson';
import {
APM_EVENTS,
CONNECTION_CHECK_OUT_FAILED,
CONNECTION_CHECK_OUT_STARTED,
CONNECTION_CHECKED_IN,
CONNECTION_CHECKED_OUT,
CONNECTION_CLOSED,
CONNECTION_CREATED,
CONNECTION_POOL_CLEARED,
CONNECTION_POOL_CLOSED,
CONNECTION_POOL_CREATED,
CONNECTION_POOL_READY,
CONNECTION_READY
} from '../constants';
import {
type AnyError,
type MongoError,
MongoInvalidArgumentError,
MongoMissingCredentialsError,
MongoNetworkError,
MongoOperationTimeoutError,
MongoRuntimeError,
MongoServerError
} from '../error';
import { type Abortable, CancellationToken, TypedEventEmitter } from '../mongo_types';
import type { Server } from '../sdam/server';
import { type TimeoutContext, TimeoutError } from '../timeout';
import {
addAbortListener,
type Callback,
kDispose,
List,
makeCounter,
noop,
now,
promiseWithResolvers
} from '../utils';
import { connect } from './connect';
import { Connection, type ConnectionEvents, type ConnectionOptions } from './connection';
import {
ConnectionCheckedInEvent,
ConnectionCheckedOutEvent,
ConnectionCheckOutFailedEvent,
ConnectionCheckOutStartedEvent,
ConnectionClosedEvent,
ConnectionCreatedEvent,
ConnectionPoolClearedEvent,
ConnectionPoolClosedEvent,
ConnectionPoolCreatedEvent,
ConnectionPoolReadyEvent,
ConnectionReadyEvent
} from './connection_pool_events';
import {
PoolClearedError,
PoolClearedOnNetworkError,
PoolClosedError,
WaitQueueTimeoutError
} from './errors';
import { ConnectionPoolMetrics } from './metrics';
/** @public */
export interface ConnectionPoolOptions extends Omit<ConnectionOptions, 'id' | 'generation'> {
/** The maximum number of connections that may be associated with a pool at a given time. This includes in use and available connections. */
maxPoolSize: number;
/** The minimum number of connections that MUST exist at any moment in a single connection pool. */
minPoolSize: number;
/** The maximum number of connections that may be in the process of being established concurrently by the connection pool. */
maxConnecting: number;
/** The maximum amount of time a connection should remain idle in the connection pool before being marked idle. */
maxIdleTimeMS: number;
/** The maximum amount of time operation execution should wait for a connection to become available. The default is 0 which means there is no limit. */
waitQueueTimeoutMS: number;
/** If we are in load balancer mode. */
loadBalanced: boolean;
/** @internal */
minPoolSizeCheckFrequencyMS?: number;
}
/** @internal */
export interface WaitQueueMember {
resolve: (conn: Connection) => void;
reject: (err: AnyError) => void;
cancelled: boolean;
checkoutTime: number;
}
/** @internal */
export const PoolState = Object.freeze({
paused: 'paused',
ready: 'ready',
closed: 'closed'
} as const);
type PoolState = (typeof PoolState)[keyof typeof PoolState];
/**
* @public
* @deprecated This interface is deprecated and will be removed in a future release as it is not used
* in the driver
*/
export interface CloseOptions {
force?: boolean;
}
/** @public */
export type ConnectionPoolEvents = {
connectionPoolCreated(event: ConnectionPoolCreatedEvent): void;
connectionPoolReady(event: ConnectionPoolReadyEvent): void;
connectionPoolClosed(event: ConnectionPoolClosedEvent): void;
connectionPoolCleared(event: ConnectionPoolClearedEvent): void;
connectionCreated(event: ConnectionCreatedEvent): void;
connectionReady(event: ConnectionReadyEvent): void;
connectionClosed(event: ConnectionClosedEvent): void;
connectionCheckOutStarted(event: ConnectionCheckOutStartedEvent): void;
connectionCheckOutFailed(event: ConnectionCheckOutFailedEvent): void;
connectionCheckedOut(event: ConnectionCheckedOutEvent): void;
connectionCheckedIn(event: ConnectionCheckedInEvent): void;
} & Omit<ConnectionEvents, 'close' | 'message'>;
/**
* A pool of connections which dynamically resizes, and emit events related to pool activity
* @internal
*/
export class ConnectionPool extends TypedEventEmitter<ConnectionPoolEvents> {
public options: Readonly<ConnectionPoolOptions>;
/** An integer representing the SDAM generation of the pool */
public generation: number;
/** A map of generations to service ids */
public serviceGenerations: Map<string, number>;
private poolState: PoolState;
private server: Server;
private connections: List<Connection>;
private pending: number;
private checkedOut: Set<Connection>;
private minPoolSizeTimer?: NodeJS.Timeout;
private connectionCounter: Generator<number>;
private cancellationToken: CancellationToken;
private waitQueue: List<WaitQueueMember>;
private metrics: ConnectionPoolMetrics;
private processingWaitQueue: boolean;
/**
* Emitted when the connection pool is created.
* @event
*/
static readonly CONNECTION_POOL_CREATED = CONNECTION_POOL_CREATED;
/**
* Emitted once when the connection pool is closed
* @event
*/
static readonly CONNECTION_POOL_CLOSED = CONNECTION_POOL_CLOSED;
/**
* Emitted each time the connection pool is cleared and it's generation incremented
* @event
*/
static readonly CONNECTION_POOL_CLEARED = CONNECTION_POOL_CLEARED;
/**
* Emitted each time the connection pool is marked ready
* @event
*/
static readonly CONNECTION_POOL_READY = CONNECTION_POOL_READY;
/**
* Emitted when a connection is created.
* @event
*/
static readonly CONNECTION_CREATED = CONNECTION_CREATED;
/**
* Emitted when a connection becomes established, and is ready to use
* @event
*/
static readonly CONNECTION_READY = CONNECTION_READY;
/**
* Emitted when a connection is closed
* @event
*/
static readonly CONNECTION_CLOSED = CONNECTION_CLOSED;
/**
* Emitted when an attempt to check out a connection begins
* @event
*/
static readonly CONNECTION_CHECK_OUT_STARTED = CONNECTION_CHECK_OUT_STARTED;
/**
* Emitted when an attempt to check out a connection fails
* @event
*/
static readonly CONNECTION_CHECK_OUT_FAILED = CONNECTION_CHECK_OUT_FAILED;
/**
* Emitted each time a connection is successfully checked out of the connection pool
* @event
*/
static readonly CONNECTION_CHECKED_OUT = CONNECTION_CHECKED_OUT;
/**
* Emitted each time a connection is successfully checked into the connection pool
* @event
*/
static readonly CONNECTION_CHECKED_IN = CONNECTION_CHECKED_IN;
constructor(server: Server, options: ConnectionPoolOptions) {
super();
this.on('error', noop);
this.options = Object.freeze({
connectionType: Connection,
...options,
maxPoolSize: options.maxPoolSize ?? 100,
minPoolSize: options.minPoolSize ?? 0,
maxConnecting: options.maxConnecting ?? 2,
maxIdleTimeMS: options.maxIdleTimeMS ?? 0,
waitQueueTimeoutMS: options.waitQueueTimeoutMS ?? 0,
minPoolSizeCheckFrequencyMS: options.minPoolSizeCheckFrequencyMS ?? 100,
autoEncrypter: options.autoEncrypter
});
if (this.options.minPoolSize > this.options.maxPoolSize) {
throw new MongoInvalidArgumentError(
'Connection pool minimum size must not be greater than maximum pool size'
);
}
this.poolState = PoolState.paused;
this.server = server;
this.connections = new List();
this.pending = 0;
this.checkedOut = new Set();
this.minPoolSizeTimer = undefined;
this.generation = 0;
this.serviceGenerations = new Map();
this.connectionCounter = makeCounter(1);
this.cancellationToken = new CancellationToken();
this.cancellationToken.setMaxListeners(Infinity);
this.waitQueue = new List();
this.metrics = new ConnectionPoolMetrics();
this.processingWaitQueue = false;
this.mongoLogger = this.server.topology.client?.mongoLogger;
this.component = 'connection';
process.nextTick(() => {
this.emitAndLog(ConnectionPool.CONNECTION_POOL_CREATED, new ConnectionPoolCreatedEvent(this));
});
}
/** The address of the endpoint the pool is connected to */
get address(): string {
return this.options.hostAddress.toString();
}
/**
* Check if the pool has been closed
*
* TODO(NODE-3263): We can remove this property once shell no longer needs it
*/
get closed(): boolean {
return this.poolState === PoolState.closed;
}
/** An integer expressing how many total connections (available + pending + in use) the pool currently has */
get totalConnectionCount(): number {
return (
this.availableConnectionCount + this.pendingConnectionCount + this.currentCheckedOutCount
);
}
/** An integer expressing how many connections are currently available in the pool. */
get availableConnectionCount(): number {
return this.connections.length;
}
get pendingConnectionCount(): number {
return this.pending;
}
get currentCheckedOutCount(): number {
return this.checkedOut.size;
}
get waitQueueSize(): number {
return this.waitQueue.length;
}
get loadBalanced(): boolean {
return this.options.loadBalanced;
}
get serverError() {
return this.server.description.error;
}
/**
* This is exposed ONLY for use in mongosh, to enable
* killing all connections if a user quits the shell with
* operations in progress.
*
* This property may be removed as a part of NODE-3263.
*/
get checkedOutConnections() {
return this.checkedOut;
}
/**
* Get the metrics information for the pool when a wait queue timeout occurs.
*/
private waitQueueErrorMetrics(): string {
return this.metrics.info(this.options.maxPoolSize);
}
/**
* Set the pool state to "ready"
*/
ready(): void {
if (this.poolState !== PoolState.paused) {
return;
}
this.poolState = PoolState.ready;
this.emitAndLog(ConnectionPool.CONNECTION_POOL_READY, new ConnectionPoolReadyEvent(this));
clearTimeout(this.minPoolSizeTimer);
this.ensureMinPoolSize();
}
/**
* Check a connection out of this pool. The connection will continue to be tracked, but no reference to it
* will be held by the pool. This means that if a connection is checked out it MUST be checked back in or
* explicitly destroyed by the new owner.
*/
async checkOut(options: { timeoutContext: TimeoutContext } & Abortable): Promise<Connection> {
const checkoutTime = now();
this.emitAndLog(
ConnectionPool.CONNECTION_CHECK_OUT_STARTED,
new ConnectionCheckOutStartedEvent(this)
);
const { promise, resolve, reject } = promiseWithResolvers<Connection>();
const timeout = options.timeoutContext.connectionCheckoutTimeout;
const waitQueueMember: WaitQueueMember = {
resolve,
reject,
cancelled: false,
checkoutTime
};
const abortListener = addAbortListener(options.signal, function () {
waitQueueMember.cancelled = true;
reject(this.reason);
});
this.waitQueue.push(waitQueueMember);
process.nextTick(() => this.processWaitQueue());
try {
timeout?.throwIfExpired();
return await (timeout ? Promise.race([promise, timeout]) : promise);
} catch (error) {
if (TimeoutError.is(error)) {
timeout?.clear();
waitQueueMember.cancelled = true;
this.emitAndLog(
ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
new ConnectionCheckOutFailedEvent(this, 'timeout', waitQueueMember.checkoutTime)
);
const timeoutError = new WaitQueueTimeoutError(
this.loadBalanced
? this.waitQueueErrorMetrics()
: 'Timed out while checking out a connection from connection pool',
this.address
);
if (options.timeoutContext.csotEnabled()) {
throw new MongoOperationTimeoutError('Timed out during connection checkout', {
cause: timeoutError
});
}
throw timeoutError;
}
throw error;
} finally {
abortListener?.[kDispose]();
timeout?.clear();
}
}
/**
* Check a connection into the pool.
*
* @param connection - The connection to check in
*/
checkIn(connection: Connection): void {
if (!this.checkedOut.has(connection)) {
return;
}
const poolClosed = this.closed;
const stale = this.connectionIsStale(connection);
const willDestroy = !!(poolClosed || stale || connection.closed);
if (!willDestroy) {
connection.markAvailable();
this.connections.unshift(connection);
}
this.checkedOut.delete(connection);
this.emitAndLog(
ConnectionPool.CONNECTION_CHECKED_IN,
new ConnectionCheckedInEvent(this, connection)
);
if (willDestroy) {
const reason = connection.closed ? 'error' : poolClosed ? 'poolClosed' : 'stale';
this.destroyConnection(connection, reason);
}
process.nextTick(() => this.processWaitQueue());
}
/**
* Clear the pool
*
* Pool reset is handled by incrementing the pool's generation count. Any existing connection of a
* previous generation will eventually be pruned during subsequent checkouts.
*/
clear(options: { serviceId?: ObjectId; interruptInUseConnections?: boolean } = {}): void {
if (this.closed) {
return;
}
// handle load balanced case
if (this.loadBalanced) {
const { serviceId } = options;
if (!serviceId) {
throw new MongoRuntimeError(
'ConnectionPool.clear() called in load balanced mode with no serviceId.'
);
}
const sid = serviceId.toHexString();
const generation = this.serviceGenerations.get(sid);
// Only need to worry if the generation exists, since it should
// always be there but typescript needs the check.
if (generation == null) {
throw new MongoRuntimeError('Service generations are required in load balancer mode.');
} else {
// Increment the generation for the service id.
this.serviceGenerations.set(sid, generation + 1);
}
this.emitAndLog(
ConnectionPool.CONNECTION_POOL_CLEARED,
new ConnectionPoolClearedEvent(this, { serviceId })
);
return;
}
// handle non load-balanced case
const interruptInUseConnections = options.interruptInUseConnections ?? false;
const oldGeneration = this.generation;
this.generation += 1;
const alreadyPaused = this.poolState === PoolState.paused;
this.poolState = PoolState.paused;
this.clearMinPoolSizeTimer();
if (!alreadyPaused) {
this.emitAndLog(
ConnectionPool.CONNECTION_POOL_CLEARED,
new ConnectionPoolClearedEvent(this, {
interruptInUseConnections
})
);
}
if (interruptInUseConnections) {
process.nextTick(() => this.interruptInUseConnections(oldGeneration));
}
this.processWaitQueue();
}
/**
* Closes all stale in-use connections in the pool with a resumable PoolClearedOnNetworkError.
*
* Only connections where `connection.generation <= minGeneration` are killed.
*/
private interruptInUseConnections(minGeneration: number) {
for (const connection of this.checkedOut) {
if (connection.generation <= minGeneration) {
connection.onError(new PoolClearedOnNetworkError(this));
this.checkIn(connection);
}
}
}
/** Close the pool */
close(): void {
if (this.closed) {
return;
}
// immediately cancel any in-flight connections
this.cancellationToken.emit('cancel');
// end the connection counter
if (typeof this.connectionCounter.return === 'function') {
this.connectionCounter.return(undefined);
}
this.poolState = PoolState.closed;
this.clearMinPoolSizeTimer();
this.processWaitQueue();
for (const conn of this.connections) {
this.emitAndLog(
ConnectionPool.CONNECTION_CLOSED,
new ConnectionClosedEvent(this, conn, 'poolClosed')
);
conn.destroy();
}
this.connections.clear();
this.emitAndLog(ConnectionPool.CONNECTION_POOL_CLOSED, new ConnectionPoolClosedEvent(this));
}
/**
* @internal
* Reauthenticate a connection
*/
async reauthenticate(connection: Connection): Promise<void> {
const authContext = connection.authContext;
if (!authContext) {
throw new MongoRuntimeError('No auth context found on connection.');
}
const credentials = authContext.credentials;
if (!credentials) {
throw new MongoMissingCredentialsError(
'Connection is missing credentials when asked to reauthenticate'
);
}
const resolvedCredentials = credentials.resolveAuthMechanism(connection.hello);
const provider = this.server.topology.client.s.authProviders.getOrCreateProvider(
resolvedCredentials.mechanism,
resolvedCredentials.mechanismProperties
);
if (!provider) {
throw new MongoMissingCredentialsError(
`Reauthenticate failed due to no auth provider for ${credentials.mechanism}`
);
}
await provider.reauth(authContext);
return;
}
/** Clear the min pool size timer */
private clearMinPoolSizeTimer(): void {
const minPoolSizeTimer = this.minPoolSizeTimer;
if (minPoolSizeTimer) {
clearTimeout(minPoolSizeTimer);
}
}
private destroyConnection(
connection: Connection,
reason: 'error' | 'idle' | 'stale' | 'poolClosed'
) {
this.emitAndLog(
ConnectionPool.CONNECTION_CLOSED,
new ConnectionClosedEvent(this, connection, reason)
);
// destroy the connection
connection.destroy();
}
private connectionIsStale(connection: Connection) {
const serviceId = connection.serviceId;
if (this.loadBalanced && serviceId) {
const sid = serviceId.toHexString();
const generation = this.serviceGenerations.get(sid);
return connection.generation !== generation;
}
return connection.generation !== this.generation;
}
private connectionIsIdle(connection: Connection) {
return !!(this.options.maxIdleTimeMS && connection.idleTime > this.options.maxIdleTimeMS);
}
/**
* Destroys a connection if the connection is perished.
*
* @returns `true` if the connection was destroyed, `false` otherwise.
*/
private destroyConnectionIfPerished(connection: Connection): boolean {
const isStale = this.connectionIsStale(connection);
const isIdle = this.connectionIsIdle(connection);
if (!isStale && !isIdle && !connection.closed) {
return false;
}
const reason = connection.closed ? 'error' : isStale ? 'stale' : 'idle';
this.destroyConnection(connection, reason);
return true;
}
private createConnection(callback: Callback<Connection>) {
const connectOptions: ConnectionOptions = {
...this.options,
id: this.connectionCounter.next().value,
generation: this.generation,
cancellationToken: this.cancellationToken,
mongoLogger: this.mongoLogger,
authProviders: this.server.topology.client.s.authProviders
};
this.pending++;
// This is our version of a "virtual" no-I/O connection as the spec requires
const connectionCreatedTime = now();
this.emitAndLog(
ConnectionPool.CONNECTION_CREATED,
new ConnectionCreatedEvent(this, { id: connectOptions.id })
);
connect(connectOptions).then(
connection => {
// The pool might have closed since we started trying to create a connection
if (this.poolState !== PoolState.ready) {
this.pending--;
connection.destroy();
callback(this.closed ? new PoolClosedError(this) : new PoolClearedError(this));
return;
}
// forward all events from the connection to the pool
for (const event of [...APM_EVENTS, Connection.CLUSTER_TIME_RECEIVED]) {
connection.on(event, (e: any) => this.emit(event, e));
}
if (this.loadBalanced) {
connection.on(Connection.PINNED, pinType => this.metrics.markPinned(pinType));
connection.on(Connection.UNPINNED, pinType => this.metrics.markUnpinned(pinType));
const serviceId = connection.serviceId;
if (serviceId) {
let generation;
const sid = serviceId.toHexString();
if ((generation = this.serviceGenerations.get(sid))) {
connection.generation = generation;
} else {
this.serviceGenerations.set(sid, 0);
connection.generation = 0;
}
}
}
connection.markAvailable();
this.emitAndLog(
ConnectionPool.CONNECTION_READY,
new ConnectionReadyEvent(this, connection, connectionCreatedTime)
);
this.pending--;
callback(undefined, connection);
},
error => {
this.pending--;
this.server.handleError(error);
this.emitAndLog(
ConnectionPool.CONNECTION_CLOSED,
new ConnectionClosedEvent(
this,
{ id: connectOptions.id, serviceId: undefined },
'error',
// TODO(NODE-5192): Remove this cast
error as MongoError
)
);
if (error instanceof MongoNetworkError || error instanceof MongoServerError) {
error.connectionGeneration = connectOptions.generation;
}
callback(error ?? new MongoRuntimeError('Connection creation failed without error'));
}
);
}
private ensureMinPoolSize() {
const minPoolSize = this.options.minPoolSize;
if (this.poolState !== PoolState.ready || minPoolSize === 0) {
return;
}
this.connections.prune(connection => this.destroyConnectionIfPerished(connection));
if (
this.totalConnectionCount < minPoolSize &&
this.pendingConnectionCount < this.options.maxConnecting
) {
// NOTE: ensureMinPoolSize should not try to get all the pending
// connection permits because that potentially delays the availability of
// the connection to a checkout request
this.createConnection((err, connection) => {
if (!err && connection) {
this.connections.push(connection);
process.nextTick(() => this.processWaitQueue());
}
if (this.poolState === PoolState.ready) {
clearTimeout(this.minPoolSizeTimer);
this.minPoolSizeTimer = setTimeout(
() => this.ensureMinPoolSize(),
this.options.minPoolSizeCheckFrequencyMS
);
}
});
} else {
clearTimeout(this.minPoolSizeTimer);
this.minPoolSizeTimer = setTimeout(
() => this.ensureMinPoolSize(),
this.options.minPoolSizeCheckFrequencyMS
);
}
}
private processWaitQueue() {
if (this.processingWaitQueue) {
return;
}
this.processingWaitQueue = true;
while (this.waitQueueSize) {
const waitQueueMember = this.waitQueue.first();
if (!waitQueueMember) {
this.waitQueue.shift();
continue;
}
if (waitQueueMember.cancelled) {
this.waitQueue.shift();
continue;
}
if (this.poolState !== PoolState.ready) {
const reason = this.closed ? 'poolClosed' : 'connectionError';
const error = this.closed ? new PoolClosedError(this) : new PoolClearedError(this);
this.emitAndLog(
ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
new ConnectionCheckOutFailedEvent(this, reason, waitQueueMember.checkoutTime, error)
);
this.waitQueue.shift();
waitQueueMember.reject(error);
continue;
}
if (!this.availableConnectionCount) {
break;
}
const connection = this.connections.shift();
if (!connection) {
break;
}
if (!this.destroyConnectionIfPerished(connection)) {
this.checkedOut.add(connection);
this.emitAndLog(
ConnectionPool.CONNECTION_CHECKED_OUT,
new ConnectionCheckedOutEvent(this, connection, waitQueueMember.checkoutTime)
);
this.waitQueue.shift();
waitQueueMember.resolve(connection);
}
}
const { maxPoolSize, maxConnecting } = this.options;
while (
this.waitQueueSize > 0 &&
this.pendingConnectionCount < maxConnecting &&
(maxPoolSize === 0 || this.totalConnectionCount < maxPoolSize)
) {
const waitQueueMember = this.waitQueue.shift();
if (!waitQueueMember || waitQueueMember.cancelled) {
continue;
}
this.createConnection((err, connection) => {
if (waitQueueMember.cancelled) {
if (!err && connection) {
this.connections.push(connection);
}
} else {
if (err) {
this.emitAndLog(
ConnectionPool.CONNECTION_CHECK_OUT_FAILED,
// TODO(NODE-5192): Remove this cast
new ConnectionCheckOutFailedEvent(
this,
'connectionError',
waitQueueMember.checkoutTime,
err as MongoError
)
);
waitQueueMember.reject(err);
} else if (connection) {
this.checkedOut.add(connection);
this.emitAndLog(
ConnectionPool.CONNECTION_CHECKED_OUT,
new ConnectionCheckedOutEvent(this, connection, waitQueueMember.checkoutTime)
);
waitQueueMember.resolve(connection);
}
}
process.nextTick(() => this.processWaitQueue());
});
}
this.processingWaitQueue = false;
}
}
/**
* A callback provided to `withConnection`
* @internal
*
* @param error - An error instance representing the error during the execution.
* @param connection - The managed connection which was checked out of the pool.
* @param callback - A function to call back after connection management is complete
*/
export type WithConnectionCallback = (
error: MongoError | undefined,
connection: Connection | undefined,
callback: Callback<Connection>
) => void;