@azure/cosmos
Version:
Microsoft Azure Cosmos DB Service Node.js SDK for NOSQL API
345 lines (344 loc) • 16.4 kB
JavaScript
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var globalPartitionEndpointManager_exports = {};
__export(globalPartitionEndpointManager_exports, {
GlobalPartitionEndpointManager: () => GlobalPartitionEndpointManager
});
module.exports = __toCommonJS(globalPartitionEndpointManager_exports);
var import_common = require("./common/index.js");
var import_index = require("./index.js");
var import_PartitionKeyRangeFailoverInfo = require("./PartitionKeyRangeFailoverInfo.js");
var import_checkURL = require("./utils/checkURL.js");
var import_time = require("./utils/time.js");
var import_typeChecks = require("./utils/typeChecks.js");
class GlobalPartitionEndpointManager {
/**
* @internal
*/
constructor(options, globalEndpointManager) {
this.globalEndpointManager = globalEndpointManager;
this.partitionKeyRangeToLocationForWrite = /* @__PURE__ */ new Map();
this.partitionKeyRangeToLocationForReadAndWrite = /* @__PURE__ */ new Map();
this.preferredLocations = options.connectionPolicy.preferredLocations;
this.preferredLocationsCount = this.preferredLocations ? this.preferredLocations.length : 0;
if (this.globalEndpointManager.lastKnownPPCBEnabled) {
this.initiateCircuitBreakerFailbackLoop();
}
}
partitionKeyRangeToLocationForWrite;
partitionKeyRangeToLocationForReadAndWrite;
preferredLocations;
preferredLocationsCount;
circuitBreakerFailbackBackgroundRefresher;
/**
* Checks eligibility of the request for partition failover and
* tries to mark the endpoint unavailable for the partition key range. Future
* requests will be routed to the next location if available.
*/
async tryPartitionLevelFailover(requestContext, diagnosticNode) {
if (!await this.isRequestEligibleForPartitionFailover(requestContext, true)) {
return false;
}
const isRequestEligibleForPerPartitionAutomaticFailover = this.isRequestEligibleForPerPartitionAutomaticFailover(requestContext);
const isRequestEligibleForPartitionLevelCircuitBreaker = this.isRequestEligibleForPartitionLevelCircuitBreaker(requestContext);
if (isRequestEligibleForPerPartitionAutomaticFailover || isRequestEligibleForPartitionLevelCircuitBreaker && await this.incrementFailureCounterAndCheckFailover(
requestContext,
isRequestEligibleForPerPartitionAutomaticFailover,
isRequestEligibleForPartitionLevelCircuitBreaker
)) {
return this.tryMarkEndpointUnavailableForPartitionKeyRange(
requestContext,
diagnosticNode,
isRequestEligibleForPerPartitionAutomaticFailover,
isRequestEligibleForPartitionLevelCircuitBreaker
);
}
return false;
}
/**
* Updates the DocumentServiceRequest routing location to point
* new a location based if a partition level failover occurred.
*/
async tryAddPartitionLevelLocationOverride(requestContext, diagnosticNode) {
if (!await this.isRequestEligibleForPartitionFailover(requestContext, false)) {
return requestContext;
}
const partitionKeyRangeId = requestContext.partitionKeyRangeId;
if (this.isRequestEligibleForPerPartitionAutomaticFailover(requestContext)) {
if (this.partitionKeyRangeToLocationForWrite.has(partitionKeyRangeId)) {
const partitionFailOver = this.partitionKeyRangeToLocationForWrite.get(partitionKeyRangeId);
requestContext.endpoint = partitionFailOver.getCurrentEndPoint();
diagnosticNode.recordEndpointResolution(requestContext.endpoint);
return requestContext;
}
} else if (this.isRequestEligibleForPartitionLevelCircuitBreaker(requestContext)) {
if (this.partitionKeyRangeToLocationForReadAndWrite.has(partitionKeyRangeId)) {
const partitionFailOver = this.partitionKeyRangeToLocationForReadAndWrite.get(partitionKeyRangeId);
const canCircuitBreakerTriggerPartitionFailOver = await partitionFailOver.canCircuitBreakerTriggerPartitionFailOver(
(0, import_common.isReadRequest)(requestContext.operationType)
);
if (canCircuitBreakerTriggerPartitionFailOver) {
requestContext.endpoint = partitionFailOver.getCurrentEndPoint();
diagnosticNode.recordEndpointResolution(requestContext.endpoint);
return requestContext;
}
}
}
return requestContext;
}
/**
* This method clears the background refresher for circuit breaker failback
* and stops the periodic checks for unhealthy endpoints.
*/
dispose() {
if (this.circuitBreakerFailbackBackgroundRefresher) {
clearTimeout(this.circuitBreakerFailbackBackgroundRefresher);
this.circuitBreakerFailbackBackgroundRefresher = void 0;
}
}
async tryMarkEndpointUnavailableForPartitionKeyRange(requestContext, diagnosticNode, isRequestEligibleForPerPartitionAutomaticFailover, isRequestEligibleForPartitionLevelCircuitBreaker) {
const partitionKeyRangeId = requestContext.partitionKeyRangeId;
const failedEndPoint = requestContext.endpoint;
const readLocations = await this.globalEndpointManager.getReadLocations();
const readEndPoints = [];
if (isRequestEligibleForPerPartitionAutomaticFailover) {
for (const location of readLocations) {
readEndPoints.push(location.databaseAccountEndpoint);
}
return this.tryAddOrUpdatePartitionFailoverInfoAndMoveToNextLocation(
partitionKeyRangeId,
failedEndPoint,
readEndPoints,
this.partitionKeyRangeToLocationForWrite,
diagnosticNode
);
} else if (isRequestEligibleForPartitionLevelCircuitBreaker) {
if (this.preferredLocations && this.preferredLocations.length > 0) {
for (const preferredLocation of this.preferredLocations) {
const location = readLocations.find(
(loc) => (0, import_checkURL.normalizeEndpoint)(loc.name) === (0, import_checkURL.normalizeEndpoint)(preferredLocation)
);
if (location) {
readEndPoints.push(location.databaseAccountEndpoint);
}
}
for (const location of readLocations) {
if (!readEndPoints.includes(location.databaseAccountEndpoint)) {
readEndPoints.push(location.databaseAccountEndpoint);
}
}
} else {
for (const location of readLocations) {
readEndPoints.push(location.databaseAccountEndpoint);
}
}
return this.tryAddOrUpdatePartitionFailoverInfoAndMoveToNextLocation(
partitionKeyRangeId,
failedEndPoint,
readEndPoints,
this.partitionKeyRangeToLocationForReadAndWrite,
diagnosticNode
);
}
return false;
}
/**
* Increments the failure counter for the specified partition and checks if the partition can fail over.
* This method is used to determine if a partition should be failed over based on the number of request failures.
*/
async incrementFailureCounterAndCheckFailover(requestContext, isRequestEligibleForPerPartitionAutomaticFailover, isRequestEligibleForPartitionLevelCircuitBreaker) {
const partitionKeyRangeId = requestContext.partitionKeyRangeId;
const failedEndPoint = requestContext.endpoint;
let partitionKeyRangeFailoverInfo;
if (isRequestEligibleForPerPartitionAutomaticFailover) {
if (!this.partitionKeyRangeToLocationForWrite.has(partitionKeyRangeId)) {
const failoverInfo = new import_PartitionKeyRangeFailoverInfo.PartitionKeyRangeFailoverInfo(failedEndPoint);
this.partitionKeyRangeToLocationForWrite.set(partitionKeyRangeId, failoverInfo);
}
partitionKeyRangeFailoverInfo = this.partitionKeyRangeToLocationForWrite.get(partitionKeyRangeId);
} else if (isRequestEligibleForPartitionLevelCircuitBreaker) {
if (!this.partitionKeyRangeToLocationForReadAndWrite.has(partitionKeyRangeId)) {
const failoverInfo = new import_PartitionKeyRangeFailoverInfo.PartitionKeyRangeFailoverInfo(failedEndPoint);
this.partitionKeyRangeToLocationForReadAndWrite.set(partitionKeyRangeId, failoverInfo);
}
partitionKeyRangeFailoverInfo = this.partitionKeyRangeToLocationForReadAndWrite.get(partitionKeyRangeId);
} else {
return false;
}
(0, import_typeChecks.assertNotUndefined)(
partitionKeyRangeFailoverInfo,
"partitionKeyRangeFailoverInfo should be set if failover flags are true."
);
const currentTimeInMilliseconds = Date.now();
await partitionKeyRangeFailoverInfo.incrementRequestFailureCounts(
(0, import_common.isReadRequest)(requestContext.operationType),
currentTimeInMilliseconds
);
return partitionKeyRangeFailoverInfo.canCircuitBreakerTriggerPartitionFailOver(
(0, import_common.isReadRequest)(requestContext.operationType)
);
}
/** Validates if the given request is eligible for partition failover. */
async isRequestEligibleForPartitionFailover(requestContext, shouldValidateFailedLocation) {
if (!requestContext || !requestContext.operationType || !requestContext.resourceType || !requestContext.partitionKeyRangeId) {
return false;
}
const canUsePartitionLevelFailoverLocations = await this.canUsePartitionLevelFailoverLocations(
requestContext.operationType,
requestContext.resourceType
);
if (!canUsePartitionLevelFailoverLocations) {
return false;
}
if (shouldValidateFailedLocation && !requestContext.endpoint) {
return false;
}
return true;
}
/** Determines if partition level failover locations can be used for the given request. */
async canUsePartitionLevelFailoverLocations(operationType, resourceType) {
const readEndPoints = await this.globalEndpointManager.getReadEndpoints();
if (readEndPoints.length <= 1) {
return false;
}
if (resourceType === import_common.ResourceType.item || resourceType === import_common.ResourceType.sproc && operationType === import_common.OperationType.Execute) {
return true;
}
return false;
}
/**
* Determines if a request is eligible for per-partition automatic failover.
* A request is eligible if it is a write request, partition level failover is enabled,
* and the global endpoint manager cannot use multiple write locations for the request.
*/
isRequestEligibleForPerPartitionAutomaticFailover(requestContext) {
return this.isPartitionLevelAutomaticFailoverEnabled() && !(0, import_common.isReadRequest)(requestContext.operationType) && !this.globalEndpointManager.canUseMultipleWriteLocations(
requestContext.resourceType,
requestContext.operationType
);
}
/**
* Determines if a request is eligible for partition-level circuit breaker.
* This method checks if partition-level circuit breaker is enabled, and if the request is a read-only request or
* the global endpoint manager can use multiple write locations for the request.
*/
isRequestEligibleForPartitionLevelCircuitBreaker(requestContext) {
const enablePartitionLevelCircuitBreaker = this.isPartitionLevelCircuitBreakerEnabled();
if (!enablePartitionLevelCircuitBreaker) {
return false;
}
if ((0, import_common.isReadRequest)(requestContext.operationType)) {
return true;
}
return this.globalEndpointManager.canUseMultipleWriteLocations(
requestContext.resourceType,
requestContext.operationType
);
}
/**
* Attempts to add or update the partition failover information and move to the next available location.
* This method checks if the current location for the partition key range has failed and updates the failover
* information to route the request to the next available location. If all locations have been tried, it removes
* the failover information for the partition key range. Return True if the failover information was successfully
* updated and the request was routed to a new location, otherwise false.
*/
async tryAddOrUpdatePartitionFailoverInfoAndMoveToNextLocation(partitionKeyRangeId, failedEndPoint, nextEndPoints, partitionKeyRangeToLocation, diagnosticNode) {
if (!partitionKeyRangeToLocation.has(partitionKeyRangeId)) {
const failoverInfo = new import_PartitionKeyRangeFailoverInfo.PartitionKeyRangeFailoverInfo(failedEndPoint);
partitionKeyRangeToLocation.set(partitionKeyRangeId, failoverInfo);
}
const partitionFailOver = partitionKeyRangeToLocation.get(partitionKeyRangeId);
if (await partitionFailOver.tryMoveNextLocation(
nextEndPoints,
failedEndPoint,
diagnosticNode,
partitionKeyRangeId
)) {
return true;
}
partitionKeyRangeToLocation.delete(partitionKeyRangeId);
return false;
}
/**
* Initiates a background loop that periodically checks for unhealthy endpoints
* and attempts to open connections to them. If a connection is successfully
* established, it initiates a failback to the original location for the partition key range.
* This is useful for scenarios where a partition key range has been marked as unavailable
* due to a circuit breaker, and we want to periodically check if the original location
* has become healthy again.
* The loop runs at a defined interval specified by Constants.StalePartitionUnavailabilityRefreshIntervalInMs.
*/
initiateCircuitBreakerFailbackLoop() {
this.circuitBreakerFailbackBackgroundRefresher = (0, import_time.startBackgroundTask)(async () => {
try {
await this.openConnectionToUnhealthyEndpointsWithFailback();
} catch (err) {
console.error("Failed to open connection to unhealthy endpoints: ", err);
}
}, import_index.Constants.StalePartitionUnavailabilityRefreshIntervalInMs);
}
/**
* Attempts to open connections to unhealthy endpoints and initiates failback if the connections are successful.
* This method checks the partition key ranges that have failed locations and tries to re-establish connections
* to those locations. If a connection is successfully re-established, it initiates a failback to the original
* location for the partition key range.
*/
async openConnectionToUnhealthyEndpointsWithFailback() {
for (const pkRange of this.partitionKeyRangeToLocationForReadAndWrite.keys()) {
const partitionFailover = this.partitionKeyRangeToLocationForReadAndWrite.get(pkRange);
if (!partitionFailover) continue;
const { firstRequestFailureTime } = await partitionFailover.snapshotPartitionFailoverTimestamps();
const now = /* @__PURE__ */ new Date();
if (now.getTime() - firstRequestFailureTime > import_index.Constants.AllowedPartitionUnavailabilityDurationInMs) {
this.partitionKeyRangeToLocationForReadAndWrite.delete(pkRange);
}
}
}
/**
* @internal
*/
changeCircuitBreakerFailbackLoop(isEnabled) {
if (isEnabled) {
if (!this.circuitBreakerFailbackBackgroundRefresher) {
this.initiateCircuitBreakerFailbackLoop();
}
} else {
if (this.circuitBreakerFailbackBackgroundRefresher) {
this.dispose();
}
}
}
/**
* Gets a value indicating whether per-partition automatic failover is currently enabled.
* @internal
*/
isPartitionLevelAutomaticFailoverEnabled() {
return this.globalEndpointManager.lastKnownPPAFEnabled;
}
/**
* Gets a value indicating whether per-partition automatic failover is currently enabled.
* @internal
*/
isPartitionLevelCircuitBreakerEnabled() {
return this.globalEndpointManager.lastKnownPPCBEnabled;
}
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
GlobalPartitionEndpointManager
});