barracuda-client-api/src/helpers/recoveryPolicy.ts

API Client to connect to Barracuda Enterprise Service Bus

import { IBarracudaClientConsumer, BarracudaConnectionStatus, IBarracudaClient, ConnectionStateChangeHandler, BarracudaQuery } from "../IBarracudaClient";
import { IBarracudaSubscriptionStatus, BarracudaBridgeReadQueryCommands, BarracudaClient } from "../BarracudaClient";
import { isDebug, logDebug, isInfo, logInfo, isWarn, logWarn, shouldLogErrors, logError } from "../logging/index";
import { BarracudaRecoveryPolicy } from "./BarracudaRecoveryPolicy";
import debounce from "lodash/debounce";

const recoveryDelay: number = 5000;

export interface RecoveryStats {
  recoveryCounter: number;
  lastRecoveryStatus: RecoveryProgress;
}

export type OnRecoveryCompleteCallBack = (recoveryStats: RecoveryStats, barracudaClient: IBarracudaClient) => void;

function strRecovery(recoveryPolicy: BarracudaRecoveryPolicy): string {
  if (typeof (recoveryPolicy) === "number") {
    return `${BarracudaRecoveryPolicy[recoveryPolicy]}(${recoveryPolicy})`;
  } else {
    return recoveryPolicy;
  }
}

let reconnectionDetectionCounter: number = 0;

/**
 * Return a clone of the supplied BCjs instance with recovery enabled.
 * Please note all constructor BarracudaConnectionProps is utilized to clone properties across BCjs instances.
 * @param {IBarracudaClientConsumer} originalBC
 * @param {BarracudaRecoveryPolicy} recoveryPolicy
 * @param {number} overrideRecoveryPeriod, The default is 5000 ms to allow a stable connection to be established.
 * @param {OnRecoveryCompleteCallBack} onRecoveryComplete
 * @returns {IBarracudaClientConsumer}
 */
export function withRecoveryPolicy(originalBC: IBarracudaClientConsumer,
                                   recoveryPolicy: BarracudaRecoveryPolicy,
                                   overrideRecoveryPeriod: number = recoveryDelay,
                                   onRecoveryComplete?: OnRecoveryCompleteCallBack): IBarracudaClientConsumer {
  const originalOnConnectionChange = originalBC.onConnectionStateChange;
  
  if (isInfo(originalBC.loglevel)) {
    logInfo(`${originalBC.bcLog(originalBC.lastConnectionProps)} adding recovery policy ${strRecovery(recoveryPolicy)} with ${overrideRecoveryPeriod}ms delay`, {
      connectionCounter: originalBC.connectionCounter,
      defaultConnectionProps: originalBC.defaultConnectionProps,
    });
  }
  
  let recoveryReconnectionCounter: number = 0;
  
  let recoverySubscriptions: IBarracudaSubscriptionStatus[] | undefined;
  const debouncedStartRecovery = debounce((bc: IBarracudaClientConsumer, onCompl?: OnRecoveryCompleteCallBack) => {
    ++recoveryReconnectionCounter;
    
    if (isInfo(bc.loglevel)) {
      logInfo(`${bc.bcLog(bc.lastConnectionProps)} reconnection detected, scheduling recovery ${strRecovery(recoveryPolicy)} in ${overrideRecoveryPeriod}ms`, {
        recoveryReconnectionCounter,
        reconnectionDetectionCounter,
        connectionCounter: bc.connectionCounter,
      });
    }
    
    const recoveryProgressPromise: Promise<RecoveryProgress> = recoverSubscriptions(bc as IBarracudaClientConsumer, recoveryPolicy, recoverySubscriptions as IBarracudaSubscriptionStatus[]);
    return recoveryProgressPromise.then((r: RecoveryProgress) => {
      if (isInfo(bc.loglevel)) {
        logInfo(`${bc.bcLog(bc.lastConnectionProps)} Recovery finished with status: ${RecoveryProgress[r]}(${r})`, {
          recoveryReconnectionCounter,
          reconnectionDetectionCounter,
          connectionCounter: bc.connectionCounter,
        });
      }
      
      switch (r) {
        case RecoveryProgress.none:
        case RecoveryProgress.inProgress:
          return;
        case RecoveryProgress.failedConnectedAckVerification:
          debouncedStartRecovery(bc, onCompl);
          return;
        case RecoveryProgress.partialRecoveryFailure:
        case RecoveryProgress.completedRecovery:
          recoverySubscriptions = undefined;
          break;
        default:
          break;
      }
      
      if (onCompl) {
        try {
          onCompl({
              recoveryCounter: recoveryReconnectionCounter,
              lastRecoveryStatus: r,
            },
            bc,
          );
        } catch (e) {
          if (shouldLogErrors(bc.loglevel)) {
            logError(`UnhandledException error in onRecoveryComplete. Handler should wrap and handle exceptions instead of bubbling to BCjs level.`, e);
          }
        }
      }
    });
  }, overrideRecoveryPeriod);
  
  const newOnConnectionStateChange: ConnectionStateChangeHandler = (state, bc) => {
    if (originalOnConnectionChange) {
      try {
        originalOnConnectionChange(state, bc);
      } catch (e) {
        // ...... original handler should have it's own policy for errors
      }
    }
    
    if (state === BarracudaConnectionStatus.reconnecting && recoveryPolicy !== BarracudaRecoveryPolicy.None) {
      if (!recoverySubscriptions) {
        recoverySubscriptions = (bc as IBarracudaClientConsumer).subscriptions;
        logInfo("recoverySubscriptions ===>", recoverySubscriptions);
      }
      
      ++reconnectionDetectionCounter;
      debouncedStartRecovery(bc as IBarracudaClientConsumer, onRecoveryComplete);
    }
  };
  
  const newBc: BarracudaClient = new BarracudaClient({
    ...originalBC.defaultConnectionProps,
    onConnectionStateChange: newOnConnectionStateChange,
  });
  
  if (isDebug(newBc.loglevel)) {
    logDebug(`${newBc.bcLog(newBc.lastConnectionProps)} new BarraucdaClient with recovery ${recoveryPolicy}`, {defaultConnectionProps: newBc.defaultConnectionProps});
  }
  
  return newBc;
}

export enum RecoveryProgress {
  none,
  inProgress,
  partialRecoveryFailure,
  completedRecovery,
  failedConnectedAckVerification,
}

let recoveryStatus: RecoveryProgress = RecoveryProgress.none;

function assertUnhandled(recoveryPolicy: never): void {
  throw new Error(`Unsupported recovery case ${recoveryPolicy}`);
}

function recoverSubscriptions(bc: IBarracudaClientConsumer, recoveryPolicy: BarracudaRecoveryPolicy, baseSubscriptions: IBarracudaSubscriptionStatus[]): Promise<RecoveryProgress> {
  if (isInfo(bc.loglevel)) {
    logInfo(`${bc.bcLog(bc.lastConnectionProps)} determining which subscriptions to reover ==> ${recoveryPolicy}`, isDebug(bc.loglevel) ? {baseSubscriptions} : undefined);
  }
  
  if (!verifyConnectedAck(bc)) {
    if (isWarn(bc.loglevel)) {
      logWarn(`${bc.bcLog(bc.lastConnectionProps)} failed verifyConnectedAck, skipping recovery.`, {
        connectionStatus: bc.connectionState,
        recoveryPolicy,
        baseSubscriptions,
      });
    }
    return Promise.resolve(RecoveryProgress.failedConnectedAckVerification);
  }
  
  switch (recoveryStatus) {
    case RecoveryProgress.none:
    case RecoveryProgress.partialRecoveryFailure:
    case RecoveryProgress.completedRecovery:
      break;
    case RecoveryProgress.inProgress:
      if (isWarn(bc.loglevel)) {
        logWarn(`${bc.bcLog(bc.lastConnectionProps)} recovery already in progress. Ignoring.`, {recoveryStatus});
      }
      return Promise.resolve(RecoveryProgress.inProgress);
  }
  
  let recoveryPolicyEnum: BarracudaRecoveryPolicy;
  switch (typeof (recoveryPolicy)) {
    case "string":
      recoveryPolicyEnum = BarracudaRecoveryPolicy[recoveryPolicy as keyof typeof BarracudaRecoveryPolicy] as BarracudaRecoveryPolicy;
      break;
    case "number":
      recoveryPolicyEnum = recoveryPolicy as BarracudaRecoveryPolicy;
      break;
  }
  
  // const previousSessionSubscriptions: IBarracudaSubscriptionStatus[] = bc.previousSessionSubscriptions;
  const previousSessionSubscriptions: IBarracudaSubscriptionStatus[] = baseSubscriptions;
  let replaySubscriptions: IBarracudaSubscriptionStatus[] = previousSessionSubscriptions;
  switch (recoveryPolicyEnum) {
    case BarracudaRecoveryPolicy.None:
      logInfo("Skipping recovery, policy set to none");
      return Promise.resolve(RecoveryProgress.none);
    case BarracudaRecoveryPolicy.ResubscribeAllInterruptedSubscriptions:
      replaySubscriptions = previousSessionSubscriptions?.filter(s => !s.unsubscribed && !s.excludeFromRecovery && s.requestCommandType)
        ?.map(s => ({
            ...s,
            requestCommandType: subscribeOnly(s.requestCommandType as BarracudaBridgeReadQueryCommands),
          }
        ));
      break;
    case BarracudaRecoveryPolicy.ReplayAllInterruptedSubscriptions:
      replaySubscriptions = previousSessionSubscriptions?.filter(s => !s.unsubscribed);
      break;
    case BarracudaRecoveryPolicy.ReplayAllRequests:
      replaySubscriptions = previousSessionSubscriptions;
      break;
    default:
      throw new Error(`Unsupported recovery case ${recoveryPolicy} --> ${recoveryPolicyEnum}`);
  }
  
  return executeSubscriptions(replaySubscriptions, bc, recoveryPolicy);
}

function executeSubscriptions(recoverSubcriptions: IBarracudaSubscriptionStatus[], bc: IBarracudaClientConsumer, recoveryPolicy: BarracudaRecoveryPolicy): Promise<RecoveryProgress> {
  if (isInfo(bc.loglevel)) {
    logInfo(`${bc.bcLog(bc.lastConnectionProps)} executeSubscriptions ==> ${recoveryPolicy}`, {
      recoverSubcriptions,
    });
  }
  
  if (!verifyConnectedAck(bc)) {
    if (isWarn(bc.loglevel)) {
      logWarn(`${bc.bcLog(bc.lastConnectionProps)} failed verifyConnectedAck, skipping recovery.`, {
        connectionStatus: bc.connectionState,
        recoveryPolicy,
      });
    }
    return Promise.resolve(RecoveryProgress.failedConnectedAckVerification);
  }
  
  if (!recoverSubcriptions || recoverSubcriptions?.length === 0) {
    if (isDebug(bc.loglevel)) {
      logDebug(`${bc.bcLog(bc.lastConnectionProps)} No subscriptions to recover.`, {
        recoverSubcriptions,
        previousSessionSubscriptions: bc.previousSessionSubscriptions,
      });
    }
    return Promise.resolve(RecoveryProgress.none);
  } else {
    const recoveryOperations = recoverSubcriptions?.filter(sub => sub.requestCommandType && sub.requestBarracudaQuery)
      ?.map(sub => {
        return bc.subscribeQuery<unknown>(sub.requestCommandType as BarracudaBridgeReadQueryCommands, sub.requestBarracudaQuery as BarracudaQuery<unknown>);
      });
    
    const allRecoveryPromises: Promise<RecoveryProgress.completedRecovery | RecoveryProgress.partialRecoveryFailure> = Promise.all(recoveryOperations)
      .then(recoveryOperationResults => {
        if (isInfo(bc.loglevel)) {
          logInfo(`${bc.bcLog(bc.lastConnectionProps)} Executing recovery ${recoveryPolicy} finished`, {recoveryOperationResults});
        }
        recoveryStatus = RecoveryProgress.completedRecovery;
        return recoveryStatus;
      })
      .catch(error => {
        if (shouldLogErrors(bc.loglevel)) {
          logError(`${bc.bcLog(bc.lastConnectionProps)} Error while executing recovery ${recoveryPolicy}`, error);
        }
        recoveryStatus = RecoveryProgress.partialRecoveryFailure;
        return recoveryStatus;
      });
    
    return allRecoveryPromises;
  }
}

function verifyConnectedAck(bc: IBarracudaClient): boolean {
  if (bc.connectionState !== BarracudaConnectionStatus.connectedAck) {
    const errMsg: string = `${bc.bcLog(bc.lastConnectionProps)} verification failed: not in ConnectedAck state [${bc.connectionState}]. Canceling recovery..`;
    if (shouldLogErrors(bc.loglevel)) {
      logError(errMsg);
    }
    return false;
  }
  return true;
}

function subscribeOnly(requestCommandType: BarracudaBridgeReadQueryCommands): BarracudaBridgeReadQueryCommands {
  switch (requestCommandType) {
    case BarracudaBridgeReadQueryCommands.snapshot:
      return BarracudaBridgeReadQueryCommands.snapshot;
    case BarracudaBridgeReadQueryCommands.snapshotAndSubscribe:
    case BarracudaBridgeReadQueryCommands.subscribe:
      return BarracudaBridgeReadQueryCommands.subscribe;
    case BarracudaBridgeReadQueryCommands.snapshotAndDiffSubscribe:
    case BarracudaBridgeReadQueryCommands.diffSubscribe:
      return BarracudaBridgeReadQueryCommands.diffSubscribe;
    default:
      return requestCommandType;
  }
}