envio/src/sources/SourceManager.res.mjs

A latency and sync speed optimized, developer friendly blockchain data indexer.

// Generated by ReScript, PLEASE EDIT WITH CARE

import * as Utils from "../Utils.res.mjs";
import * as Hrtime from "../bindings/Hrtime.res.mjs";
import * as Source from "./Source.res.mjs";
import * as Logging from "../Logging.res.mjs";
import * as FetchState from "../FetchState.res.mjs";
import * as Prometheus from "../Prometheus.res.mjs";
import * as ErrorHandling from "../ErrorHandling.res.mjs";
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";

function getActiveSource(sourceManager) {
  return sourceManager.activeSource;
}

function getRateLimitTimeMs(sourceManager) {
  let startMs = sourceManager.activeRateLimitStartMs;
  return sourceManager.committedRateLimitTimeMs + (
    startMs !== undefined ? Date.now() - startMs : 0.0
  );
}

function isRateLimited(sourceManager) {
  return Stdlib_Option.isSome(sourceManager.activeRateLimitStartMs);
}

function getRateLimitResetInMs(sourceManager) {
  let resetAt = sourceManager.activeRateLimitResetAtMs;
  if (resetAt === undefined) {
    return;
  }
  let remaining = resetAt - Date.now();
  if (remaining > 0.0) {
    return remaining;
  }
}

function startRateLimitTimeout(sourceManager, resetMs) {
  let now = Date.now();
  if (sourceManager.rateLimitWaiters === 0) {
    sourceManager.activeRateLimitStartMs = now;
  }
  let resetAt = now + resetMs;
  let existing = sourceManager.activeRateLimitResetAtMs;
  sourceManager.activeRateLimitResetAtMs = existing !== undefined ? Primitive_float.max(existing, resetAt) : resetAt;
  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters + 1 | 0;
}

async function waitForRateLimitReset(sourceManager, resetMs, retry, logger) {
  let waitMs = Primitive_int.min(resetMs, 300000);
  let log = retry >= 2 ? Logging.childWarn : Logging.childTrace;
  log(logger, {
    msg: `HyperSync source is rate-limited — not critical, the indexer will retry in ` + (waitMs / 1000 | 0).toString() + `s. For higher limits upgrade your plan at https://envio.dev/app/api-tokens.`,
    retry: retry,
    waitMs: waitMs
  });
  startRateLimitTimeout(sourceManager, waitMs);
  await Utils.delay(waitMs);
  sourceManager.rateLimitWaiters = sourceManager.rateLimitWaiters - 1 | 0;
  if (sourceManager.rateLimitWaiters !== 0) {
    return;
  }
  let startMs = sourceManager.activeRateLimitStartMs;
  if (startMs !== undefined) {
    sourceManager.committedRateLimitTimeMs = sourceManager.committedRateLimitTimeMs + Date.now() - startMs;
    sourceManager.activeRateLimitStartMs = undefined;
  }
  sourceManager.activeRateLimitResetAtMs = undefined;
}

function onReorg(sourceManager, rollbackTargetBlock) {
  sourceManager.sourcesState.forEach(param => {
    let cb = param.source.onReorg;
    if (cb !== undefined) {
      return cb(rollbackTargetBlock);
    }
  });
}

function getSourceRole(sourceFor, isRealtime, hasRealtime) {
  if (isRealtime) {
    switch (sourceFor) {
      case "Sync" :
        if (hasRealtime) {
          return "Secondary";
        } else {
          return "Primary";
        }
      case "Fallback" :
        return "Secondary";
      case "Realtime" :
        return "Primary";
    }
  } else {
    switch (sourceFor) {
      case "Sync" :
        return "Primary";
      case "Fallback" :
        return "Secondary";
      case "Realtime" :
        return;
    }
  }
}

function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative, maxRetryInterval) {
  return retry => {
    let backoff = retry === 0 ? 1 : retry * backoffMultiplicative | 0;
    return Primitive_int.min(initialRetryInterval * backoff | 0, maxRetryInterval);
  };
}

function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
  let newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
  let newBlockStallTimeoutRealtime = newBlockStallTimeoutRealtimeOpt !== undefined ? newBlockStallTimeoutRealtimeOpt : 20000;
  let stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
  let reducedPollingInterval = reducedPollingIntervalOpt !== undefined ? reducedPollingIntervalOpt : 60000;
  let recoveryTimeout = recoveryTimeoutOpt !== undefined ? recoveryTimeoutOpt : 60000.0;
  let getHeightRetryInterval = getHeightRetryIntervalOpt !== undefined ? getHeightRetryIntervalOpt : makeGetHeightRetryInterval(1000, 2, 60000);
  let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
  let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
  let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
  Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
  Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
  return {
    sourcesState: sources.map(source => ({
      source: source,
      knownHeight: 0,
      unsubscribe: undefined,
      pendingHeightResolvers: [],
      disabled: false,
      lastFailedAt: undefined
    })),
    statusStart: Hrtime.makeTimer(),
    status: "Idle",
    maxPartitionConcurrency: maxPartitionConcurrency,
    newBlockStallTimeout: newBlockStallTimeout,
    newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
    stalledPollingInterval: stalledPollingInterval,
    reducedPollingInterval: reducedPollingInterval,
    getHeightRetryInterval: getHeightRetryInterval,
    activeSource: initialActiveSource,
    waitingForNewBlockStateId: undefined,
    fetchingPartitionsCount: 0,
    recoveryTimeout: recoveryTimeout,
    hasRealtime: hasRealtime,
    committedRateLimitTimeMs: 0.0,
    rateLimitWaiters: 0,
    activeRateLimitStartMs: undefined,
    activeRateLimitResetAtMs: undefined
  };
}

function trackNewStatus(sourceManager, newStatus) {
  let match = sourceManager.status;
  let promCounter;
  switch (match) {
    case "Idle" :
      promCounter = Prometheus.IndexingIdleTime.counter;
      break;
    case "WaitingForNewBlock" :
      promCounter = Prometheus.IndexingSourceWaitingTime.counter;
      break;
    case "Querieng" :
      promCounter = Prometheus.IndexingQueryTime.counter;
      break;
  }
  Prometheus.SafeCounter.handleFloat(promCounter, sourceManager.activeSource.chain, Hrtime.toSecondsFloat(Hrtime.timeSince(sourceManager.statusStart)));
  sourceManager.statusStart = Hrtime.makeTimer();
  sourceManager.status = newStatus;
}

async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
  let nextQuery = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0);
  if (typeof nextQuery !== "object") {
    switch (nextQuery) {
      case "WaitingForNewBlock" :
        let waitingStateId = sourceManager.waitingForNewBlockStateId;
        if (waitingStateId !== undefined && waitingStateId >= stateId) {
          return;
        }
        trackNewStatus(sourceManager, "WaitingForNewBlock");
        sourceManager.waitingForNewBlockStateId = stateId;
        let knownHeight = await waitForNewBlock(fetchState.knownHeight);
        let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
        if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
          trackNewStatus(sourceManager, "Idle");
          sourceManager.waitingForNewBlockStateId = undefined;
          return onNewBlock(knownHeight);
        } else {
          return;
        }
      case "ReachedMaxConcurrency" :
      case "NothingToQuery" :
        return;
    }
  } else {
    let queries = nextQuery._0;
    FetchState.startFetchingQueries(fetchState, queries);
    sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
    Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
    trackNewStatus(sourceManager, "Querieng");
    await Promise.all(queries.map(q => {
      let promise = executeQuery(q);
      promise.then(param => {
        sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
        Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
        if (sourceManager.fetchingPartitionsCount === 0) {
          return trackNewStatus(sourceManager, "Idle");
        }
      });
      return promise;
    }));
    return;
  }
}

function disableSource(sourceManager, sourceState) {
  if (sourceState.disabled) {
    return false;
  }
  sourceState.disabled = true;
  let unsubscribe = sourceState.unsubscribe;
  if (unsubscribe !== undefined) {
    unsubscribe();
  }
  if (sourceState.source.sourceFor === "Realtime") {
    let hasOtherRealtime = sourceManager.sourcesState.some(s => {
      if (s !== sourceState && !s.disabled) {
        return s.source.sourceFor === "Realtime";
      } else {
        return false;
      }
    });
    sourceManager.hasRealtime = hasOtherRealtime;
  }
  return true;
}

async function getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isRealtime, status, logger, reducedPolling) {
  let source = sourceState.source;
  let initialHeight = sourceState.knownHeight;
  let newHeight = {
    contents: initialHeight
  };
  let retry = 0;
  while (newHeight.contents <= knownHeight && status.contents !== "Done") {
    let match = sourceState.unsubscribe;
    if (match !== undefined) {
      let subscriptionPromise = new Promise((resolve, _reject) => {
        sourceState.pendingHeightResolvers.push(resolve);
      });
      let half = stallTimeout / 2 | 0;
      let pollingFallback = Utils.delay(half + (Math.random() * half | 0) | 0).then(async () => {
        Logging.childTrace(logger, {
          msg: "onHeight subscription stale, switching to polling fallback",
          source: source.name,
          chainId: source.chain
        });
        let h = initialHeight;
        while (h <= knownHeight && newHeight.contents <= initialHeight) {
          try {
            h = await source.getHeightOrThrow();
          } catch (exn) {
            
          }
          if (h <= knownHeight && newHeight.contents <= initialHeight) {
            await Utils.delay(source.pollingInterval);
          }
        };
        return h;
      });
      let height = await Promise.race([
        subscriptionPromise,
        pollingFallback
      ]);
      if (height > initialHeight) {
        newHeight.contents = height;
      }
    } else {
      try {
        let height$1 = await source.getHeightOrThrow();
        newHeight.contents = height$1;
        if (height$1 <= knownHeight) {
          retry = 0;
          let createSubscription = source.createHeightSubscription;
          let exit = 0;
          if (createSubscription !== undefined && isRealtime) {
            let unsubscribe = createSubscription(newHeight => {
              if (newHeight <= sourceState.knownHeight) {
                return;
              }
              sourceState.knownHeight = newHeight;
              let resolvers = sourceState.pendingHeightResolvers;
              sourceState.pendingHeightResolvers = [];
              resolvers.forEach(resolve => resolve(newHeight));
            });
            sourceState.unsubscribe = unsubscribe;
          } else {
            exit = 1;
          }
          if (exit === 1) {
            let pollingInterval = reducedPolling ? sourceManager.reducedPollingInterval : (
                status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval
              );
            await Utils.delay(pollingInterval);
          }
        }
      } catch (raw_exn) {
        let exn = Primitive_exceptions.internalToException(raw_exn);
        let retryInterval = sourceManager.getHeightRetryInterval(retry);
        Logging.childTrace(logger, {
          msg: `Height retrieval from ` + source.name + ` source failed. Retrying in ` + retryInterval.toString() + `ms.`,
          source: source.name,
          err: Utils.prettifyExn(exn)
        });
        retry = retry + 1 | 0;
        await Utils.delay(retryInterval);
      }
    }
  };
  if (newHeight.contents > initialHeight) {
    Prometheus.SourceHeight.set(source.name, source.chain, newHeight.contents);
  }
  return newHeight.contents;
}

function compareByOldestFailure(a, b) {
  let match = a.lastFailedAt;
  let match$1 = b.lastFailedAt;
  if (match !== undefined) {
    if (match$1 !== undefined) {
      if (match < match$1) {
        return -1;
      } else if (match > match$1) {
        return 1;
      } else {
        return 0;
      }
    } else {
      return 1;
    }
  } else if (match$1 !== undefined) {
    return -1;
  } else {
    return 0;
  }
}

function getNextSources(sourceManager, isRealtime, excludedSources) {
  let now = Date.now();
  let workingPrimarySources = [];
  let allPrimarySources = [];
  let workingSecondarySources = [];
  for (let i = 0, i_finish = sourceManager.sourcesState.length; i < i_finish; ++i) {
    let sourceState = sourceManager.sourcesState[i];
    if (!sourceState.disabled) {
      let isExcluded = excludedSources !== undefined ? Primitive_option.valFromOption(excludedSources).has(sourceState) : false;
      if (!isExcluded) {
        let failedAt = sourceState.lastFailedAt;
        let isWorking = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
        let match = getSourceRole(sourceState.source.sourceFor, isRealtime, sourceManager.hasRealtime);
        if (match !== undefined) {
          if (match === "Primary") {
            allPrimarySources.push(sourceState);
            if (isWorking) {
              workingPrimarySources.push(sourceState);
            }
          } else if (isWorking) {
            workingSecondarySources.push(sourceState);
          }
        }
      }
    }
  }
  if (workingPrimarySources.length !== 0) {
    return workingPrimarySources;
  } else if (workingSecondarySources.length !== 0) {
    return workingSecondarySources;
  } else {
    allPrimarySources.sort(compareByOldestFailure);
    return allPrimarySources;
  }
}

function getNextSource(sourceManager, isRealtime, excludedSources) {
  let sources = getNextSources(sourceManager, isRealtime, excludedSources);
  let first = sources[0];
  if (first === undefined) {
    return;
  }
  if (first.source === sourceManager.activeSource) {
    return first;
  }
  let result = sources.find(s => s.source === sourceManager.activeSource);
  if (result !== undefined) {
    return result;
  } else {
    return sources[0];
  }
}

async function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling) {
  let sourcesState = sourceManager.sourcesState;
  let logger = Logging.createChild({
    chainId: sourceManager.activeSource.chain,
    knownHeight: knownHeight
  });
  if (reducedPolling) {
    Logging.childTrace(logger, `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s). Chain is caught up, waiting for other chains to backfill.`);
  } else {
    Logging.childTrace(logger, "Initiating check for new blocks.");
  }
  let mainSources = getNextSources(sourceManager, isRealtime, undefined);
  let status = {
    contents: "Active"
  };
  let stallTimeout = reducedPolling ? (sourceManager.reducedPollingInterval << 1) : (
      isRealtime ? sourceManager.newBlockStallTimeoutRealtime : sourceManager.newBlockStallTimeout
    );
  let match = await Promise.race(mainSources.map(async sourceState => [
    sourceState.source,
    await getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isRealtime, status, logger, reducedPolling)
  ]).concat([Utils.delay(stallTimeout).then(() => {
      let fallbackSources = [];
      sourcesState.forEach(sourceState => {
        if (!sourceState.disabled && !mainSources.includes(sourceState) && Stdlib_Option.isSome(getSourceRole(sourceState.source.sourceFor, isRealtime, sourceManager.hasRealtime))) {
          fallbackSources.push(sourceState);
          return;
        }
      });
      if (status.contents !== "Done") {
        status.contents = "Stalled";
        if (fallbackSources.length !== 0) {
          Logging.childWarn(logger, `No new blocks detected within ` + (stallTimeout / 1000 | 0).toString() + `s. Continuing polling with secondary RPC sources from the configuration.`);
        } else {
          Logging.childWarn(logger, `No new blocks detected within ` + (stallTimeout / 1000 | 0).toString() + `s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync`);
        }
      }
      return Promise.race(fallbackSources.map(async sourceState => [
        sourceState.source,
        await getSourceNewHeight(sourceManager, sourceState, knownHeight, stallTimeout, isRealtime, status, logger, reducedPolling)
      ]));
    })]));
  let newBlockHeight = match[1];
  let source = match[0];
  sourceManager.activeSource = source;
  let log = status.contents === "Stalled" ? Logging.childInfo : Logging.childTrace;
  log(logger, {
    msg: `New blocks successfully found.`,
    source: source.name,
    newBlockHeight: newBlockHeight
  });
  status.contents = "Done";
  return newBlockHeight;
}

async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
  let excludedSourcesRef;
  let toBlockRef = query.toBlock;
  let responseRef;
  let retryRef = 0;
  while (Stdlib_Option.isNone(responseRef)) {
    let s = getNextSource(sourceManager, isRealtime, excludedSourcesRef);
    let sourceState;
    if (s !== undefined) {
      if (s.source !== sourceManager.activeSource) {
        let logger = Logging.createChild({
          chainId: sourceManager.activeSource.chain
        });
        Logging.childInfo(logger, {
          msg: "Switching data-source",
          source: s.source.name,
          previousSource: sourceManager.activeSource.name,
          fromBlock: query.fromBlock
        });
      }
      sourceState = s;
    } else {
      let logger$1 = Logging.createChild({
        chainId: sourceManager.activeSource.chain
      });
      sourceState = ErrorHandling.mkLogAndRaise(logger$1, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
    }
    sourceManager.activeSource = sourceState.source;
    let source = sourceState.source;
    let toBlock = toBlockRef;
    let retry = retryRef;
    let logger$2 = Logging.createChild({
      chainId: source.chain,
      logType: "Block Range Query",
      partitionId: query.partitionId,
      source: source.name,
      fromBlock: query.fromBlock,
      toBlock: toBlock,
      addresses: FetchState.addressesByContractNameCount(query.addressesByContractName),
      retry: retry
    });
    try {
      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingAddresses, knownHeight, query.partitionId, query.selection, retry, logger$2);
      Logging.childTrace(logger$2, {
        msg: "Fetched block range from server",
        toBlock: response.latestFetchedBlockNumber,
        numEvents: response.parsedQueueItems.length,
        stats: response.stats
      });
      sourceState.lastFailedAt = undefined;
      responseRef = response;
    } catch (raw_error) {
      let error = Primitive_exceptions.internalToException(raw_error);
      if (error.RE_EXN_ID === Source.RateLimited) {
        await waitForRateLimitReset(sourceManager, error.resetMs, retry, logger$2);
        retryRef = retryRef + 1 | 0;
      } else if (error.RE_EXN_ID === Source.GetItemsError) {
        let error$1 = error._1;
        let exit = 0;
        switch (error$1.TAG) {
          case "UnsupportedSelection" :
          case "FailedGettingFieldSelection" :
            exit = 1;
            break;
          case "FailedGettingItems" :
            let match = error$1.retry;
            let attemptedToBlock = error$1.attemptedToBlock;
            let exn = error$1.exn;
            switch (match.TAG) {
              case "WithSuggestedToBlock" :
                let toBlock$1 = match.toBlock;
                Logging.childTrace(logger$2, {
                  msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
                  toBlock: attemptedToBlock,
                  suggestedToBlock: toBlock$1
                });
                toBlockRef = toBlock$1;
                retryRef = 0;
                break;
              case "WithBackoff" :
                let backoffMillis = match.backoffMillis;
                let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
                log(logger$2, {
                  msg: match.message,
                  toBlock: attemptedToBlock,
                  backOffMilliseconds: backoffMillis,
                  retry: retry,
                  err: Utils.prettifyExn(exn)
                });
                let shouldSwitch = retry !== 0 && retry !== 1 ? retry % 2 === 0 : false;
                if (shouldSwitch) {
                  let now = Date.now();
                  sourceState.lastFailedAt = now;
                  let nextSource = getNextSource(sourceManager, isRealtime, excludedSourcesRef);
                  let hasWorkingAlternative;
                  if (nextSource !== undefined) {
                    let failedAt = nextSource.lastFailedAt;
                    hasWorkingAlternative = failedAt !== undefined ? now - failedAt >= sourceManager.recoveryTimeout : true;
                  } else {
                    hasWorkingAlternative = false;
                  }
                  if (!hasWorkingAlternative) {
                    await Utils.delay(Primitive_int.min(backoffMillis, 60000));
                  }
                } else {
                  await Utils.delay(Primitive_int.min(backoffMillis, 60000));
                }
                retryRef = retryRef + 1 | 0;
                break;
              case "ImpossibleForTheQuery" :
                let s$1 = excludedSourcesRef;
                let excludedSources;
                if (s$1 !== undefined) {
                  excludedSources = Primitive_option.valFromOption(s$1);
                } else {
                  let s$2 = new Set();
                  excludedSourcesRef = Primitive_option.some(s$2);
                  excludedSources = s$2;
                }
                excludedSources.add(sourceState);
                Logging.childWarn(logger$2, {
                  msg: match.message + " - Attempting another source",
                  toBlock: attemptedToBlock,
                  err: Utils.prettifyExn(exn)
                });
                retryRef = 0;
                break;
            }
            break;
        }
        if (exit === 1) {
          let notAlreadyDisabled = disableSource(sourceManager, sourceState);
          if (notAlreadyDisabled) {
            switch (error$1.TAG) {
              case "UnsupportedSelection" :
                Logging.childError(logger$2, error$1.message);
                break;
              case "FailedGettingFieldSelection" :
                Logging.childError(logger$2, {
                  msg: error$1.message,
                  err: Utils.prettifyExn(error$1.exn),
                  blockNumber: error$1.blockNumber,
                  logIndex: error$1.logIndex
                });
                break;
              case "FailedGettingItems" :
                break;
            }
          }
          retryRef = 0;
        }
      } else {
        ErrorHandling.mkLogAndRaise(logger$2, "Failed to fetch block Range", error);
      }
    }
  };
  return responseRef;
}

async function getBlockHashes(sourceManager, blockNumbers, isRealtime) {
  let responseRef;
  let retryRef = 0;
  while (Stdlib_Option.isNone(responseRef)) {
    let s = getNextSource(sourceManager, isRealtime, undefined);
    let sourceState;
    if (s !== undefined) {
      sourceState = s;
    } else {
      let logger = Logging.createChild({
        chainId: sourceManager.activeSource.chain
      });
      sourceState = ErrorHandling.mkLogAndRaise(logger, "No data-sources available for fetching block hashes.", null);
    }
    sourceManager.activeSource = sourceState.source;
    let source = sourceState.source;
    let retry = retryRef;
    let logger$1 = Logging.createChild({
      chainId: source.chain,
      logType: "Block Hash Query",
      source: source.name,
      retry: retry
    });
    try {
      let res = await source.getBlockHashes(blockNumbers, logger$1);
      if (res.TAG === "Ok") {
        sourceState.lastFailedAt = undefined;
        responseRef = res._0;
      } else {
        throw res._0;
      }
    } catch (raw_exn) {
      let exn = Primitive_exceptions.internalToException(raw_exn);
      if (exn.RE_EXN_ID === Source.RateLimited) {
        await waitForRateLimitReset(sourceManager, exn.resetMs, retry, logger$1);
        retryRef = retryRef + 1 | 0;
      } else {
        let backoffMillis = retry !== 0 ? 1000 * retry | 0 : 500;
        let log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
        log(logger$1, {
          msg: "Failed to fetch block hashes. Retrying.",
          retry: retry,
          backOffMilliseconds: backoffMillis,
          err: Utils.prettifyExn(exn)
        });
        let shouldSwitch = retry !== 0 && retry !== 1 ? retry % 2 === 0 : false;
        if (shouldSwitch) {
          sourceState.lastFailedAt = Date.now();
        }
        await Utils.delay(Primitive_int.min(backoffMillis, 60000));
        retryRef = retryRef + 1 | 0;
      }
    }
  };
  return responseRef;
}

export {
  getSourceRole,
  make,
  getActiveSource,
  onReorg,
  fetchNext,
  waitForNewBlock,
  executeQuery,
  makeGetHeightRetryInterval,
  getBlockHashes,
  getRateLimitTimeMs,
  isRateLimited,
  getRateLimitResetInMs,
}
/* Utils Not a pure module */