envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
401 lines (380 loc) • 17.2 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
;
var Caml = require("rescript/lib/js/caml.js");
var Utils = require("../Utils.res.js");
var Hrtime = require("../bindings/Hrtime.res.js");
var Js_exn = require("rescript/lib/js/js_exn.js");
var Source = require("./Source.res.js");
var Logging = require("../Logging.res.js");
var Belt_Array = require("rescript/lib/js/belt_Array.js");
var FetchState = require("../FetchState.res.js");
var Prometheus = require("../Prometheus.res.js");
var Belt_Option = require("rescript/lib/js/belt_Option.js");
var ErrorHandling = require("../ErrorHandling.res.js");
var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
function getActiveSource(sourceManager) {
return sourceManager.activeSource;
}
function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative, maxRetryInterval) {
return function (retry) {
var backoff = retry === 0 ? 1 : Math.imul(retry, backoffMultiplicative);
return Caml.int_min(Math.imul(initialRetryInterval, backoff), maxRetryInterval);
};
}
function make(sources, maxPartitionConcurrency, newBlockFallbackStallTimeoutOpt, stalledPollingIntervalOpt, getHeightRetryIntervalOpt) {
var newBlockFallbackStallTimeout = newBlockFallbackStallTimeoutOpt !== undefined ? newBlockFallbackStallTimeoutOpt : 20000;
var stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
var getHeightRetryInterval = getHeightRetryIntervalOpt !== undefined ? getHeightRetryIntervalOpt : makeGetHeightRetryInterval(1000, 2, 60000);
var source = sources.find(function (source) {
return source.sourceFor === "Sync";
});
var initialActiveSource = source !== undefined ? source : Js_exn.raiseError("Invalid configuration, no data-source for historical sync provided");
Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
return {
sources: new Set(sources),
statusStart: Hrtime.makeTimer(),
status: "Idle",
maxPartitionConcurrency: maxPartitionConcurrency,
newBlockFallbackStallTimeout: newBlockFallbackStallTimeout,
stalledPollingInterval: stalledPollingInterval,
getHeightRetryInterval: getHeightRetryInterval,
activeSource: initialActiveSource,
waitingForNewBlockStateId: undefined,
fetchingPartitionsCount: 0
};
}
function trackNewStatus(sourceManager, newStatus) {
var match = sourceManager.status;
var promCounter;
switch (match) {
case "Idle" :
promCounter = Prometheus.IndexingIdleTime.counter;
break;
case "WaitingForNewBlock" :
promCounter = Prometheus.IndexingSourceWaitingTime.counter;
break;
case "Querieng" :
promCounter = Prometheus.IndexingQueryTime.counter;
break;
}
Prometheus.SafeCounter.incrementMany(promCounter, sourceManager.activeSource.chain, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(sourceManager.statusStart))));
sourceManager.statusStart = Hrtime.makeTimer();
sourceManager.status = newStatus;
}
async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQuery, waitForNewBlock, onNewBlock, stateId) {
var queries = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0, currentBlockHeight, stateId);
if (typeof queries !== "object") {
switch (queries) {
case "WaitingForNewBlock" :
var waitingStateId = sourceManager.waitingForNewBlockStateId;
if (waitingStateId !== undefined && waitingStateId >= stateId) {
return ;
}
trackNewStatus(sourceManager, "WaitingForNewBlock");
sourceManager.waitingForNewBlockStateId = stateId;
var currentBlockHeight$1 = await waitForNewBlock(currentBlockHeight);
var waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
trackNewStatus(sourceManager, "Idle");
sourceManager.waitingForNewBlockStateId = undefined;
return onNewBlock(currentBlockHeight$1);
} else {
return ;
}
case "ReachedMaxConcurrency" :
case "NothingToQuery" :
return ;
}
} else {
var queries$1 = queries._0;
FetchState.startFetchingQueries(fetchState, queries$1, stateId);
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries$1.length | 0;
Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
trackNewStatus(sourceManager, "Querieng");
await Promise.all(Belt_Array.map(queries$1, (function (q) {
var promise = executeQuery(q);
promise.then(function (param) {
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
if (sourceManager.fetchingPartitionsCount === 0) {
return trackNewStatus(sourceManager, "Idle");
}
});
return promise;
})));
return ;
}
}
async function getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger) {
var newHeight = 0;
var retry = 0;
while(newHeight <= currentBlockHeight && status.contents !== "Done") {
try {
var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
source: source.name,
chainId: source.chain
});
var height = await source.getHeightOrThrow();
endTimer();
newHeight = height;
if (height <= currentBlockHeight) {
retry = 0;
var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
await Utils.delay(pollingInterval);
}
}
catch (raw_exn){
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
var retryInterval = sourceManager.getHeightRetryInterval(retry);
Logging.childTrace(logger, {
msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
source: source.name,
err: Utils.prettifyExn(exn)
});
retry = retry + 1 | 0;
await Utils.delay(retryInterval);
}
};
Prometheus.SourceHeight.set(source.name, source.chain, newHeight);
return newHeight;
}
async function waitForNewBlock(sourceManager, currentBlockHeight) {
var logger = Logging.createChild({
chainId: sourceManager.activeSource.chain,
currentBlockHeight: currentBlockHeight
});
Logging.childTrace(logger, "Initiating check for new blocks.");
var syncSources = [];
var fallbackSources = [];
sourceManager.sources.forEach(function (source) {
if (source.sourceFor === "Sync" || source === sourceManager.activeSource) {
syncSources.push(source);
} else {
fallbackSources.push(source);
}
});
var status = {
contents: "Active"
};
var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (source) {
return [
source,
await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
];
})), [Utils.delay(sourceManager.newBlockFallbackStallTimeout).then(function () {
if (status.contents !== "Done") {
status.contents = "Stalled";
if (fallbackSources.length !== 0) {
Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Continuing polling with fallback RPC sources from the configuration.");
} else {
Logging.childWarn(logger, "No new blocks detected within " + String(sourceManager.newBlockFallbackStallTimeout / 1000 | 0) + "s. Polling will continue at a reduced rate. For better reliability, refer to our RPC fallback guide: https://docs.envio.dev/docs/HyperIndex/rpc-sync");
}
}
return Promise.race(Belt_Array.map(fallbackSources, (async function (source) {
return [
source,
await getSourceNewHeight(sourceManager, source, currentBlockHeight, status, logger)
];
})));
})]));
var newBlockHeight = match[1];
var source = match[0];
sourceManager.activeSource = source;
var log = status.contents === "Stalled" ? Logging.childInfo : Logging.childTrace;
log(logger, {
msg: "New blocks successfully found.",
source: source.name,
newBlockHeight: newBlockHeight
});
status.contents = "Done";
return newBlockHeight;
}
function getNextSyncSource(sourceManager, initialSource, currentSource, attemptFallbacksOpt) {
var attemptFallbacks = attemptFallbacksOpt !== undefined ? attemptFallbacksOpt : false;
var before = [];
var after = [];
var hasActive = {
contents: false
};
sourceManager.sources.forEach(function (source) {
if (source === currentSource) {
hasActive.contents = true;
return ;
}
var match = source.sourceFor;
var tmp;
tmp = match === "Sync" ? true : attemptFallbacks || source === initialSource;
if (tmp) {
(
hasActive.contents ? after : before
).push(source);
return ;
}
});
var s = Belt_Array.get(after, 0);
if (s !== undefined) {
return s;
}
var s$1 = Belt_Array.get(before, 0);
if (s$1 !== undefined) {
return s$1;
} else {
return currentSource;
}
}
async function executeQuery(sourceManager, query, currentBlockHeight) {
var match = query.target;
var toBlockRef;
toBlockRef = typeof match !== "object" ? undefined : (
match.TAG === "EndBlock" ? match.toBlock : match.toBlock
);
var responseRef;
var retryRef = 0;
var initialSource = sourceManager.activeSource;
var sourceRef = initialSource;
var shouldUpdateActiveSource = false;
while(Belt_Option.isNone(responseRef)) {
var source = sourceRef;
var toBlock = toBlockRef;
var retry = retryRef;
var logger = Logging.createChild({
chainId: source.chain,
logType: "Block Range Query",
partitionId: query.partitionId,
source: source.name,
fromBlock: query.fromBlock,
toBlock: toBlock,
addresses: FetchState.addressesByContractNameCount(query.addressesByContractName),
retry: retry
});
try {
var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts, currentBlockHeight, query.partitionId, query.selection, retry, logger);
Logging.childTrace(logger, {
msg: "Fetched block range from server",
toBlock: response.latestFetchedBlockNumber,
numEvents: response.parsedQueueItems.length,
stats: response.stats
});
responseRef = response;
}
catch (raw_error){
var error = Caml_js_exceptions.internalToOCamlException(raw_error);
if (error.RE_EXN_ID === Source.GetItemsError) {
var error$1 = error._1;
var exit = 0;
switch (error$1.TAG) {
case "UnsupportedSelection" :
case "FailedGettingFieldSelection" :
exit = 1;
break;
case "FailedGettingItems" :
var match$1 = error$1.retry;
var attemptedToBlock = error$1.attemptedToBlock;
var exn = error$1.exn;
switch (match$1.TAG) {
case "WithSuggestedToBlock" :
var toBlock$1 = match$1.toBlock;
Logging.childTrace(logger, {
msg: "Failed getting data for the block range. Immediately retrying with the suggested block range from response.",
toBlock: attemptedToBlock,
suggestedToBlock: toBlock$1
});
toBlockRef = toBlock$1;
retryRef = 0;
break;
case "WithBackoff" :
var backoffMillis = match$1.backoffMillis;
var attemptFallbacks = retry >= 10;
var nextSource = !(retry === 0 || retry === 1) && retry % 2 === 0 ? getNextSyncSource(sourceManager, initialSource, source, attemptFallbacks) : source;
var log = retry >= 4 ? Logging.childWarn : Logging.childTrace;
log(logger, {
msg: match$1.message,
toBlock: attemptedToBlock,
backOffMilliseconds: backoffMillis,
retry: retry,
err: Utils.prettifyExn(exn)
});
var shouldSwitch = nextSource !== source;
if (shouldSwitch) {
Logging.childInfo(logger, {
msg: "Switching to another data-source",
source: nextSource.name
});
sourceRef = nextSource;
shouldUpdateActiveSource = true;
} else {
await Utils.delay(backoffMillis < 60000 ? backoffMillis : 60000);
}
retryRef = retryRef + 1 | 0;
break;
case "ImpossibleForTheQuery" :
var nextSource$1 = getNextSyncSource(sourceManager, initialSource, source, true);
var hasAnotherSource = nextSource$1 !== initialSource;
Logging.childWarn(logger, {
msg: match$1.message + (
hasAnotherSource ? " - Attempting to another source" : ""
),
toBlock: attemptedToBlock,
err: Utils.prettifyExn(exn)
});
if (hasAnotherSource) {
sourceRef = nextSource$1;
shouldUpdateActiveSource = false;
retryRef = 0;
} else {
ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
}
break;
}
break;
}
if (exit === 1) {
var nextSource$2 = getNextSyncSource(sourceManager, initialSource, source, undefined);
var notAlreadyDeleted = sourceManager.sources.delete(source);
if (notAlreadyDeleted) {
switch (error$1.TAG) {
case "UnsupportedSelection" :
Logging.childError(logger, error$1.message);
break;
case "FailedGettingFieldSelection" :
Logging.childError(logger, {
msg: error$1.message,
err: Utils.prettifyExn(error$1.exn),
blockNumber: error$1.blockNumber,
logIndex: error$1.logIndex
});
break;
case "FailedGettingItems" :
break;
}
}
if (nextSource$2 === source) {
ErrorHandling.mkLogAndRaise(logger, "The indexer doesn't have data-sources which can continue fetching. Please, check the error logs or reach out to the Envio team.", null);
} else {
Logging.childInfo(logger, {
msg: "Switching to another data-source",
source: nextSource$2.name
});
sourceRef = nextSource$2;
shouldUpdateActiveSource = true;
retryRef = 0;
}
}
} else {
ErrorHandling.mkLogAndRaise(logger, "Failed to fetch block Range", error);
}
}
};
if (shouldUpdateActiveSource) {
sourceManager.activeSource = sourceRef;
}
return responseRef;
}
exports.make = make;
exports.getActiveSource = getActiveSource;
exports.fetchNext = fetchNext;
exports.waitForNewBlock = waitForNewBlock;
exports.executeQuery = executeQuery;
exports.makeGetHeightRetryInterval = makeGetHeightRetryInterval;
/* Utils Not a pure module */