envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
699 lines (667 loc) • 29.6 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
;
var Rpc = require("./Rpc.res.js");
var Caml = require("rescript/lib/js/caml.js");
var Rest = require("../vendored/Rest.res.js");
var Time = require("../Time.res.js");
var Viem = require("../bindings/Viem.res.js");
var Utils = require("../Utils.res.js");
var Ethers = require("../bindings/Ethers.res.js");
var Hrtime = require("../bindings/Hrtime.res.js");
var Js_exn = require("rescript/lib/js/js_exn.js");
var Source = require("./Source.res.js");
var Logging = require("../Logging.res.js");
var $$Promise = require("../bindings/Promise.res.js");
var Belt_Int = require("rescript/lib/js/belt_Int.js");
var Belt_Array = require("rescript/lib/js/belt_Array.js");
var FetchState = require("../FetchState.res.js");
var LazyLoader = require("../LazyLoader.res.js");
var Belt_Option = require("rescript/lib/js/belt_Option.js");
var Caml_option = require("rescript/lib/js/caml_option.js");
var EventRouter = require("./EventRouter.res.js");
var LogSelection = require("../LogSelection.res.js");
var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
var S$RescriptSchema = require("rescript-schema/src/S.res.js");
var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
var QueryTimout = /* @__PURE__ */Caml_exceptions.create("RpcSource.QueryTimout");
function getKnownBlock(provider, blockNumber) {
return provider.getBlock(blockNumber).then(function (blockNullable) {
if (blockNullable == null) {
return Promise.reject(Js_exn.raiseError("RPC returned null for blockNumber " + String(blockNumber)));
} else {
return Promise.resolve(blockNullable);
}
});
}
async function getKnownBlockWithBackoff(provider, sourceName, chain, blockNumber, backoffMsOnFailure) {
try {
return await getKnownBlock(provider, blockNumber);
}
catch (raw_err){
var err = Caml_js_exceptions.internalToOCamlException(raw_err);
Logging.warn({
err: err,
msg: "Issue while running fetching batch of events from the RPC. Will wait " + String(backoffMsOnFailure) + "ms and try again.",
source: sourceName,
chainId: chain,
type: "EXPONENTIAL_BACKOFF"
});
await Time.resolvePromiseAfterDelay(backoffMsOnFailure);
return await getKnownBlockWithBackoff(provider, sourceName, chain, blockNumber, (backoffMsOnFailure << 1));
}
}
var suggestedRangeRegExp = /retry with the range (\d+)-(\d+)/;
var blockRangeLimitRegExp = /limited to a (\d+) blocks range/;
var alchemyRangeRegExp = /up to a (\d+) block range/;
var cloudflareRangeRegExp = /Max range: (\d+)/;
var thirdwebRangeRegExp = /Maximum allowed number of requested blocks is (\d+)/;
var blockpiRangeRegExp = /limited to (\d+) block/;
var baseRangeRegExp = /block range too large/;
var maxAllowedBlocksRegExp = /maximum allowed is (\d+) blocks/;
var blastPaidRegExp = /exceeds the range allowed for your plan \(\d+ > (\d+)\)/;
var chainstackRegExp = /Block range limit exceeded./;
var coinbaseRegExp = /please limit the query to at most (\d+) blocks/;
var publicNodeRegExp = /maximum block range: (\d+)/;
var hyperliquidRegExp = /query exceeds max block range (\d+)/;
function getSuggestedBlockIntervalFromExn(exn) {
if (exn.RE_EXN_ID !== Js_exn.$$Error) {
return ;
}
try {
var message = exn._1.error.message;
S$RescriptSchema.assertOrThrow(message, S$RescriptSchema.string);
var extractBlockRange = function (execResult, isMaxRange) {
if (execResult.length !== 2) {
return ;
}
var blockRangeLimit = execResult[1];
if (blockRangeLimit === null || blockRangeLimit === undefined) {
return ;
}
var blockRangeLimit$1 = Belt_Int.fromString(blockRangeLimit);
if (blockRangeLimit$1 !== undefined && blockRangeLimit$1 > 0) {
return [
blockRangeLimit$1,
isMaxRange
];
}
};
var execResult = suggestedRangeRegExp.exec(message);
if (execResult !== null) {
if (execResult.length !== 3) {
return ;
}
var fromBlock = execResult[1];
if (fromBlock === null || fromBlock === undefined) {
return ;
}
var toBlock = execResult[2];
if (toBlock === null || toBlock === undefined) {
return ;
}
var match = Belt_Int.fromString(fromBlock);
var match$1 = Belt_Int.fromString(toBlock);
if (match !== undefined && match$1 !== undefined && match$1 >= match) {
return [
(match$1 - match | 0) + 1 | 0,
false
];
} else {
return ;
}
}
var execResult$1 = blockRangeLimitRegExp.exec(message);
if (execResult$1 !== null) {
return extractBlockRange(execResult$1, true);
}
var execResult$2 = alchemyRangeRegExp.exec(message);
if (execResult$2 !== null) {
return extractBlockRange(execResult$2, true);
}
var execResult$3 = cloudflareRangeRegExp.exec(message);
if (execResult$3 !== null) {
return extractBlockRange(execResult$3, true);
}
var execResult$4 = thirdwebRangeRegExp.exec(message);
if (execResult$4 !== null) {
return extractBlockRange(execResult$4, true);
}
var execResult$5 = blockpiRangeRegExp.exec(message);
if (execResult$5 !== null) {
return extractBlockRange(execResult$5, true);
}
var execResult$6 = maxAllowedBlocksRegExp.exec(message);
if (execResult$6 !== null) {
return extractBlockRange(execResult$6, true);
}
var match$2 = baseRangeRegExp.exec(message);
if (match$2 !== null) {
return [
2000,
true
];
}
var execResult$7 = blastPaidRegExp.exec(message);
if (execResult$7 !== null) {
return extractBlockRange(execResult$7, true);
}
var match$3 = chainstackRegExp.exec(message);
if (match$3 !== null) {
return [
10000,
true
];
}
var execResult$8 = coinbaseRegExp.exec(message);
if (execResult$8 !== null) {
return extractBlockRange(execResult$8, true);
}
var execResult$9 = publicNodeRegExp.exec(message);
if (execResult$9 !== null) {
return extractBlockRange(execResult$9, true);
}
var execResult$10 = hyperliquidRegExp.exec(message);
if (execResult$10 !== null) {
return extractBlockRange(execResult$10, true);
} else {
return ;
}
}
catch (exn$1){
return ;
}
}
var maxSuggestedBlockIntervalKey = "max";
function getNextPage(fromBlock, toBlock, addresses, topicQuery, loadBlock, sc, provider, mutSuggestedBlockIntervals, partitionId) {
var queryTimoutPromise = Time.resolvePromiseAfterDelay(sc.queryTimeoutMillis).then(function () {
return Promise.reject({
RE_EXN_ID: QueryTimout,
_1: "Query took longer than " + String(sc.queryTimeoutMillis / 1000 | 0) + " seconds"
});
});
var latestFetchedBlockPromise = loadBlock(toBlock);
var logsPromise = provider.getLogs(Ethers.CombinedFilter.toFilter({
address: addresses,
topics: topicQuery,
fromBlock: fromBlock,
toBlock: toBlock
})).then(async function (logs) {
return {
logs: logs,
latestFetchedBlock: await latestFetchedBlockPromise
};
});
return $$Promise.$$catch(Promise.race([
queryTimoutPromise,
logsPromise
]), (function (err) {
var match = getSuggestedBlockIntervalFromExn(err);
if (match !== undefined) {
var nextBlockIntervalTry = match[0];
mutSuggestedBlockIntervals[match[1] ? maxSuggestedBlockIntervalKey : partitionId] = nextBlockIntervalTry;
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "FailedGettingItems",
exn: err,
attemptedToBlock: toBlock,
retry: {
TAG: "WithSuggestedToBlock",
toBlock: (fromBlock + nextBlockIntervalTry | 0) - 1 | 0
}
},
Error: new Error()
};
}
var executedBlockInterval = (toBlock - fromBlock | 0) + 1 | 0;
var nextBlockIntervalTry$1 = executedBlockInterval * sc.backoffMultiplicative | 0;
mutSuggestedBlockIntervals[partitionId] = nextBlockIntervalTry$1;
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "FailedGettingItems",
exn: err,
attemptedToBlock: toBlock,
retry: {
TAG: "WithBackoff",
message: "Failed getting data for the block range. Will try smaller block range for the next attempt.",
backoffMillis: sc.backoffMillis
}
},
Error: new Error()
};
}));
}
function getSelectionConfig(selection, chain) {
var staticTopicSelections = [];
var dynamicEventFilters = [];
Belt_Array.forEach(selection.eventConfigs, (function (param) {
var s = param.getEventFiltersOrThrow(chain);
if (s.TAG === "Static") {
Caml_splice_call.spliceObjApply(staticTopicSelections, "push", [s._0]);
return ;
}
dynamicEventFilters.push(s._0);
}));
var match = LogSelection.compressTopicSelections(staticTopicSelections);
var len = match.length;
var getLogSelectionOrThrow;
if (len !== 1) {
if (len !== 0) {
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you."
},
Error: new Error()
};
}
var len$1 = dynamicEventFilters.length;
if (len$1 !== 1) {
if (len$1 !== 0) {
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you."
},
Error: new Error()
};
}
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "Invalid events configuration for the partition. Nothing to fetch. Please, report to the Envio team."
},
Error: new Error()
};
}
var dynamicEventFilter = dynamicEventFilters[0];
if (selection.eventConfigs.length === 1) {
var eventConfig = selection.eventConfigs[0];
getLogSelectionOrThrow = (function (addressesByContractName) {
var addresses = FetchState.addressesByContractNameGetAll(addressesByContractName);
var match = dynamicEventFilter(addresses);
if (match.length !== 1) {
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "RPC data-source currently doesn't support an array of event filters. Please, create a GitHub issue if it's a blocker for you."
},
Error: new Error()
};
}
var topicSelection = match[0];
return {
addresses: eventConfig.isWildcard ? undefined : addresses,
topicQuery: Rpc.GetLogs.mapTopicQuery(topicSelection)
};
});
} else {
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you."
},
Error: new Error()
};
}
} else {
var topicSelection = match[0];
if (dynamicEventFilters.length !== 0) {
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "UnsupportedSelection",
message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you."
},
Error: new Error()
};
}
var topicQuery = Rpc.GetLogs.mapTopicQuery(topicSelection);
getLogSelectionOrThrow = (function (addressesByContractName) {
var addresses = FetchState.addressesByContractNameGetAll(addressesByContractName);
return {
addresses: addresses.length !== 0 ? addresses : undefined,
topicQuery: topicQuery
};
});
}
return {
getLogSelectionOrThrow: getLogSelectionOrThrow
};
}
function memoGetSelectionConfig(chain) {
var cache = new WeakMap();
return function (selection) {
var c = cache.get(selection);
if (c !== undefined) {
return c;
}
var c$1 = getSelectionConfig(selection, chain);
cache.set(selection, c$1);
return c$1;
};
}
function makeThrowingGetEventBlock(getBlock) {
return async function (log) {
return await getBlock(log.blockNumber);
};
}
function makeThrowingGetEventTransaction(getTransactionFields) {
var fnsCache = new WeakMap();
return function (log, transactionSchema) {
var fn = fnsCache.get(transactionSchema);
var tmp;
if (fn !== undefined) {
tmp = fn;
} else {
var transactionSchema$1 = S$RescriptSchema.removeTypeValidation(transactionSchema);
var match = transactionSchema$1.t;
var transactionFieldItems;
transactionFieldItems = typeof match !== "object" || match.TAG !== "object" ? Js_exn.raiseError("Unexpected internal error: transactionSchema is not an object") : match.items;
var parseOrThrowReadableError = function (data) {
try {
return S$RescriptSchema.parseOrThrow(data, transactionSchema$1);
}
catch (raw_error){
var error = Caml_js_exceptions.internalToOCamlException(raw_error);
if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
var error$1 = error._1;
return Js_exn.raiseError("Invalid transaction field \"" + S$RescriptSchema.Path.toArray(error$1.path).join(".") + "\" found in the RPC response. Error: " + S$RescriptSchema.$$Error.reason(error$1));
}
throw error;
}
};
var fn$1;
var exit = 0;
var len = transactionFieldItems.length;
if (len >= 3) {
exit = 1;
} else {
switch (len) {
case 0 :
fn$1 = (function (param) {
return Promise.resolve({});
});
break;
case 1 :
var match$1 = transactionFieldItems[0];
switch (match$1.location) {
case "hash" :
exit = 2;
break;
case "transactionIndex" :
fn$1 = (function (log) {
return Promise.resolve(parseOrThrowReadableError(log));
});
break;
default:
exit = 1;
}
break;
case 2 :
var match$2 = transactionFieldItems[0];
switch (match$2.location) {
case "hash" :
var match$3 = transactionFieldItems[1];
exit = match$3.location === "transactionIndex" ? 2 : 1;
break;
case "transactionIndex" :
var match$4 = transactionFieldItems[1];
exit = match$4.location === "hash" ? 2 : 1;
break;
default:
exit = 1;
}
break;
}
}
switch (exit) {
case 1 :
fn$1 = (function (log) {
return getTransactionFields(log).then(parseOrThrowReadableError);
});
break;
case 2 :
fn$1 = (function (log) {
return Promise.resolve(parseOrThrowReadableError({
hash: log.transactionHash,
transactionIndex: log.transactionIndex
}));
});
break;
}
fnsCache.set(transactionSchema$1, fn$1);
tmp = fn$1;
}
return tmp(log);
};
}
function sanitizeUrl(url) {
var regex = /https?:\/\/([^\/?]+).*/;
var result = regex.exec(url);
if (result === null) {
return ;
}
var host = Belt_Array.get(result, 1);
if (host !== undefined) {
return Caml_option.nullable_to_opt(Caml_option.valFromOption(host));
}
}
function make(param) {
var eventRouter = param.eventRouter;
var chain = param.chain;
var url = param.url;
var syncConfig = param.syncConfig;
var host = sanitizeUrl(url);
var urlHost = host !== undefined ? host : Js_exn.raiseError("EE109: The RPC url \"" + url + "\" is incorrect format. The RPC url needs to start with either http:// or https://");
var name = "RPC (" + urlHost + ")";
var provider = Ethers.JsonRpcProvider.make(url, chain);
var getSelectionConfig = memoGetSelectionConfig(chain);
var mutSuggestedBlockIntervals = {};
var transactionLoader = LazyLoader.make((function (transactionHash) {
return provider.getTransaction(transactionHash);
}), (function (am, exn) {
Logging.error({
err: exn,
msg: "EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in " + String(am._retryDelayMillis / 1000 | 0) + " seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the \"suggestedFix\" in the metadata of this command",
source: name,
chainId: chain,
metadata: {
asyncTaskName: "transactionLoader: fetching transaction data - `getTransaction` rpc call",
suggestedFix: "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint."
}
});
}), undefined, undefined, undefined, undefined);
var blockLoader = LazyLoader.make((function (blockNumber) {
return getKnownBlockWithBackoff(provider, name, chain, blockNumber, 1000);
}), (function (am, exn) {
Logging.error({
err: exn,
msg: "EE1100: Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in " + String(am._retryDelayMillis / 1000 | 0) + " seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the \"suggestedFix\" in the metadata of this command",
source: name,
chainId: chain,
metadata: {
asyncTaskName: "blockLoader: fetching block data - `getBlock` rpc call",
suggestedFix: "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint."
}
});
}), undefined, undefined, undefined, undefined);
var getEventBlockOrThrow = makeThrowingGetEventBlock(function (blockNumber) {
return LazyLoader.get(blockLoader, blockNumber);
});
var getEventTransactionOrThrow = makeThrowingGetEventTransaction(Ethers.JsonRpcProvider.makeGetTransactionFields(function (__x) {
return LazyLoader.get(transactionLoader, __x);
}));
var contractNameAbiMapping = {};
Belt_Array.forEach(param.contracts, (function (contract) {
contractNameAbiMapping[contract.name] = contract.abi;
}));
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, currentBlockHeight, partitionId, selection, param, param$1) {
var startFetchingBatchTimeRef = Hrtime.makeTimer();
var maxSuggestedBlockInterval = mutSuggestedBlockIntervals[maxSuggestedBlockIntervalKey];
var suggestedBlockInterval = maxSuggestedBlockInterval !== undefined ? maxSuggestedBlockInterval : Belt_Option.getWithDefault(mutSuggestedBlockIntervals[partitionId], syncConfig.initialBlockInterval);
var toBlock$1 = toBlock !== undefined && toBlock < currentBlockHeight ? toBlock : currentBlockHeight;
var suggestedToBlock = Caml.int_max(Caml.int_min((fromBlock + suggestedBlockInterval | 0) - 1 | 0, toBlock$1), fromBlock);
var firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader, fromBlock - 1 | 0).then(function (res) {
return res;
}) : Promise.resolve(undefined);
var match = getSelectionConfig(selection);
var match$1 = match.getLogSelectionOrThrow(addressesByContractName);
var match$2 = await getNextPage(fromBlock, suggestedToBlock, match$1.addresses, match$1.topicQuery, (function (blockNumber) {
return LazyLoader.get(blockLoader, blockNumber);
}), syncConfig, provider, mutSuggestedBlockIntervals, partitionId);
var latestFetchedBlock = match$2.latestFetchedBlock;
var executedBlockInterval = (suggestedToBlock - fromBlock | 0) + 1 | 0;
if (executedBlockInterval >= suggestedBlockInterval && !Utils.Dict.has(mutSuggestedBlockIntervals, maxSuggestedBlockIntervalKey)) {
mutSuggestedBlockIntervals[partitionId] = Caml.int_min(executedBlockInterval + syncConfig.accelerationAdditive | 0, syncConfig.intervalCeiling);
}
var parsedQueueItems = await Promise.all(Belt_Array.keepMap(match$2.logs, (function (log) {
var topic0 = log.topics[0];
var eventConfig = EventRouter.get(eventRouter, EventRouter.getEvmEventId(topic0, log.topics.length), log.address, log.blockNumber, indexingContracts);
if (eventConfig === undefined) {
return ;
}
var blockNumber = log.blockNumber;
var logIndex = log.index;
return Caml_option.some((async function () {
var match;
try {
match = await Promise.all([
getEventBlockOrThrow(log),
getEventTransactionOrThrow(log, eventConfig.transactionSchema)
]);
}
catch (raw_exn){
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "FailedGettingFieldSelection",
exn: exn,
blockNumber: blockNumber,
logIndex: logIndex,
message: "Failed getting selected fields. Please double-check your RPC provider returns correct data."
},
Error: new Error()
};
}
var block = match[0];
var decodedEvent;
try {
decodedEvent = Viem.parseLogOrThrow(contractNameAbiMapping, eventConfig.contractName, log.topics, log.data);
}
catch (raw_exn$1){
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
throw {
RE_EXN_ID: Source.GetItemsError,
_1: {
TAG: "FailedParsingItems",
exn: exn$1,
blockNumber: blockNumber,
logIndex: logIndex,
message: "Failed to parse event with viem, please double-check your ABI."
},
Error: new Error()
};
}
return {
kind: 0,
eventConfig: eventConfig,
timestamp: block.timestamp,
chain: chain,
blockNumber: block.number,
logIndex: log.index,
event: {
params: decodedEvent.args,
chainId: chain,
srcAddress: log.address,
logIndex: log.index,
transaction: match[1],
block: block
}
};
})());
})));
var optFirstBlockParent = await firstBlockParentPromise;
var totalTimeElapsed = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(startFetchingBatchTimeRef)));
var reorgGuard_rangeLastBlock = {
blockHash: latestFetchedBlock.hash,
blockNumber: latestFetchedBlock.number
};
var reorgGuard_prevRangeLastBlock = Belt_Option.map(optFirstBlockParent, (function (b) {
return {
blockHash: b.hash,
blockNumber: b.number
};
}));
var reorgGuard = {
rangeLastBlock: reorgGuard_rangeLastBlock,
prevRangeLastBlock: reorgGuard_prevRangeLastBlock
};
return {
currentBlockHeight: currentBlockHeight,
reorgGuard: reorgGuard,
parsedQueueItems: parsedQueueItems,
fromBlockQueried: fromBlock,
latestFetchedBlockNumber: latestFetchedBlock.number,
latestFetchedBlockTimestamp: latestFetchedBlock.timestamp,
stats: {
"total time elapsed (ms)": totalTimeElapsed
}
};
};
var getBlockHashes = function (blockNumbers, _currentlyUnusedLogger) {
return $$Promise.$$catch(Promise.all(Belt_Array.map(blockNumbers, (function (blockNum) {
return LazyLoader.get(blockLoader, blockNum);
}))).then(function (blocks) {
return {
TAG: "Ok",
_0: Belt_Array.map(blocks, (function (b) {
return {
blockHash: b.hash,
blockNumber: b.number,
blockTimestamp: b.timestamp
};
}))
};
}), (function (exn) {
return Promise.resolve({
TAG: "Error",
_0: exn
});
}));
};
var client = Rest.client(url, undefined);
return {
name: name,
sourceFor: param.sourceFor,
chain: chain,
poweredByHyperSync: false,
pollingInterval: 1000,
getBlockHashes: getBlockHashes,
getHeightOrThrow: (function () {
return Rest.$$fetch(Rpc.GetBlockHeight.route, undefined, client);
}),
getItemsOrThrow: getItemsOrThrow
};
}
exports.QueryTimout = QueryTimout;
exports.getKnownBlock = getKnownBlock;
exports.getKnownBlockWithBackoff = getKnownBlockWithBackoff;
exports.getSuggestedBlockIntervalFromExn = getSuggestedBlockIntervalFromExn;
exports.maxSuggestedBlockIntervalKey = maxSuggestedBlockIntervalKey;
exports.getNextPage = getNextPage;
exports.getSelectionConfig = getSelectionConfig;
exports.memoGetSelectionConfig = memoGetSelectionConfig;
exports.makeThrowingGetEventBlock = makeThrowingGetEventBlock;
exports.makeThrowingGetEventTransaction = makeThrowingGetEventTransaction;
exports.sanitizeUrl = sanitizeUrl;
exports.make = make;
/* Rpc Not a pure module */