UNPKG

envio

Version:

A latency and sync speed optimized, developer friendly blockchain data indexer.

1,131 lines (1,102 loc) • 69.3 kB
// Generated by ReScript, PLEASE EDIT WITH CARE import * as Utils from "./Utils.res.mjs"; import * as Js_math from "@rescript/runtime/lib/es6/Js_math.js"; import * as Logging from "./Logging.res.mjs"; import * as Prometheus from "./Prometheus.res.mjs"; import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js"; import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js"; import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js"; import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js"; import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js"; function deriveEffectiveStartBlock(registrationBlock, contractStartBlock) { return Primitive_int.max(Primitive_int.max(registrationBlock, 0), Stdlib_Option.getOr(contractStartBlock, 0)); } function getMinHistoryRange(p) { let match = p.prevQueryRange; let match$1 = p.prevPrevQueryRange; if (match !== 0 && match$1 !== 0) { return match < match$1 ? match : match$1; } } function getMinQueryRange(partitions) { let min = 0; for (let i = 0, i_finish = partitions.length; i < i_finish; ++i) { let p = partitions[i]; let a = p.prevQueryRange; let b = p.prevPrevQueryRange; if (a > 0 && (min === 0 || a < min)) { min = a; } if (b > 0 && (min === 0 || b < min)) { min = b; } } return min; } function count(optimizedPartitions) { return optimizedPartitions.idsInAscOrder.length; } function getOrThrow(optimizedPartitions, partitionId) { let p = optimizedPartitions.entities[partitionId]; if (p !== undefined) { return p; } else { return Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId); } } function mergePartitionsAtBlock(p1, p2, potentialMergeBlock, contractName, maxAddrInPartition, nextPartitionIndexRef) { let combinedAddresses = p1.addressesByContractName[contractName].concat(p2.addressesByContractName[contractName]); let p1Below = p1.latestFetchedBlock.blockNumber < potentialMergeBlock; let p2Below = p2.latestFetchedBlock.blockNumber < potentialMergeBlock; let completed = []; let continuingBase; if (p1Below) { if (p2Below) { completed.push({ id: p1.id, latestFetchedBlock: p1.latestFetchedBlock, selection: p1.selection, addressesByContractName: p1.addressesByContractName, mergeBlock: potentialMergeBlock, dynamicContract: p1.dynamicContract, mutPendingQueries: p1.mutPendingQueries, prevQueryRange: p1.prevQueryRange, prevPrevQueryRange: p1.prevPrevQueryRange, latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock }); completed.push({ id: p2.id, latestFetchedBlock: p2.latestFetchedBlock, selection: p2.selection, addressesByContractName: p2.addressesByContractName, mergeBlock: potentialMergeBlock, dynamicContract: p2.dynamicContract, mutPendingQueries: p2.mutPendingQueries, prevQueryRange: p2.prevQueryRange, prevPrevQueryRange: p2.prevPrevQueryRange, latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock }); let newId = nextPartitionIndexRef.contents.toString(); nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0; let minRange = getMinQueryRange([ p1, p2 ]); continuingBase = { id: newId, latestFetchedBlock: { blockNumber: potentialMergeBlock, blockTimestamp: 0 }, selection: p1.selection, addressesByContractName: {}, mergeBlock: undefined, dynamicContract: contractName, mutPendingQueries: [], prevQueryRange: minRange, prevPrevQueryRange: minRange, latestBlockRangeUpdateBlock: 0 }; } else { completed.push({ id: p1.id, latestFetchedBlock: p1.latestFetchedBlock, selection: p1.selection, addressesByContractName: p1.addressesByContractName, mergeBlock: potentialMergeBlock, dynamicContract: p1.dynamicContract, mutPendingQueries: p1.mutPendingQueries, prevQueryRange: p1.prevQueryRange, prevPrevQueryRange: p1.prevPrevQueryRange, latestBlockRangeUpdateBlock: p1.latestBlockRangeUpdateBlock }); continuingBase = p2; } } else if (p2Below) { completed.push({ id: p2.id, latestFetchedBlock: p2.latestFetchedBlock, selection: p2.selection, addressesByContractName: p2.addressesByContractName, mergeBlock: potentialMergeBlock, dynamicContract: p2.dynamicContract, mutPendingQueries: p2.mutPendingQueries, prevQueryRange: p2.prevQueryRange, prevPrevQueryRange: p2.prevPrevQueryRange, latestBlockRangeUpdateBlock: p2.latestBlockRangeUpdateBlock }); continuingBase = p1; } else { continuingBase = p1; } if (combinedAddresses.length > maxAddrInPartition) { let addressesFull = combinedAddresses.slice(0, maxAddrInPartition); let addressesRest = combinedAddresses.slice(maxAddrInPartition); let abcFull = {}; abcFull[contractName] = addressesFull; let abcRest = {}; abcRest[contractName] = addressesRest; completed.push({ id: continuingBase.id, latestFetchedBlock: continuingBase.latestFetchedBlock, selection: continuingBase.selection, addressesByContractName: abcFull, mergeBlock: continuingBase.mergeBlock, dynamicContract: continuingBase.dynamicContract, mutPendingQueries: continuingBase.mutPendingQueries, prevQueryRange: continuingBase.prevQueryRange, prevPrevQueryRange: continuingBase.prevPrevQueryRange, latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock }); let restId = nextPartitionIndexRef.contents.toString(); nextPartitionIndexRef.contents = nextPartitionIndexRef.contents + 1 | 0; completed.push({ id: restId, latestFetchedBlock: continuingBase.latestFetchedBlock, selection: continuingBase.selection, addressesByContractName: abcRest, mergeBlock: continuingBase.mergeBlock, dynamicContract: continuingBase.dynamicContract, mutPendingQueries: [], prevQueryRange: continuingBase.prevQueryRange, prevPrevQueryRange: continuingBase.prevPrevQueryRange, latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock }); return completed; } let abc = {}; abc[contractName] = combinedAddresses; completed.push({ id: continuingBase.id, latestFetchedBlock: continuingBase.latestFetchedBlock, selection: continuingBase.selection, addressesByContractName: abc, mergeBlock: continuingBase.mergeBlock, dynamicContract: continuingBase.dynamicContract, mutPendingQueries: continuingBase.mutPendingQueries, prevQueryRange: continuingBase.prevQueryRange, prevPrevQueryRange: continuingBase.prevPrevQueryRange, latestBlockRangeUpdateBlock: continuingBase.latestBlockRangeUpdateBlock }); return completed; } function ascSortFn(a, b) { return Primitive_int.compare(a.latestFetchedBlock.blockNumber, b.latestFetchedBlock.blockNumber); } function make(partitions, maxAddrInPartition, nextPartitionIndex, dynamicContracts) { let newPartitions = []; let mergingPartitions = {}; let nextPartitionIndexRef = { contents: nextPartitionIndex }; for (let idx = 0, idx_finish = partitions.length; idx < idx_finish; ++idx) { let p = partitions[idx]; let exit = 0; if (p.dynamicContract !== undefined && p.mergeBlock === undefined && p.selection.dependsOnAddresses) { let contractName = p.dynamicContract; let pAddressesCount = p.addressesByContractName[contractName].length; let match = Utils.$$Array.last(p.mutPendingQueries); let potentialMergeBlock; if (match !== undefined) { let toBlock = match.toBlock; potentialMergeBlock = toBlock !== undefined && match.isChunk ? toBlock : undefined; } else { potentialMergeBlock = p.latestFetchedBlock.blockNumber; } if (potentialMergeBlock !== undefined && pAddressesCount < maxAddrInPartition) { let partitionsByMergeBlock = Utils.Dict.getOrInsertEmptyDict(mergingPartitions, contractName); let existingPartition = partitionsByMergeBlock[potentialMergeBlock]; if (existingPartition !== undefined) { let result = mergePartitionsAtBlock(existingPartition, p, potentialMergeBlock, contractName, maxAddrInPartition, nextPartitionIndexRef); for (let i = 0, i_finish = result.length - 2 | 0; i <= i_finish; ++i) { newPartitions.push(result[i]); } partitionsByMergeBlock[potentialMergeBlock] = Utils.$$Array.lastUnsafe(result); } else { partitionsByMergeBlock[potentialMergeBlock] = p; } } else { newPartitions.push(p); } } else { exit = 1; } if (exit === 1) { newPartitions.push(p); } } let merginDynamicContracts = Object.keys(mergingPartitions); for (let idx$1 = 0, idx_finish$1 = merginDynamicContracts.length; idx$1 < idx_finish$1; ++idx$1) { let contractName$1 = merginDynamicContracts[idx$1]; let partitionsByMergeBlock$1 = mergingPartitions[contractName$1]; let ascPartitionKeys = Object.keys(partitionsByMergeBlock$1); if (ascPartitionKeys[ascPartitionKeys.length - 1 | 0] === "-1") { ascPartitionKeys.unshift(ascPartitionKeys.pop()); } let currentPRef = partitionsByMergeBlock$1[Utils.$$Array.firstUnsafe(ascPartitionKeys)]; let currentPMergeBlockRef = Stdlib_Int.fromString(Utils.$$Array.firstUnsafe(ascPartitionKeys), undefined); let nextJdx = 1; while (nextJdx < ascPartitionKeys.length) { let nextKey = ascPartitionKeys[nextJdx]; let currentP = currentPRef; let nextP = partitionsByMergeBlock$1[nextKey]; let nextPMergeBlock = Stdlib_Int.fromString(nextKey, undefined); let currentPMergeBlock = currentPMergeBlockRef; let isTooFar = (currentPMergeBlock + 20000 | 0) < nextPMergeBlock; if (isTooFar) { newPartitions.push(currentP); currentPRef = nextP; currentPMergeBlockRef = nextPMergeBlock; } else { let result$1 = mergePartitionsAtBlock(nextP, currentP, nextPMergeBlock, contractName$1, maxAddrInPartition, nextPartitionIndexRef); for (let i$1 = 0, i_finish$1 = result$1.length - 2 | 0; i$1 <= i_finish$1; ++i$1) { newPartitions.push(result$1[i$1]); } currentPRef = Utils.$$Array.lastUnsafe(result$1); currentPMergeBlockRef = nextPMergeBlock; } nextJdx = nextJdx + 1 | 0; }; newPartitions.push(currentPRef); } newPartitions.sort(ascSortFn); let partitionsCount = newPartitions.length; let idsInAscOrder = Array(partitionsCount); let entities = {}; for (let idx$2 = 0; idx$2 < partitionsCount; ++idx$2) { let p$1 = newPartitions[idx$2]; idsInAscOrder[idx$2] = p$1.id; entities[p$1.id] = p$1; } return { idsInAscOrder: idsInAscOrder, entities: entities, maxAddrInPartition: maxAddrInPartition, nextPartitionIndex: nextPartitionIndexRef.contents, dynamicContracts: dynamicContracts }; } function consumeFetchedQueries(mutPendingQueries, initialLatestFetchedBlock) { let latestFetchedBlock = initialLatestFetchedBlock; while ((() => { let tmp = false; if (mutPendingQueries.length !== 0) { let pq = Utils.$$Array.firstUnsafe(mutPendingQueries); tmp = pq.fetchedBlock !== undefined && pq.fromBlock <= (latestFetchedBlock.blockNumber + 1 | 0); } return tmp; })()) { let removedQuery = mutPendingQueries.shift(); latestFetchedBlock = removedQuery.fetchedBlock; }; return latestFetchedBlock; } function getPendingQueryOrThrow(p, fromBlock) { let idxRef = 0; let pendingQueryRef; while (idxRef < p.mutPendingQueries.length && pendingQueryRef === undefined) { let pq = p.mutPendingQueries[idxRef]; if (pq.fromBlock === fromBlock) { pendingQueryRef = pq; } idxRef = idxRef + 1 | 0; }; let pq$1 = pendingQueryRef; if (pq$1 !== undefined) { return pq$1; } else { return Stdlib_JsError.throwWithMessage(`Pending query not found for partition ` + p.id + ` fromBlock ` + fromBlock.toString()); } } function handleQueryResponse(optimizedPartitions, query, knownHeight, latestFetchedBlock) { let partitionId = query.partitionId; let p = optimizedPartitions.entities[partitionId]; let p$1 = p !== undefined ? p : Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId); let mutEntities = Utils.Dict.shallowCopy(optimizedPartitions.entities); let pendingQuery = getPendingQueryOrThrow(p$1, query.fromBlock); pendingQuery.fetchedBlock = latestFetchedBlock; let blockRange = (latestFetchedBlock.blockNumber - query.fromBlock | 0) + 1 | 0; let shouldUpdateBlockRange = false; if (latestFetchedBlock.blockNumber > p$1.latestBlockRangeUpdateBlock) { let queryToBlock = query.toBlock; let tmp; if (queryToBlock !== undefined) { if (latestFetchedBlock.blockNumber >= queryToBlock) { let minHistoryRange = getMinHistoryRange(p$1); tmp = minHistoryRange !== undefined ? ((queryToBlock - query.fromBlock | 0) + 1 | 0) >= minHistoryRange : false; } else { tmp = true; } } else { tmp = latestFetchedBlock.blockNumber < (knownHeight - 10 | 0); } shouldUpdateBlockRange = tmp; } let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p$1.prevQueryRange; let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p$1.prevQueryRange : p$1.prevPrevQueryRange; let mutPendingQueries = p$1.mutPendingQueries; let latestFetchedBlock$1 = p$1.latestFetchedBlock; while ((() => { let tmp = false; if (mutPendingQueries.length !== 0) { let pq = Utils.$$Array.firstUnsafe(mutPendingQueries); tmp = pq.fetchedBlock !== undefined && pq.fromBlock <= (latestFetchedBlock$1.blockNumber + 1 | 0); } return tmp; })()) { let removedQuery = mutPendingQueries.shift(); latestFetchedBlock$1 = removedQuery.fetchedBlock; }; let updatedLatestFetchedBlock = latestFetchedBlock$1; let mergeBlock = p$1.mergeBlock; let partitionReachedMergeBlock = mergeBlock !== undefined ? updatedLatestFetchedBlock.blockNumber >= mergeBlock : false; if (partitionReachedMergeBlock) { Utils.Dict.deleteInPlace(mutEntities, p$1.id); } else { let updatedMainPartition_id = p$1.id; let updatedMainPartition_selection = p$1.selection; let updatedMainPartition_addressesByContractName = p$1.addressesByContractName; let updatedMainPartition_mergeBlock = p$1.mergeBlock; let updatedMainPartition_dynamicContract = p$1.dynamicContract; let updatedMainPartition_mutPendingQueries = p$1.mutPendingQueries; let updatedMainPartition_latestBlockRangeUpdateBlock = shouldUpdateBlockRange ? latestFetchedBlock.blockNumber : p$1.latestBlockRangeUpdateBlock; let updatedMainPartition = { id: updatedMainPartition_id, latestFetchedBlock: updatedLatestFetchedBlock, selection: updatedMainPartition_selection, addressesByContractName: updatedMainPartition_addressesByContractName, mergeBlock: updatedMainPartition_mergeBlock, dynamicContract: updatedMainPartition_dynamicContract, mutPendingQueries: updatedMainPartition_mutPendingQueries, prevQueryRange: updatedPrevQueryRange, prevPrevQueryRange: updatedPrevPrevQueryRange, latestBlockRangeUpdateBlock: updatedMainPartition_latestBlockRangeUpdateBlock }; mutEntities[p$1.id] = updatedMainPartition; } return make(Object.values(mutEntities), optimizedPartitions.maxAddrInPartition, optimizedPartitions.nextPartitionIndex, optimizedPartitions.dynamicContracts); } function getLatestFullyFetchedBlock(optimizedPartitions) { let id = optimizedPartitions.idsInAscOrder[0]; if (id !== undefined) { return optimizedPartitions.entities[id].latestFetchedBlock; } } let OptimizedPartitions = { count: count, getOrThrow: getOrThrow, mergePartitionsAtBlock: mergePartitionsAtBlock, tooFarBlockRange: 20000, ascSortFn: ascSortFn, make: make, consumeFetchedQueries: consumeFetchedQueries, getPendingQueryOrThrow: getPendingQueryOrThrow, handleQueryResponse: handleQueryResponse, getLatestFullyFetchedBlock: getLatestFullyFetchedBlock }; function bufferBlockNumber(param) { let latestOnBlockBlockNumber = param.latestOnBlockBlockNumber; let optimizedPartitions = param.optimizedPartitions; let id = optimizedPartitions.idsInAscOrder[0]; let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined; if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock.blockNumber) { return latestFullyFetchedBlock.blockNumber; } else { return latestOnBlockBlockNumber; } } function bufferBlock(param) { let latestOnBlockBlockNumber = param.latestOnBlockBlockNumber; let optimizedPartitions = param.optimizedPartitions; let id = optimizedPartitions.idsInAscOrder[0]; let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined; if (latestFullyFetchedBlock !== undefined && latestOnBlockBlockNumber >= latestFullyFetchedBlock.blockNumber) { return latestFullyFetchedBlock; } else { return { blockNumber: latestOnBlockBlockNumber, blockTimestamp: 0 }; } } function compareBufferItem(a, b) { let blockOrdering = Primitive_int.compare(a.blockNumber, b.blockNumber); if (blockOrdering === 0) { return Primitive_int.compare(a.logIndex, b.logIndex); } else { return blockOrdering; } } function numAddresses(fetchState) { return Utils.Dict.size(fetchState.indexingAddresses); } function appendOnBlockItems(mutItems, onBlockConfigs, indexerStartBlock, fromBlock, maxBlockNumber, targetBufferSize) { let newItemsCounter = 0; let latestOnBlockBlockNumber = fromBlock; while (latestOnBlockBlockNumber < maxBlockNumber && newItemsCounter <= targetBufferSize) { let blockNumber = latestOnBlockBlockNumber + 1 | 0; latestOnBlockBlockNumber = blockNumber; for (let configIdx = 0, configIdx_finish = onBlockConfigs.length; configIdx < configIdx_finish; ++configIdx) { let onBlockConfig = onBlockConfigs[configIdx]; let startBlock = onBlockConfig.startBlock; let handlerStartBlock = startBlock !== undefined ? startBlock : indexerStartBlock; let tmp = false; if (blockNumber >= handlerStartBlock) { let endBlock = onBlockConfig.endBlock; tmp = endBlock !== undefined ? blockNumber <= endBlock : true; } if (tmp && Primitive_int.mod_(blockNumber - handlerStartBlock | 0, onBlockConfig.interval) === 0) { mutItems.push({ kind: 1, onBlockConfig: onBlockConfig, blockNumber: blockNumber, logIndex: 16777216 + onBlockConfig.index | 0 }); newItemsCounter = newItemsCounter + 1 | 0; } } }; return latestOnBlockBlockNumber; } function updateInternal(fetchState, optimizedPartitionsOpt, indexingAddressesOpt, mutItems, blockLagOpt, knownHeightOpt) { let optimizedPartitions = optimizedPartitionsOpt !== undefined ? optimizedPartitionsOpt : fetchState.optimizedPartitions; let indexingAddresses = indexingAddressesOpt !== undefined ? indexingAddressesOpt : fetchState.indexingAddresses; let blockLag = blockLagOpt !== undefined ? blockLagOpt : fetchState.blockLag; let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : fetchState.knownHeight; let mutItemsRef = mutItems; let onBlockConfigs = fetchState.onBlockConfigs; let latestOnBlockBlockNumber; if (onBlockConfigs.length !== 0) { let mutItems$1 = mutItemsRef; let item = ( mutItems$1 !== undefined ? mutItems$1 : fetchState.buffer )[fetchState.targetBufferSize - 1 | 0]; let maxBlockNumber; if (item !== undefined) { maxBlockNumber = item.blockNumber; } else { let id = optimizedPartitions.idsInAscOrder[0]; let latestFullyFetchedBlock = id !== undefined ? optimizedPartitions.entities[id].latestFetchedBlock : undefined; maxBlockNumber = latestFullyFetchedBlock !== undefined ? latestFullyFetchedBlock.blockNumber : knownHeight; } let mutItems$2 = mutItemsRef; let mutItems$3 = mutItems$2 !== undefined ? mutItems$2 : fetchState.buffer.slice(); mutItemsRef = mutItems$3; latestOnBlockBlockNumber = appendOnBlockItems(mutItems$3, onBlockConfigs, fetchState.startBlock, fetchState.latestOnBlockBlockNumber, maxBlockNumber, fetchState.targetBufferSize); } else { latestOnBlockBlockNumber = knownHeight; } let mutItems$4 = mutItemsRef; let updatedFetchState_startBlock = fetchState.startBlock; let updatedFetchState_endBlock = fetchState.endBlock; let updatedFetchState_normalSelection = fetchState.normalSelection; let updatedFetchState_contractConfigs = fetchState.contractConfigs; let updatedFetchState_chainId = fetchState.chainId; let updatedFetchState_buffer = mutItems$4 !== undefined ? (mutItems$4.sort(compareBufferItem), mutItems$4) : fetchState.buffer; let updatedFetchState_targetBufferSize = fetchState.targetBufferSize; let updatedFetchState_onBlockConfigs = fetchState.onBlockConfigs; let updatedFetchState_firstEventBlock = fetchState.firstEventBlock; let updatedFetchState = { optimizedPartitions: optimizedPartitions, startBlock: updatedFetchState_startBlock, endBlock: updatedFetchState_endBlock, normalSelection: updatedFetchState_normalSelection, indexingAddresses: indexingAddresses, contractConfigs: updatedFetchState_contractConfigs, chainId: updatedFetchState_chainId, latestOnBlockBlockNumber: latestOnBlockBlockNumber, blockLag: blockLag, buffer: updatedFetchState_buffer, targetBufferSize: updatedFetchState_targetBufferSize, onBlockConfigs: updatedFetchState_onBlockConfigs, knownHeight: knownHeight, firstEventBlock: updatedFetchState_firstEventBlock }; Prometheus.IndexingPartitions.set(optimizedPartitions.idsInAscOrder.length, fetchState.chainId); Prometheus.IndexingBufferSize.set(updatedFetchState_buffer.length, fetchState.chainId); Prometheus.IndexingBufferBlockNumber.set(bufferBlockNumber(updatedFetchState), fetchState.chainId); if (indexingAddresses !== fetchState.indexingAddresses) { Prometheus.IndexingAddresses.set(Utils.Dict.size(indexingAddresses), fetchState.chainId); } return updatedFetchState; } function warnDifferentContractType(fetchState, existingContract, dc) { let logger = Logging.createChild({ chainId: fetchState.chainId, contractAddress: dc.address, existingContractType: existingContract.contractName, newContractType: dc.contractName }); Logging.childWarn(logger, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`); } function addressesByContractNameCount(addressesByContractName) { let numAddresses = { contents: 0 }; Utils.Dict.forEach(addressesByContractName, addresses => { numAddresses.contents = numAddresses.contents + addresses.length | 0; }); return numAddresses.contents; } function addressesByContractNameGetAll(addressesByContractName) { let all = []; Utils.Dict.forEach(addressesByContractName, addresses => { for (let idx = 0, idx_finish = addresses.length; idx < idx_finish; ++idx) { all.push(addresses[idx]); } }); return all; } function createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContracts, normalSelection, maxAddrInPartition, nextPartitionIndex, existingPartitions, progressBlockNumber) { let nextPartitionIndexRef = nextPartitionIndex; let dynamicPartitions = []; let nonDynamicPartitions = []; let contractNames = Object.keys(registeringContractsByContract); for (let cIdx = 0, cIdx_finish = contractNames.length; cIdx < cIdx_finish; ++cIdx) { let contractName = contractNames[cIdx]; let registeringContracts = registeringContractsByContract[contractName]; let addresses = Object.keys(registeringContracts); let isDynamic = dynamicContracts.has(contractName); let partitions = isDynamic ? dynamicPartitions : nonDynamicPartitions; let byStartBlock = {}; for (let jdx = 0, jdx_finish = addresses.length; jdx < jdx_finish; ++jdx) { let address = addresses[jdx]; let indexingContract = registeringContracts[address]; Utils.Dict.push(byStartBlock, indexingContract.effectiveStartBlock.toString(), address); } let ascKeys = Object.keys(byStartBlock); let initialKey = Utils.$$Array.firstUnsafe(ascKeys); let startBlockRef = Stdlib_Int.fromString(initialKey, undefined); let addressesRef = byStartBlock[initialKey]; for (let idx = 0, idx_finish = ascKeys.length; idx < idx_finish; ++idx) { let maybeNextStartBlockKey = ascKeys[idx + 1 | 0]; let shouldAllocateNewPartition; if (maybeNextStartBlockKey !== undefined) { let nextStartBlock = Stdlib_Int.fromString(maybeNextStartBlockKey, undefined); let shouldJoinCurrentStartBlock = (nextStartBlock - startBlockRef | 0) < 20000; if (shouldJoinCurrentStartBlock) { addressesRef = addressesRef.concat(byStartBlock[maybeNextStartBlockKey]); shouldAllocateNewPartition = false; } else { shouldAllocateNewPartition = true; } } else { shouldAllocateNewPartition = true; } if (shouldAllocateNewPartition) { let latestFetchedBlock_blockNumber = Primitive_int.max(startBlockRef - 1 | 0, progressBlockNumber); let latestFetchedBlock = { blockNumber: latestFetchedBlock_blockNumber, blockTimestamp: 0 }; while (addressesRef.length !== 0) { let pAddresses = addressesRef.slice(0, maxAddrInPartition); addressesRef = addressesRef.slice(maxAddrInPartition); let addressesByContractName = {}; addressesByContractName[contractName] = pAddresses; partitions.push({ id: nextPartitionIndexRef.toString(), latestFetchedBlock: latestFetchedBlock, selection: normalSelection, addressesByContractName: addressesByContractName, mergeBlock: undefined, dynamicContract: isDynamic ? contractName : undefined, mutPendingQueries: [], prevQueryRange: 0, prevPrevQueryRange: 0, latestBlockRangeUpdateBlock: 0 }); nextPartitionIndexRef = nextPartitionIndexRef + 1 | 0; }; if (maybeNextStartBlockKey !== undefined) { startBlockRef = Stdlib_Int.fromString(maybeNextStartBlockKey, undefined); addressesRef = byStartBlock[maybeNextStartBlockKey]; } } } } let mergedNonDynamic = []; if (nonDynamicPartitions.length !== 0) { nonDynamicPartitions.sort(ascSortFn); let currentPRef = nonDynamicPartitions[0]; let nextIdx = 1; while (nextIdx < nonDynamicPartitions.length) { let nextP = nonDynamicPartitions[nextIdx]; let currentP = currentPRef; let currentPBlock = currentP.latestFetchedBlock.blockNumber; let nextPBlock = nextP.latestFetchedBlock.blockNumber; let totalCount = addressesByContractNameCount(currentP.addressesByContractName) + addressesByContractNameCount(nextP.addressesByContractName) | 0; if (totalCount > maxAddrInPartition) { mergedNonDynamic.push(currentP); currentPRef = nextP; } else { let mergedAddresses = Utils.Dict.shallowCopy(nextP.addressesByContractName); let currentContractNames = Object.keys(currentP.addressesByContractName); for (let jdx$1 = 0, jdx_finish$1 = currentContractNames.length; jdx$1 < jdx_finish$1; ++jdx$1) { let cn = currentContractNames[jdx$1]; let currentAddrs = currentP.addressesByContractName[cn]; let existingAddrs = mergedAddresses[cn]; if (existingAddrs !== undefined) { mergedAddresses[cn] = existingAddrs.concat(currentAddrs); } else { mergedAddresses[cn] = currentAddrs; } } let isTooFar = (currentPBlock + 20000 | 0) < nextPBlock; if (isTooFar) { mergedNonDynamic.push({ id: currentP.id, latestFetchedBlock: currentP.latestFetchedBlock, selection: currentP.selection, addressesByContractName: currentP.addressesByContractName, mergeBlock: currentPBlock < nextPBlock ? nextPBlock : undefined, dynamicContract: currentP.dynamicContract, mutPendingQueries: currentP.mutPendingQueries, prevQueryRange: currentP.prevQueryRange, prevPrevQueryRange: currentP.prevPrevQueryRange, latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock }); currentPRef = { id: nextP.id, latestFetchedBlock: nextP.latestFetchedBlock, selection: nextP.selection, addressesByContractName: mergedAddresses, mergeBlock: nextP.mergeBlock, dynamicContract: nextP.dynamicContract, mutPendingQueries: nextP.mutPendingQueries, prevQueryRange: nextP.prevQueryRange, prevPrevQueryRange: nextP.prevPrevQueryRange, latestBlockRangeUpdateBlock: nextP.latestBlockRangeUpdateBlock }; } else { currentPRef = { id: currentP.id, latestFetchedBlock: currentP.latestFetchedBlock, selection: currentP.selection, addressesByContractName: mergedAddresses, mergeBlock: currentP.mergeBlock, dynamicContract: currentP.dynamicContract, mutPendingQueries: currentP.mutPendingQueries, prevQueryRange: currentP.prevQueryRange, prevPrevQueryRange: currentP.prevPrevQueryRange, latestBlockRangeUpdateBlock: currentP.latestBlockRangeUpdateBlock }; } } nextIdx = nextIdx + 1 | 0; }; mergedNonDynamic.push(currentPRef); } let mergedPartitions = mergedNonDynamic.concat(dynamicPartitions); return make(existingPartitions.concat(mergedPartitions), maxAddrInPartition, nextPartitionIndexRef, dynamicContracts); } function registerDynamicContracts(fetchState, items) { if (Utils.$$Array.isEmpty(fetchState.normalSelection.eventConfigs)) { Stdlib_JsError.throwWithMessage("Invalid configuration. No events to fetch for the dynamic contract registration."); } let indexingAddresses = fetchState.indexingAddresses; let registeringContractsByContract = {}; let earliestRegisteringEventBlockNumber = Infinity; let noEventsAddresses = {}; let registeringAddresses = {}; for (let itemIdx = 0, itemIdx_finish = items.length; itemIdx < itemIdx_finish; ++itemIdx) { let item = items[itemIdx]; let dcs = item.dcs; if (dcs !== undefined) { let idx = 0; while (idx < dcs.length) { let dc = dcs[idx]; let shouldRemove = false; let match = fetchState.contractConfigs[dc.contractName]; if (match !== undefined) { let dcWithStartBlock_address = dc.address; let dcWithStartBlock_contractName = dc.contractName; let dcWithStartBlock_registrationBlock = dc.registrationBlock; let dcWithStartBlock_effectiveStartBlock = deriveEffectiveStartBlock(dc.registrationBlock, match.startBlock); let dcWithStartBlock = { address: dcWithStartBlock_address, contractName: dcWithStartBlock_contractName, registrationBlock: dcWithStartBlock_registrationBlock, effectiveStartBlock: dcWithStartBlock_effectiveStartBlock }; let existingContract = indexingAddresses[dc.address]; if (existingContract !== undefined) { if (existingContract.contractName !== dc.contractName) { warnDifferentContractType(fetchState, existingContract, dcWithStartBlock); } else if (existingContract.effectiveStartBlock > dcWithStartBlock_effectiveStartBlock) { let logger = Logging.createChild({ chainId: fetchState.chainId, contractAddress: dc.address, existingBlockNumber: existingContract.effectiveStartBlock, newBlockNumber: dcWithStartBlock_effectiveStartBlock }); Logging.childWarn(logger, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`); } shouldRemove = true; } else { let registeringContract = registeringAddresses[dc.address]; let shouldUpdate = registeringContract !== undefined ? ( registeringContract.contractName !== dc.contractName ? (warnDifferentContractType(fetchState, registeringContract, dcWithStartBlock), false) : false ) : true; if (shouldUpdate) { earliestRegisteringEventBlockNumber = Primitive_int.min(earliestRegisteringEventBlockNumber, dcWithStartBlock_effectiveStartBlock); Utils.Dict.getOrInsertEmptyDict(registeringContractsByContract, dc.contractName)[dc.address] = dcWithStartBlock; registeringAddresses[dc.address] = dcWithStartBlock; } else { shouldRemove = true; } } } else { let dcAsIndexingAddress_address = dc.address; let dcAsIndexingAddress_contractName = dc.contractName; let dcAsIndexingAddress_registrationBlock = dc.registrationBlock; let dcAsIndexingAddress_effectiveStartBlock = deriveEffectiveStartBlock(dc.registrationBlock, undefined); let dcAsIndexingAddress = { address: dcAsIndexingAddress_address, contractName: dcAsIndexingAddress_contractName, registrationBlock: dcAsIndexingAddress_registrationBlock, effectiveStartBlock: dcAsIndexingAddress_effectiveStartBlock }; let existingContract$1 = indexingAddresses[dc.address]; if (existingContract$1 !== undefined) { if (existingContract$1.contractName !== dc.contractName) { warnDifferentContractType(fetchState, existingContract$1, dcAsIndexingAddress); } shouldRemove = true; } else { let existingContract$2 = registeringAddresses[dc.address]; if (existingContract$2 !== undefined) { if (existingContract$2.contractName !== dc.contractName) { warnDifferentContractType(fetchState, existingContract$2, dcAsIndexingAddress); } shouldRemove = true; } else { let logger$1 = Logging.createChild({ chainId: fetchState.chainId, contractAddress: dc.address, contractName: dc.contractName }); Logging.childWarn(logger$1, `Persisting contract registration without fetching: Contract doesn't have any events to fetch. It'll be picked up on restart if you add events for the contract.`); noEventsAddresses[dc.address] = dcAsIndexingAddress; registeringAddresses[dc.address] = dcAsIndexingAddress; } } } if (shouldRemove) { dcs.splice(idx, 1); } else { idx = idx + 1 | 0; } }; } } let dcContractNamesToStore = Object.keys(registeringContractsByContract); let hasNoEventsUpdates = !Utils.Dict.isEmpty(noEventsAddresses); if (dcContractNamesToStore.length !== 0) { let newPartitions = []; let newIndexingAddresses = Utils.Dict.shallowCopy(indexingAddresses); let dynamicContractsRef = fetchState.optimizedPartitions.dynamicContracts; let mutExistingPartitions = Object.values(fetchState.optimizedPartitions.entities); for (let idx$1 = 0, idx_finish = dcContractNamesToStore.length; idx$1 < idx_finish; ++idx$1) { let contractName = dcContractNamesToStore[idx$1]; if (!dynamicContractsRef.has(contractName)) { dynamicContractsRef = Utils.$$Set.immutableAdd(dynamicContractsRef, contractName); for (let idx$2 = 0, idx_finish$1 = mutExistingPartitions.length; idx$2 < idx_finish$1; ++idx$2) { let p = mutExistingPartitions[idx$2]; let addresses = p.addressesByContractName[contractName]; if (addresses !== undefined && p.selection.dependsOnAddresses && p.mergeBlock === undefined) { let allPartitionContractNames = Object.keys(p.addressesByContractName); if (allPartitionContractNames.length !== 1) { let isFetching = p.mutPendingQueries.length !== 0; if (!isFetching) { let newPartitionId = (fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0).toString(); let restAddressesByContractName = Utils.Dict.shallowCopy(p.addressesByContractName); Utils.Dict.deleteInPlace(restAddressesByContractName, contractName); mutExistingPartitions[idx$2] = { id: p.id, latestFetchedBlock: p.latestFetchedBlock, selection: p.selection, addressesByContractName: restAddressesByContractName, mergeBlock: p.mergeBlock, dynamicContract: p.dynamicContract, mutPendingQueries: p.mutPendingQueries, prevQueryRange: p.prevQueryRange, prevPrevQueryRange: p.prevPrevQueryRange, latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock }; let addressesByContractName = {}; addressesByContractName[contractName] = addresses; newPartitions.push({ id: newPartitionId, latestFetchedBlock: p.latestFetchedBlock, selection: fetchState.normalSelection, addressesByContractName: addressesByContractName, mergeBlock: undefined, dynamicContract: contractName, mutPendingQueries: p.mutPendingQueries, prevQueryRange: p.prevQueryRange, prevPrevQueryRange: p.prevPrevQueryRange, latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock }); } } else { mutExistingPartitions[idx$2] = { id: p.id, latestFetchedBlock: p.latestFetchedBlock, selection: p.selection, addressesByContractName: p.addressesByContractName, mergeBlock: p.mergeBlock, dynamicContract: contractName, mutPendingQueries: p.mutPendingQueries, prevQueryRange: p.prevQueryRange, prevPrevQueryRange: p.prevPrevQueryRange, latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock }; } } } } let registeringContracts = registeringContractsByContract[contractName]; Object.assign(newIndexingAddresses, registeringContracts); } Object.assign(newIndexingAddresses, noEventsAddresses); let optimizedPartitions = createPartitionsFromIndexingAddresses(registeringContractsByContract, dynamicContractsRef, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, fetchState.optimizedPartitions.nextPartitionIndex + newPartitions.length | 0, mutExistingPartitions.concat(newPartitions), 0); return updateInternal(fetchState, optimizedPartitions, newIndexingAddresses, undefined, undefined, undefined); } if (!hasNoEventsUpdates) { return fetchState; } let newIndexingContracts = Utils.Dict.shallowCopy(indexingAddresses); Object.assign(newIndexingContracts, noEventsAddresses); return updateInternal(fetchState, undefined, newIndexingContracts, undefined, undefined, undefined); } function handleQueryResult(fetchState, query, latestFetchedBlock, newItems) { let newItems$1 = newItems.filter(item => { if (item.kind !== 0) { return true; } let filter = item.eventConfig.clientAddressFilter; if (filter !== undefined) { return filter(item.event, item.blockNumber, fetchState.indexingAddresses); } else { return true; } }); return updateInternal(fetchState, handleQueryResponse(fetchState.optimizedPartitions, query, fetchState.knownHeight, latestFetchedBlock), undefined, newItems$1.length !== 0 ? fetchState.buffer.concat(newItems$1) : undefined, undefined, undefined); } function startFetchingQueries(param, queries) { let optimizedPartitions = param.optimizedPartitions; for (let qIdx = 0, qIdx_finish = queries.length; qIdx < qIdx_finish; ++qIdx) { let q = queries[qIdx]; let partitionId = q.partitionId; let p = optimizedPartitions.entities[partitionId]; let p$1 = p !== undefined ? p : Stdlib_JsError.throwWithMessage(`Unexpected case: Couldn't find partition ` + partitionId); let pq = { fromBlock: q.fromBlock, toBlock: q.toBlock, isChunk: q.isChunk, fetchedBlock: undefined }; let inserted = false; let i = 0; while (i < p$1.mutPendingQueries.length && !inserted) { if (p$1.mutPendingQueries[i].fromBlock > q.fromBlock) { p$1.mutPendingQueries.splice(i, 0, pq); inserted = true; } i = i + 1 | 0; }; if (!inserted) { p$1.mutPendingQueries.push(pq); } } } function pushQueriesForRange(queries, partitionId, rangeFromBlock, rangeEndBlock, maxQueryBlockNumber, maybeChunkRange, selection, addressesByContractName, indexingAddresses) { if (rangeFromBlock > maxQueryBlockNumber) { return; } if (rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock) { return; } if (maybeChunkRange !== undefined) { let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber; let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8); if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0, isChunk: true, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock + chunkSize | 0, toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0, isChunk: true, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } else { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: rangeEndBlock, isChunk: rangeEndBlock !== undefined, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } return; } queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: rangeEndBlock, isChunk: false, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } function getNextQuery(fetchState, concurrencyLimit) { let knownHeight = fetchState.knownHeight; let blockLag = fetchState.blockLag; let indexingAddresses = fetchState.indexingAddresses; let optimizedPartitions = fetchState.optimizedPartitions; let headBlockNumber = knownHeight - blockLag | 0; if (headBlockNumber <= 0) { return "WaitingForNewBlock"; } if (concurrencyLimit === 0) { return "ReachedMaxConcurrency"; } let isOnBlockBehindTheHead = fetchState.latestOnBlockBlockNumber < headBlockNumber; let endBlock = fetchState.endBlock; let shouldWaitForNewBlock = ( endBlock !== undefined ? headBlockNumber < endBlock : true ) && !isOnBlockBehindTheHead; let item = fetchState.buffer[fetchState.targetBufferSize - 1 | 0]; let maxQueryBlockNumber = item !== undefined ? Primitive_int.min(item.blockNumber, knownHeight) : knownHeight; let queries = []; let partitionsCount = optimizedPartitions.idsInAscOrder.length; let idxRef = 0; while (idxRef < partitionsCount) { let idx = idxRef; let partitionId = optimizedPartitions.idsInAscOrder[idx]; let p = optimizedPartitions.entities[partitionId]; let isBehindTheHead = p.latestFetchedBlock.blockNumber < headBlockNumber; let hasPendingQueries = Utils.$$Array.notEmpty(p.mutPendingQueries); if (hasPendingQueries || isBehindTheHead) { shouldWaitForNewBlock = false; } let queryEndBlock = Utils.$$Math.minOptInt(fetchState.endBlock, p.mergeBlock); let queryEndBlock$1 = blockLag !== 0 ? Utils.$$Math.minOptInt(headBlockNumber, queryEndBlock) : queryEndBlock; let match = maxQueryBlockNumber < knownHeight; let queryEndBlock$2 = queryEndBlock$1 !== undefined ? ( match ? Primitive_int.min(maxQueryBlockNumber, queryEndBlock$1) : queryEndBlock$1 ) : ( match ? maxQueryBlockNumber : queryEndBlock$1 ); let maybeChunkRange = getMinHistoryRange(p); let cursor = p.latestFetchedBlock.blockNumber + 1 | 0; let canContinue = true; let pqIdx = 0; while (pqIdx < p.mutPendingQueries.length && canContinue) { let pq = p.mutPendingQueries[pqIdx]; if (pq.fromBlock > cursor) { let addressesByContractName = p.addressesByContractName; let selection = p.selection; let rangeEndBlock = Utils.$$Math.minOptInt(pq.fromBlock - 1 | 0, queryEndBlock$2); let rangeFromBlock = cursor; if (rangeFromBlock <= maxQueryBlockNumber) { let exit = 0; if (!(rangeEndBlock !== undefined && rangeFromBlock > rangeEndBlock)) { exit = 1; } if (exit === 1) { if (maybeChunkRange !== undefined) { let maxBlock = rangeEndBlock !== undefined ? rangeEndBlock : maxQueryBlockNumber; let chunkSize = Js_math.ceil_int(maybeChunkRange * 1.8); if (((rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0) <= maxBlock) { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: (rangeFromBlock + chunkSize | 0) - 1 | 0, isChunk: true, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock + chunkSize | 0, toBlock: (rangeFromBlock + (chunkSize << 1) | 0) - 1 | 0, isChunk: true, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } else { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: rangeEndBlock, isChunk: rangeEndBlock !== undefined, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } } else { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock, toBlock: rangeEndBlock, isChunk: false, selection: selection, addressesByContractName: addressesByContractName, indexingAddresses: indexingAddresses }); } } } } let toBlock = pq.toBlock; if (toBlock !== undefined && pq.isChunk) { let match$1 = pq.fetchedBlock; if (match$1 !== undefined) { let blockNumber = match$1.blockNumber; cursor = blockNumber < toBlock ? blockNumber + 1 | 0 : toBlock + 1 | 0; } else { cursor = toBlock + 1 | 0; } } else { canContinue = false; } pqIdx = pqIdx + 1 | 0; }; if (canContinue) { let addressesByContractName$1 = p.addressesByContractName; let selection$1 = p.selection; let rangeFromBlock$1 = cursor; if (rangeFromBlock$1 <= maxQueryBlockNumber) { let exit$1 = 0; if (!(queryEndBlock$2 !== undefined && rangeFromBlock$1 > queryEndBlock$2)) { exit$1 = 1; } if (exit$1 === 1) { if (maybeChunkRange !== undefined) { let maxBlock$1 = queryEndBlock$2 !== undefined ? queryEndBlock$2 : maxQueryBlockNumber; let chunkSize$1 = Js_math.ceil_int(maybeChunkRange * 1.8); if (((rangeFromBlock$1 + (chunkSize$1 << 1) | 0) - 1 | 0) <= maxBlock$1) { queries.push({ partitionId: partitionId, fromBlock: rangeFromBlock$1, toBlock: (rangeFromBlock$1 + chunkSize$1 | 0) - 1 | 0, isChunk: true, selection: selectio