UNPKG

envio

Version:

A latency and sync speed optimized, developer friendly blockchain data indexer.

655 lines (638 loc) • 24.5 kB
// Generated by ReScript, PLEASE EDIT WITH CARE import * as Env from "./Env.res.mjs"; import * as Tui from "./tui/Tui.res.mjs"; import * as Envio from "./Envio.res.mjs"; import * as Utils from "./Utils.res.mjs"; import * as Config from "./Config.res.mjs"; import * as Logging from "./Logging.res.mjs"; import Express from "express"; import * as Process from "process"; import * as ChainMap from "./ChainMap.res.mjs"; import * as PgStorage from "./PgStorage.res.mjs"; import * as FetchState from "./FetchState.res.mjs"; import * as Prometheus from "./Prometheus.res.mjs"; import * as GlobalState from "./GlobalState.res.mjs"; import * as Persistence from "./Persistence.res.mjs"; import * as PromClient from "prom-client"; import Yargs from "yargs/yargs"; import * as ChainFetcher from "./ChainFetcher.res.mjs"; import * as ChainManager from "./ChainManager.res.mjs"; import * as ErrorHandling from "./ErrorHandling.res.mjs"; import * as HandlerLoader from "./HandlerLoader.res.mjs"; import * as InMemoryStore from "./InMemoryStore.res.mjs"; import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js"; import * as SourceManager from "./sources/SourceManager.res.mjs"; import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js"; import * as Helpers from "yargs/helpers"; import * as RollbackCommit from "./RollbackCommit.res.mjs"; import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js"; import * as HandlerRegister from "./HandlerRegister.res.mjs"; import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js"; import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs"; import * as GlobalStateManager from "./GlobalStateManager.res.mjs"; import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js"; import * as MarkBatchProcessedAdapter from "./adapters/MarkBatchProcessedAdapter.res.mjs"; let chainDataSchema = S$RescriptSchema.schema(s => ({ chainId: s.m(S$RescriptSchema.float), poweredByHyperSync: s.m(S$RescriptSchema.bool), firstEventBlockNumber: s.m(S$RescriptSchema.option(S$RescriptSchema.int)), latestProcessedBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)), timestampCaughtUpToHeadOrEndblock: s.m(S$RescriptSchema.option(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined))), numEventsProcessed: s.m(S$RescriptSchema.float), latestFetchedBlockNumber: s.m(S$RescriptSchema.int), currentBlockHeight: s.m(S$RescriptSchema.int), numBatchesFetched: s.m(S$RescriptSchema.int), startBlock: s.m(S$RescriptSchema.int), endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)), numAddresses: s.m(S$RescriptSchema.int) })); let stateSchema = S$RescriptSchema.union([ S$RescriptSchema.literal({ status: "disabled" }), S$RescriptSchema.literal({ status: "initializing" }), S$RescriptSchema.schema(s => ({ status: "active", envioVersion: s.m(S$RescriptSchema.string), chains: s.m(S$RescriptSchema.array(chainDataSchema)), indexerStartTime: s.m(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined)), isPreRegisteringDynamicContracts: false, rollbackOnReorg: s.m(S$RescriptSchema.bool) })) ]); let blockRangeSchema = S$RescriptSchema.strict(S$RescriptSchema.object(s => ({ _gte: s.f("_gte", S$RescriptSchema.option(S$RescriptSchema.int)), _lte: s.f("_lte", S$RescriptSchema.option(S$RescriptSchema.int)), _every: s.f("_every", S$RescriptSchema.Option.getOr(S$RescriptSchema.option(S$RescriptSchema.intMin(S$RescriptSchema.int, 1, undefined)), 1)) }))); let defaultBlockRange = { _gte: undefined, _lte: undefined, _every: 1 }; let globalGsManagerRef = { contents: undefined }; let globalPersistenceRef = { contents: undefined }; function getInitialChainState(chainId) { let persistence = globalPersistenceRef.contents; if (persistence === undefined) { return; } let initialState = persistence.storageStatus; if (typeof initialState !== "object" || initialState.TAG === "Initializing") { return; } else { return initialState._0.chains.find(c => c.id === chainId); } } function buildChainsObject(config) { let chainIds = []; let chains = Object.create(null); ChainMap.values(config.chainMap).forEach(chainConfig => { let chainIdStr = chainConfig.id.toString(); chainIds.push(chainConfig.id); let chainObj = Object.create(null); Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(chainObj, "id", { enumerable: true, value: chainConfig.id }), "startBlock", { enumerable: true, get: () => { let chainState = getInitialChainState(chainConfig.id); if (chainState !== undefined) { return chainState.startBlock; } else { return chainConfig.startBlock; } } }), "endBlock", { enumerable: true, get: () => { let match = getInitialChainState(chainConfig.id); if (match === undefined) { return chainConfig.endBlock; } let eb = match.endBlock; if (eb !== undefined) { return eb; } else { return chainConfig.endBlock; } } }), "name", { enumerable: true, value: chainConfig.name }), "isRealtime", { enumerable: true, get: () => { let gsManager = globalGsManagerRef.contents; if (gsManager !== undefined) { return GlobalStateManager.getState(Primitive_option.valFromOption(gsManager)).chainManager.isRealtime; } else if (Env.updateSyncTimeOnRestart) { return false; } else { return ChainMap.values(config.chainMap).every(c => { let chainState = getInitialChainState(c.id); if (chainState !== undefined) { return Stdlib_Option.isSome(chainState.timestampCaughtUpToHeadOrEndblock); } else { return false; } }); } } }); chainConfig.contracts.forEach(contract => { let contractObj = Object.create(null); Object.defineProperty(Object.defineProperty(Object.defineProperty(contractObj, "name", { enumerable: true, value: contract.name }), "abi", { enumerable: true, value: contract.abi }), "addresses", { enumerable: true, get: () => { let gsManager = globalGsManagerRef.contents; if (gsManager !== undefined) { let state = GlobalStateManager.getState(Primitive_option.valFromOption(gsManager)); let chain = ChainMap.Chain.makeUnsafe(chainConfig.id); let chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain); let indexingAddresses = chainFetcher.fetchState.indexingAddresses; let addresses = []; let values = Object.values(indexingAddresses); for (let idx = 0, idx_finish = values.length; idx < idx_finish; ++idx) { let indexingContract = values[idx]; if (indexingContract.contractName === contract.name) { addresses.push(indexingContract.address); } } return addresses; } let chainState = getInitialChainState(chainConfig.id); if (chainState === undefined) { return contract.addresses; } let addresses$1 = contract.addresses.slice(); chainState.indexingAddresses.forEach(dc => { if (dc.contractName === contract.name) { addresses$1.push(dc.address); return; } }); return addresses$1; } }); Object.defineProperty(chainObj, contract.name, { enumerable: true, value: contractObj }); }); Object.defineProperty(chains, chainIdStr, { enumerable: true, value: chainObj }); if (chainConfig.name !== chainIdStr) { Object.defineProperty(chains, chainConfig.name, { enumerable: false, value: chainObj }); return; } }); return [ chains, chainIds ]; } function getGlobalIndexer() { let parseIdentityConfig = identityConfig => { let match; if (typeof identityConfig.contract === "string") { match = [ identityConfig.contract, identityConfig.event ]; } else { let event = identityConfig.event; match = [ event.contract, event._0 ]; } let wildcard = identityConfig.wildcard; let where = identityConfig.where; let eventOptions = wildcard !== undefined || where !== undefined ? ({ wildcard: wildcard, where: where }) : undefined; return [ match[0], match[1], eventOptions ]; }; let onEventFn = (identityConfig, handler) => { HandlerRegister.throwIfFinishedRegistration("onEvent"); let match = parseIdentityConfig(identityConfig); HandlerRegister.setHandler(match[0], match[1], handler, match[2], undefined); }; let parseSvmIdentityConfig = identityConfig => { let match; if (typeof identityConfig.program === "string") { match = [ identityConfig.program, identityConfig.instruction ]; } else { let inst = identityConfig.instruction; match = [ inst.contract, inst._0 ]; } let where = identityConfig.where; let eventOptions = where !== undefined ? ({ where: where }) : undefined; return [ match[0], match[1], eventOptions ]; }; let onInstructionFn = (identityConfig, handler) => { HandlerRegister.throwIfFinishedRegistration("onInstruction"); let match = parseSvmIdentityConfig(identityConfig); HandlerRegister.setHandler(match[0], match[1], args => handler({ instruction: args.event, context: args.context }), match[2], undefined); }; let contractRegisterFn = (identityConfig, handler) => { HandlerRegister.throwIfFinishedRegistration("contractRegister"); let match = parseIdentityConfig(identityConfig); HandlerRegister.setContractRegister(match[0], match[1], handler, match[2], undefined); }; let onRollbackCommitFn = callback => { HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit"); RollbackCommit.register(callback); }; let extractRange = (filter, name, ecosystem) => { try { let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema); if (inner !== undefined) { return S$RescriptSchema.parseOrThrow(Primitive_option.valFromOption(inner), blockRangeSchema); } else { return defaultBlockRange; } } catch (raw_exn) { let exn = Primitive_exceptions.internalToException(raw_exn); if (exn.RE_EXN_ID === S$RescriptSchema.Raised) { return Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` returned an invalid filter: ` + Utils.prettifyExn(exn._1)); } throw exn; } }; let onBlockFn = (rawOptions, handler) => { HandlerRegister.throwIfFinishedRegistration("onBlock"); let config = Config.loadWithoutRegistrations(); let ecosystem = config.ecosystem; let match = buildChainsObject(config); let chains = match[0]; let name = rawOptions.name; let logger = Logging.createChild({ onBlock: name }); let w = rawOptions.where; let where = w === undefined || w === null ? undefined : ( typeof w === "function" ? rawOptions.where : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` expected \`where\` to be a function or omitted, but got ` + typeof w + `.`) ); let matchedAny = { contents: false }; ChainMap.values(config.chainMap).forEach(chainConfig => { let chainId = chainConfig.id; let chainObj = chains[chainId.toString()]; let result = where !== undefined ? where({ chain: chainObj }) : true; let match = result === true ? [ true, defaultBlockRange ] : ( result === false ? [ false, defaultBlockRange ] : ( typeof result === "object" && !Array.isArray(result) && result !== null ? [ true, extractRange(result, name, ecosystem) ] : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` predicate returned an invalid value of type ` + typeof result + `. Expected boolean or a filter object.`) ) ); if (!match[0]) { return; } let range = match[1]; matchedAny.contents = true; HandlerRegister.registerOnBlock(name, chainId, range._every, range._gte, range._lte, handler); }); if (!matchedAny.contents) { return Logging.childWarn(logger, `\`indexer.` + ecosystem.onBlockMethodName + `\` matched 0 chains. Check the \`where\` predicate.`); } }; let keysMemo = { contents: undefined }; let getKeys = () => { let k = keysMemo.contents; if (k !== undefined) { return k; } let match = Config.loadWithoutRegistrations().ecosystem.name; let keys; let exit = 0; switch (match) { case "evm" : case "fuel" : exit = 1; break; case "svm" : keys = [ "name", "description", "chainIds", "chains", "onInstruction", "onSlot", "~internalAndWillBeRemovedSoon_onRollbackCommit" ]; break; } if (exit === 1) { keys = [ "name", "description", "chainIds", "chains", "onEvent", "contractRegister", "onBlock", "~internalAndWillBeRemovedSoon_onRollbackCommit" ]; } keysMemo.contents = keys; return keys; }; let get = prop => { switch (prop) { case "chainIds" : return buildChainsObject(Config.loadWithoutRegistrations())[1]; case "chains" : return buildChainsObject(Config.loadWithoutRegistrations())[0]; case "contractRegister" : return contractRegisterFn; case "description" : return Config.loadWithoutRegistrations().description; case "name" : return Config.loadWithoutRegistrations().name; case "onEvent" : return onEventFn; case "onInstruction" : return onInstructionFn; case "onBlock" : case "onSlot" : return onBlockFn; case "~internalAndWillBeRemovedSoon_onRollbackCommit" : return onRollbackCommitFn; default: return Stdlib_JsError.throwWithMessage(`Field \`` + prop + `\` does not exist on \`indexer\`. Available fields: ` + getKeys().join(", ") + `.`); } }; let traps_get = (target, prop) => { if (typeof prop === "string") { return get(prop); } else { return target[prop]; } }; let traps_ownKeys = param => getKeys(); let traps_getOwnPropertyDescriptor = (param, prop) => { if (typeof prop === "string" && getKeys().includes(prop)) { return { value: get(prop), enumerable: true, configurable: true }; } }; let traps = { get: traps_get, ownKeys: traps_ownKeys, getOwnPropertyDescriptor: traps_getOwnPropertyDescriptor }; return new Proxy(Object.create(null), traps); } function startServer(getState, ctx, isDevelopmentMode) { let app = Express(); let consoleCorsMiddleware = (req, res, next) => { let origin = req.headers["origin"]; if (origin !== undefined && (origin === Env.prodEnvioAppUrl || origin === Env.envioAppUrl)) { res.setHeader("Access-Control-Allow-Origin", origin); } res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"); res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); if (req.method === "OPTIONS") { res.sendStatus(200); return; } else { return next(); } }; app.use("/console", consoleCorsMiddleware); app.use("/metrics", consoleCorsMiddleware); app.use("/metrics/runtime", consoleCorsMiddleware); app.get("/healthz", (_req, res) => { res.sendStatus(200); }); app.get("/console/state", (_req, res) => { let state = isDevelopmentMode ? getState() : ({ status: "disabled" }); res.json(S$RescriptSchema.reverseConvertToJsonOrThrow(state, stateSchema)); }); app.post("/console/syncCache", (_req, res) => { if (isDevelopmentMode) { Persistence.getInitializedStorageOrThrow(ctx.persistence).dumpEffectCache().then(() => { res.json(true); }); } else { res.json(false); } }); let runtimeRegistry = new PromClient.Registry(); PromClient.collectDefaultMetrics({ register: runtimeRegistry }); app.get("/metrics", (_req, res) => { res.set("Content-Type", PromClient.register.contentType); PromClient.register.metrics().then(metrics => res.end(metrics)); }); app.get("/metrics/runtime", (_req, res) => { res.set("Content-Type", runtimeRegistry.contentType); runtimeRegistry.metrics().then(metrics => res.end(metrics)); }); let server = app.listen(Env.serverPort); server.on("error", err => { let code = err.code; if (code === "EADDRINUSE") { Logging.error(`Port ` + Env.serverPort.toString() + ` is already in use. To fix this either:` + (`\n 1. Kill the process using the port: lsof -ti :` + Env.serverPort.toString() + ` | xargs kill -9`) + `\n 2. Use a different port by setting the ENVIO_INDEXER_PORT environment variable: ENVIO_INDEXER_PORT=9899 envio start`); } else { Logging.errorWithExn(err, "Failed to start indexer server"); } Process.exit(1); }); } function getEnvioInfo() { return Config.stripSensitiveData(Config.getPublicConfigJson()); } async function migrate(reset) { let config = Config.loadWithoutRegistrations(); let persistence = PgStorage.makePersistenceFromConfig(config, undefined); await Persistence.init(persistence, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset); return await persistence.storage.close(); } async function dropSchema() { let config = Config.loadWithoutRegistrations(); let persistence = PgStorage.makePersistenceFromConfig(config, undefined); await persistence.storage.reset(); return await persistence.storage.close(); } async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockOpt, patchConfig) { let reset = resetOpt !== undefined ? resetOpt : false; let isTest = isTestOpt !== undefined ? isTestOpt : false; let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false; let mainArgs = Yargs(Helpers.hideBin(process.argv)).argv; let off = mainArgs["tui-off"]; let explicitTui = off !== undefined ? !off : Env.tuiEnvVar; let shouldUseTui = isTest ? false : ( explicitTui !== undefined ? explicitTui : !Envio.isNonInteractive() ); let configWithoutRegistrations = Config.loadWithoutRegistrations(); let isDevelopmentMode = !isTest && configWithoutRegistrations.isDev; let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(configWithoutRegistrations, undefined); globalPersistenceRef.contents = persistence$1; await Persistence.init(persistence$1, ChainMap.values(configWithoutRegistrations.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset); let match = await HandlerLoader.registerAllHandlers(configWithoutRegistrations); let registrations = match[1]; let config = match[0]; let config$1 = isTest ? ({ name: config.name, description: config.description, handlers: config.handlers, contractHandlers: config.contractHandlers, shouldRollbackOnReorg: false, shouldSaveFullHistory: config.shouldSaveFullHistory, storage: config.storage, chainMap: config.chainMap, defaultChain: config.defaultChain, ecosystem: config.ecosystem, enableRawEvents: config.enableRawEvents, maxAddrInPartition: config.maxAddrInPartition, batchSize: config.batchSize, lowercaseAddresses: config.lowercaseAddresses, isDev: config.isDev, userEntitiesByName: config.userEntitiesByName, userEntities: config.userEntities, allEntities: config.allEntities, allEnums: config.allEnums }) : config; let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1; let onError = errHandler => { ErrorHandling.log(errHandler); Process.exit(1); }; let ctx_inMemoryStore = InMemoryStore.make(persistence$1.allEntities, Persistence.getInitializedState(persistence$1).checkpointId, persistence$1, config$2, exn => onError(ErrorHandling.make(exn, undefined, "Failed writing batch to the database"))); let ctx = { registrations: registrations, config: config$2, persistence: persistence$1, inMemoryStore: ctx_inMemoryStore }; let envioVersion = Utils.EnvioPackage.value.version; Prometheus.Info.set(envioVersion); Prometheus.ProcessStartTimeSeconds.set(); Prometheus.RollbackEnabled.set(config$2.shouldRollbackOnReorg); if (!isTest) { startServer(() => { let gsManager = globalGsManagerRef.contents; if (gsManager === undefined) { return { status: "initializing" }; } let state = GlobalStateManager.getState(Primitive_option.valFromOption(gsManager)); let chains = ChainMap.values(state.chainManager.chainFetchers).map(cf => { let latestFetchedBlockNumber = Primitive_int.max(FetchState.bufferBlockNumber(cf.fetchState), 0); let knownHeight = ChainFetcher.hasProcessedToEndblock(cf) ? Stdlib_Option.getOr(cf.fetchState.endBlock, cf.fetchState.knownHeight) : cf.fetchState.knownHeight; return { chainId: cf.chainConfig.id, poweredByHyperSync: SourceManager.getActiveSource(cf.sourceManager).poweredByHyperSync, firstEventBlockNumber: cf.fetchState.firstEventBlock, latestProcessedBlock: cf.committedProgressBlockNumber === -1 ? undefined : cf.committedProgressBlockNumber, timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock, numEventsProcessed: cf.numEventsProcessed, latestFetchedBlockNumber: latestFetchedBlockNumber, currentBlockHeight: knownHeight, numBatchesFetched: 0, startBlock: cf.fetchState.startBlock, endBlock: cf.fetchState.endBlock, numAddresses: FetchState.numAddresses(cf.fetchState) }; }); return { status: "active", envioVersion: envioVersion, chains: chains, indexerStartTime: state.indexerStartTime, isPreRegisteringDynamicContracts: false, rollbackOnReorg: config$2.shouldRollbackOnReorg }; }, ctx, isDevelopmentMode); } let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations, undefined); let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError); let gsManager = GlobalStateManager.make(globalState, GlobalState.makeReducers(undefined, undefined, undefined, MarkBatchProcessedAdapter.make(ctx_inMemoryStore)), exn => onError(ErrorHandling.make(exn, undefined, "Indexer has failed with an unexpected error"))); if (shouldUseTui) { Tui.start(() => GlobalStateManager.getState(gsManager)); } globalGsManagerRef.contents = Primitive_option.some(gsManager); GlobalStateManager.dispatchTask(gsManager, { TAG: "NextQuery", _0: "CheckAllChains" }); return GlobalStateManager.dispatchTask(gsManager, "ProcessEventBatch"); } export { chainDataSchema, stateSchema, blockRangeSchema, defaultBlockRange, globalGsManagerRef, globalPersistenceRef, getInitialChainState, buildChainsObject, getGlobalIndexer, startServer, getEnvioInfo, migrate, dropSchema, start, } /* chainDataSchema Not a pure module */