envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
655 lines (638 loc) • 24.5 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
import * as Env from "./Env.res.mjs";
import * as Tui from "./tui/Tui.res.mjs";
import * as Envio from "./Envio.res.mjs";
import * as Utils from "./Utils.res.mjs";
import * as Config from "./Config.res.mjs";
import * as Logging from "./Logging.res.mjs";
import Express from "express";
import * as Process from "process";
import * as ChainMap from "./ChainMap.res.mjs";
import * as PgStorage from "./PgStorage.res.mjs";
import * as FetchState from "./FetchState.res.mjs";
import * as Prometheus from "./Prometheus.res.mjs";
import * as GlobalState from "./GlobalState.res.mjs";
import * as Persistence from "./Persistence.res.mjs";
import * as PromClient from "prom-client";
import Yargs from "yargs/yargs";
import * as ChainFetcher from "./ChainFetcher.res.mjs";
import * as ChainManager from "./ChainManager.res.mjs";
import * as ErrorHandling from "./ErrorHandling.res.mjs";
import * as HandlerLoader from "./HandlerLoader.res.mjs";
import * as InMemoryStore from "./InMemoryStore.res.mjs";
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
import * as SourceManager from "./sources/SourceManager.res.mjs";
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
import * as Helpers from "yargs/helpers";
import * as RollbackCommit from "./RollbackCommit.res.mjs";
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
import * as HandlerRegister from "./HandlerRegister.res.mjs";
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
import * as GlobalStateManager from "./GlobalStateManager.res.mjs";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
import * as MarkBatchProcessedAdapter from "./adapters/MarkBatchProcessedAdapter.res.mjs";
let chainDataSchema = S$RescriptSchema.schema(s => ({
chainId: s.m(S$RescriptSchema.float),
poweredByHyperSync: s.m(S$RescriptSchema.bool),
firstEventBlockNumber: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
latestProcessedBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
timestampCaughtUpToHeadOrEndblock: s.m(S$RescriptSchema.option(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined))),
numEventsProcessed: s.m(S$RescriptSchema.float),
latestFetchedBlockNumber: s.m(S$RescriptSchema.int),
currentBlockHeight: s.m(S$RescriptSchema.int),
numBatchesFetched: s.m(S$RescriptSchema.int),
startBlock: s.m(S$RescriptSchema.int),
endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
numAddresses: s.m(S$RescriptSchema.int)
}));
let stateSchema = S$RescriptSchema.union([
S$RescriptSchema.literal({
status: "disabled"
}),
S$RescriptSchema.literal({
status: "initializing"
}),
S$RescriptSchema.schema(s => ({
status: "active",
envioVersion: s.m(S$RescriptSchema.string),
chains: s.m(S$RescriptSchema.array(chainDataSchema)),
indexerStartTime: s.m(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined)),
isPreRegisteringDynamicContracts: false,
rollbackOnReorg: s.m(S$RescriptSchema.bool)
}))
]);
let blockRangeSchema = S$RescriptSchema.strict(S$RescriptSchema.object(s => ({
_gte: s.f("_gte", S$RescriptSchema.option(S$RescriptSchema.int)),
_lte: s.f("_lte", S$RescriptSchema.option(S$RescriptSchema.int)),
_every: s.f("_every", S$RescriptSchema.Option.getOr(S$RescriptSchema.option(S$RescriptSchema.intMin(S$RescriptSchema.int, 1, undefined)), 1))
})));
let defaultBlockRange = {
_gte: undefined,
_lte: undefined,
_every: 1
};
let globalGsManagerRef = {
contents: undefined
};
let globalPersistenceRef = {
contents: undefined
};
function getInitialChainState(chainId) {
let persistence = globalPersistenceRef.contents;
if (persistence === undefined) {
return;
}
let initialState = persistence.storageStatus;
if (typeof initialState !== "object" || initialState.TAG === "Initializing") {
return;
} else {
return initialState._0.chains.find(c => c.id === chainId);
}
}
function buildChainsObject(config) {
let chainIds = [];
let chains = Object.create(null);
ChainMap.values(config.chainMap).forEach(chainConfig => {
let chainIdStr = chainConfig.id.toString();
chainIds.push(chainConfig.id);
let chainObj = Object.create(null);
Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(chainObj, "id", {
enumerable: true,
value: chainConfig.id
}), "startBlock", {
enumerable: true,
get: () => {
let chainState = getInitialChainState(chainConfig.id);
if (chainState !== undefined) {
return chainState.startBlock;
} else {
return chainConfig.startBlock;
}
}
}), "endBlock", {
enumerable: true,
get: () => {
let match = getInitialChainState(chainConfig.id);
if (match === undefined) {
return chainConfig.endBlock;
}
let eb = match.endBlock;
if (eb !== undefined) {
return eb;
} else {
return chainConfig.endBlock;
}
}
}), "name", {
enumerable: true,
value: chainConfig.name
}), "isRealtime", {
enumerable: true,
get: () => {
let gsManager = globalGsManagerRef.contents;
if (gsManager !== undefined) {
return GlobalStateManager.getState(Primitive_option.valFromOption(gsManager)).chainManager.isRealtime;
} else if (Env.updateSyncTimeOnRestart) {
return false;
} else {
return ChainMap.values(config.chainMap).every(c => {
let chainState = getInitialChainState(c.id);
if (chainState !== undefined) {
return Stdlib_Option.isSome(chainState.timestampCaughtUpToHeadOrEndblock);
} else {
return false;
}
});
}
}
});
chainConfig.contracts.forEach(contract => {
let contractObj = Object.create(null);
Object.defineProperty(Object.defineProperty(Object.defineProperty(contractObj, "name", {
enumerable: true,
value: contract.name
}), "abi", {
enumerable: true,
value: contract.abi
}), "addresses", {
enumerable: true,
get: () => {
let gsManager = globalGsManagerRef.contents;
if (gsManager !== undefined) {
let state = GlobalStateManager.getState(Primitive_option.valFromOption(gsManager));
let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
let chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
let indexingAddresses = chainFetcher.fetchState.indexingAddresses;
let addresses = [];
let values = Object.values(indexingAddresses);
for (let idx = 0, idx_finish = values.length; idx < idx_finish; ++idx) {
let indexingContract = values[idx];
if (indexingContract.contractName === contract.name) {
addresses.push(indexingContract.address);
}
}
return addresses;
}
let chainState = getInitialChainState(chainConfig.id);
if (chainState === undefined) {
return contract.addresses;
}
let addresses$1 = contract.addresses.slice();
chainState.indexingAddresses.forEach(dc => {
if (dc.contractName === contract.name) {
addresses$1.push(dc.address);
return;
}
});
return addresses$1;
}
});
Object.defineProperty(chainObj, contract.name, {
enumerable: true,
value: contractObj
});
});
Object.defineProperty(chains, chainIdStr, {
enumerable: true,
value: chainObj
});
if (chainConfig.name !== chainIdStr) {
Object.defineProperty(chains, chainConfig.name, {
enumerable: false,
value: chainObj
});
return;
}
});
return [
chains,
chainIds
];
}
function getGlobalIndexer() {
let parseIdentityConfig = identityConfig => {
let match;
if (typeof identityConfig.contract === "string") {
match = [
identityConfig.contract,
identityConfig.event
];
} else {
let event = identityConfig.event;
match = [
event.contract,
event._0
];
}
let wildcard = identityConfig.wildcard;
let where = identityConfig.where;
let eventOptions = wildcard !== undefined || where !== undefined ? ({
wildcard: wildcard,
where: where
}) : undefined;
return [
match[0],
match[1],
eventOptions
];
};
let onEventFn = (identityConfig, handler) => {
HandlerRegister.throwIfFinishedRegistration("onEvent");
let match = parseIdentityConfig(identityConfig);
HandlerRegister.setHandler(match[0], match[1], handler, match[2], undefined);
};
let parseSvmIdentityConfig = identityConfig => {
let match;
if (typeof identityConfig.program === "string") {
match = [
identityConfig.program,
identityConfig.instruction
];
} else {
let inst = identityConfig.instruction;
match = [
inst.contract,
inst._0
];
}
let where = identityConfig.where;
let eventOptions = where !== undefined ? ({
where: where
}) : undefined;
return [
match[0],
match[1],
eventOptions
];
};
let onInstructionFn = (identityConfig, handler) => {
HandlerRegister.throwIfFinishedRegistration("onInstruction");
let match = parseSvmIdentityConfig(identityConfig);
HandlerRegister.setHandler(match[0], match[1], args => handler({
instruction: args.event,
context: args.context
}), match[2], undefined);
};
let contractRegisterFn = (identityConfig, handler) => {
HandlerRegister.throwIfFinishedRegistration("contractRegister");
let match = parseIdentityConfig(identityConfig);
HandlerRegister.setContractRegister(match[0], match[1], handler, match[2], undefined);
};
let onRollbackCommitFn = callback => {
HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit");
RollbackCommit.register(callback);
};
let extractRange = (filter, name, ecosystem) => {
try {
let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema);
if (inner !== undefined) {
return S$RescriptSchema.parseOrThrow(Primitive_option.valFromOption(inner), blockRangeSchema);
} else {
return defaultBlockRange;
}
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
return Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` returned an invalid filter: ` + Utils.prettifyExn(exn._1));
}
throw exn;
}
};
let onBlockFn = (rawOptions, handler) => {
HandlerRegister.throwIfFinishedRegistration("onBlock");
let config = Config.loadWithoutRegistrations();
let ecosystem = config.ecosystem;
let match = buildChainsObject(config);
let chains = match[0];
let name = rawOptions.name;
let logger = Logging.createChild({
onBlock: name
});
let w = rawOptions.where;
let where = w === undefined || w === null ? undefined : (
typeof w === "function" ? rawOptions.where : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` expected \`where\` to be a function or omitted, but got ` + typeof w + `.`)
);
let matchedAny = {
contents: false
};
ChainMap.values(config.chainMap).forEach(chainConfig => {
let chainId = chainConfig.id;
let chainObj = chains[chainId.toString()];
let result = where !== undefined ? where({
chain: chainObj
}) : true;
let match = result === true ? [
true,
defaultBlockRange
] : (
result === false ? [
false,
defaultBlockRange
] : (
typeof result === "object" && !Array.isArray(result) && result !== null ? [
true,
extractRange(result, name, ecosystem)
] : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` predicate returned an invalid value of type ` + typeof result + `. Expected boolean or a filter object.`)
)
);
if (!match[0]) {
return;
}
let range = match[1];
matchedAny.contents = true;
HandlerRegister.registerOnBlock(name, chainId, range._every, range._gte, range._lte, handler);
});
if (!matchedAny.contents) {
return Logging.childWarn(logger, `\`indexer.` + ecosystem.onBlockMethodName + `\` matched 0 chains. Check the \`where\` predicate.`);
}
};
let keysMemo = {
contents: undefined
};
let getKeys = () => {
let k = keysMemo.contents;
if (k !== undefined) {
return k;
}
let match = Config.loadWithoutRegistrations().ecosystem.name;
let keys;
let exit = 0;
switch (match) {
case "evm" :
case "fuel" :
exit = 1;
break;
case "svm" :
keys = [
"name",
"description",
"chainIds",
"chains",
"onInstruction",
"onSlot",
"~internalAndWillBeRemovedSoon_onRollbackCommit"
];
break;
}
if (exit === 1) {
keys = [
"name",
"description",
"chainIds",
"chains",
"onEvent",
"contractRegister",
"onBlock",
"~internalAndWillBeRemovedSoon_onRollbackCommit"
];
}
keysMemo.contents = keys;
return keys;
};
let get = prop => {
switch (prop) {
case "chainIds" :
return buildChainsObject(Config.loadWithoutRegistrations())[1];
case "chains" :
return buildChainsObject(Config.loadWithoutRegistrations())[0];
case "contractRegister" :
return contractRegisterFn;
case "description" :
return Config.loadWithoutRegistrations().description;
case "name" :
return Config.loadWithoutRegistrations().name;
case "onEvent" :
return onEventFn;
case "onInstruction" :
return onInstructionFn;
case "onBlock" :
case "onSlot" :
return onBlockFn;
case "~internalAndWillBeRemovedSoon_onRollbackCommit" :
return onRollbackCommitFn;
default:
return Stdlib_JsError.throwWithMessage(`Field \`` + prop + `\` does not exist on \`indexer\`. Available fields: ` + getKeys().join(", ") + `.`);
}
};
let traps_get = (target, prop) => {
if (typeof prop === "string") {
return get(prop);
} else {
return target[prop];
}
};
let traps_ownKeys = param => getKeys();
let traps_getOwnPropertyDescriptor = (param, prop) => {
if (typeof prop === "string" && getKeys().includes(prop)) {
return {
value: get(prop),
enumerable: true,
configurable: true
};
}
};
let traps = {
get: traps_get,
ownKeys: traps_ownKeys,
getOwnPropertyDescriptor: traps_getOwnPropertyDescriptor
};
return new Proxy(Object.create(null), traps);
}
function startServer(getState, ctx, isDevelopmentMode) {
let app = Express();
let consoleCorsMiddleware = (req, res, next) => {
let origin = req.headers["origin"];
if (origin !== undefined && (origin === Env.prodEnvioAppUrl || origin === Env.envioAppUrl)) {
res.setHeader("Access-Control-Allow-Origin", origin);
}
res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
if (req.method === "OPTIONS") {
res.sendStatus(200);
return;
} else {
return next();
}
};
app.use("/console", consoleCorsMiddleware);
app.use("/metrics", consoleCorsMiddleware);
app.use("/metrics/runtime", consoleCorsMiddleware);
app.get("/healthz", (_req, res) => {
res.sendStatus(200);
});
app.get("/console/state", (_req, res) => {
let state = isDevelopmentMode ? getState() : ({
status: "disabled"
});
res.json(S$RescriptSchema.reverseConvertToJsonOrThrow(state, stateSchema));
});
app.post("/console/syncCache", (_req, res) => {
if (isDevelopmentMode) {
Persistence.getInitializedStorageOrThrow(ctx.persistence).dumpEffectCache().then(() => {
res.json(true);
});
} else {
res.json(false);
}
});
let runtimeRegistry = new PromClient.Registry();
PromClient.collectDefaultMetrics({
register: runtimeRegistry
});
app.get("/metrics", (_req, res) => {
res.set("Content-Type", PromClient.register.contentType);
PromClient.register.metrics().then(metrics => res.end(metrics));
});
app.get("/metrics/runtime", (_req, res) => {
res.set("Content-Type", runtimeRegistry.contentType);
runtimeRegistry.metrics().then(metrics => res.end(metrics));
});
let server = app.listen(Env.serverPort);
server.on("error", err => {
let code = err.code;
if (code === "EADDRINUSE") {
Logging.error(`Port ` + Env.serverPort.toString() + ` is already in use. To fix this either:` + (`\n 1. Kill the process using the port: lsof -ti :` + Env.serverPort.toString() + ` | xargs kill -9`) + `\n 2. Use a different port by setting the ENVIO_INDEXER_PORT environment variable: ENVIO_INDEXER_PORT=9899 envio start`);
} else {
Logging.errorWithExn(err, "Failed to start indexer server");
}
Process.exit(1);
});
}
function getEnvioInfo() {
return Config.stripSensitiveData(Config.getPublicConfigJson());
}
async function migrate(reset) {
let config = Config.loadWithoutRegistrations();
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
await Persistence.init(persistence, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset);
return await persistence.storage.close();
}
async function dropSchema() {
let config = Config.loadWithoutRegistrations();
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
await persistence.storage.reset();
return await persistence.storage.close();
}
async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockOpt, patchConfig) {
let reset = resetOpt !== undefined ? resetOpt : false;
let isTest = isTestOpt !== undefined ? isTestOpt : false;
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
let mainArgs = Yargs(Helpers.hideBin(process.argv)).argv;
let off = mainArgs["tui-off"];
let explicitTui = off !== undefined ? !off : Env.tuiEnvVar;
let shouldUseTui = isTest ? false : (
explicitTui !== undefined ? explicitTui : !Envio.isNonInteractive()
);
let configWithoutRegistrations = Config.loadWithoutRegistrations();
let isDevelopmentMode = !isTest && configWithoutRegistrations.isDev;
let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(configWithoutRegistrations, undefined);
globalPersistenceRef.contents = persistence$1;
await Persistence.init(persistence$1, ChainMap.values(configWithoutRegistrations.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset);
let match = await HandlerLoader.registerAllHandlers(configWithoutRegistrations);
let registrations = match[1];
let config = match[0];
let config$1 = isTest ? ({
name: config.name,
description: config.description,
handlers: config.handlers,
contractHandlers: config.contractHandlers,
shouldRollbackOnReorg: false,
shouldSaveFullHistory: config.shouldSaveFullHistory,
storage: config.storage,
chainMap: config.chainMap,
defaultChain: config.defaultChain,
ecosystem: config.ecosystem,
enableRawEvents: config.enableRawEvents,
maxAddrInPartition: config.maxAddrInPartition,
batchSize: config.batchSize,
lowercaseAddresses: config.lowercaseAddresses,
isDev: config.isDev,
userEntitiesByName: config.userEntitiesByName,
userEntities: config.userEntities,
allEntities: config.allEntities,
allEnums: config.allEnums
}) : config;
let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1;
let onError = errHandler => {
ErrorHandling.log(errHandler);
Process.exit(1);
};
let ctx_inMemoryStore = InMemoryStore.make(persistence$1.allEntities, Persistence.getInitializedState(persistence$1).checkpointId, persistence$1, config$2, exn => onError(ErrorHandling.make(exn, undefined, "Failed writing batch to the database")));
let ctx = {
registrations: registrations,
config: config$2,
persistence: persistence$1,
inMemoryStore: ctx_inMemoryStore
};
let envioVersion = Utils.EnvioPackage.value.version;
Prometheus.Info.set(envioVersion);
Prometheus.ProcessStartTimeSeconds.set();
Prometheus.RollbackEnabled.set(config$2.shouldRollbackOnReorg);
if (!isTest) {
startServer(() => {
let gsManager = globalGsManagerRef.contents;
if (gsManager === undefined) {
return {
status: "initializing"
};
}
let state = GlobalStateManager.getState(Primitive_option.valFromOption(gsManager));
let chains = ChainMap.values(state.chainManager.chainFetchers).map(cf => {
let latestFetchedBlockNumber = Primitive_int.max(FetchState.bufferBlockNumber(cf.fetchState), 0);
let knownHeight = ChainFetcher.hasProcessedToEndblock(cf) ? Stdlib_Option.getOr(cf.fetchState.endBlock, cf.fetchState.knownHeight) : cf.fetchState.knownHeight;
return {
chainId: cf.chainConfig.id,
poweredByHyperSync: SourceManager.getActiveSource(cf.sourceManager).poweredByHyperSync,
firstEventBlockNumber: cf.fetchState.firstEventBlock,
latestProcessedBlock: cf.committedProgressBlockNumber === -1 ? undefined : cf.committedProgressBlockNumber,
timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock,
numEventsProcessed: cf.numEventsProcessed,
latestFetchedBlockNumber: latestFetchedBlockNumber,
currentBlockHeight: knownHeight,
numBatchesFetched: 0,
startBlock: cf.fetchState.startBlock,
endBlock: cf.fetchState.endBlock,
numAddresses: FetchState.numAddresses(cf.fetchState)
};
});
return {
status: "active",
envioVersion: envioVersion,
chains: chains,
indexerStartTime: state.indexerStartTime,
isPreRegisteringDynamicContracts: false,
rollbackOnReorg: config$2.shouldRollbackOnReorg
};
}, ctx, isDevelopmentMode);
}
let chainManager = ChainManager.makeFromDbState(Persistence.getInitializedState(persistence$1), config$2, registrations, undefined);
let globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
let gsManager = GlobalStateManager.make(globalState, GlobalState.makeReducers(undefined, undefined, undefined, MarkBatchProcessedAdapter.make(ctx_inMemoryStore)), exn => onError(ErrorHandling.make(exn, undefined, "Indexer has failed with an unexpected error")));
if (shouldUseTui) {
Tui.start(() => GlobalStateManager.getState(gsManager));
}
globalGsManagerRef.contents = Primitive_option.some(gsManager);
GlobalStateManager.dispatchTask(gsManager, {
TAG: "NextQuery",
_0: "CheckAllChains"
});
return GlobalStateManager.dispatchTask(gsManager, "ProcessEventBatch");
}
export {
chainDataSchema,
stateSchema,
blockRangeSchema,
defaultBlockRange,
globalGsManagerRef,
globalPersistenceRef,
getInitialChainState,
buildChainsObject,
getGlobalIndexer,
startServer,
getEnvioInfo,
migrate,
dropSchema,
start,
}
/* chainDataSchema Not a pure module */