envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
664 lines (645 loc) • 25.7 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
import * as Env from "./Env.res.mjs";
import * as Main from "./Main.res.mjs";
import * as Table from "./db/Table.res.mjs";
import * as Utils from "./Utils.res.mjs";
import * as Config from "./Config.res.mjs";
import * as Logging from "./Logging.res.mjs";
import * as Process from "process";
import * as ChainMap from "./ChainMap.res.mjs";
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
import * as Persistence from "./Persistence.res.mjs";
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
import * as EntityFilter from "./db/EntityFilter.res.mjs";
import * as InternalTable from "./db/InternalTable.res.mjs";
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
import * as SimulateItems from "./SimulateItems.res.mjs";
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
import * as Worker_threads from "worker_threads";
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
import * as TestIndexerProxyStorage from "./TestIndexerProxyStorage.res.mjs";
function toIndexingAddress(dc) {
return {
address: Config.EnvioAddresses.getAddress(dc),
contractName: dc.contract_name,
registrationBlock: dc.registration_block
};
}
function handleLoad(state, tableName, filter) {
let entityConfig = state.entityConfigs[tableName];
if (entityConfig === undefined) {
return [];
}
let entityDict = Stdlib_Option.getOr(state.entities[tableName], {});
let results = [];
let parseLeaf = (fieldName, fieldValue, isArray) => {
let queryField = Table.queryFields(entityConfig.table)[fieldName];
let queryField$1 = queryField !== undefined ? queryField : Stdlib_JsError.throwWithMessage(`Field ` + fieldName + ` not found in entity ` + tableName);
return S$RescriptSchema.convertOrThrow(fieldValue, isArray ? queryField$1.arrayFieldSchema : queryField$1.fieldSchema);
};
let filter$1 = EntityFilter.mapValues(filter, parseLeaf);
Object.values(entityDict).forEach(entity => {
if (!EntityFilter.matches(filter$1, entity)) {
return;
}
let jsonEntity = S$RescriptSchema.reverseConvertToJsonOrThrow(entity, entityConfig.schema);
results.push(jsonEntity);
});
return results;
}
function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChainIds, checkpointBlockNumbers, checkpointEventsProcessed) {
let changesByCheckpoint = {};
updatedEntities.forEach(param => {
let entityName = param.entityName;
let dict = state.entities[entityName];
let entityDict;
if (dict !== undefined) {
entityDict = dict;
} else {
let dict$1 = {};
state.entities[entityName] = dict$1;
entityDict = dict$1;
}
let entityConfig = state.entityConfigs[entityName];
let processChange = change => {
if (change.type === "SET") {
let parsedEntity = S$RescriptSchema.parseOrThrow(change.entity, entityConfig.schema);
entityDict[change.entityId] = parsedEntity;
let checkpointKey = change.checkpointId.toString();
let changes = changesByCheckpoint[checkpointKey];
let entityChanges;
if (changes !== undefined) {
entityChanges = changes;
} else {
let changes$1 = {};
changesByCheckpoint[checkpointKey] = changes$1;
entityChanges = changes$1;
}
let change$1 = entityChanges[entityName];
let entityChange;
if (change$1 !== undefined) {
entityChange = change$1;
} else {
let change_sets = [];
let change_deleted = [];
let change$2 = {
sets: change_sets,
deleted: change_deleted
};
entityChanges[entityName] = change$2;
entityChange = change$2;
}
entityChange.sets.push(parsedEntity);
return;
}
let entityId = change.entityId;
Stdlib_Dict.$$delete(entityDict, entityId);
let checkpointKey$1 = change.checkpointId.toString();
let changes$2 = changesByCheckpoint[checkpointKey$1];
let entityChanges$1;
if (changes$2 !== undefined) {
entityChanges$1 = changes$2;
} else {
let changes$3 = {};
changesByCheckpoint[checkpointKey$1] = changes$3;
entityChanges$1 = changes$3;
}
let change$3 = entityChanges$1[entityName];
let entityChange$1;
if (change$3 !== undefined) {
entityChange$1 = change$3;
} else {
let change_sets$1 = [];
let change_deleted$1 = [];
let change$4 = {
sets: change_sets$1,
deleted: change_deleted$1
};
entityChanges$1[entityName] = change$4;
entityChange$1 = change$4;
}
entityChange$1.deleted.push(entityId);
};
param.changes.forEach(processChange);
});
for (let i = 0, i_finish = checkpointIds.length; i < i_finish; ++i) {
let checkpointId = checkpointIds[i];
let change = {};
state.progressBlockByChain[checkpointChainIds[i].toString()] = checkpointBlockNumbers[i];
change["block"] = checkpointBlockNumbers[i];
change["chainId"] = checkpointChainIds[i];
change["eventsProcessed"] = checkpointEventsProcessed[i];
let checkpointKey = checkpointId.toString();
let entityChanges = changesByCheckpoint[checkpointKey];
if (entityChanges !== undefined) {
Object.entries(entityChanges).forEach(param => {
let match = param[1];
let deleted = match.deleted;
let sets = match.sets;
let entityName = param[0];
if (entityName === Config.EnvioAddresses.name) {
let entityObj = {};
if (sets.length !== 0) {
let simplifiedSets = sets.map(entity => ({
address: Config.EnvioAddresses.getAddress(entity),
contract: entity.contract_name
}));
entityObj["sets"] = simplifiedSets;
}
change["addresses"] = entityObj;
return;
}
let entityObj$1 = {};
if (sets.length !== 0) {
entityObj$1["sets"] = sets;
}
if (deleted.length !== 0) {
entityObj$1["deleted"] = deleted;
}
change[entityName] = entityObj$1;
});
}
state.processChanges.push(change);
}
}
function makeInitialState(config, processConfigChains, indexingAddressesByChain) {
let chainKeys = Object.keys(processConfigChains);
let chains = chainKeys.map(chainIdStr => {
let chainId = Stdlib_Option.getOr(Stdlib_Int.fromString(chainIdStr, undefined), 0);
let chain = ChainMap.Chain.makeUnsafe(chainId);
if (!ChainMap.has(config.chainMap, chain)) {
Stdlib_JsError.throwWithMessage(`Chain ` + chainIdStr + ` is not configured in config.yaml`);
}
let processChainConfig = processConfigChains[chainIdStr];
let indexingAddresses = Stdlib_Option.getOr(indexingAddressesByChain[chainIdStr], []);
return {
id: chainId,
startBlock: processChainConfig.startBlock,
endBlock: processChainConfig.endBlock,
maxReorgDepth: 0,
progressBlockNumber: -1,
numEventsProcessed: 0,
firstEventBlockNumber: undefined,
timestampCaughtUpToHeadOrEndblock: undefined,
indexingAddresses: indexingAddresses,
sourceBlockNumber: Stdlib_Option.getOr(processChainConfig.endBlock, 0)
};
});
return {
cleanRun: true,
cache: {},
chains: chains,
checkpointId: InternalTable.Checkpoints.initialCheckpointId,
reorgCheckpoints: [],
envioInfo: Config.stripSensitiveData(Config.getPublicConfigJson())
};
}
let rawChainConfigSchema = S$RescriptSchema.schema(s => ({
startBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.int)),
simulate: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.json(false))))
}));
let processConfigSchema = S$RescriptSchema.schema(s => ({
chains: s.m(S$RescriptSchema.dict(rawChainConfigSchema))
}));
function getSimulateEndBlock(simulateItems, config, startBlock) {
let maxBlock = {
contents: startBlock
};
simulateItems.forEach(rawJson => {
let blockJson = rawJson.block;
if (blockJson == null) {
return;
}
let n = Stdlib_Option.flatMap(blockJson[config.ecosystem.blockNumberName], v => {
if (v == null) {
return;
} else {
return Primitive_option.some(v);
}
});
if (n !== undefined && n > maxBlock.contents) {
maxBlock.contents = n;
return;
}
});
return maxBlock.contents;
}
function parseBlockRange(chainIdStr, config, rawChainConfig, progressBlock) {
let id = Stdlib_Int.fromString(chainIdStr, undefined);
let chainId = id !== undefined ? id : Stdlib_JsError.throwWithMessage(`Invalid chain ID "` + chainIdStr + `": expected a numeric chain ID`);
let chain = ChainMap.Chain.makeUnsafe(chainId);
if (!ChainMap.has(config.chainMap, chain)) {
Stdlib_JsError.throwWithMessage(`Chain ` + chainIdStr + ` is not configured in config.yaml`);
}
let configChain = ChainMap.get(config.chainMap, chain);
let sb = rawChainConfig.startBlock;
let startBlock = sb !== undefined ? sb : (
progressBlock !== undefined ? progressBlock + 1 | 0 : configChain.startBlock
);
let eb = rawChainConfig.endBlock;
let endBlock = eb !== undefined ? eb : (
Stdlib_Option.isSome(rawChainConfig.simulate) ? getSimulateEndBlock(Stdlib_Option.getOrThrow(rawChainConfig.simulate, undefined), config, startBlock) : undefined
);
if (startBlock < configChain.startBlock) {
Stdlib_JsError.throwWithMessage(`Invalid block range for chain ` + chainIdStr + `: startBlock (` + startBlock.toString() + `) is less than config.startBlock (` + configChain.startBlock.toString() + `). ` + (`Either use startBlock >= ` + configChain.startBlock.toString() + ` or create a new test indexer with createTestIndexer().`));
}
let match = configChain.endBlock;
if (endBlock !== undefined && match !== undefined && endBlock > match) {
Stdlib_JsError.throwWithMessage(`Invalid block range for chain ` + chainIdStr + `: endBlock (` + endBlock.toString() + `) exceeds config.endBlock (` + match.toString() + `). ` + (`Either use endBlock <= ` + match.toString() + ` or create a new test indexer with createTestIndexer().`));
}
if (progressBlock !== undefined && startBlock <= progressBlock) {
Stdlib_JsError.throwWithMessage(`Invalid block range for chain ` + chainIdStr + `: startBlock (` + startBlock.toString() + `) must be greater than previously processed endBlock (` + progressBlock.toString() + `). ` + (`Either use startBlock > ` + progressBlock.toString() + ` or create a new test indexer with createTestIndexer().`));
}
return {
startBlock: startBlock,
endBlock: endBlock
};
}
function getEntityFromState(state, entityConfig, entityId, methodName) {
if (state.processInProgress) {
Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.` + methodName + `() while indexer.process() is running. ` + "Wait for process() to complete before accessing entities directly.");
}
let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
return entityDict[entityId];
}
function makeEntityGet(state, entityConfig) {
return entityId => Promise.resolve(getEntityFromState(state, entityConfig, entityId, "get"));
}
function makeEntityGetOrThrow(state, entityConfig) {
return (entityId, message) => {
let entity = getEntityFromState(state, entityConfig, entityId, "getOrThrow");
if (entity !== undefined) {
return Promise.resolve(entity);
} else {
return Stdlib_JsError.throwWithMessage(message !== undefined ? message : `Entity ` + entityConfig.name + ` with id ` + entityId + ` not found`);
}
};
}
function makeEntitySet(state, entityConfig) {
return entity => {
if (state.processInProgress) {
Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.set() while indexer.process() is running. ` + "Wait for process() to complete before modifying entities directly.");
}
let dict = state.entities[entityConfig.name];
let entityDict;
if (dict !== undefined) {
entityDict = dict;
} else {
let dict$1 = {};
state.entities[entityConfig.name] = dict$1;
entityDict = dict$1;
}
entityDict[entity.id] = entity;
};
}
function makeEntityGetAll(state, entityConfig) {
return () => {
if (state.processInProgress) {
Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.getAll() while indexer.process() is running. ` + "Wait for process() to complete before accessing entities directly.");
}
let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
return Promise.resolve(Object.values(entityDict));
};
}
function makeCreateTestIndexer(config, workerPath) {
return () => {
let allEntities = config.allEntities;
let entities = {};
let entityConfigs = {};
allEntities.forEach(entityConfig => {
entities[entityConfig.name] = {};
entityConfigs[entityConfig.name] = entityConfig;
});
let envioAddressesDict = entities[Config.EnvioAddresses.name];
ChainMap.values(config.chainMap).forEach(chainConfig => {
chainConfig.contracts.forEach(contract => {
contract.addresses.forEach(address => {
let entity_id = Config.EnvioAddresses.makeId(chainConfig.id, address);
let entity_chain_id = chainConfig.id;
let entity_contract_name = contract.name;
let entity = {
id: entity_id,
chain_id: entity_chain_id,
registration_block: -1,
registration_log_index: -1,
contract_name: entity_contract_name
};
envioAddressesDict[entity_id] = entity;
});
});
});
let state = {
processInProgress: false,
progressBlockByChain: {},
entities: entities,
entityConfigs: entityConfigs,
processChanges: []
};
let entityOpsDict = {};
allEntities.forEach(entityConfig => {
if (entityConfig.name !== Config.EnvioAddresses.name) {
entityOpsDict[entityConfig.name] = {
get: makeEntityGet(state, entityConfig),
getAll: makeEntityGetAll(state, entityConfig),
getOrThrow: makeEntityGetOrThrow(state, entityConfig),
set: makeEntitySet(state, entityConfig)
};
return;
}
});
let chainIds = [];
let chains = Object.create(null);
ChainMap.values(config.chainMap).forEach(chainConfig => {
let chainIdStr = chainConfig.id.toString();
chainIds.push(chainConfig.id);
let chainObj = Object.create(null);
Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(chainObj, "id", {
enumerable: true,
value: chainConfig.id
}), "startBlock", {
enumerable: true,
value: chainConfig.startBlock
}), "endBlock", {
enumerable: true,
value: chainConfig.endBlock
}), "name", {
enumerable: true,
value: chainConfig.name
}), "isRealtime", {
enumerable: true,
value: false
});
chainConfig.contracts.forEach(contract => {
let contractObj = Object.create(null);
Object.defineProperty(Object.defineProperty(Object.defineProperty(contractObj, "name", {
enumerable: true,
value: contract.name
}), "abi", {
enumerable: true,
value: contract.abi
}), "addresses", {
enumerable: true,
get: () => {
if (state.processInProgress) {
Stdlib_JsError.throwWithMessage(`Cannot access ` + contract.name + `.addresses while indexer.process() is running. ` + "Wait for process() to complete before reading contract addresses.");
}
let addresses = [];
let dcDict = state.entities[Config.EnvioAddresses.name];
if (dcDict !== undefined) {
Object.values(dcDict).forEach(entity => {
if (entity.contract_name === contract.name && entity.chain_id === chainConfig.id) {
addresses.push(Config.EnvioAddresses.getAddress(entity));
return;
}
});
}
return addresses;
}
});
Object.defineProperty(chainObj, contract.name, {
enumerable: true,
value: contractObj
});
});
Object.defineProperty(chains, chainIdStr, {
enumerable: true,
value: chainObj
});
if (chainConfig.name !== chainIdStr) {
Object.defineProperty(chains, chainConfig.name, {
enumerable: false,
value: chainObj
});
return;
}
});
let result = {};
result["chainIds"] = chainIds;
result["chains"] = chains;
Object.entries(entityOpsDict).forEach(param => {
result[param[0]] = param[1];
});
result["process"] = processConfig => {
if (state.processInProgress) {
Stdlib_JsError.throwWithMessage("createTestIndexer process is already running. Only one process call is allowed at a time");
}
let parsedConfig;
try {
parsedConfig = S$RescriptSchema.parseOrThrow(processConfig, processConfigSchema);
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
parsedConfig = Stdlib_JsError.throwWithMessage(`Invalid processConfig: ` + Utils.prettifyExn(exn._1));
} else {
throw exn;
}
}
let rawChains = parsedConfig.chains;
let chainKeys = Object.keys(rawChains);
if (chainKeys.length === 0) {
Stdlib_JsError.throwWithMessage("createTestIndexer requires at least one chain to be defined");
}
let sortedChainKeys = chainKeys.slice();
sortedChainKeys.sort((a, b) => {
let aId = Stdlib_Option.getOr(Stdlib_Int.fromString(a, undefined), 0);
let bId = Stdlib_Option.getOr(Stdlib_Int.fromString(b, undefined), 0);
return Primitive_int.compare(aId, bId);
});
let chainEntries = sortedChainKeys.map(chainIdStr => {
let rawChainConfig = rawChains[chainIdStr];
let id = Stdlib_Int.fromString(chainIdStr, undefined);
let chainId = id !== undefined ? id : Stdlib_JsError.throwWithMessage(`Invalid chain ID "` + chainIdStr + `": expected a numeric chain ID`);
let processChainConfig = parseBlockRange(chainIdStr, config, rawChainConfig, state.progressBlockByChain[chainIdStr]);
return [
chainIdStr,
chainId,
rawChainConfig,
processChainConfig
];
});
state.processChanges = [];
let runChainWorker = param => {
let processChainConfig = param[3];
let rawChainConfig = param[2];
let chainId = param[1];
let chains = {};
chains[param[0]] = processChainConfig;
let indexingAddressesByChain = {};
let dcDict = state.entities[Config.EnvioAddresses.name];
if (dcDict !== undefined) {
Object.values(dcDict).forEach(entity => {
let dcChainIdStr = entity.chain_id.toString();
let arr = indexingAddressesByChain[dcChainIdStr];
let contracts;
if (arr !== undefined) {
contracts = arr;
} else {
let arr$1 = [];
indexingAddressesByChain[dcChainIdStr] = arr$1;
contracts = arr$1;
}
contracts.push(toIndexingAddress(entity));
});
}
let initialState = makeInitialState(config, chains, indexingAddressesByChain);
return new Promise((resolve, reject) => {
let workerData_startBlock = processChainConfig.startBlock;
let workerData_endBlock = processChainConfig.endBlock;
let workerData_simulate = rawChainConfig.simulate;
let workerData = {
chainId: chainId,
startBlock: workerData_startBlock,
endBlock: workerData_endBlock,
simulate: workerData_simulate,
initialState: initialState
};
let worker;
try {
worker = new Worker_threads.Worker(workerPath, {
workerData: workerData,
env: process.env
});
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
reject(exn);
throw exn;
}
worker.on("message", msg => {
let respond = data => {
worker.postMessage({
id: msg.id,
payload: {
type: "response",
data: data
}
});
};
let match = msg.payload;
if (match.type === "load") {
return respond(handleLoad(state, match.tableName, match.filter));
}
handleWriteBatch(state, match.updatedEntities, match.checkpointIds, match.checkpointChainIds, match.checkpointBlockNumbers, match.checkpointEventsProcessed);
respond(null);
});
worker.on("error", err => {
worker.terminate();
reject(err);
});
worker.on("exit", code => {
if (code !== 0) {
return reject(new Error(`Worker exited with code ` + code.toString()));
} else {
return resolve();
}
});
});
};
state.processInProgress = true;
let runChains = idx => {
if (idx >= chainEntries.length) {
state.processInProgress = false;
return Promise.resolve({
changes: state.processChanges
});
} else {
return runChainWorker(chainEntries[idx]).then(() => runChains(idx + 1 | 0));
}
};
return Stdlib_Promise.$$catch(runChains(0), err => {
state.processInProgress = false;
return Promise.reject(Utils.prettifyExn(err));
});
};
return result;
};
}
function initTestWorker() {
if (Worker_threads.isMainThread) {
Stdlib_JsError.throwWithMessage("initTestWorker must be called from a worker thread");
}
let port = Worker_threads.parentPort;
let parentPort = (port == null) ? Stdlib_JsError.throwWithMessage("initTestWorker: No parent port available") : port;
let workerData = Worker_threads.workerData;
if (workerData == null) {
Logging.error("TestIndexerWorker: No worker data provided");
Process.exit(1);
return;
}
let simulate = workerData.simulate;
let endBlock = workerData.endBlock;
let chainIdStr = workerData.chainId.toString();
let exitAfterFirstEventBlock = Stdlib_Option.isNone(endBlock);
let resolvedChainDict = {};
resolvedChainDict["startBlock"] = workerData.startBlock;
if (endBlock !== undefined) {
resolvedChainDict["endBlock"] = endBlock;
}
if (simulate !== undefined) {
resolvedChainDict["simulate"] = simulate;
}
let resolvedChainsDict = {};
resolvedChainsDict[chainIdStr] = resolvedChainDict;
let processConfig = {
chains: resolvedChainsDict
};
let proxy = TestIndexerProxyStorage.make(parentPort, workerData.initialState);
let storage = TestIndexerProxyStorage.makeStorage(proxy);
let config = Config.loadWithoutRegistrations();
let persistence = Persistence.make(config.userEntities, config.allEnums, storage);
if (Env.userLogLevel !== undefined) {
} else {
Logging.setLogLevel("silent");
}
let patchConfig = (config, _registrations) => {
let config$1 = SimulateItems.patchConfig(config, processConfig);
if (exitAfterFirstEventBlock) {
return {
name: config$1.name,
description: config$1.description,
handlers: config$1.handlers,
contractHandlers: config$1.contractHandlers,
shouldRollbackOnReorg: config$1.shouldRollbackOnReorg,
shouldSaveFullHistory: config$1.shouldSaveFullHistory,
storage: config$1.storage,
chainMap: config$1.chainMap,
defaultChain: config$1.defaultChain,
ecosystem: config$1.ecosystem,
enableRawEvents: config$1.enableRawEvents,
maxAddrInPartition: config$1.maxAddrInPartition,
batchSize: 1,
lowercaseAddresses: config$1.lowercaseAddresses,
isDev: config$1.isDev,
userEntitiesByName: config$1.userEntitiesByName,
userEntities: config$1.userEntities,
allEntities: config$1.allEntities,
allEnums: config$1.allEnums
};
} else {
return config$1;
}
};
Main.start(persistence, undefined, true, exitAfterFirstEventBlock, patchConfig);
}
export {
toIndexingAddress,
handleLoad,
handleWriteBatch,
makeInitialState,
rawChainConfigSchema,
processConfigSchema,
getSimulateEndBlock,
parseBlockRange,
getEntityFromState,
makeEntityGet,
makeEntityGetOrThrow,
makeEntitySet,
makeEntityGetAll,
makeCreateTestIndexer,
initTestWorker,
}
/* rawChainConfigSchema Not a pure module */