envio/src/InMemoryStore.res.mjs

A latency and sync speed optimized, developer friendly blockchain data indexer.

// Generated by ReScript, PLEASE EDIT WITH CARE

import * as Env from "./Env.res.mjs";
import * as Table from "./db/Table.res.mjs";
import * as Utils from "./Utils.res.mjs";
import * as Config from "./Config.res.mjs";
import * as Logging from "./Logging.res.mjs";
import * as Internal from "./Internal.res.mjs";
import * as Throttler from "./Throttler.res.mjs";
import * as Prometheus from "./Prometheus.res.mjs";
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
import * as ErrorHandling from "./ErrorHandling.res.mjs";
import * as InMemoryTable from "./InMemoryTable.res.mjs";
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
import * as RollbackCommit from "./RollbackCommit.res.mjs";
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";

let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("InMemoryStore.EntityTables.UndefinedEntity");

function make(entities) {
  let init = {};
  entities.forEach(entityConfig => {
    init[entityConfig.name] = InMemoryTable.Entity.make();
  });
  return init;
}

function get(self, entityName) {
  let table = self[entityName];
  if (table !== undefined) {
    return table;
  } else {
    return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
      RE_EXN_ID: UndefinedEntity,
      entityName: entityName
    });
  }
}

let EntityTables = {
  UndefinedEntity: UndefinedEntity,
  make: make,
  get: get
};

function make$1(entities, committedCheckpointIdOpt, persistence, config, onError) {
  let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
  let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
  let chainMetaThrottler = Throttler.make(intervalMillis, Logging.createChild({
    context: "Throttler for chain metadata writes",
    intervalMillis: intervalMillis
  }));
  return {
    allEntities: entities,
    entities: make(entities),
    effects: {},
    rollback: undefined,
    committedCheckpointId: committedCheckpointId,
    processedCheckpointId: committedCheckpointId,
    processedBatches: [],
    isProcessing: false,
    processedBatchesCount: 0,
    writeFiber: undefined,
    hasFailedWrite: false,
    onError: onError,
    commitWaiters: [],
    persistence: persistence,
    config: config,
    chainMeta: {},
    chainMetaDirty: false,
    chainMetaThrottler: chainMetaThrottler
  };
}

function getEffectInMemTable(inMemoryStore, effect) {
  let key = effect.name;
  let table = inMemoryStore.effects[key];
  if (table !== undefined) {
    return table;
  }
  let table$1 = {
    idsToStore: [],
    invalidationsCount: 0,
    dict: {},
    changesCount: 0,
    effect: effect
  };
  inMemoryStore.effects[key] = table$1;
  return table$1;
}

function hasEffectOutput(inMemTable, key) {
  let match = inMemTable.dict[key];
  if (match !== undefined) {
    return match.type === "SET";
  } else {
    return false;
  }
}

function getEffectOutputUnsafe(inMemTable, key) {
  let match = inMemTable.dict[key];
  if (match !== undefined && match.type === "SET") {
    return match.entity;
  }
  return undefined;
}

function setEffectOutput(inMemTable, checkpointId, cacheKey, output, shouldCache) {
  let match = inMemTable.dict[cacheKey];
  if (match !== undefined) {
    
  } else {
    inMemTable.changesCount = inMemTable.changesCount + 1;
  }
  inMemTable.dict[cacheKey] = {
    type: "SET",
    entityId: cacheKey,
    entity: output,
    checkpointId: checkpointId
  };
  if (shouldCache) {
    inMemTable.idsToStore.push(cacheKey);
    return;
  }
}

function initEffectOutputFromDb(inMemTable, cacheKey, output) {
  if (Stdlib_Option.isNone(inMemTable.dict[cacheKey])) {
    inMemTable.changesCount = inMemTable.changesCount + 1;
    inMemTable.dict[cacheKey] = {
      type: "SET",
      entityId: cacheKey,
      entity: output,
      checkpointId: Internal.loadedFromDbCheckpointId
    };
    return;
  }
}

function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb) {
  let keysToDelete = [];
  Utils.Dict.forEachWithKey(inMemTable.dict, (change, key) => {
    let checkpointId = change.checkpointId;
    if (checkpointId <= committedCheckpointId && !(keepLoadedFromDb && checkpointId === Internal.loadedFromDbCheckpointId)) {
      keysToDelete.push(key);
      return;
    }
  });
  keysToDelete.forEach(key => Utils.Dict.deleteInPlace(inMemTable.dict, key));
  inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
}

function getInMemTable(inMemoryStore, entityConfig) {
  return get(inMemoryStore.entities, entityConfig.name);
}

function isRollingBack(inMemoryStore) {
  return inMemoryStore.rollback !== undefined;
}

function getChangesCount(inMemoryStore) {
  let total = {
    contents: 0
  };
  inMemoryStore.allEntities.forEach(entityConfig => {
    total.contents = total.contents + getInMemTable(inMemoryStore, entityConfig).changesCount;
  });
  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
    total.contents = total.contents + inMemTable.changesCount;
  });
  inMemoryStore.processedBatches.forEach(batch => {
    total.contents = total.contents + batch.totalBatchSize;
  });
  return total.contents;
}

function wakeCommitWaiters(inMemoryStore) {
  let waiters = inMemoryStore.commitWaiters;
  inMemoryStore.commitWaiters = [];
  waiters.forEach(resolve => resolve());
}

function waitForCommit(inMemoryStore) {
  return new Promise((resolve, param) => {
    inMemoryStore.commitWaiters.push(resolve);
  });
}

function drainBatchRun(inMemoryStore) {
  let all = inMemoryStore.processedBatches;
  let isInReorgThreshold = all[0].isInReorgThreshold;
  let rest = [];
  let progressedChainsById = {};
  let totalBatchSize = {
    contents: 0
  };
  let items = [];
  let checkpointIds = [];
  let checkpointChainIds = [];
  let checkpointBlockNumbers = [];
  let checkpointBlockHashes = [];
  let checkpointEventsProcessed = [];
  all.forEach(batch => {
    if (Utils.$$Array.isEmpty(rest) && batch.isInReorgThreshold === isInReorgThreshold) {
      Utils.Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, key) => {
        progressedChainsById[key] = chainAfterBatch;
      });
      totalBatchSize.contents = totalBatchSize.contents + batch.totalBatchSize | 0;
      items.push(...batch.items);
      checkpointIds.push(...batch.checkpointIds);
      checkpointChainIds.push(...batch.checkpointChainIds);
      checkpointBlockNumbers.push(...batch.checkpointBlockNumbers);
      checkpointBlockHashes.push(...batch.checkpointBlockHashes);
      checkpointEventsProcessed.push(...batch.checkpointEventsProcessed);
    } else {
      rest.push(batch);
    }
  });
  inMemoryStore.processedBatches = rest;
  return {
    totalBatchSize: totalBatchSize.contents,
    items: items,
    progressedChainsById: progressedChainsById,
    isInReorgThreshold: isInReorgThreshold,
    checkpointIds: checkpointIds,
    checkpointChainIds: checkpointChainIds,
    checkpointBlockNumbers: checkpointBlockNumbers,
    checkpointBlockHashes: checkpointBlockHashes,
    checkpointEventsProcessed: checkpointEventsProcessed
  };
}

function snapshotEffects(inMemoryStore, cache) {
  let acc = [];
  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
    let idsToStore = inMemTable.idsToStore;
    let invalidationsCount = inMemTable.invalidationsCount;
    let dict = inMemTable.dict;
    let effect = inMemTable.effect;
    if (idsToStore.length !== 0) {
      let items = Stdlib_Array.filterMap(idsToStore, id => {
        let match = dict[id];
        if (match.type === "SET") {
          return {
            id: id,
            output: match.entity
          };
        }
      });
      let effectName = effect.name;
      let c = cache[effectName];
      let effectCacheRecord;
      if (c !== undefined) {
        effectCacheRecord = c;
      } else {
        let c$1 = {
          effectName: effectName,
          count: 0
        };
        cache[effectName] = c$1;
        effectCacheRecord = c$1;
      }
      let shouldInitialize = effectCacheRecord.count === 0;
      effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
      acc.push({
        effect: effect,
        items: items,
        shouldInitialize: shouldInitialize
      });
    }
    inMemTable.idsToStore = [];
    inMemTable.invalidationsCount = 0;
  });
  return acc;
}

async function runOneWrite(inMemoryStore, persistence, config) {
  let match = persistence.storageStatus;
  let cache;
  cache = typeof match !== "object" || match.TAG === "Initializing" ? Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`) : match._0.cache;
  let chainMetaData = inMemoryStore.chainMetaDirty ? (inMemoryStore.chainMetaDirty = false, Utils.Dict.shallowCopy(inMemoryStore.chainMeta)) : undefined;
  let match$1 = inMemoryStore.processedBatches;
  if (match$1.length === 0) {
    if (chainMetaData !== undefined) {
      return await persistence.storage.setChainMeta(chainMetaData);
    } else {
      return;
    }
  }
  let committedCheckpointId = inMemoryStore.committedCheckpointId;
  let batch = drainBatchRun(inMemoryStore);
  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
  let rollback = inMemoryStore.rollback;
  inMemoryStore.rollback = undefined;
  let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
    let table = getInMemTable(inMemoryStore, entityConfig);
    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
    if (Utils.$$Array.isEmpty(changes)) {
      return;
    } else {
      return {
        entityConfig: entityConfig,
        changes: changes
      };
    }
  });
  let updatedEffectsCache = snapshotEffects(inMemoryStore, cache);
  await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
  inMemoryStore.committedCheckpointId = upToCheckpointId;
  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
  }
}

function hasPendingWrite(inMemoryStore) {
  if (Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
    return true;
  } else {
    return inMemoryStore.chainMetaDirty;
  }
}

async function runWriteLoop(inMemoryStore) {
  while (hasPendingWrite(inMemoryStore) && !inMemoryStore.hasFailedWrite) {
    try {
      await runOneWrite(inMemoryStore, inMemoryStore.persistence, inMemoryStore.config);
      wakeCommitWaiters(inMemoryStore);
    } catch (raw_exn) {
      let exn = Primitive_exceptions.internalToException(raw_exn);
      inMemoryStore.hasFailedWrite = true;
      inMemoryStore.onError(exn);
    }
  };
  inMemoryStore.writeFiber = undefined;
  return wakeCommitWaiters(inMemoryStore);
}

function kick(inMemoryStore) {
  if (Stdlib_Option.isNone(inMemoryStore.writeFiber) && !inMemoryStore.hasFailedWrite && hasPendingWrite(inMemoryStore)) {
    inMemoryStore.writeFiber = runWriteLoop(inMemoryStore);
    return;
  }
}

function metaFieldsEqual(a, b) {
  if (Primitive_object.equal(a.first_event_block, b.first_event_block) && a.buffer_block === b.buffer_block && a._is_hyper_sync === b._is_hyper_sync) {
    return Primitive_object.equal(Stdlib_Option.map(Primitive_option.fromNull(a.ready_at), prim => prim.getTime()), Stdlib_Option.map(Primitive_option.fromNull(b.ready_at), prim => prim.getTime()));
  } else {
    return false;
  }
}

function setChainMeta(inMemoryStore, chainsData) {
  Utils.Dict.forEachWithKey(chainsData, (meta, chainId) => {
    let prev = inMemoryStore.chainMeta[chainId];
    let changed = prev !== undefined ? !metaFieldsEqual(meta, prev) : true;
    if (changed) {
      inMemoryStore.chainMeta[chainId] = meta;
      inMemoryStore.chainMetaDirty = true;
      return;
    }
  });
  if (inMemoryStore.chainMetaDirty) {
    return Throttler.schedule(inMemoryStore.chainMetaThrottler, () => {
      kick(inMemoryStore);
      return Promise.resolve();
    });
  }
}

function commitBatch(inMemoryStore, batch) {
  inMemoryStore.processedBatches.push(batch);
  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
  if (checkpointId !== undefined) {
    inMemoryStore.processedCheckpointId = checkpointId;
  }
  kick(inMemoryStore);
}

function dropCommitted(inMemoryStore, keepLoadedFromDb) {
  let committedCheckpointId = inMemoryStore.committedCheckpointId;
  inMemoryStore.allEntities.forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(getInMemTable(inMemoryStore, entityConfig), committedCheckpointId, keepLoadedFromDb));
  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
}

async function awaitCapacity(inMemoryStore) {
  if (!inMemoryStore.hasFailedWrite && getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
    dropCommitted(inMemoryStore, true);
    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
      dropCommitted(inMemoryStore, false);
    }
    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
      kick(inMemoryStore);
      await waitForCommit(inMemoryStore);
      return await awaitCapacity(inMemoryStore);
    } else {
      return;
    }
  }
}

async function flush(inMemoryStore) {
  if (inMemoryStore.hasFailedWrite) {
    return;
  }
  kick(inMemoryStore);
  let fiber = inMemoryStore.writeFiber;
  if (fiber !== undefined) {
    await fiber;
    return await flush(inMemoryStore);
  }
}

async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId) {
  inMemoryStore.entities = make(inMemoryStore.allEntities);
  inMemoryStore.effects = {};
  inMemoryStore.rollback = {
    targetCheckpointId: rollbackTargetCheckpointId,
    diffCheckpointId: rollbackDiffCheckpointId,
    progressBlockNumberByChainId: progressBlockNumberByChainId
  };
  let deletedEntities = {};
  let setEntities = {};
  await Promise.all(persistence.allEntities.map(async entityConfig => {
    let entityTable = getInMemTable(inMemoryStore, entityConfig);
    let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
    match[0].forEach(data => {
      Utils.Dict.push(deletedEntities, entityConfig.name, data.id);
      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
        type: "DELETE",
        entityId: data.id,
        checkpointId: rollbackDiffCheckpointId
      });
    });
    let restoredEntities = S$RescriptSchema.parseOrThrow(match[1], Table.pgRowsSchema(entityConfig.table));
    restoredEntities.forEach(entity => {
      Utils.Dict.push(setEntities, entityConfig.name, entity.id);
      InMemoryTable.Entity.set(entityTable, inMemoryStore.committedCheckpointId, {
        type: "SET",
        entityId: entity.id,
        entity: entity,
        checkpointId: rollbackDiffCheckpointId
      });
    });
  }));
  return {
    deletedEntities: deletedEntities,
    setEntities: setEntities
  };
}

function setBatchDcs(inMemoryStore, batch) {
  let inMemTable = getInMemTable(inMemoryStore, Config.EnvioAddresses.entityConfig);
  let itemIdx = 0;
  for (let checkpoint = 0, checkpoint_finish = batch.checkpointIds.length; checkpoint < checkpoint_finish; ++checkpoint) {
    let checkpointId = batch.checkpointIds[checkpoint];
    let chainId = batch.checkpointChainIds[checkpoint];
    let checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpoint];
    for (let idx = 0; idx < checkpointEventsProcessed; ++idx) {
      let item = batch.items[itemIdx + idx | 0];
      let dcs = item.dcs;
      if (dcs !== undefined) {
        for (let dcIdx = 0, dcIdx_finish = dcs.length; dcIdx < dcIdx_finish; ++dcIdx) {
          let dc = dcs[dcIdx];
          let entity_id = Config.EnvioAddresses.makeId(chainId, dc.address);
          let entity_registration_block = item.blockNumber;
          let entity_registration_log_index = item.logIndex;
          let entity_contract_name = dc.contractName;
          let entity = {
            id: entity_id,
            chain_id: chainId,
            registration_block: entity_registration_block,
            registration_log_index: entity_registration_log_index,
            contract_name: entity_contract_name
          };
          InMemoryTable.Entity.set(inMemTable, inMemoryStore.committedCheckpointId, {
            type: "SET",
            entityId: entity_id,
            entity: entity,
            checkpointId: checkpointId
          });
        }
      }
    }
    itemIdx = itemIdx + checkpointEventsProcessed | 0;
  }
}

let keepLatestChangesLimit = Env.inMemoryObjectsTarget;

export {
  EntityTables,
  make$1 as make,
  keepLatestChangesLimit,
  getEffectInMemTable,
  hasEffectOutput,
  getEffectOutputUnsafe,
  setEffectOutput,
  initEffectOutputFromDb,
  dropCommittedEffects,
  getInMemTable,
  isRollingBack,
  getChangesCount,
  wakeCommitWaiters,
  waitForCommit,
  drainBatchRun,
  snapshotEffects,
  runOneWrite,
  hasPendingWrite,
  runWriteLoop,
  kick,
  metaFieldsEqual,
  setChainMeta,
  commitBatch,
  dropCommitted,
  awaitCapacity,
  flush,
  prepareRollbackDiff,
  setBatchDcs,
}
/* Env Not a pure module */