envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
239 lines (231 loc) • 11.2 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
import * as Table from "./db/Table.res.mjs";
import * as Utils from "./Utils.res.mjs";
import * as Hrtime from "./bindings/Hrtime.res.mjs";
import * as Logging from "./Logging.res.mjs";
import * as Prometheus from "./Prometheus.res.mjs";
import * as LoadManager from "./LoadManager.res.mjs";
import * as Persistence from "./Persistence.res.mjs";
import * as EntityFilter from "./db/EntityFilter.res.mjs";
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
import * as ErrorHandling from "./ErrorHandling.res.mjs";
import * as InMemoryStore from "./InMemoryStore.res.mjs";
import * as InMemoryTable from "./InMemoryTable.res.mjs";
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
function loadById(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, entityId) {
let key = entityConfig.name + `.get`;
let inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
let load = async (idsToLoad, param) => {
let storage = Persistence.getInitializedStorageOrThrow(persistence);
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
let dbEntities;
try {
dbEntities = await storage.loadOrThrow({
operator: "in",
fieldName: Table.idFieldName,
fieldValue: idsToLoad
}, entityConfig.table);
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
if (exn.RE_EXN_ID === Persistence.StorageError) {
dbEntities = ErrorHandling.mkLogAndRaise(Logging.getItemLogger(item), exn.message, exn.reason);
} else {
throw exn;
}
}
let entitiesMap = {};
for (let idx = 0, idx_finish = dbEntities.length; idx < idx_finish; ++idx) {
let entity = dbEntities[idx];
entitiesMap[entity.id] = entity;
}
idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, inMemoryStore.committedCheckpointId, entityId, entitiesMap[entityId]));
return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
};
return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, hash => hash in inMemTable.latestEntityChangeById, InMemoryTable.Entity.getUnsafe(inMemTable));
}
function callEffect(effect, arg, inMemTable, timerRef, onError) {
let effectName = effect.name;
let hadActiveCalls = effect.activeCallsCount > 0;
effect.activeCallsCount = effect.activeCallsCount + 1 | 0;
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
if (hadActiveCalls) {
let elapsed = Hrtime.secondsBetween(effect.prevCallStartTimerRef, timerRef);
if (elapsed > 0) {
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, elapsed);
}
}
effect.prevCallStartTimerRef = timerRef;
return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => {
effect.activeCallsCount = effect.activeCallsCount - 1 | 0;
Prometheus.SafeGauge.handleInt(Prometheus.EffectCalls.activeCallsCount, effectName, effect.activeCallsCount);
let newTimer = Hrtime.makeTimer();
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.timeCounter, effectName, Hrtime.secondsBetween(effect.prevCallStartTimerRef, newTimer));
effect.prevCallStartTimerRef = newTimer;
Prometheus.SafeCounter.increment(Prometheus.EffectCalls.totalCallsCount, effectName);
Prometheus.SafeCounter.handleFloat(Prometheus.EffectCalls.sumTimeCounter, effectName, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
});
}
function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
let effectName = effect.name;
let timerRef = Hrtime.makeTimer();
let promises = [];
let state = effect.rateLimit;
if (state !== undefined) {
let now = Date.now();
if (now >= state.windowStartTime + state.durationMs) {
state.availableCalls = state.callsPerDuration;
state.windowStartTime = now;
state.nextWindowPromise = undefined;
}
let immediateCount = Math.min(state.availableCalls, effectArgs.length);
let immediateArgs = effectArgs.slice(0, immediateCount);
let queuedArgs = effectArgs.slice(immediateCount);
state.availableCalls = state.availableCalls - immediateCount | 0;
for (let idx = 0, idx_finish = immediateArgs.length; idx < idx_finish; ++idx) {
promises.push(callEffect(effect, immediateArgs[idx], inMemTable, timerRef, onError));
}
if (immediateCount > 0 && isFromQueue) {
state.queueCount = state.queueCount - immediateCount | 0;
Prometheus.EffectQueueCount.set(state.queueCount, effectName);
}
if (Utils.$$Array.notEmpty(queuedArgs)) {
if (!isFromQueue) {
state.queueCount = state.queueCount + queuedArgs.length | 0;
Prometheus.EffectQueueCount.set(state.queueCount, effectName);
}
let millisUntilReset = {
contents: 0
};
let p = state.nextWindowPromise;
let nextWindowPromise;
if (p !== undefined) {
nextWindowPromise = p;
} else {
millisUntilReset.contents = state.windowStartTime + state.durationMs - now | 0;
let p$1 = Utils.delay(millisUntilReset.contents);
state.nextWindowPromise = p$1;
nextWindowPromise = p$1;
}
promises.push(nextWindowPromise.then(() => {
if (millisUntilReset.contents > 0) {
Prometheus.SafeCounter.handleFloat(Prometheus.EffectQueueCount.timeCounter, effectName, millisUntilReset.contents / 1000);
}
return executeWithRateLimit(effect, queuedArgs, inMemTable, onError, true);
}));
}
} else {
for (let idx$1 = 0, idx_finish$1 = effectArgs.length; idx$1 < idx_finish$1; ++idx$1) {
promises.push(callEffect(effect, effectArgs[idx$1], inMemTable, timerRef, onError));
}
}
return Promise.all(promises);
}
function loadEffect(loadManager, persistence, effect, effectArgs, inMemoryStore, shouldGroup, item) {
let effectName = effect.name;
let key = effectName + `.effect`;
let inMemTable = InMemoryStore.getEffectInMemTable(inMemoryStore, effect);
let load = async (args, onError) => {
let idsToLoad = args.map(arg => arg.cacheKey);
let idsFromCache = new Set();
let match = persistence.storageStatus;
let tmp;
tmp = typeof match !== "object" || match.TAG === "Initializing" ? false : effectName in match._0.cache;
if (tmp) {
let storage = Persistence.getInitializedStorageOrThrow(persistence);
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
let match$1 = effect.storageMeta;
let outputSchema = match$1.outputSchema;
let dbEntities;
try {
dbEntities = await storage.loadOrThrow({
operator: "in",
fieldName: Table.idFieldName,
fieldValue: idsToLoad
}, match$1.table);
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
Logging.childWarn(Logging.getItemLogger(item), {
msg: `Failed to load cache effect cache. The indexer will continue working, but the effect will not be able to use the cache.`,
err: Utils.prettifyExn(exn),
effect: effectName
});
dbEntities = [];
}
dbEntities.forEach(dbEntity => {
try {
let output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
idsFromCache.add(dbEntity.id);
return InMemoryStore.initEffectOutputFromDb(inMemTable, dbEntity.id, output);
} catch (raw_error) {
let error = Primitive_exceptions.internalToException(raw_error);
if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
inMemTable.invalidationsCount = inMemTable.invalidationsCount + 1 | 0;
Prometheus.EffectCacheInvalidationsCount.increment(effectName);
return Logging.childTrace(Logging.getItemLogger(item), {
msg: "Invalidated effect cache",
input: dbEntity.id,
effect: effectName,
err: S$RescriptSchema.$$Error.message(error._1)
});
}
throw error;
}
});
Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
}
let remainingCallsCount = idsToLoad.length - idsFromCache.size | 0;
if (remainingCallsCount <= 0) {
return;
}
let argsToCall = [];
for (let idx = 0, idx_finish = args.length; idx < idx_finish; ++idx) {
let arg = args[idx];
if (!idsFromCache.has(arg.cacheKey)) {
argsToCall.push(arg);
}
}
if (Utils.$$Array.notEmpty(argsToCall)) {
return await executeWithRateLimit(effect, argsToCall, inMemTable, onError, false);
}
};
return LoadManager.call(loadManager, effectArgs, key, load, args => args.cacheKey, shouldGroup, hash => InMemoryStore.hasEffectOutput(inMemTable, hash), hash => InMemoryStore.getEffectOutputUnsafe(inMemTable, hash));
}
function loadByFilter(loadManager, persistence, entityConfig, inMemoryStore, shouldGroup, item, filter) {
let key = EntityFilter.toOperationKey(filter, entityConfig.name);
let inMemTable = InMemoryStore.getInMemTable(inMemoryStore, entityConfig);
let load = async (filters, param) => {
let storage = Persistence.getInitializedStorageOrThrow(persistence);
let timerRef = Prometheus.StorageLoad.startOperation(storage.name, key);
let size = {
contents: 0
};
filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
let queries = EntityFilter.merge(filters);
await Promise.all(queries.map(async filter => {
try {
let entities = await storage.loadOrThrow(filter, entityConfig.table);
entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, inMemoryStore.committedCheckpointId, entity.id, entity));
size.contents = size.contents + entities.length | 0;
return;
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
if (exn.RE_EXN_ID === Persistence.StorageError) {
return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Logging.getItemLogger(item), {
operation: key,
params: EntityFilter.getParams(filter)
}), exn.message, exn.reason);
}
throw exn;
}
}));
return Prometheus.StorageLoad.endOperation(timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
};
return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
}
export {
loadById,
loadByFilter,
loadEffect,
}
/* Table Not a pure module */