envio
Version:
A latency and sync speed optimized, developer friendly blockchain data indexer.
392 lines (378 loc) • 15 kB
JavaScript
// Generated by ReScript, PLEASE EDIT WITH CARE
import * as Env from "../Env.res.mjs";
import * as Table from "../db/Table.res.mjs";
import * as Utils from "../Utils.res.mjs";
import * as Logging from "../Logging.res.mjs";
import * as Persistence from "../Persistence.res.mjs";
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
import * as EntityHistory from "../db/EntityHistory.res.mjs";
import * as InternalTable from "../db/InternalTable.res.mjs";
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
function getClickHouseFieldType(fieldType, isNullable, isArray) {
let baseType;
if (typeof fieldType !== "object") {
switch (fieldType) {
case "Boolean" :
baseType = "Bool";
break;
case "Uint32" :
baseType = "UInt32";
break;
case "UInt52" :
case "UInt64" :
baseType = "UInt64";
break;
case "Number" :
baseType = "Float64";
break;
case "Int32" :
case "Serial" :
baseType = "Int32";
break;
case "BigSerial" :
baseType = "Int64";
break;
case "String" :
case "Json" :
baseType = "String";
break;
case "Date" :
baseType = "DateTime64(3, 'UTC')";
break;
}
} else {
switch (fieldType.type) {
case "BigInt" :
let precision = fieldType.precision;
baseType = precision !== undefined && precision <= 38 ? `Decimal(` + precision.toString() + `,0)` : "String";
break;
case "BigDecimal" :
let config = fieldType.config;
if (config !== undefined) {
let scale = config[1];
let precision$1 = config[0];
baseType = precision$1 > 38 || scale > precision$1 ? "String" : `Decimal(` + precision$1.toString() + `,` + scale.toString() + `)`;
} else {
baseType = "String";
}
break;
case "Enum" :
let config$1 = fieldType.config;
let variantsLength = config$1.variants.length;
let enumType = variantsLength <= 127 ? "Enum8" : "Enum16";
let enumValues = config$1.variants.map(variant => `'` + variant + `'`).join(", ");
baseType = enumType + `(` + enumValues + `)`;
break;
case "Entity" :
baseType = "String";
break;
}
}
let baseType$1 = isArray ? `Array(` + baseType + `)` : baseType;
if (isNullable) {
return `Nullable(` + baseType$1 + `)`;
} else {
return baseType$1;
}
}
function makeClickHouseEntitySchema(table) {
return S$RescriptSchema.object(s => {
let dict = {};
table.fields.forEach(field => {
if (field.TAG !== "Field") {
return;
}
let f = field._0;
let fieldName = Table.getClickHouseDbFieldName(f);
let match = f.fieldType;
let fieldSchema;
if (typeof match !== "object") {
switch (match) {
case "UInt52" :
let uint52Schema = S$RescriptSchema.preprocess(S$RescriptSchema.float, param => ({
p: unknown => parseFloat(unknown)
}));
fieldSchema = f.isNullable ? S$RescriptSchema.$$null(uint52Schema) : (
f.isArray ? S$RescriptSchema.array(uint52Schema) : uint52Schema
);
break;
case "Date" :
let dateSchema = Utils.Schema.clickHouseDate;
fieldSchema = f.isNullable ? S$RescriptSchema.$$null(dateSchema) : (
f.isArray ? S$RescriptSchema.array(dateSchema) : dateSchema
);
break;
default:
fieldSchema = f.fieldSchema;
}
} else {
fieldSchema = f.fieldSchema;
}
dict[Table.getApiFieldName(f)] = s.f(fieldName, fieldSchema);
});
return dict;
});
}
let logger = Logging.createChild({
context: "ClickHouse"
});
async function insertWithRetry(client, table, values, format, retriesOpt) {
let retries = retriesOpt !== undefined ? retriesOpt : 8;
try {
return await client.insert({
table: table,
values: values,
format: format
});
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
if (retries > 0) {
let delayMs = Math.min(1000, 100 + ((900 * (8 - retries | 0) | 0) / 7 | 0) | 0);
if (values.length > 1) {
Logging.childWarn(logger, {
msg: "ClickHouse insert failed, splitting batch in half and retrying",
table: table,
batchSize: values.length,
retriesLeft: retries,
err: Utils.prettifyExn(exn)
});
await Utils.delay(delayMs);
let mid = (values.length >> 1);
let first = values.slice(0, mid);
let second = values.slice(mid);
await insertWithRetry(client, table, first, format, retries - 1 | 0);
return await insertWithRetry(client, table, second, format, retries - 1 | 0);
}
Logging.childWarn(logger, {
msg: "ClickHouse insert failed, retrying after delay",
table: table,
retriesLeft: retries,
err: Utils.prettifyExn(exn)
});
await Utils.delay(delayMs);
return await insertWithRetry(client, table, values, format, retries - 1 | 0);
}
throw exn;
}
}
async function setCheckpointsOrThrow(client, batch, database) {
let checkpointsCount = batch.checkpointIds.length;
if (checkpointsCount === 0) {
return;
}
let checkpointRows = [];
for (let idx = 0; idx < checkpointsCount; ++idx) {
checkpointRows.push([
batch.checkpointIds[idx].toString(),
batch.checkpointChainIds[idx],
batch.checkpointBlockNumbers[idx],
batch.checkpointBlockHashes[idx],
batch.checkpointEventsProcessed[idx]
]);
}
try {
return await insertWithRetry(client, database + `.\`` + InternalTable.Checkpoints.table.tableName + `\``, checkpointRows, "JSONCompactEachRow", undefined);
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
throw {
RE_EXN_ID: Persistence.StorageError,
message: `Failed to insert checkpoints into ClickHouse table "` + InternalTable.Checkpoints.table.tableName + `"`,
reason: Utils.prettifyExn(exn),
Error: new Error()
};
}
}
async function setUpdatesOrThrow(client, cache, changes, entityConfig, database) {
if (changes.length === 0) {
return;
}
let cached = cache.get(entityConfig);
let match;
if (cached !== undefined) {
match = cached;
} else {
let cached_tableName = database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\``;
let cached_convertOrThrow = S$RescriptSchema.compile(S$RescriptSchema.array(S$RescriptSchema.union([
EntityHistory.makeSetUpdateSchema(makeClickHouseEntitySchema(entityConfig.table)),
S$RescriptSchema.object(s => {
s.tag(EntityHistory.changeFieldName, "DELETE");
return {
type: "DELETE",
entityId: s.f(Table.idFieldName, S$RescriptSchema.string),
checkpointId: s.f(EntityHistory.checkpointIdFieldName, EntityHistory.unsafeCheckpointIdSchema)
};
})
])), "Output", "Json", "Sync", false);
let cached$1 = {
tableName: cached_tableName,
convertOrThrow: cached_convertOrThrow
};
cache.set(entityConfig, cached$1);
match = cached$1;
}
let tableName = match.tableName;
try {
let values = match.convertOrThrow(changes);
return await insertWithRetry(client, tableName, values, "JSONEachRow", undefined);
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
throw {
RE_EXN_ID: Persistence.StorageError,
message: `Failed to insert items into ClickHouse table "` + tableName + `"`,
reason: Utils.prettifyExn(exn),
Error: new Error()
};
}
}
function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt) {
let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
let fieldDefinitions = Stdlib_Array.filterMap(entityConfig.table.fields, field => {
if (field.TAG !== "Field") {
return;
}
let field$1 = field._0;
let fieldName = Table.getClickHouseDbFieldName(field$1);
let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray);
return `\`` + fieldName + `\` ` + clickHouseType;
});
return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\` (
` + fieldDefinitions.join(",\n ") + `,
\`` + EntityHistory.checkpointIdFieldName + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
\`` + EntityHistory.changeFieldName + `\` ` + getClickHouseFieldType({
type: "Enum",
config: EntityHistory.RowAction.config
}, false, false) + `
)
ENGINE = ` + tableEngine + `
ORDER BY (` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
}
function makeCreateCheckpointsTableQuery(database, replicatedOpt) {
let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` (
\`` + "id" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
\`` + "chain_id" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
\`` + "block_number" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
\`` + "block_hash" + `\` ` + getClickHouseFieldType("String", true, false) + `,
\`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `
)
ENGINE = ` + tableEngine + `
ORDER BY (` + "id" + `)`;
}
function makeCreateViewQuery(entityConfig, database) {
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
let checkpointsTableName = InternalTable.Checkpoints.table.tableName;
let entityFields = Stdlib_Array.filterMap(entityConfig.table.fields, field => {
if (field.TAG !== "Field") {
return;
}
let fieldName = Table.getClickHouseDbFieldName(field._0);
return `\`` + fieldName + `\``;
}).join(", ");
return `CREATE VIEW IF NOT EXISTS ` + database + `.\`` + entityConfig.name + `\` AS
SELECT ` + entityFields + `
FROM (
SELECT ` + entityFields + `, \`` + EntityHistory.changeFieldName + `\`
FROM ` + database + `.\`` + historyTableName + `\`
WHERE \`` + EntityHistory.checkpointIdFieldName + `\` <= (SELECT max(` + "id" + `) FROM ` + database + `.\`` + checkpointsTableName + `\`)
ORDER BY \`` + EntityHistory.checkpointIdFieldName + `\` DESC
LIMIT 1 BY \`` + Table.idFieldName + `\`
)
WHERE \`` + EntityHistory.changeFieldName + `\` = '` + "SET" + `'`;
}
async function initialize(client, database, entities, param) {
try {
let replicated = Env.ClickHouse.replicated();
let databaseEngine = Env.ClickHouse.databaseEngine();
let databaseEngineClause = databaseEngine !== undefined ? ` ENGINE = ` + databaseEngine : "";
let onClusterClause = replicated ? ` ON CLUSTER '{cluster}'` : "";
if (databaseEngine !== undefined) {
let expectedEngineName = databaseEngine.split("(")[0].trim();
let existingResult = await client.query({
query: `SELECT engine FROM system.databases WHERE name = '` + database + `'`
});
let rows = (await existingResult.json()).data;
let row = rows[0];
if (row !== undefined) {
let row$1 = Primitive_option.valFromOption(row);
if (row$1.engine !== expectedEngineName) {
Stdlib_JsError.throwWithMessage(`ClickHouse database "` + database + `" exists with engine "` + row$1.engine + `" but ENVIO_CLICKHOUSE_DATABASE_ENGINE specifies "` + expectedEngineName + `". Drop the database manually to change its engine.`);
}
}
}
await client.exec({
query: `TRUNCATE DATABASE IF EXISTS ` + database + onClusterClause
});
await client.exec({
query: `CREATE DATABASE IF NOT EXISTS ` + database + onClusterClause + databaseEngineClause
});
await client.exec({
query: `USE ` + database
});
await Promise.all(entities.map(entityConfig => client.exec({
query: makeCreateHistoryTableQuery(entityConfig, database, replicated)
})));
await client.exec({
query: makeCreateCheckpointsTableQuery(database, replicated)
});
await Promise.all(entities.map(entityConfig => client.exec({
query: makeCreateViewQuery(entityConfig, database)
})));
return Logging.trace("ClickHouse storage initialization completed successfully");
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
Logging.errorWithExn(exn, "Failed to initialize ClickHouse storage");
return Stdlib_JsError.throwWithMessage("ClickHouse initialization failed");
}
}
async function resume(client, database, checkpointId) {
try {
try {
await client.exec({
query: `USE ` + database
});
} catch (raw_exn) {
let exn = Primitive_exceptions.internalToException(raw_exn);
Logging.errorWithExn(exn, `ClickHouse storage database "` + database + `" not found. Please run 'envio start -r' to reinitialize the indexer (it'll also drop Postgres database).`);
Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
}
let tablesResult = await client.query({
query: `SHOW TABLES FROM ` + database + ` LIKE '` + EntityHistory.historyTablePrefix + `%'`
});
let tables = (await tablesResult.json()).data;
await Promise.all(tables.map(table => {
let tableName = table.name;
return client.exec({
query: `ALTER TABLE ` + database + `.\`` + tableName + `\` DELETE WHERE \`` + EntityHistory.checkpointIdFieldName + `\` > ` + checkpointId.toString()
});
}));
return await client.exec({
query: `DELETE FROM ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` WHERE \`` + Table.idFieldName + `\` > ` + checkpointId.toString()
});
} catch (raw_exn$1) {
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
if (exn$1.RE_EXN_ID === Persistence.StorageError) {
throw exn$1;
}
Logging.errorWithExn(exn$1, "Failed to resume ClickHouse storage");
return Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
}
}
export {
getClickHouseFieldType,
makeClickHouseEntitySchema,
logger,
insertWithRetry,
setCheckpointsOrThrow,
setUpdatesOrThrow,
makeCreateHistoryTableQuery,
makeCreateCheckpointsTableQuery,
makeCreateViewQuery,
initialize,
resume,
}
/* logger Not a pure module */