@ceramicnetwork/core
Version:
Typescript implementation of the Ceramic protocol
403 lines • 16.8 kB
JavaScript
import { CID } from 'multiformats/cid';
import cloneDeep from 'lodash.clonedeep';
import { StreamUtils, EnvironmentUtils, UnreachableCaseError, base64urlToJSON, toCID, } from '@ceramicnetwork/common';
import { StreamID } from '@ceramicnetwork/streamid';
import { ServiceMetrics as Metrics } from '@ceramicnetwork/observability';
import { MsgType, } from './pubsub/pubsub-message.js';
import { Pubsub } from './pubsub/pubsub.js';
import { empty } from 'rxjs';
import { MessageBus } from './pubsub/message-bus.js';
import { LRUCache } from 'least-recent';
import { PubsubKeepalive } from './pubsub/pubsub-keepalive.js';
import { PubsubRateLimit } from './pubsub/pubsub-ratelimit.js';
import { TaskQueue } from './ancillary/task-queue.js';
import { CARFactory, CarBlock } from 'cartonne';
import all from 'it-all';
import { IPFS_CACHE_HIT, IPFS_CACHE_MISS, IPLDRecordsCache } from './store/ipld-records-cache.js';
import { NodeMetrics } from '@ceramicnetwork/node-metrics';
const IPFS_GET_RETRIES = 3;
const DEFAULT_IPFS_GET_SYNC_TIMEOUT = 30000;
const DEFAULT_IPFS_GET_LOCAL_TIMEOUT = 1000;
const IPFS_MAX_COMMIT_SIZE = 256000;
const IPFS_RESUBSCRIBE_INTERVAL_DELAY = 1000 * 15;
const IPFS_NO_MESSAGE_INTERVAL = 1000 * 60 * 1;
const MAX_PUBSUB_PUBLISH_INTERVAL = 60 * 1000;
const MAX_INTERVAL_WITHOUT_KEEPALIVE = 24 * 60 * 60 * 1000;
const IPFS_CACHE_SIZE = process.env.CERAMIC_AUDIT_EVENT_PERSISTENCE == 'true' ? 1 : 1024;
const IPFS_OFFLINE_GET_TIMEOUT = 200;
const PUBSUB_CACHE_SIZE = 500;
const ERROR_IPFS_TIMEOUT = 'ipfs_timeout';
const ERROR_STORING_COMMIT = 'error_storing_commit';
const COMMITS_STORED = 'commits_stored';
const IMPORT_CAR_INIT_EVENT_REQUESTED = 'import_car_init_event_requested';
const IMPORT_CAR_STORE_EVENT_REQUESTED = 'import_car_store_event_requested';
const IMPORT_CAR_INIT_EVENT_TIME = 'import_car_init_event_time';
const IMPORT_CAR_STORE_EVENT_TIME = 'import_car_store_event_time';
const CREATE_CAR_INIT_EVENT_TIME = 'create_car_init_event_time';
const CREATE_CAR_STORE_EVENT_TIME = 'create_car_store_event_time';
function messageTypeToString(type) {
switch (type) {
case MsgType.UPDATE:
return 'Update';
case MsgType.QUERY:
return 'Query';
case MsgType.RESPONSE:
return 'Response';
case MsgType.KEEPALIVE:
return 'Keepalive';
default:
throw new UnreachableCaseError(type, `Unsupported message type`);
}
}
export class CommitSizeError extends Error {
constructor(cid, size) {
super(`${cid} commit size ${size} exceeds the maximum block size of ${IPFS_MAX_COMMIT_SIZE}`);
}
}
function restrictBlockSize(block, cid) {
const size = block.byteLength;
if (size > IPFS_MAX_COMMIT_SIZE)
throw new CommitSizeError(cid, size);
}
export class Dispatcher {
constructor(_ipfs, topic, repository, _logger, _pubsubLogger, _shutdownSignal, enableSync, maxQueriesPerSecond, recon, tasks = new TaskQueue()) {
this._ipfs = _ipfs;
this.topic = topic;
this.repository = repository;
this._logger = _logger;
this._pubsubLogger = _pubsubLogger;
this._shutdownSignal = _shutdownSignal;
this.recon = recon;
this.tasks = tasks;
this.pubsubCache = new LRUCache(PUBSUB_CACHE_SIZE);
const rustCeramic = EnvironmentUtils.useRustCeramic();
this.enableSync = rustCeramic ? false : enableSync;
if (this.enableSync) {
const pubsub = new Pubsub(_ipfs, topic, IPFS_RESUBSCRIBE_INTERVAL_DELAY, IPFS_NO_MESSAGE_INTERVAL, _pubsubLogger, _logger, tasks);
this.messageBus = new MessageBus(new PubsubRateLimit(new PubsubKeepalive(pubsub, _ipfs, MAX_PUBSUB_PUBLISH_INTERVAL, MAX_INTERVAL_WITHOUT_KEEPALIVE), _logger, maxQueriesPerSecond), !this.enableSync);
}
this.ipldCache = new IPLDRecordsCache(IPFS_CACHE_SIZE);
this.carFactory = new CARFactory();
for (const codec of this._ipfs.codecs.listCodecs()) {
this.carFactory.codecs.add(codec);
this.ipldCache.codecs.add(codec);
}
if (this.enableSync) {
this._ipfsTimeout = DEFAULT_IPFS_GET_SYNC_TIMEOUT;
}
else {
this._ipfsTimeout = DEFAULT_IPFS_GET_LOCAL_TIMEOUT;
}
}
async init() {
if (this.enableSync) {
this.messageBus.subscribe(this.handleMessage.bind(this));
}
}
get shutdownSignal() {
return this._shutdownSignal;
}
async ipfsNodeStatus() {
const ipfsId = await this._ipfs.id();
const peerId = ipfsId.id.toString();
const multiAddrs = ipfsId.addresses.map(String);
return { peerId, addresses: multiAddrs };
}
async storeRecord(record) {
return this._shutdownSignal
.abortable((signal) => this._ipfs.dag.put(record, { signal: signal }))
.then((cid) => {
const cidv13 = toCID(cid.toString());
this.ipldCache.setRecord(cidv13, record);
return cidv13;
});
}
async getIpfsBlock(cid) {
const found = this.ipldCache.get(cid);
if (found) {
Metrics.count(IPFS_CACHE_HIT, 1);
return new CarBlock(cid, found.block);
}
else {
Metrics.count(IPFS_CACHE_MISS, 1);
const bytes = await this._shutdownSignal.abortable((signal) => {
return this._ipfs.block.get(cid, { signal, offline: !this.enableSync });
});
this.ipldCache.setBlock(cid, bytes);
return new CarBlock(cid, bytes);
}
}
importCAR(car, streamId) {
const useRecon = EnvironmentUtils.useRustCeramic() && (streamId.type === 2 || streamId.type === 3);
if (useRecon) {
return this._shutdownSignal
.abortable(async (signal) => {
await this.recon.put(car, { signal });
})
.catch((e) => {
throw new Error(`Error while storing car to Recon for stream ${streamId.toString()}: ${e}`);
});
}
return this._shutdownSignal
.abortable(async (signal) => {
await all(this._ipfs.dag.import(car, { signal, pinRoots: false }));
})
.catch((e) => {
throw new Error(`Error while storing car to IPFS for stream ${streamId.toString()}: ${e}`);
});
}
_createCAR(data) {
const carFile = this.carFactory.build();
if (StreamUtils.isSignedCommitContainer(data)) {
const { jws, linkedBlock, cacaoBlock } = data;
if (cacaoBlock) {
const decodedProtectedHeader = base64urlToJSON(data.jws.signatures[0].protected);
const capCID = CID.parse(decodedProtectedHeader.cap.replace('ipfs://', ''));
carFile.blocks.put(new CarBlock(capCID, cacaoBlock));
restrictBlockSize(cacaoBlock, capCID);
this.ipldCache.set(capCID, {
record: carFile.get(capCID),
block: cacaoBlock,
});
}
const payloadCID = jws.link;
carFile.blocks.put(new CarBlock(payloadCID, linkedBlock));
restrictBlockSize(linkedBlock, jws.link);
this.ipldCache.set(payloadCID, {
record: carFile.get(payloadCID),
block: linkedBlock,
});
const cid = carFile.put(jws, { codec: 'dag-jose', hasher: 'sha2-256', isRoot: true });
const cidBlock = carFile.blocks.get(cid).payload;
restrictBlockSize(cidBlock, cid);
this.ipldCache.set(cid, {
record: carFile.get(cid),
block: cidBlock,
});
return carFile;
}
const cid = carFile.put(data, { isRoot: true });
const cidBlock = carFile.blocks.get(cid).payload;
restrictBlockSize(cidBlock, cid);
this.ipldCache.set(cid, {
record: carFile.get(cid),
block: cidBlock,
});
return carFile;
}
async storeInitEvent(data, streamType) {
try {
const timeStartCreate = Date.now();
const car = this._createCAR(data);
const timeEndCreate = Date.now();
Metrics.observe(CREATE_CAR_INIT_EVENT_TIME, timeEndCreate - timeStartCreate);
const streamId = new StreamID(streamType, car.roots[0]);
Metrics.count(IMPORT_CAR_INIT_EVENT_REQUESTED, 1);
const timeStartImport = Date.now();
await this.importCAR(car, streamId);
const timeEndImport = Date.now();
Metrics.observe(IMPORT_CAR_INIT_EVENT_TIME, timeEndImport - timeStartImport);
Metrics.count(COMMITS_STORED, 1);
return car.roots[0];
}
catch (e) {
this._logger.err(`Error while storing init event: ${e}`);
Metrics.count(ERROR_STORING_COMMIT, 1);
NodeMetrics.recordError(ERROR_STORING_COMMIT);
throw e;
}
}
async storeEvent(data, streamId) {
try {
const timeStartCreate = Date.now();
const car = this._createCAR(data);
const timeEndCreate = Date.now();
Metrics.observe(CREATE_CAR_STORE_EVENT_TIME, timeEndCreate - timeStartCreate);
Metrics.count(IMPORT_CAR_STORE_EVENT_REQUESTED, 1);
const timeStartImport = Date.now();
await this.importCAR(car, streamId);
const timeEndImport = Date.now();
Metrics.observe(IMPORT_CAR_STORE_EVENT_TIME, timeEndImport - timeStartImport);
Metrics.count(COMMITS_STORED, 1);
return car.roots[0];
}
catch (e) {
this._logger.err(`Error while storing event for stream ${streamId.toString()}: ${e}`);
Metrics.count(ERROR_STORING_COMMIT, 1);
NodeMetrics.recordError(ERROR_STORING_COMMIT);
throw e;
}
}
async retrieveCommit(cid, streamId) {
try {
return await this.getFromIpfs(cid);
}
catch (e) {
if (streamId) {
this._logger.err(`Error while loading commit CID ${cid.toString()} from IPFS for stream ${streamId.toString()}: ${e}`);
}
else {
this._logger.err(`Error while loading commit CID ${cid.toString()} from IPFS: ${e}`);
}
throw e;
}
}
async retrieveFromIPFS(cid, path) {
try {
return await this.getFromIpfs(cid, path);
}
catch (e) {
this._logger.err(`Error while loading CID ${cid.toString()} from IPFS: ${e}`);
throw e;
}
}
async cidExistsInLocalIPFSStore(cid) {
const asCid = typeof cid === 'string' ? CID.parse(cid) : cid;
try {
const result = await this._ipfs.dag.get(asCid, {
offline: true,
timeout: IPFS_OFFLINE_GET_TIMEOUT,
});
return result != null;
}
catch (err) {
this._logger.warn(`Error loading CID ${cid.toString()} from local IPFS node: ${err}`);
return false;
}
}
async getFromIpfs(cid, path) {
const asCid = typeof cid === 'string' ? CID.parse(cid) : cid;
const resolutionPath = `${asCid}${path || ''}`;
const fromCache = this.ipldCache.getWithResolutionPath(resolutionPath);
if (fromCache) {
Metrics.count(IPFS_CACHE_HIT, 1);
return cloneDeep(fromCache.record);
}
Metrics.count(IPFS_CACHE_MISS, 1);
let result = null;
for (let retries = IPFS_GET_RETRIES - 1; retries >= 0 && result == null; retries--) {
try {
let blockCid = asCid;
if (path) {
const resolution = await this._shutdownSignal.abortable((signal) => this._ipfs.dag.resolve(asCid, {
timeout: this._ipfsTimeout,
path: path,
signal: signal,
offline: this.recon.enabled,
}));
blockCid = toCID(resolution.cid.toString());
}
const codec = await this._ipfs.codecs.getCodec(blockCid.code);
const block = await this._shutdownSignal.abortable((signal) => this._ipfs.block.get(blockCid, {
timeout: this._ipfsTimeout,
signal: signal,
offline: !this.enableSync || this.recon.enabled,
}));
restrictBlockSize(block, blockCid);
result = codec.decode(block);
this.ipldCache.setWithResolutionPath(resolutionPath, {
record: result,
block: block,
});
return cloneDeep(result);
}
catch (err) {
if (err.code == 'ERR_TIMEOUT' ||
err.name == 'TimeoutError' ||
err.message == 'Request timed out') {
this._logger.warn(`Timeout error while loading CID ${asCid.toString()} from IPFS. ${retries} retries remain`);
Metrics.count(ERROR_IPFS_TIMEOUT, 1);
NodeMetrics.recordError(ERROR_IPFS_TIMEOUT);
if (retries > 0) {
continue;
}
else {
throw new Error(`Timeout error while loading CID ${asCid.toString()} from IPFS: ${err}`);
}
}
throw err;
}
}
}
publishTip(streamId, tip, model) {
if (!this.enableSync) {
return empty().subscribe();
}
return this.publish({ typ: MsgType.UPDATE, stream: streamId, tip, model });
}
async handleMessage(message) {
try {
switch (message.typ) {
case MsgType.UPDATE:
await this._handleUpdateMessage(message);
break;
case MsgType.QUERY:
await this._handleQueryMessage(message);
break;
case MsgType.RESPONSE:
await this._handleResponseMessage(message);
break;
case MsgType.KEEPALIVE:
break;
default:
throw new UnreachableCaseError(message, `Unsupported message type`);
}
}
catch (e) {
this._logger.err(`Error while processing ${messageTypeToString(message.typ)} message from pubsub: ${e}`);
this._logger.err(e);
}
}
async _handleTip(tip, streamId, model) {
if (this.pubsubCache.get(tip.toString()) === streamId.toString()) {
return;
}
this.pubsubCache.set(tip.toString(), streamId.toString());
await this.repository.handleUpdateFromNetwork(streamId, tip, model);
}
async _handleUpdateMessage(message) {
if (!this.enableSync) {
return;
}
const { stream: streamId, tip, model } = message;
return this._handleTip(tip, streamId, model);
}
async _handleQueryMessage(message) {
const { stream: streamId, id } = message;
const streamState = await this.repository.streamState(streamId);
if (streamState) {
const tip = streamState.log[streamState.log.length - 1].cid;
const tipMap = new Map().set(streamId.toString(), tip);
this.publish({ typ: MsgType.RESPONSE, id, tips: tipMap });
}
}
async _handleResponseMessage(message) {
if (!this.enableSync) {
return;
}
const { id: queryId, tips } = message;
const outstandingQuery = this.messageBus.outstandingQueries.queryMap.get(queryId);
const expectedStreamID = outstandingQuery?.streamID;
if (expectedStreamID) {
const newTip = tips.get(expectedStreamID.toString());
if (!newTip) {
throw new Error("Response to query with ID '" +
queryId +
"' is missing expected new tip for StreamID '" +
expectedStreamID +
"'");
}
return this._handleTip(newTip, expectedStreamID);
}
}
async close() {
if (this.enableSync) {
this.messageBus.unsubscribe();
}
await this.tasks.onIdle();
}
publish(message) {
return this.messageBus.next(message);
}
}
//# sourceMappingURL=dispatcher.js.map