@holographxyz/cli
Version:
Holograph operator CLI
482 lines (481 loc) • 24.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const fs = tslib_1.__importStar(require("fs-extra"));
const path = tslib_1.__importStar(require("node:path"));
const inquirer = tslib_1.__importStar(require("inquirer"));
const core_1 = require("@oclif/core");
const bignumber_1 = require("@ethersproject/bignumber");
const environment_1 = require("@holographxyz/environment");
const networks_1 = require("@holographxyz/networks");
const config_1 = require("../../utils/config");
const network_monitor_1 = require("../../utils/network-monitor");
const utils_1 = require("../../utils/utils");
const validation_1 = require("../../utils/validation");
const operator_job_1 = require("../../utils/operator-job");
const healthcheck_1 = require("../../base-commands/healthcheck");
/**
* Operator
* Description: The primary command for operating jobs on the Holograph network.
*/
class Operator extends operator_job_1.OperatorJobAwareCommand {
static description = 'Listen for jobs and execute jobs.';
static examples = ['$ <%= config.bin %> <%= command.id %> --networks goerli fuji mumbai --mode=auto --sync'];
static flags = {
mode: core_1.Flags.string({
description: 'The mode in which to run the operator',
options: Object.values(network_monitor_1.OperatorMode),
char: 'm',
}),
sync: core_1.Flags.boolean({
description: 'Start from last saved block position instead of latest block position',
default: false,
}),
unsafePassword: core_1.Flags.string({
description: 'Enter the plain text password for the wallet in the holograph cli config',
}),
...network_monitor_1.networksFlag,
...healthcheck_1.HealthCheck.flags,
};
/**
* Operator class variables
*/
operatorMode = network_monitor_1.OperatorMode.listen;
environment;
jobsFile;
/**
* Command Entry Point
*/
async run() {
const { flags } = await this.parse(Operator);
// Check the flags
const enableHealthCheckServer = flags.healthCheck;
const healthCheckPort = flags.healthCheckPort;
const syncFlag = flags.sync;
const unsafePassword = flags.unsafePassword;
this.operatorMode =
network_monitor_1.OperatorMode[(await (0, validation_1.checkOptionFlag)(Object.values(network_monitor_1.OperatorMode), flags.mode, 'Select the mode in which to run the operator'))];
this.log(`Operator mode: ${this.operatorMode}`);
this.log('Loading user configurations...');
const { environment, userWallet, configFile } = await (0, config_1.ensureConfigFileIsValid)(this.config.configDir, unsafePassword, true);
this.environment = environment;
this.log('User configurations loaded.');
this.networkMonitor = new network_monitor_1.NetworkMonitor({
parent: this,
configFile,
networks: flags.networks,
debug: this.debug,
processTransactions: this.processTransactions,
userWallet,
lastBlockFilename: 'operator-blocks.json',
});
this.jobsFile = path.join(this.config.configDir, this.networkMonitor.environment + '.operator-job-details.json');
// Load the last block height from the file
this.networkMonitor.latestBlockHeight = await this.networkMonitor.loadLastBlocks(this.config.configDir);
// Check if the operator has previous missed blocks
let canSync = false;
const lastBlockKeys = Object.keys(this.networkMonitor.latestBlockHeight);
for (let i = 0, l = lastBlockKeys.length; i < l; i++) {
if (this.networkMonitor.latestBlockHeight[lastBlockKeys[i]] > 0) {
canSync = true;
break;
}
}
if (canSync && !syncFlag) {
const syncPrompt = await inquirer.prompt([
{
name: 'shouldSync',
message: 'Operator has previous (missed) blocks that can be synced. Would you like to sync?',
type: 'confirm',
default: true,
},
]);
if (syncPrompt.shouldSync === false) {
this.networkMonitor.latestBlockHeight = {};
this.networkMonitor.currentBlockHeight = {};
}
}
this.operatorStatus.address = userWallet.address.toLowerCase();
this.networkMonitor.exitCallback = this.exitCallback.bind(this);
core_1.CliUx.ux.action.start(`Starting operator in mode: ${network_monitor_1.OperatorMode[this.operatorMode]}`);
await this.networkMonitor.run(true, undefined, this.filterBuilder);
core_1.CliUx.ux.action.stop('🚀');
// check if file exists
if (await fs.pathExists(this.jobsFile)) {
// if file exists, need to add it to list of jobs to process
this.operatorJobs = (await fs.readJson(this.jobsFile));
// need to check each job and make sure it's still valid
for (const jobHash of Object.keys(this.operatorJobs)) {
this.operatorJobs[jobHash].gasLimit = bignumber_1.BigNumber.from(this.operatorJobs[jobHash].gasLimit);
this.operatorJobs[jobHash].gasPrice = bignumber_1.BigNumber.from(this.operatorJobs[jobHash].gasPrice);
this.operatorJobs[jobHash].jobDetails.startTimestamp = bignumber_1.BigNumber.from(this.operatorJobs[jobHash].jobDetails.startTimestamp);
// if job is still valid, it will stay in object, otherwise it will be removed
await this.checkJobStatus(jobHash);
}
}
for (const network of this.networkMonitor.networks) {
// instantiate all network operator job watchers
setTimeout(this.processOperatorJobs.bind(this, network), 60000);
}
// Start health check server on port 6000
// Can be used to monitor that the operator is online and running
if (enableHealthCheckServer) {
await this.config.runHook('healthCheck', { networkMonitor: this.networkMonitor, healthCheckPort });
}
}
exitCallback() {
const jobs = this.operatorJobs;
for (const jobHash of Object.keys(jobs)) {
jobs[jobHash].gasLimit = bignumber_1.BigNumber.from(jobs[jobHash].gasLimit).toHexString();
jobs[jobHash].gasPrice = bignumber_1.BigNumber.from(jobs[jobHash].gasPrice).toHexString();
jobs[jobHash].jobDetails.startTimestamp = bignumber_1.BigNumber.from(jobs[jobHash].jobDetails.startTimestamp).toHexString();
}
fs.writeFileSync(this.jobsFile, JSON.stringify(jobs, undefined, 2));
}
/**
* Build the filters to search for events via the network monitor
*/
async filterBuilder() {
this.networkMonitor.filters = [
// want to catch AvailableOperatorJob event instead of watching LZ Endpoint from address
{
type: network_monitor_1.FilterType.from,
match: this.networkMonitor.LAYERZERO_RECEIVERS,
networkDependant: true,
},
// want to also catch FinishedOperatorJob and FailedOperatorJob event
// for now we catch all calls to HolographOperator with function executeJob
{
type: network_monitor_1.FilterType.functionSig,
match: (0, utils_1.functionSignature)('executeJob(bytes)'),
networkDependant: false,
},
];
if (this.environment === environment_1.Environment.localhost) {
this.networkMonitor.filters.push({
type: network_monitor_1.FilterType.to,
match: this.networkMonitor.bridgeAddress,
networkDependant: false,
});
}
// for first time init, get operator status details
for (const network of this.networkMonitor.networks) {
this.operatorStatus.active[network] = false;
this.operatorStatus.currentPod[network] = 0;
this.operatorStatus.podIndex[network] = 0;
this.operatorStatus.podSize[network] = 0;
await this.updateOperatorStatus(network);
}
}
/**
* Process the transactions in each block job
*/
async processTransactions(job, transactions) {
if (transactions.length > 0) {
for (const transaction of transactions) {
const tags = [];
tags.push(transaction.blockNumber, this.networkMonitor.randomTag());
const to = transaction.to?.toLowerCase();
const from = transaction.from?.toLowerCase();
if (to === this.networkMonitor.bridgeAddress) {
// this only triggers in localhost environment
this.networkMonitor.structuredLog(job.network, `handleBridgeOutEvent ${networks_1.networks[job.network].explorer}/tx/${transaction.hash}`, tags);
await this.handleBridgeOutEvent(transaction, job.network, tags);
}
else if (to === this.networkMonitor.operatorAddress) {
// use this to speed up logic for getting AvailableOperatorJob event
this.networkMonitor.structuredLog(job.network, `handleBridgeInEvent ${networks_1.networks[job.network].explorer}/tx/${transaction.hash}`, tags);
await this.handleBridgeInEvent(transaction, job.network, tags);
}
else if (from === this.networkMonitor.LAYERZERO_RECEIVERS[job.network]) {
this.networkMonitor.structuredLog(job.network, `handleAvailableOperatorJobEvent ${networks_1.networks[job.network].explorer}/tx/${transaction.hash}`, tags);
await this.handleAvailableOperatorJobEvent(transaction, job.network, tags);
}
else if (transaction.data?.slice(0, 10).startsWith((0, utils_1.functionSignature)('executeJob(bytes)'))) {
this.networkMonitor.structuredLog(job.network, `handleBridgeInEvent ${networks_1.networks[job.network].explorer}/tx/${transaction.hash}`, tags);
await this.handleBridgeInEvent(transaction, job.network, tags);
}
else {
this.networkMonitor.structuredLog(job.network, `irrelevant transaction`, tags);
}
}
}
}
async handleBridgeOutEvent(transaction, network, tags) {
const receipt = await this.networkMonitor.getTransactionReceipt({
network,
transactionHash: transaction.hash,
attempts: 10,
canFail: true,
});
if (receipt === null) {
throw new Error(`Could not get receipt for ${transaction.hash}`);
}
if (receipt.status === 1) {
this.networkMonitor.structuredLog(network, `Checking for job hash`, tags);
const operatorJobHash = this.networkMonitor.decodeCrossChainMessageSentEvent(receipt, this.networkMonitor.operatorAddress);
if (operatorJobHash === undefined) {
this.networkMonitor.structuredLog(network, `No CrossChainMessageSent event found`, tags);
}
else {
const bridgeTransaction = await this.networkMonitor.bridgeContract.interface.parseTransaction({
data: transaction.data,
});
const args = this.networkMonitor.decodeLzEvent(receipt, this.networkMonitor.lzEndpointAddress[network]);
const jobHash = utils_1.web3.utils.keccak256(args[2]);
this.networkMonitor.structuredLog(network, `Bridge request found for job hash ${jobHash}`, tags);
// adding this double check for just in case
if (this.environment === environment_1.Environment.localhost) {
await this.executeLzPayload((0, networks_1.getNetworkByHolographId)(bridgeTransaction.args[0]).key, jobHash, [args[0], args[1], 0, args[2]], tags);
}
}
}
else {
this.networkMonitor.structuredLog(network, `Transaction failed, ignoring it`, tags);
}
}
async handleBridgeInEvent(transaction, network, tags) {
const receipt = await this.networkMonitor.getTransactionReceipt({
network,
transactionHash: transaction.hash,
attempts: 10,
canFail: true,
});
if (receipt === null) {
throw new Error(`Could not get receipt for ${transaction.hash}`);
}
if (receipt.status === 1) {
this.networkMonitor.structuredLog(network, `Checking for executeJob function`, tags);
const parsedTransaction = this.networkMonitor.operatorContract.interface.parseTransaction(transaction);
if (parsedTransaction.name === 'executeJob') {
this.networkMonitor.structuredLog(network, `Extracting bridgeInRequest from transaction`, tags);
const args = Object.values(parsedTransaction.args);
const operatorJobPayload = args === undefined ? undefined : args[0];
const operatorJobHash = operatorJobPayload === undefined ? undefined : (0, utils_1.sha3)(operatorJobPayload);
if (operatorJobHash === undefined) {
this.networkMonitor.structuredLog(network, `Could not find bridgeInRequest in ${transaction.hash}`, tags);
}
else {
this.networkMonitor.structuredLog(network, `Operator executed job ${operatorJobHash}`, tags);
// remove job from operatorJobs if it exists
if (operatorJobHash in this.operatorJobs) {
this.networkMonitor.structuredLog(network, `Removing job from list of available jobs`, tags);
delete this.operatorJobs[operatorJobHash];
}
// update operator details, in case operator was selected for a job, or any data changed
this.networkMonitor.structuredLog(network, `Updating operator status`, tags);
await this.updateOperatorStatus(network);
}
}
else {
this.networkMonitor.structuredLog(network, `Function call was ${parsedTransaction.name} and not executeJob`, tags);
}
}
else {
this.networkMonitor.structuredLog(network, `Transaction failed, ignoring it`, tags);
}
}
/**
* Handle the AvailableOperatorJob event from the LayerZero contract when one is picked up while processing transactions
*/
async handleAvailableOperatorJobEvent(transaction, network, tags) {
const receipt = await this.networkMonitor.getTransactionReceipt({
network,
transactionHash: transaction.hash,
attempts: 30,
canFail: true,
});
if (receipt === null) {
throw new Error(`Could not get receipt for ${transaction.hash}`);
}
if (receipt.status === 1) {
this.networkMonitor.structuredLog(network, `Checking for job hash`, tags);
const operatorJobPayloadData = this.networkMonitor.decodeAvailableOperatorJobEvent(receipt, this.networkMonitor.operatorAddress);
const operatorJobHash = operatorJobPayloadData === undefined ? undefined : operatorJobPayloadData[0];
const operatorJobPayload = operatorJobPayloadData === undefined ? undefined : operatorJobPayloadData[1];
if (operatorJobHash === undefined) {
this.networkMonitor.structuredLog(network, `No AvailableOperatorJob event found`, tags);
}
else {
this.networkMonitor.structuredLog(network, `Found a new job ${operatorJobHash}`, tags);
// first update operator details, in case operator was selected for a job, or any data changed
this.networkMonitor.structuredLog(network, `Updating operator status`, tags);
await this.updateOperatorStatus(network);
// then add operator job to internal list of jobs to monitor and work on
this.networkMonitor.structuredLog(network, `Adding job to list of available jobs`, tags);
await this.decodeOperatorJob(network, operatorJobHash, operatorJobPayload, tags);
}
}
else {
this.networkMonitor.structuredLog(network, `Transaction failed, ignoring it`, tags);
}
}
processOperatorJob = async (network, jobHash, tags) => {
// if success then pass back payload hash to remove it from list
if (await this.executeJob(jobHash, tags)) {
// check job status just in case
await this.checkJobStatus(jobHash);
// job was a success
this.processOperatorJobs(network, jobHash);
}
else {
// check job status just in case
await this.checkJobStatus(jobHash);
// job failed, gotta try again
this.processOperatorJobs(network);
}
Promise.resolve();
};
processOperatorJobs = (network, jobHash) => {
const tags = [this.networkMonitor.randomTag()];
if (jobHash !== undefined && jobHash !== '' && jobHash in this.operatorJobs) {
delete this.operatorJobs[jobHash];
}
const gasPricing = this.networkMonitor.gasPrices[network];
let highestGas = bignumber_1.BigNumber.from('0');
const now = Date.now();
// update wait times really quickly
this.updateJobTimes();
// DO LOGIC HERE FOR FINDING VALID JOB
const jobs = [];
// extract jobs for network
for (const job of Object.values(this.operatorJobs)) {
if (job.network === network) {
jobs.push(job);
}
}
// sort jobs based on target time, to prioritize ones that need to be finished first
jobs.sort((a, b) => {
return a.targetTime - b.targetTime;
});
const candidates = [];
for (const job of jobs) {
// check that time is within scope
if (job.targetTime < now) {
// add to list of candidates
candidates.push(job);
// find highest gas candidate first
if (bignumber_1.BigNumber.from(job.gasPrice).gt(highestGas)) {
highestGas = bignumber_1.BigNumber.from(job.gasPrice);
}
}
}
if (candidates.length > 0) {
// sort candidates by gas priority
// returning highest gas first
candidates.sort((a, b) => {
return bignumber_1.BigNumber.from(b.gasPrice).sub(bignumber_1.BigNumber.from(a.gasPrice)).toNumber();
});
const compareGas = gasPricing.isEip1559 ? gasPricing.maxFeePerGas : gasPricing.gasPrice;
let foundCandidate = false;
for (const candidate of candidates) {
if (candidate.jobDetails.operator === utils_1.zeroAddress || bignumber_1.BigNumber.from(candidate.gasPrice).gte(compareGas)) {
this.networkMonitor.structuredLog(network, `Sending job ${candidate.hash} for execution`, tags);
// have a valid job to do right away
this.processOperatorJob(network, candidate.hash, tags);
foundCandidate = true;
break;
}
}
if (!foundCandidate) {
setTimeout(this.processOperatorJobs.bind(this, network), 1000);
}
}
else {
setTimeout(this.processOperatorJobs.bind(this, network), 1000);
}
};
/**
* Execute the job
*/
async executeJob(jobHash, tags) {
// quickly check that job is still valid
await this.checkJobStatus(jobHash);
if (jobHash in this.operatorJobs) {
const job = this.operatorJobs[jobHash];
const network = job.network;
let operate = this.operatorMode === network_monitor_1.OperatorMode.auto;
if (this.operatorMode === network_monitor_1.OperatorMode.manual) {
const operatorPrompt = await inquirer.prompt([
{
name: 'shouldContinue',
message: `A job is available for execution, would you like to operate?\n`,
type: 'confirm',
default: true,
},
]);
operate = operatorPrompt.shouldContinue;
}
if (operate) {
const gasPricing = this.networkMonitor.gasPrices[network];
const gasPrice = gasPricing.isEip1559 ? gasPricing.maxFeePerGas : gasPricing.gasPrice;
const receipt = await this.networkMonitor.executeTransaction({
network,
tags,
contract: this.networkMonitor.operatorContract,
methodName: 'executeJob',
args: [job.payload],
gasPrice: gasPrice,
gasLimit: bignumber_1.BigNumber.from(job.gasLimit).mul(bignumber_1.BigNumber.from('2')),
canFail: false,
waitForReceipt: true,
interval: 1000,
});
if (receipt !== null && receipt.status === 1) {
delete this.operatorJobs[jobHash];
}
return receipt !== null;
}
this.networkMonitor.structuredLog(network, 'Available job will not be executed', tags);
return false;
}
return true;
}
/**
* Execute the lz message payload on the destination network
*/
async executeLzPayload(network, jobHash, args, tags) {
// If the operator is in listen mode, payloads will not be executed
// If the operator is in manual mode, the payload must be manually executed
// If the operator is in auto mode, the payload will be executed automatically
let operate = this.operatorMode === network_monitor_1.OperatorMode.auto;
if (this.operatorMode === network_monitor_1.OperatorMode.manual) {
const operatorPrompt = await inquirer.prompt([
{
name: 'shouldContinue',
message: `A transaction appeared on ${network} for execution, would you like to operate?\n`,
type: 'confirm',
default: true,
},
]);
operate = operatorPrompt.shouldContinue;
}
if (operate) {
const data = (await this.networkMonitor.messagingModuleContract
.connect(this.networkMonitor.localhostWallets[network])
.populateTransaction.lzReceive(...args)).data;
let estimatedGas;
try {
estimatedGas = await this.networkMonitor.lzEndpointContract[network].estimateGas.adminCall(this.networkMonitor.messagingModuleAddress, data);
}
catch {
this.networkMonitor.structuredLog(network, 'Job is not valid/available for ' + jobHash, tags);
}
if (estimatedGas !== undefined) {
this.networkMonitor.structuredLog(network, 'Sending cross-chain message for ' + jobHash, tags);
const tx = await this.networkMonitor.lzEndpointContract[network].adminCall(this.networkMonitor.messagingModuleAddress, data);
const receipt = await tx.wait();
if (receipt.status === 1) {
this.networkMonitor.structuredLog(network, 'Sent cross-chain message for ' + jobHash, tags);
}
else {
this.networkMonitor.structuredLog(network, 'Failed sending cross-chain message for ' + jobHash, tags);
}
}
}
else {
this.networkMonitor.structuredLog(network, 'Dropped potential payload to execute', tags);
}
}
}
exports.default = Operator;