UNPKG

@kyve/core-beta

Version:

🚀 The base KYVE node implementation.

1,794 lines (1,434 loc) • 51.5 kB
import { Logger } from "tslog"; import { DataItem, generateIndexPairs, ICompression, IStorageProvider, Node, sha256, } from "../src/index"; import { runCache } from "../src/methods/main/runCache"; import { genesis_pool } from "./mocks/constants"; import { client } from "./mocks/client.mock"; import { lcd } from "./mocks/lcd.mock"; import { TestCacheProvider } from "./mocks/cache.mock"; import { setupMetrics } from "../src/methods"; import { register } from "prom-client"; import { TestRuntime } from "./mocks/runtime.mock"; import { TestNormalStorageProvider } from "./mocks/storageProvider.mock"; import { TestNormalCompression } from "./mocks/compression.mock"; /* TEST CASES - multiple sources tests * start caching from a pool with multiple sources which is in genesis state * start caching from a pool with multiple sources which has a bundle proposal ongoing * continue caching from a pool with multiple sources which has a bundle proposal ongoing * start caching from a pool with multiple sources where last bundle proposal was dropped * start caching from a pool with multiple sources where getDataItem fails once * start caching from a pool with multiple sources where getDataItem fails multiple times * start caching from multiple sources but sources return different result * start caching from multiple sources but sources but validateDataItem fails */ describe("multiple sources tests", () => { let core: Node; let processExit: jest.Mock<never, never>; let setTimeoutMock: jest.Mock; let storageProvider: IStorageProvider; let compression: ICompression; beforeEach(() => { core = new Node(new TestRuntime()); core["cacheProvider"] = new TestCacheProvider(); // mock storage provider storageProvider = new TestNormalStorageProvider(); core["storageProviderFactory"] = jest .fn() .mockResolvedValue(storageProvider); // mock compression compression = new TestNormalCompression(); core["compressionFactory"] = jest.fn().mockReturnValue(compression); // mock process.exit processExit = jest.fn<never, never>(); process.exit = processExit; // mock setTimeout setTimeoutMock = jest .fn() .mockImplementation( ( callback: (args: void) => void, ms?: number | undefined ): NodeJS.Timeout => { callback(); return null as any; } ); global.setTimeout = setTimeoutMock as any; // mock logger core.logger = new Logger(); core.logger.info = jest.fn(); core.logger.debug = jest.fn(); core.logger.warn = jest.fn(); core.logger.error = jest.fn(); core["poolId"] = 0; core["staker"] = "test_staker"; core["poolConfig"] = { sources: [ "https://rpc.api.moonbeam.network", "https://moonbeam.api.onfinality.io/public", "https://moonbeam.public.blastapi.io", ], }; core.client = client(); core.lcd = lcd(); core["continueRound"] = jest .fn() .mockReturnValueOnce(true) .mockReturnValue(false); core["waitForCacheContinuation"] = jest.fn(); setupMetrics.call(core); }); afterEach(() => { // reset prometheus register.clear(); }); test("start caching from a pool with multiple sources which is in genesis state", async () => { // ARRANGE core.pool = { ...genesis_pool, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { const item = { key: n.toString(), value: `${n}-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, n.toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.del).toHaveBeenCalledTimes(0); expect(cacheProvider.drop).toHaveBeenCalledTimes(1); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), core.poolConfig.sources[s], b.toString() ); n++; } } expect(runtime.transformDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: b.toString(), value: `${b}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: b.toString(), value: `${b}-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } // we only call getNextKey max_bundle_size - 1 because // the pool is in genesis state and therefore start_key // is used for the first time expect(runtime.nextKey).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) - 1 ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) - 1; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from a pool with multiple sources which has a bundle proposal ongoing", async () => { // ARRANGE core.pool = { ...genesis_pool, data: { ...genesis_pool.data, current_key: "99", current_index: "100", }, bundle_proposal: { ...genesis_pool.bundle_proposal, storage_id: "test_storage_id", uploader: "test_staker", next_uploader: "test_staker", data_size: "123456789", data_hash: "test_bundle_hash", bundle_size: "50", from_key: "100", to_key: "149", bundle_summary: "test_summary", updated_at: "0", voters_valid: ["test_staker"], }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 50 ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 50; n++) { const item = { key: (n + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${ n + parseInt(genesis_pool.data.max_bundle_size) }-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 50 ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 50; n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString() ); } expect(cacheProvider.del).toHaveBeenCalledTimes(100); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.del).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.drop).toHaveBeenCalledTimes(0); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( (parseInt(genesis_pool.data.max_bundle_size) + 50) * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size) + 50; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), core.poolConfig.sources[s], (b + parseInt(genesis_pool.data.max_bundle_size)).toString() ); n++; } } expect(runtime.transformDataItem).toHaveBeenCalledTimes( (parseInt(genesis_pool.data.max_bundle_size) + 50) * core.poolConfig.sources.length ); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size) + 50; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${b + parseInt(genesis_pool.data.max_bundle_size)}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( (parseInt(genesis_pool.data.max_bundle_size) + 50) * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size) + 50; b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${ b + parseInt(genesis_pool.data.max_bundle_size) }-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } expect(runtime.nextKey).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 50 ); // here we subtract the key - 1 because we start using the // current key for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 50; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size) - 1).toString() ); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("continue caching from a pool with multiple sources which has a bundle proposal ongoing", async () => { // ARRANGE core["cacheProvider"].exists = jest .fn() .mockResolvedValueOnce(true) .mockResolvedValueOnce(true) .mockResolvedValueOnce(true) .mockResolvedValue(false); core.pool = { ...genesis_pool, data: { ...genesis_pool.data, current_key: "99", current_index: "100", }, bundle_proposal: { ...genesis_pool.bundle_proposal, storage_id: "test_storage_id", uploader: "test_staker", next_uploader: "test_staker", data_size: "123456789", data_hash: "test_bundle_hash", bundle_size: "3", from_key: "100", to_key: "102", bundle_summary: "test_summary", updated_at: "0", voters_valid: ["test_staker"], }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { const item = { key: (n + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${ n + parseInt(genesis_pool.data.max_bundle_size) + 3 }-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 3 ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 3; n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString() ); } expect(cacheProvider.del).toHaveBeenCalledTimes(100); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.del).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.drop).toHaveBeenCalledTimes(0); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), core.poolConfig.sources[s], (b + parseInt(genesis_pool.data.max_bundle_size) + 3).toString() ); n++; } } expect(runtime.transformDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${b + parseInt(genesis_pool.data.max_bundle_size) + 3}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${ b + parseInt(genesis_pool.data.max_bundle_size) + 3 }-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } expect(runtime.nextKey).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 3 ); // here we subtract the key - 1 because we start using the // current key for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 3; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith( n + 1, (n + 100 - 1).toString() ); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from a pool with multiple sources where last bundle proposal was dropped", async () => { // ARRANGE core.pool = { ...genesis_pool, data: { ...genesis_pool.data, current_key: "99", current_index: "100", }, bundle_proposal: { ...genesis_pool.bundle_proposal, storage_id: "", uploader: "", next_uploader: "test_staker", data_size: "0", data_hash: "", bundle_size: "0", from_key: "", to_key: "", bundle_summary: "", updated_at: "0", }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { const item = { key: (n + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${ n + parseInt(genesis_pool.data.max_bundle_size) }-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString() ); } expect(cacheProvider.del).toHaveBeenCalledTimes(100); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.del).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.drop).toHaveBeenCalledTimes(1); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), core.poolConfig.sources[s], (b + parseInt(genesis_pool.data.max_bundle_size)).toString() ); n++; } } expect(runtime.transformDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${b + parseInt(genesis_pool.data.max_bundle_size)}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < parseInt(genesis_pool.data.max_bundle_size); b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size)).toString(), value: `${ b + parseInt(genesis_pool.data.max_bundle_size) }-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } expect(runtime.nextKey).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); // here we subtract the key - 1 because we start using the // current key for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size) - 1).toString() ); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from a pool with multiple sources where getDataItem fails once", async () => { // ARRANGE core["runtime"].getDataItem = jest .fn() .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockRejectedValueOnce(new Error("network error")) .mockImplementation((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ); core.pool = { ...genesis_pool, data: { ...genesis_pool.data, max_bundle_size: "2", }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes(2); for (let n = 0; n < 2; n++) { const item = { key: n.toString(), value: `${n}-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, n.toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes(2); for (let n = 0; n < 2; n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.del).toHaveBeenCalledTimes(0); expect(cacheProvider.drop).toHaveBeenCalledTimes(1); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( 2 * core.poolConfig.sources.length + 1 ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 1, expect.any(Node), core.poolConfig.sources[0], "0" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 2, expect.any(Node), core.poolConfig.sources[1], "0" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 3, expect.any(Node), core.poolConfig.sources[2], "0" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 4, expect.any(Node), core.poolConfig.sources[0], "1" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 5, expect.any(Node), core.poolConfig.sources[1], "1" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 5, expect.any(Node), core.poolConfig.sources[1], "1" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 6, expect.any(Node), core.poolConfig.sources[2], "1" ); expect(runtime.transformDataItem).toHaveBeenCalledTimes(6); let n = 1; for (let b = 0; b < 2; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: b.toString(), value: `${b}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( 2 * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < 2; b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: b.toString(), value: `${b}-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } // we only call getNextKey max_bundle_size - 1 because // the pool is in genesis state and therefore start_key // is used for the first time expect(runtime.nextKey).toHaveBeenCalledTimes(2 - 1); for (let n = 0; n < 2 - 1; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from a pool with multiple sources where getDataItem fails multiple times", async () => { // ARRANGE core["runtime"].getDataItem = jest .fn() .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockRejectedValueOnce(new Error("network error")) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockRejectedValueOnce(new Error("network error")) .mockImplementation((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ); core["cacheProvider"].exists = jest .fn() .mockResolvedValueOnce(true) .mockResolvedValueOnce(true) .mockResolvedValueOnce(true) .mockResolvedValue(false); core.pool = { ...genesis_pool, data: { ...genesis_pool.data, current_key: "99", current_index: "100", }, bundle_proposal: { ...genesis_pool.bundle_proposal, storage_id: "test_storage_id", uploader: "test_staker", next_uploader: "test_staker", data_size: "123456789", data_hash: "test_bundle_hash", bundle_size: "3", from_key: "100", to_key: "102", bundle_summary: "test_summary", updated_at: "0", voters_valid: ["test_staker"], }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { const item = { key: (n + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${ n + parseInt(genesis_pool.data.max_bundle_size) + 3 }-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 3 ); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 3; n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith( n + 1, (n + parseInt(genesis_pool.data.max_bundle_size)).toString() ); } expect(cacheProvider.del).toHaveBeenCalledTimes(100); for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size); n++) { expect(cacheProvider.del).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.drop).toHaveBeenCalledTimes(0); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenNthCalledWith( 1, expect.any(Node), core.poolConfig.sources[0], "103" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 2, expect.any(Node), core.poolConfig.sources[1], "103" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 3, expect.any(Node), core.poolConfig.sources[2], "103" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 4, expect.any(Node), core.poolConfig.sources[0], "104" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 5, expect.any(Node), core.poolConfig.sources[1], "104" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 6, expect.any(Node), core.poolConfig.sources[2], "104" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 7, expect.any(Node), core.poolConfig.sources[0], "104" ); expect(runtime.getDataItem).toHaveBeenNthCalledWith( 8, expect.any(Node), core.poolConfig.sources[2], "104" ); // ... expect(runtime.transformDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < 2; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${b + parseInt(genesis_pool.data.max_bundle_size) + 3}-value`, }; expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } expect(runtime.validateDataItem).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) * core.poolConfig.sources.length ); const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; for (let b = 0; b < 2; b++) { for (let p = 0; p < pairs.length; p++) { const item = { key: (b + parseInt(genesis_pool.data.max_bundle_size) + 3).toString(), value: `${ b + parseInt(genesis_pool.data.max_bundle_size) + 3 }-value-transform`, }; expect(runtime.validateDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), item, item ); n++; } } expect(runtime.nextKey).toHaveBeenCalledTimes( parseInt(genesis_pool.data.max_bundle_size) + 3 ); // here we subtract the key - 1 because we start using the // current key for (let n = 0; n < parseInt(genesis_pool.data.max_bundle_size) + 3; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith( n + 1, (n + 100 - 1).toString() ); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from multiple sources but sources return different result", async () => { // ARRANGE core["runtime"].getDataItem = jest .fn() .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ) .mockImplementationOnce((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value-invalid`, }) ) .mockImplementation((core: Node, source: string, key: string) => Promise.resolve({ key, value: `${key}-value`, }) ); core.pool = { ...genesis_pool, data: { ...genesis_pool.data, max_bundle_size: "3", }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes(1); for (let n = 0; n < 1; n++) { const item = { key: n.toString(), value: `${n}-value-transform`, }; expect(cacheProvider.put).toHaveBeenNthCalledWith( n + 1, n.toString(), item ); } expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes(2); for (let n = 0; n < 2; n++) { expect(cacheProvider.exists).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(cacheProvider.del).toHaveBeenCalledTimes(0); expect(cacheProvider.drop).toHaveBeenCalledTimes(1); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( 2 * core.poolConfig.sources.length ); let n = 1; for (let b = 0; b < 2; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenNthCalledWith( n, expect.any(Node), core.poolConfig.sources[s], b.toString() ); n++; } } expect(runtime.transformDataItem).toHaveBeenCalledTimes( 2 * core.poolConfig.sources.length ); n = 1; for (let b = 0; b < 2; b++) { for (let s = 0; s < core.poolConfig.sources.length; s++) { let item; if (n === 5) { item = { key: b.toString(), value: `${b}-value-invalid`, }; } else { item = { key: b.toString(), value: `${b}-value`, }; } expect(runtime.transformDataItem).toHaveBeenNthCalledWith(n, item); n++; } } const pairs = generateIndexPairs(core.poolConfig.sources.length); n = 1; expect(runtime.validateDataItem).toHaveBeenCalledTimes(4); expect(runtime.validateDataItem).toHaveBeenNthCalledWith( 1, expect.any(Node), { key: `0`, value: `0-value-transform`, }, { key: `0`, value: `0-value-transform`, } ); expect(runtime.validateDataItem).toHaveBeenNthCalledWith( 2, expect.any(Node), { key: `0`, value: `0-value-transform`, }, { key: `0`, value: `0-value-transform`, } ); expect(runtime.validateDataItem).toHaveBeenNthCalledWith( 3, expect.any(Node), { key: `0`, value: `0-value-transform`, }, { key: `0`, value: `0-value-transform`, } ); expect(runtime.validateDataItem).toHaveBeenNthCalledWith( 4, expect.any(Node), { key: `1`, value: `1-value-transform`, }, { key: `1`, value: `1-value-invalid-transform`, } ); // we only call getNextKey max_bundle_size - 1 because // the pool is in genesis state and therefore start_key // is used for the first time expect(runtime.nextKey).toHaveBeenCalledTimes(2 - 1); for (let n = 0; n < 2 - 1; n++) { expect(runtime.nextKey).toHaveBeenNthCalledWith(n + 1, n.toString()); } expect(runtime.summarizeDataBundle).toHaveBeenCalledTimes(0); // ======================== // ASSERT NODEJS INTERFACES // ======================== // assert that only one round ran expect(core["waitForCacheContinuation"]).toHaveBeenCalledTimes(1); // TODO: assert timeouts }); test("start caching from multiple sources but sources but validateDataItem fails", async () => { // ARRANGE core["runtime"].validateDataItem = jest .fn() .mockImplementationOnce( async ( core: Node, proposedDataItem: DataItem, validationDataItem: DataItem ) => { const proposedDataItemHash = sha256( Buffer.from(JSON.stringify(proposedDataItem)) ); const validationDataItemHash = sha256( Buffer.from(JSON.stringify(validationDataItem)) ); return proposedDataItemHash === validationDataItemHash; } ) .mockRejectedValueOnce(new Error()) .mockImplementation( async ( core: Node, proposedDataItem: DataItem, validationDataItem: DataItem ) => { const proposedDataItemHash = sha256( Buffer.from(JSON.stringify(proposedDataItem)) ); const validationDataItemHash = sha256( Buffer.from(JSON.stringify(validationDataItem)) ); return proposedDataItemHash === validationDataItemHash; } ); core.pool = { ...genesis_pool, data: { ...genesis_pool.data, max_bundle_size: "3", }, } as any; // ACT await runCache.call(core); // ASSERT const txs = core["client"].kyve.bundles.v1beta1; const queries = core["lcd"].kyve.query.v1beta1; const cacheProvider = core["cacheProvider"]; const runtime = core["runtime"]; // ======================== // ASSERT CLIENT INTERFACES // ======================== expect(txs.claimUploaderRole).toHaveBeenCalledTimes(0); expect(txs.voteBundleProposal).toHaveBeenCalledTimes(0); expect(txs.submitBundleProposal).toHaveBeenCalledTimes(0); expect(txs.skipUploaderRole).toHaveBeenCalledTimes(0); // ===================== // ASSERT LCD INTERFACES // ===================== expect(queries.canVote).toHaveBeenCalledTimes(0); expect(queries.canPropose).toHaveBeenCalledTimes(0); // ========================= // ASSERT STORAGE INTERFACES // ========================= expect(storageProvider.saveBundle).toHaveBeenCalledTimes(0); expect(storageProvider.retrieveBundle).toHaveBeenCalledTimes(0); // ======================= // ASSERT CACHE INTERFACES // ======================= expect(cacheProvider.put).toHaveBeenCalledTimes(0); expect(cacheProvider.get).toHaveBeenCalledTimes(0); expect(cacheProvider.exists).toHaveBeenCalledTimes(1); expect(cacheProvider.exists).toHaveBeenLastCalledWith("0"); expect(cacheProvider.del).toHaveBeenCalledTimes(0); expect(cacheProvider.drop).toHaveBeenCalledTimes(1); // ============================= // ASSERT COMPRESSION INTERFACES // ============================= expect(compression.compress).toHaveBeenCalledTimes(0); expect(compression.decompress).toHaveBeenCalledTimes(0); // ========================= // ASSERT RUNTIME INTERFACES // ========================= expect(runtime.getDataItem).toHaveBeenCalledTimes( core.poolConfig.sources.length ); let n = 1; for (let s = 0; s < core.poolConfig.sources.length; s++) { expect(runtime.getDataItem).toHaveBeenN