@temporalio/common
Version:
Common library for code that's used across the Client, Worker, and/or Workflow
149 lines • 6.61 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExternalStorageRunner = void 0;
/**
* Engine that drives External Storage `store` and `retrieve` operations.
*
* @module
*/
const proto_1 = require("@temporalio/proto");
const extstore_1 = require("../converter/extstore");
const errors_1 = require("../errors");
const extstore_helpers_1 = require("./extstore-helpers");
const PayloadProto = proto_1.temporal.api.common.v1.Payload;
/**
* Drives External Storage operations against a configured {@link ExternalStorage}.
*
* @internal
* @experimental
*/
class ExternalStorageRunner {
externalStorage;
constructor(externalStorage) {
this.externalStorage = externalStorage;
}
/**
* Replace each payload above the configured size threshold with a reference payload.
* Payloads below the threshold (or that the selector keeps inline) pass through
* unchanged. Order is preserved.
*/
async store(payloads, options = {}) {
if (payloads.length === 0)
return payloads;
const { driverSelector, payloadSizeThreshold } = this.externalStorage;
const { batchSignal, batchController } = makeBatchSignal(options.abortSignal);
const storeCtx = { abortSignal: batchSignal, target: options.target };
const driverGroups = new Map();
for (const [i, payload] of payloads.entries()) {
const size = payloadProtoSize(payload);
if (size < payloadSizeThreshold)
continue;
const selected = driverSelector(storeCtx, payload);
if (selected === null)
continue;
if (this.externalStorage.getDriver(selected.name) !== selected) {
throw new errors_1.ValueError(`Driver '${selected.name}' returned by driverSelector is not registered in ExternalStorage.drivers`);
}
let group = driverGroups.get(selected.name);
if (group === undefined) {
group = { driver: selected, items: [] };
driverGroups.set(selected.name, group);
}
group.items.push({ index: i, payload, size });
}
if (driverGroups.size === 0)
return payloads;
const result = payloads.slice();
await runWithAbortOnFirstError(batchController, [...driverGroups.values()], async (group) => {
const claims = await group.driver.store(storeCtx, group.items.map((it) => it.payload));
if (claims.length !== group.items.length) {
throw new errors_1.ValueError(`Driver '${group.driver.name}' returned ${claims.length} claims for ${group.items.length} payloads`);
}
for (const [j, claim] of claims.entries()) {
const item = group.items[j];
result[item.index] = (0, extstore_helpers_1.encodeReferencePayload)({
driverName: group.driver.name,
claim,
sizeBytes: item.size,
});
}
});
return result;
}
/**
* Replace each reference payload in `payloads` with the payload bytes returned by the
* named driver. Non-reference payloads are passed through unchanged. Order is preserved.
*/
async retrieve(payloads, options = {}) {
if (payloads.length === 0)
return payloads;
const { batchSignal, batchController } = makeBatchSignal(options.abortSignal);
const retrieveCtx = { abortSignal: batchSignal };
const driverGroups = new Map();
for (const [i, payload] of payloads.entries()) {
if (!(0, extstore_helpers_1.isReferencePayload)(payload))
continue;
const decoded = (0, extstore_helpers_1.decodeReferencePayload)(payload);
const driver = this.externalStorage.getDriver(decoded.driverName);
if (driver === null) {
throw new errors_1.ValueError(`No driver registered with name '${decoded.driverName}'`);
}
let group = driverGroups.get(decoded.driverName);
if (group === undefined) {
group = { driver, items: [] };
driverGroups.set(decoded.driverName, group);
}
group.items.push({ index: i, claim: new extstore_1.StorageDriverClaim(decoded.claimData) });
}
if (driverGroups.size === 0)
return payloads;
const result = payloads.slice();
await runWithAbortOnFirstError(batchController, [...driverGroups.values()], async (group) => {
const retrieved = await group.driver.retrieve(retrieveCtx, group.items.map((it) => it.claim));
if (retrieved.length !== group.items.length) {
throw new errors_1.ValueError(`Driver '${group.driver.name}' returned ${retrieved.length} payloads for ${group.items.length} claims`);
}
for (const [j, retrievedPayload] of retrieved.entries()) {
const item = group.items[j];
result[item.index] = retrievedPayload;
}
});
return result;
}
}
exports.ExternalStorageRunner = ExternalStorageRunner;
// ============================================================================
// Internal helpers
// ============================================================================
function payloadProtoSize(payload) {
return PayloadProto.encode(payload).len;
}
/**
* Builds an internal controller composed with the caller's signal, so a failing
* driver call can abort its siblings.
*/
function makeBatchSignal(abortSignal) {
const batchController = new AbortController();
const batchSignal = abortSignal ? AbortSignal.any([batchController.signal, abortSignal]) : batchController.signal;
return { batchSignal, batchController };
}
/**
* Run `task(item)` for each item in parallel. As soon as any task rejects,
* signal `controller.abort(reason)` so siblings can cancel mid-flight. Awaits
* all tasks regardless of outcome and re-throws the first rejection.
*/
async function runWithAbortOnFirstError(controller, items, task) {
const promises = items.map(task);
for (const p of promises) {
p.catch((reason) => {
if (!controller.signal.aborted)
controller.abort(reason);
});
}
const settled = await Promise.allSettled(promises);
for (const outcome of settled) {
if (outcome.status === 'rejected')
throw outcome.reason;
}
}
//# sourceMappingURL=external-storage-runner.js.map