durable-utils
Version:
Utilities for Cloudflare Durable Objects and Workers, including SQL migrations, sharding and retry utilities, and more.
538 lines (530 loc) • 19.9 kB
JavaScript
;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var index_exports = {};
__export(index_exports, {
DOSharding: () => do_sharding_exports,
DOUtils: () => do_utils_exports,
Retries: () => retries_exports,
SQLMigrations: () => sql_migrations_exports
});
module.exports = __toCommonJS(index_exports);
// src/sql-migrations.ts
var sql_migrations_exports = {};
__export(sql_migrations_exports, {
SQLSchemaMigrations: () => SQLSchemaMigrations
});
var SQLSchemaMigrations = class {
#_config;
#_migrations;
#_lastMigrationMonotonicId = -1;
constructor(config) {
this.#_config = config;
const migrations = [...config.migrations];
migrations.sort((a, b) => a.idMonotonicInc - b.idMonotonicInc);
const idSeen = /* @__PURE__ */ new Set();
migrations.forEach((m) => {
if (m.idMonotonicInc < 0) {
throw new Error(`migration ID cannot be negative: ${m.idMonotonicInc}`);
}
if (idSeen.has(m.idMonotonicInc)) {
throw new Error(`duplicate migration ID detected: ${m.idMonotonicInc}`);
}
idSeen.add(m.idMonotonicInc);
});
this.#_migrations = migrations;
}
/**
* This is a quick check based on the in memory tracker of last migration ran,
* therefore this always returns `true` until `runAll` runs once.
* @returns True if the `migrations` list provided has not been ran in full yet.
*/
hasMigrationsToRun() {
if (!this.#_migrations.length) {
return false;
}
return this.#_lastMigrationMonotonicId !== this.#_migrations[this.#_migrations.length - 1].idMonotonicInc;
}
/**
* Runs all the migrations that haven't already ran. The `idMonotonicInc` of each migration is used
* to track which migrations ran or not. New migrations should always have higher `idMonotonicInc`
* than older ones!
*
* @param sqlGen An optional callback function to generate the SQL statement of a given migration at runtime.
* If the migration entry already has a valid `sql` statement this callback is NOT called.
* @returns The numbers of rows read and written throughout the migration execution.
*/
async runAll(sqlGen) {
const result = {
rowsRead: 0,
rowsWritten: 0
};
if (!this.hasMigrationsToRun()) {
return result;
}
this.#_lastMigrationMonotonicId = await this.#_config.doStorage.get(this.#_lastMigrationIDKeyName()) ?? -1;
let idx = 0, sz = this.#_migrations.length;
while (idx < sz && this.#_migrations[idx].idMonotonicInc <= this.#_lastMigrationMonotonicId) {
idx += 1;
}
if (idx >= sz) {
return result;
}
const doSql = this.#_config.doStorage.sql;
const migrationsToRun = this.#_migrations.slice(idx);
await this.#_config.doStorage.transaction(async () => {
let _lastMigrationMonotonicId = this.#_lastMigrationMonotonicId;
migrationsToRun.forEach((migration) => {
let query = migration.sql ?? sqlGen?.(migration.idMonotonicInc);
if (!query) {
throw new Error(`migration with neither 'sql' nor 'sqlGen' provided: ${migration.idMonotonicInc}`);
}
const cursor = doSql.exec(query);
let _ = cursor.toArray();
result.rowsRead += cursor.rowsRead;
result.rowsWritten += cursor.rowsWritten;
_lastMigrationMonotonicId = migration.idMonotonicInc;
});
this.#_lastMigrationMonotonicId = _lastMigrationMonotonicId;
await this.#_config.doStorage.put(this.#_lastMigrationIDKeyName(), this.#_lastMigrationMonotonicId);
});
return result;
}
#_lastMigrationIDKeyName() {
return this.#_config.keyNameTrackingLastMigrationID ?? "__sql_migrations_lastID";
}
};
// src/do-sharding.ts
var do_sharding_exports = {};
__export(do_sharding_exports, {
FixedShardedDO: () => FixedShardedDO,
StaticShardedDO: () => StaticShardedDO
});
// node_modules/js-xxhash/dist/esm/xxHash32.js
var PRIME32_1 = 2654435761;
var PRIME32_2 = 2246822519;
var PRIME32_3 = 3266489917;
var PRIME32_4 = 668265263;
var PRIME32_5 = 374761393;
var encoder;
function xxHash32(input, seed = 0) {
const buffer = typeof input === "string" ? (encoder ??= new TextEncoder()).encode(input) : input;
const b = buffer;
let acc = seed + PRIME32_5 & 4294967295;
let offset = 0;
if (b.length >= 16) {
const accN = [
seed + PRIME32_1 + PRIME32_2 & 4294967295,
seed + PRIME32_2 & 4294967295,
seed + 0 & 4294967295,
seed - PRIME32_1 & 4294967295
];
const b2 = buffer;
const limit2 = b2.length - 16;
let lane = 0;
for (offset = 0; (offset & 4294967280) <= limit2; offset += 4) {
const i = offset;
const laneN0 = b2[i + 0] + (b2[i + 1] << 8);
const laneN1 = b2[i + 2] + (b2[i + 3] << 8);
const laneNP = laneN0 * PRIME32_2 + (laneN1 * PRIME32_2 << 16);
let acc2 = accN[lane] + laneNP & 4294967295;
acc2 = acc2 << 13 | acc2 >>> 19;
const acc0 = acc2 & 65535;
const acc1 = acc2 >>> 16;
accN[lane] = acc0 * PRIME32_1 + (acc1 * PRIME32_1 << 16) & 4294967295;
lane = lane + 1 & 3;
}
acc = (accN[0] << 1 | accN[0] >>> 31) + (accN[1] << 7 | accN[1] >>> 25) + (accN[2] << 12 | accN[2] >>> 20) + (accN[3] << 18 | accN[3] >>> 14) & 4294967295;
}
acc = acc + buffer.length & 4294967295;
const limit = buffer.length - 4;
for (; offset <= limit; offset += 4) {
const i = offset;
const laneN0 = b[i + 0] + (b[i + 1] << 8);
const laneN1 = b[i + 2] + (b[i + 3] << 8);
const laneP = laneN0 * PRIME32_3 + (laneN1 * PRIME32_3 << 16);
acc = acc + laneP & 4294967295;
acc = acc << 17 | acc >>> 15;
acc = (acc & 65535) * PRIME32_4 + ((acc >>> 16) * PRIME32_4 << 16) & 4294967295;
}
for (; offset < b.length; ++offset) {
const lane = b[offset];
acc = acc + lane * PRIME32_5;
acc = acc << 11 | acc >>> 21;
acc = (acc & 65535) * PRIME32_1 + ((acc >>> 16) * PRIME32_1 << 16) & 4294967295;
}
acc = acc ^ acc >>> 15;
acc = ((acc & 65535) * PRIME32_2 & 4294967295) + ((acc >>> 16) * PRIME32_2 << 16);
acc = acc ^ acc >>> 13;
acc = ((acc & 65535) * PRIME32_3 & 4294967295) + ((acc >>> 16) * PRIME32_3 << 16);
acc = acc ^ acc >>> 16;
return acc < 0 ? acc + 4294967296 : acc;
}
// src/do-utils.ts
var do_utils_exports = {};
__export(do_utils_exports, {
isErrorRetryable: () => isErrorRetryable,
stubByName: () => stubByName
});
function stubByName(doNamespace, name, options) {
const doId = doNamespace.idFromName(name);
return doNamespace.get(doId, options);
}
function isErrorRetryable(err) {
return err?.retryable && !err?.overloaded;
}
// src/retries.ts
var retries_exports = {};
__export(retries_exports, {
jitterBackoff: () => jitterBackoff,
tryN: () => tryN,
tryWhile: () => tryWhile
});
function jitterBackoff(attempt, baseDelayMs, maxDelayMs) {
const attemptUpperBoundMs = Math.min(2 ** attempt * baseDelayMs, maxDelayMs);
return Math.floor(Math.random() * attemptUpperBoundMs);
}
async function tryN(n, fn, optionsOrFunction, optionsOld) {
if (n <= 0) {
throw new Error("n must be greater than 0");
}
n = Math.floor(n);
let options;
if (typeof optionsOrFunction !== "function") {
options = optionsOrFunction ?? {};
} else {
options = {
...optionsOld,
isRetryable: optionsOrFunction
};
}
return await tryWhile(
fn,
(err, nextAttempt) => {
return nextAttempt <= n && (options.isRetryable?.(err, nextAttempt) ?? true);
},
options
);
}
async function tryWhile(fn, isRetryable, options) {
const baseDelayMs = Math.floor(options?.baseDelayMs ?? 100);
const maxDelayMs = Math.floor(options?.maxDelayMs ?? 3e3);
if (baseDelayMs <= 0 || maxDelayMs <= 0) {
throw new Error("baseDelayMs and maxDelayMs must be greater than 0");
}
if (baseDelayMs >= maxDelayMs) {
throw new Error("baseDelayMs must be less than maxDelayMs");
}
let attempt = 1;
while (true) {
try {
return await fn(attempt);
} catch (err) {
if (options?.verbose) {
console.info({
message: "tryWhile",
attempt,
error: String(err),
errorProps: err
});
}
attempt += 1;
if (!isRetryable(err, attempt)) {
throw err;
}
const delay = jitterBackoff(attempt, baseDelayMs, maxDelayMs);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
// src/do-sharding.ts
var GOLDEN_RATIO = 2654435761;
var StaticShardedDO = class {
#doNamespace;
#options;
constructor(doNamespace, options) {
if (options.numShards <= 0) {
throw new Error("Invalid number of shards, must be greater than 0");
}
if (options.concurrency !== void 0 && options.concurrency <= 0) {
throw new Error("Invalid number of subrequests, must be greater than 0");
}
if (options.shardGroupName !== void 0 || options.prefixName !== void 0) {
if (options.shardGroupName !== void 0 && options.prefixName !== void 0) {
throw new Error("Cannot use both shardGroupName and prefixName options together");
}
options.shardGroupName = (options.shardGroupName ?? options.prefixName ?? "").trim();
if (!options.shardGroupName) {
throw new Error("Invalid shard group name, must not be empty or only whitespace");
}
}
this.#doNamespace = doNamespace;
this.#options = {
...options,
concurrency: options.concurrency ?? 10,
// WARNING: Never change the default value of this otherwise there will be data loss!
shardGroupName: options.shardGroupName ?? "fixed-sharded-do"
};
}
/**
* The number of shards used to spread out the load. Immutable after creation.
*/
get N() {
return this.#options.numShards;
}
/**
* Execute a single request against the Durable Object responsible for the given partitionKey.
* @param partitionKey The partition key is used to determine the shard.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @param options The options to control the execution like retries.
* @returns An array of the results from the given `doer` callback function for each shard,
* Each item in the array will be a `TryResult` object with the `ok` property indicating success or failure.
*/
async tryOne(partitionKey, doer, options) {
const shard = this.#shardForPartitionKey(partitionKey);
const result = await this.#makeTryShard(doer, options)(shard);
return result;
}
/**
* Execute a single request against the Durable Object responsible for the given partitionKey.
* @param partitionKey The partition key is used to determine the shard.
* @param doer The callback function to execute with the Durable Object stub for the chosen shard.
* @param options The options to control the execution like retries.
* @returns The result of the given `doer` callback function. If `doer()` throws the error is propagated.
*/
async one(partitionKey, doer, options) {
const response = await this.tryOne(partitionKey, doer, options);
if (!response.ok) {
throw response.error;
}
return response.result;
}
/**
* Execute a request to the shards selected by the `options.filterFn` callback, concurrently.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @param options The options to control the execution like retries and filtering of shards.
* @returns An array of the results from the given `doer` callback function for each shard,
* Each item in the array will be a `TryResult` object with the `ok` property indicating success or failure.
* Only shards that pass the `options.filterFn` will be returned.
*/
async trySome(doer, options) {
const responses = await this.#pipelineRequests(
this.#options.concurrency,
this.#options.numShards,
{ ...options, earlyReturn: false },
this.#makeTryShard(doer, options)
);
return Array.from(responses.results.values(), (r) => r).sort((a, b) => a.shard - b.shard);
}
/**
* Execute a request to the shards selected by the `options.filterFn` callback, concurrently.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @param options The options to control the execution like retries and filtering of shards.
* @returns An array of results from the given `doer` callback function for each filtered shard.
* Only shards that pass the `options.filterFn` will be returned.
* In case of an error, the function will throw the error immediately.
*/
async some(doer, options) {
const responses = await this.#pipelineRequests(
this.#options.concurrency,
this.#options.numShards,
{ ...options, earlyReturn: true },
this.#makeTryShard(doer, options)
);
return [...responses.results.values()].sort((a, b) => a.shard - b.shard).map((r) => {
if (!r.ok) {
throw r.error;
}
return r.result;
});
}
/**
* Execute a request to each of the shards, concurrently.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @param options The options to control the execution like retries.
* @returns An array of the results from the given `doer` callback function for each shard,
* Each item in the array will be a `TryResult` object with the `ok` property indicating success or failure.
*/
async tryAll(doer, options) {
if (this.#options.numShards > 1e3) {
throw new Error(
`Too many shards [${this.#options.numShards}], Cloudflare Workers only supports up to 1000 subrequests.`
);
}
return await this.trySome(doer, { ...options, filterFn: (_shard) => true });
}
/**
* Execute a request to each of the shards, concurrently.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @param options The options to control the execution like retries.
* @returns An array of results from the given `doer` callback function for each shard.
* In case of an error, the function will throw the error immediately.
*/
async all(doer, options) {
if (this.#options.numShards > 1e3) {
throw new Error(
`Too many shards [${this.#options.numShards}], Cloudflare Workers only supports up to 1000 subrequests.`
);
}
return await this.some(doer, { ...options, filterFn: (_shard) => true });
}
//////////////////////////////////////////////////////
// DEPRECATED
/**
* Execute a request to each of the shards, concurrently.
* @deprecated Use `all` or `tryAll` instead.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @returns An async generator of results from the given `doer` callback function for each shard.
* In case of an error, the function will throw the error immediately.
*/
async *genAll(doer) {
if (this.#options.numShards > 1e3) {
throw new Error(
`Too many shards [${this.#options.numShards}], Cloudflare Workers only supports up to 1000 subrequests.`
);
}
for await (const result of this.#genPipelineRequests(
this.#options.concurrency,
this.#options.numShards,
async (shard) => {
const stub = this.#stub(shard);
return await doer(stub, shard);
}
)) {
yield result;
}
}
/**
* Execute a request to each of the shards, concurrently.
* @deprecated Use `tryAll` instead.
* @param doer The callback function to execute with the Durable Object stub for each shard.
* @returns An array of results from the given `doer` callback function for each shard,
* and an array of errors for each shard that failed.
* Items in the results array will be `undefined` if the shard failed, and similarly for the errors array.
*/
async allMaybe(doer) {
if (this.#options.numShards > 1e3) {
throw new Error(
`Too many shards [${this.#options.numShards}], Cloudflare Workers only supports up to 1000 subrequests.`
);
}
const results = await this.#pipelineRequests(
this.#options.concurrency,
this.#options.numShards,
{ filterFn: () => true, earlyReturn: false },
this.#makeTryShard(doer)
);
const finalResults = {
results: Array.from({ length: this.#options.numShards }),
errors: Array.from({ length: this.#options.numShards }),
hasErrors: results.hasErrors
};
results.results.forEach((r) => {
if (r.ok) {
finalResults.results[r.shard] = r.result;
} else {
finalResults.errors[r.shard] = r.error;
}
});
return finalResults;
}
//////////////////////////////////////////////////////
// INTERNAL IMPLEMENTATION
async #pipelineRequests(concurrency, n, tryOptions, tryShardDoer) {
const results = /* @__PURE__ */ new Map();
let i = 0;
let hasErrors = false;
const next = async () => {
if (i >= n) {
return;
}
const shard = i++;
if (!tryOptions.filterFn(shard)) {
return await next();
}
const shardResult = await tryShardDoer(shard);
results.set(shard, shardResult);
if (!shardResult.ok) {
if (tryOptions.earlyReturn) {
throw shardResult.error;
}
hasErrors = true;
}
return await next();
};
await Promise.all(
Array(concurrency).fill(0).map(() => next())
);
return { results, hasErrors };
}
async *#genPipelineRequests(concurrency, n, doer) {
const results = Array(n).fill(null);
let idxAwait = 0;
for (let shard = 0; shard < n; shard++) {
if (shard < concurrency) {
results[shard] = doer(shard);
} else {
yield await results[idxAwait];
idxAwait++;
results[shard] = doer(shard);
}
}
for (; idxAwait < n; idxAwait++) {
yield await results[idxAwait];
}
}
#makeTryShard(doer, tryOptions) {
return async (shard) => {
const stub = this.#stub(shard);
try {
const result = await tryWhile(
async () => await doer(stub, shard),
(e, attempt) => {
return !!tryOptions?.shouldRetry?.(e, attempt, shard);
}
);
return { ok: true, shard, result };
} catch (e) {
return { ok: false, shard, error: e };
}
};
}
#shardForPartitionKey(partitionKey) {
const shard = xxHash32(partitionKey, GOLDEN_RATIO) % this.#options.numShards;
return shard;
}
#stub(shard) {
return stubByName(this.#doNamespace, `${this.#options.shardGroupName}-${shard}`, this.#stubOptions(shard));
}
#stubOptions(shard) {
const options = {};
if (this.#options.shardLocationHintFn) {
options.locationHint = this.#options.shardLocationHintFn(shard);
}
return options;
}
};
var FixedShardedDO = StaticShardedDO;
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
DOSharding,
DOUtils,
Retries,
SQLMigrations
});