typed-tasks
Version:
A type-safe abstraction for Google Cloud Tasks
275 lines (270 loc) • 8.46 kB
JavaScript
// src/factory.ts
import "zod";
// src/constants.ts
var MINUTE_SECONDS = 60;
var HOUR_SECONDS = 60 * MINUTE_SECONDS;
var DAY_SECONDS = 24 * HOUR_SECONDS;
var defaultHandlerOptions = {
memory: "512MiB",
timeoutSeconds: 30 * MINUTE_SECONDS,
// 30 minutes (maximum allowed)
vpcConnector: void 0,
// Queue congestion control settings
rateLimits: {
maxDispatchesPerSecond: 500,
maxConcurrentDispatches: 1e3
},
// Retry configuration for failed tasks
retryConfig: {
maxAttempts: 10,
minBackoffSeconds: 10,
maxBackoffSeconds: HOUR_SECONDS,
// 1 hour
maxRetrySeconds: 0
// unlimited
}
};
// src/handler.ts
import { onTaskDispatched } from "firebase-functions/tasks";
import { got } from "get-or-throw";
import "zod";
function createTaskHandlerFactory(schemas, region, globalOptions = defaultHandlerOptions) {
return ({
queueName,
options = {},
handler
}) => {
const mergedOptions = {
...defaultHandlerOptions,
...globalOptions,
...options,
rateLimits: {
...defaultHandlerOptions.rateLimits,
...globalOptions.rateLimits ?? {},
...options.rateLimits ?? {}
},
retryConfig: {
...defaultHandlerOptions.retryConfig,
...globalOptions.retryConfig ?? {},
...options.retryConfig ?? {}
}
};
const { memory, timeoutSeconds, vpcConnector, rateLimits, retryConfig } = mergedOptions;
const taskHandler = onTaskDispatched(
{
region,
vpcConnector,
cpu: 1,
memory,
timeoutSeconds,
rateLimits,
retryConfig
},
async ({ data }) => {
const schema = got(schemas, queueName);
const result = schema.safeParse(data);
if (!result.success) {
console.error(
new Error(`Zod validation error for queue ${queueName}`),
result.error.flatten()
);
return;
}
return handler(result.data);
}
);
return taskHandler;
};
}
// src/scheduler.ts
import crypto from "crypto";
import pRetry, { AbortError } from "p-retry";
function generateTaskNameFromPayload(data, deduplicationWindowSeconds) {
const dataString = typeof data === "string" ? data : JSON.stringify(data);
const baseHash = crypto.createHash("md5").update(dataString).digest("hex");
if (deduplicationWindowSeconds && deduplicationWindowSeconds > 0) {
const currentTime = Date.now();
const windowBoundary = Math.floor(
currentTime / (deduplicationWindowSeconds * 1e3)
);
return `${baseHash}-${windowBoundary}`;
}
return baseHash;
}
function createSchedulerFactory(tasksClient, projectId, region, taskRegistry) {
return (queueName) => {
return async (data, options) => {
const taskConfig = taskRegistry.get(queueName);
const deduplicationWindowSeconds = taskConfig?.deduplicationWindowSeconds;
const targetRegion = region;
const parent = tasksClient.queuePath(projectId, targetRegion, queueName);
const serviceAccountEmail = `${projectId}@appspot.gserviceaccount.com`;
let scheduleTimeSeconds;
const useDeduplication = !!taskConfig?.useDeduplication || !!deduplicationWindowSeconds && deduplicationWindowSeconds > 0;
let finalTaskName = options?.taskName;
if (useDeduplication && !finalTaskName) {
finalTaskName = generateTaskNameFromPayload(
data,
deduplicationWindowSeconds
);
} else if (finalTaskName && deduplicationWindowSeconds && deduplicationWindowSeconds > 0) {
const currentTime = Date.now();
const windowBoundary = Math.floor(
currentTime / (deduplicationWindowSeconds * 1e3)
);
finalTaskName = `${finalTaskName}-${windowBoundary}`;
}
try {
if (deduplicationWindowSeconds && deduplicationWindowSeconds > 0) {
scheduleTimeSeconds = Math.floor(Date.now() / 1e3) + deduplicationWindowSeconds;
} else if (options?.delaySeconds && options.delaySeconds > 0) {
scheduleTimeSeconds = Math.floor(Date.now() / 1e3) + options.delaySeconds;
}
const body = Buffer.from(JSON.stringify({ data })).toString("base64");
const task = {
httpRequest: {
httpMethod: "POST",
url: `https://${targetRegion}-${projectId}.cloudfunctions.net/${queueName}`,
oidcToken: {
serviceAccountEmail
},
headers: {
"content-type": "application/json"
},
body
}
};
if (finalTaskName) {
task.name = tasksClient.taskPath(
projectId,
targetRegion,
queueName,
finalTaskName
);
}
if (scheduleTimeSeconds) {
task.scheduleTime = {
seconds: scheduleTimeSeconds
};
}
await pRetry(
async () => {
try {
return await tasksClient.createTask({ parent, task });
} catch (error) {
if (error instanceof Error && error.message.includes("ALREADY_EXISTS")) {
throw new AbortError(error.message);
}
throw error;
}
},
{
retries: 5,
// Maximum number of retry attempts
factor: 2,
// Exponential backoff factor
minTimeout: 1e3,
// Initial retry delay (1 second)
maxTimeout: 1e4,
// Maximum retry delay (10 seconds)
randomize: true,
// Add jitter to prevent thundering herd
onFailedAttempt: (error) => {
if (!(error instanceof AbortError)) {
console.warn(
`Task scheduling attempt ${String(error.attemptNumber)} failed for ${String(queueName)}. ${String(error.retriesLeft)} retries left.`,
error.message
);
}
}
}
);
} catch (error) {
if (error instanceof Error && error.message.includes("ALREADY_EXISTS")) {
console.info(`Skipping task ${finalTaskName}`, { data });
return;
}
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(
new Error(
`Failed to schedule task ${String(queueName)} after multiple retries: ${errorMessage}`
)
);
throw error;
}
};
};
}
// src/task-registry.ts
function createTaskRegistry() {
return /* @__PURE__ */ new Map();
}
// src/factory.ts
function isSchemaDefinition(definition) {
return typeof definition === "function" || "parse" in definition;
}
function createTypedTasks({
client,
definitions,
projectId,
region,
options = {}
}) {
const globalHandlerOptions = {
...defaultHandlerOptions,
...options,
rateLimits: {
...defaultHandlerOptions.rateLimits,
...options.rateLimits ?? {}
},
retryConfig: {
...defaultHandlerOptions.retryConfig,
...options.retryConfig ?? {}
}
};
const taskRegistry = createTaskRegistry();
const schemas = Object.fromEntries(
Object.entries(definitions).map(([key, value]) => {
if (isSchemaDefinition(value)) {
return [key, value];
} else {
return [key, value.schema];
}
})
);
Object.entries(definitions).forEach(([queueName, definition]) => {
if (!isSchemaDefinition(definition) && definition.options) {
const deduplicationWindowSeconds = definition.options.deduplicationWindowSeconds;
const useDeduplication = !!definition.options.useDeduplication || !!deduplicationWindowSeconds && deduplicationWindowSeconds > 0;
taskRegistry.set(queueName, {
deduplicationWindowSeconds,
useDeduplication
});
}
});
const schedulerFactory = createSchedulerFactory(
client,
projectId,
region,
taskRegistry
);
const handlerFactory = createTaskHandlerFactory(
schemas,
region,
globalHandlerOptions
);
const tasksProxy = {
createScheduler: schedulerFactory,
createHandler: (config) => {
return handlerFactory(config);
}
};
Object.keys(definitions).forEach((queueName) => {
tasksProxy[queueName] = true;
});
return tasksProxy;
}
export {
createTypedTasks
};
//# sourceMappingURL=index.js.map