bullmq
Version:
Queue for messages and jobs based on Redis
465 lines • 17.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.addJobScheduler = void 0;
const content = `--[[
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
Input:
KEYS[1] 'repeat' key
KEYS[2] 'delayed' key
KEYS[3] 'wait' key
KEYS[4] 'paused' key
KEYS[5] 'meta' key
KEYS[6] 'prioritized' key
KEYS[7] 'marker' key
KEYS[8] 'id' key
KEYS[9] 'events' key
KEYS[10] 'pc' priority counter
KEYS[11] 'active' key
ARGV[1] next milliseconds
ARGV[2] msgpacked options
[1] name
[2] tz?
[3] patten?
[4] endDate?
[5] every?
ARGV[3] jobs scheduler id
ARGV[4] Json stringified template data
ARGV[5] mspacked template opts
ARGV[6] msgpacked delayed opts
ARGV[7] timestamp
ARGV[8] prefix key
ARGV[9] producer key
Output:
repeatableKey - OK
]]
local rcall = redis.call
local repeatKey = KEYS[1]
local delayedKey = KEYS[2]
local waitKey = KEYS[3]
local pausedKey = KEYS[4]
local metaKey = KEYS[5]
local prioritizedKey = KEYS[6]
local eventsKey = KEYS[9]
local nextMillis = ARGV[1]
local jobSchedulerId = ARGV[3]
local templateOpts = cmsgpack.unpack(ARGV[5])
local prefixKey = ARGV[8]
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Adds a delayed job to the queue by doing the following:
- Creates a new job key with the job data.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
]]
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Function to return the next delayed job timestamp.
]]
local function getNextDelayedTimestamp(delayedKey)
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
if #result then
local nextTimestamp = tonumber(result[2])
if nextTimestamp ~= nil then
return nextTimestamp / 0x1000
end
end
end
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Replace the score of the marker with the newest known
-- next timestamp.
rcall("ZADD", markerKey, nextTimestamp, "1")
end
end
--[[
Bake in the job id first 12 bits into the timestamp
to guarantee correct execution order of delayed jobs
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
]]
local function getDelayedScore(delayedKey, timestamp, delay)
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
local minScore = delayedTimestamp * 0x1000
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
minScore, "WITHSCORES","LIMIT", 0, 1)
if #result then
local currentMaxScore = tonumber(result[2])
if currentMaxScore ~= nil then
if currentMaxScore >= maxScore then
return maxScore, delayedTimestamp
else
return currentMaxScore + 1, delayedTimestamp
end
end
end
return minScore, delayedTimestamp
end
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
maxEvents, markerKey, delay)
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- mark that a delayed job is available
addDelayMarkerIfNeeded(markerKey, delayedKey)
end
--[[
Function to add job considering priority.
]]
-- Includes
--[[
Add marker if needed when a job is available.
]]
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
if not isPausedOrMaxed then
rcall("ZADD", markerKey, 0, "0")
end
end
--[[
Function to get priority score.
]]
local function getPriorityScore(priority, priorityCounterKey)
local prioCounter = rcall("INCR", priorityCounterKey)
return priority * 0x100000000 + prioCounter % 0x100000000
end
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
isPausedOrMaxed)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function isQueuePaused(queueMetaKey)
return rcall("HEXISTS", queueMetaKey, "paused") == 1
end
--[[
Function to store a job
]]
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
parentKey, parentData, repeatJobKey)
local jsonOpts = cjson.encode(opts)
local delay = opts['delay'] or 0
local priority = opts['priority'] or 0
local debounceId = opts['de'] and opts['de']['id']
local optionalValues = {}
if parentKey ~= nil then
table.insert(optionalValues, "parentKey")
table.insert(optionalValues, parentKey)
table.insert(optionalValues, "parent")
table.insert(optionalValues, parentData)
end
if repeatJobKey then
table.insert(optionalValues, "rjk")
table.insert(optionalValues, repeatJobKey)
end
if debounceId then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
"timestamp", timestamp, "delay", delay, "priority", priority,
unpack(optionalValues))
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
return delay, priority
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
if queueAttributes[1] then
return pausedKey, true
else
if queueAttributes[2] then
local activeCount = rcall("LLEN", activeKey)
if activeCount >= tonumber(queueAttributes[2]) then
return waitKey, true
else
return waitKey, false
end
end
end
return waitKey, false
end
--[[
Function to add job in target list and add marker if needed.
]]
-- Includes
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
rcall(pushCmd, targetKey, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
data, jobSchedulerId)
local opts = cmsgpack.unpack(rawOpts)
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
opts, timestamp, nil, nil, jobSchedulerId)
if delay ~= 0 then
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
else
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
-- Standard or priority add
if priority == 0 then
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
else
-- Priority add
addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounter, isPausedOrMaxed)
end
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", jobId)
end
end
--[[
Function to get max events value or set by default 10000.
]]
local function getOrSetMaxEvents(metaKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if not maxEvents then
maxEvents = 10000
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
end
return maxEvents
end
--[[
Function to remove job.
]]
-- Includes
--[[
Function to remove deduplication key if needed
when a job is being removed.
]]
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
jobKey, jobId)
local deduplicationId = rcall("HGET", jobKey, "deid")
if deduplicationId then
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
local currentJobId = rcall('GET', deduplicationKey)
if currentJobId and currentJobId == jobId then
return rcall("DEL", deduplicationKey)
end
end
end
--[[
Function to remove job keys.
]]
local function removeJobKeys(jobKey)
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
end
--[[
Check if this job has a parent. If so we will just remove it from
the parent child list, but if it is the last child we should move the parent to "wait/paused"
which requires code from "moveToFinished"
]]
-- Includes
--[[
Functions to destructure job key.
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
]]
local getJobIdFromKey = function (jobKey)
return string.match(jobKey, ".*:(.*)")
end
local getJobKeyPrefix = function (jobKey, jobId)
return string.sub(jobKey, 0, #jobKey - #jobId)
end
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
parentPrefix .. "wait", parentPrefix .. "paused")
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
if emitEvent then
local parentEventStream = parentPrefix .. "events"
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
end
end
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
if parentKey then
local parentDependenciesKey = parentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(parentKey)
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then -- remove parent in same queue
if parentPrefix == baseKey then
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
removeJobKeys(parentKey)
if debounceId then
rcall("DEL", parentPrefix .. "de:" .. debounceId)
end
else
_moveParentToWait(parentPrefix, parentId)
end
else
_moveParentToWait(parentPrefix, parentId, true)
end
end
end
return true
end
else
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
local missedParentKey = parentAttributes[1]
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
and (rcall("EXISTS", missedParentKey) == 1)) then
local parentDependenciesKey = missedParentKey .. ":dependencies"
local result = rcall("SREM", parentDependenciesKey, jobKey)
if result > 0 then
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
if pendingDependencies == 0 then
local parentId = getJobIdFromKey(missedParentKey)
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
if numRemovedElements == 1 then
if hard then
if parentPrefix == baseKey then
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
removeJobKeys(missedParentKey)
if parentAttributes[2] then
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
end
else
_moveParentToWait(parentPrefix, parentId)
end
else
_moveParentToWait(parentPrefix, parentId, true)
end
end
end
return true
end
end
end
return false
end
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
local jobKey = baseKey .. jobId
removeParentDependencyKey(jobKey, hard, nil, baseKey)
if shouldRemoveDeduplicationKey then
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
end
removeJobKeys(jobKey)
end
--[[
Function to store a job scheduler
]]
local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMillis, opts,
templateData, templateOpts)
rcall("ZADD", repeatKey, nextMillis, schedulerId)
local optionalValues = {}
if opts['tz'] then
table.insert(optionalValues, "tz")
table.insert(optionalValues, opts['tz'])
end
if opts['limit'] then
table.insert(optionalValues, "limit")
table.insert(optionalValues, opts['limit'])
end
if opts['pattern'] then
table.insert(optionalValues, "pattern")
table.insert(optionalValues, opts['pattern'])
end
if opts['endDate'] then
table.insert(optionalValues, "endDate")
table.insert(optionalValues, opts['endDate'])
end
if opts['every'] then
table.insert(optionalValues, "every")
table.insert(optionalValues, opts['every'])
end
if opts['offset'] then
table.insert(optionalValues, "offset")
table.insert(optionalValues, opts['offset'])
end
local jsonTemplateOpts = cjson.encode(templateOpts)
if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
table.insert(optionalValues, "opts")
table.insert(optionalValues, jsonTemplateOpts)
end
if templateData and templateData ~= '{}' then
table.insert(optionalValues, "data")
table.insert(optionalValues, templateData)
end
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
end
-- If we are overriding a repeatable job we must delete the delayed job for
-- the next iteration.
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
local maxEvents = getOrSetMaxEvents(metaKey)
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
metaKey, eventsKey)
if rcall("ZSCORE", delayedKey, jobId) then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
rcall("ZREM", delayedKey, jobId)
return true
elseif rcall("ZSCORE", prioritizedKey, jobId) then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
rcall("ZREM", prioritizedKey, jobId)
return true
else
local pausedOrWaitKey = waitKey
if isQueuePaused(metaKey) then
pausedOrWaitKey = pausedKey
end
if rcall("LREM", pausedOrWaitKey, 1, jobId) > 0 then
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
return true
end
end
return false
end
if rcall("EXISTS", nextDelayedJobKey) == 1 then
if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
nextDelayedJobId, metaKey, eventsKey) then
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", nextDelayedJobId)
return nextDelayedJobId .. "" -- convert to string
end
end
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
if prevMillis then
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
local currentDelayedJobKey = schedulerKey .. ":" .. prevMillis
if currentJobId ~= nextDelayedJobId and rcall("EXISTS", currentDelayedJobKey) == 1 then
removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
currentJobId, metaKey, eventsKey)
end
end
local schedulerOpts = cmsgpack.unpack(ARGV[2])
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
rcall("INCR", KEYS[8])
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[6], waitKey, pausedKey,
KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
if ARGV[9] ~= "" then
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
end
return nextDelayedJobId .. "" -- convert to string
`;
exports.addJobScheduler = {
name: 'addJobScheduler',
content,
keys: 11,
};
//# sourceMappingURL=addJobScheduler-11.js.map