bullmq
Version:
Queue for messages and jobs based on Redis
274 lines • 9.87 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.updateJobScheduler = void 0;
const content = `--[[
Updates a job scheduler and adds next delayed job
Input:
KEYS[1] 'repeat' key
KEYS[2] 'delayed'
KEYS[3] 'wait' key
KEYS[4] 'paused' key
KEYS[5] 'meta'
KEYS[6] 'prioritized' key
KEYS[7] 'marker',
KEYS[8] 'id'
KEYS[9] events stream key
KEYS[10] 'pc' priority counter
KEYS[11] producer key
KEYS[12] 'active' key
ARGV[1] next milliseconds
ARGV[2] jobs scheduler id
ARGV[3] Json stringified delayed data
ARGV[4] msgpacked delayed opts
ARGV[5] timestamp
ARGV[6] prefix key
ARGV[7] producer id
Output:
next delayed job id - OK
]]
local rcall = redis.call
local repeatKey = KEYS[1]
local delayedKey = KEYS[2]
local waitKey = KEYS[3]
local pausedKey = KEYS[4]
local metaKey = KEYS[5]
local prioritizedKey = KEYS[6]
local nextMillis = ARGV[1]
local jobSchedulerId = ARGV[2]
local timestamp = ARGV[5]
local prefixKey = ARGV[6]
local producerId = ARGV[7]
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Adds a delayed job to the queue by doing the following:
- Creates a new job key with the job data.
- adds to delayed zset.
- Emits a global event 'delayed' if the job is delayed.
]]
-- Includes
--[[
Add delay marker if needed.
]]
-- Includes
--[[
Function to return the next delayed job timestamp.
]]
local function getNextDelayedTimestamp(delayedKey)
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
if #result then
local nextTimestamp = tonumber(result[2])
if nextTimestamp ~= nil then
return nextTimestamp / 0x1000
end
end
end
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
if nextTimestamp ~= nil then
-- Replace the score of the marker with the newest known
-- next timestamp.
rcall("ZADD", markerKey, nextTimestamp, "1")
end
end
--[[
Bake in the job id first 12 bits into the timestamp
to guarantee correct execution order of delayed jobs
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
]]
local function getDelayedScore(delayedKey, timestamp, delay)
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
local minScore = delayedTimestamp * 0x1000
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
minScore, "WITHSCORES","LIMIT", 0, 1)
if #result then
local currentMaxScore = tonumber(result[2])
if currentMaxScore ~= nil then
if currentMaxScore >= maxScore then
return maxScore, delayedTimestamp
else
return currentMaxScore + 1, delayedTimestamp
end
end
end
return minScore, delayedTimestamp
end
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
maxEvents, markerKey, delay)
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
rcall("ZADD", delayedKey, score, jobId)
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
"jobId", jobId, "delay", delayedTimestamp)
-- mark that a delayed job is available
addDelayMarkerIfNeeded(markerKey, delayedKey)
end
--[[
Function to add job considering priority.
]]
-- Includes
--[[
Add marker if needed when a job is available.
]]
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
if not isPausedOrMaxed then
rcall("ZADD", markerKey, 0, "0")
end
end
--[[
Function to get priority score.
]]
local function getPriorityScore(priority, priorityCounterKey)
local prioCounter = rcall("INCR", priorityCounterKey)
return priority * 0x100000000 + prioCounter % 0x100000000
end
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
isPausedOrMaxed)
local score = getPriorityScore(priority, priorityCounterKey)
rcall("ZADD", prioritizedKey, score, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function isQueuePaused(queueMetaKey)
return rcall("HEXISTS", queueMetaKey, "paused") == 1
end
--[[
Function to store a job
]]
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
parentKey, parentData, repeatJobKey)
local jsonOpts = cjson.encode(opts)
local delay = opts['delay'] or 0
local priority = opts['priority'] or 0
local debounceId = opts['de'] and opts['de']['id']
local optionalValues = {}
if parentKey ~= nil then
table.insert(optionalValues, "parentKey")
table.insert(optionalValues, parentKey)
table.insert(optionalValues, "parent")
table.insert(optionalValues, parentData)
end
if repeatJobKey then
table.insert(optionalValues, "rjk")
table.insert(optionalValues, repeatJobKey)
end
if debounceId then
table.insert(optionalValues, "deid")
table.insert(optionalValues, debounceId)
end
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
"timestamp", timestamp, "delay", delay, "priority", priority,
unpack(optionalValues))
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
return delay, priority
end
--[[
Function to check for the meta.paused key to decide if we are paused or not
(since an empty list and !EXISTS are not really the same).
]]
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
if queueAttributes[1] then
return pausedKey, true
else
if queueAttributes[2] then
local activeCount = rcall("LLEN", activeKey)
if activeCount >= tonumber(queueAttributes[2]) then
return waitKey, true
else
return waitKey, false
end
end
end
return waitKey, false
end
--[[
Function to add job in target list and add marker if needed.
]]
-- Includes
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
rcall(pushCmd, targetKey, jobId)
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
end
local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
data, jobSchedulerId)
local opts = cmsgpack.unpack(rawOpts)
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
opts, timestamp, nil, nil, jobSchedulerId)
if delay ~= 0 then
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
else
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
-- Standard or priority add
if priority == 0 then
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
else
-- Priority add
addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounter, isPausedOrMaxed)
end
-- Emit waiting event
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", jobId)
end
end
--[[
Function to get max events value or set by default 10000.
]]
local function getOrSetMaxEvents(metaKey)
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
if not maxEvents then
maxEvents = 10000
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
end
return maxEvents
end
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
-- Validate that scheduler exists.
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
if prevMillis then
local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
if producerId == currentDelayedJobId then
local eventsKey = KEYS[9]
local maxEvents = getOrSetMaxEvents(metaKey)
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
rcall("HINCRBY", schedulerKey, "ic", 1)
rcall("INCR", KEYS[8])
-- TODO: remove this workaround in next breaking change,
-- all job-schedulers must save job data
local templateData = schedulerAttributes[2] or ARGV[3]
if templateData and templateData ~= '{}' then
rcall("HSET", schedulerKey, "data", templateData)
end
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey,
KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
-- TODO: remove this workaround in next breaking change
if KEYS[11] ~= "" then
rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
end
return nextDelayedJobId .. "" -- convert to string
else
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
"duplicated", "jobId", nextDelayedJobId)
end
end
end
`;
exports.updateJobScheduler = {
name: 'updateJobScheduler',
content,
keys: 12,
};
//# sourceMappingURL=updateJobScheduler-12.js.map