@egeria/job
Version:
Egeria's job system
306 lines (283 loc) • 10.5 kB
JavaScript
/*
.--. .-'. .--. .--. .--. .--. .`-. .--.
:::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\
' `--' `.-' `--' `--' `--' `-.' `--' `
Egeria - She bestows Knowledge and Wisdom
Copyright (C) 2016-2019 MySidesTheyAreGone <mysidestheyaregone@protonmail.com>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
.--. .-'. .--. .--. .--. .--. .`-. .--.
:::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\::::::::.\
' `--' `.-' `--' `--' `--' `-.' `--' `
*/
module.exports = function (queue, pluginManager, auth, storage, cache, state) {
const R = require('ramda')
const T = require('@egeria/tools')
const Ajv = require('ajv')
const Knowledge = require('@egeria/knowledge')
const S = require('node-schedule')
const Duration = require('luxon').Duration
const ajv = new Ajv({ $data: true })
require('ajv-keywords')(ajv)
const err = (message, origin, data) => T.err(state.get('prefix'), message, origin, data)
const pipePluginLog = (w) => T.wrapLog(state, null, w)
const { logInfo, logSilly, logError } = T.setupLogging()
function requires (thing, plugin) {
return R.contains(thing, R.defaultTo([], plugin.get('runtime', 'requires')))
}
async function destructPlugin (plugin) {
let runtime = plugin.get('runtime')
if (requires('destruction', plugin) && !R.isNil(runtime)) {
return runtime.destruct(plugin)
} else {
return true
}
}
async function loadPlugin (plugin) {
let pluginName = plugin.get('plugin')
let action = 'While destroying an old instance of plugin "' + pluginName + '"'
try {
await destructPlugin(plugin)
} catch (e) {
throw err(action, e)
}
action = 'While building plugin ' + pluginName
let runtime
try {
runtime = await pluginManager.load(pluginName)
} catch (e) {
throw err(action, e)
}
logSilly(state, 'Plugin ' + pluginName + ' loaded')
plugin.set('runtime', runtime)
return plugin
}
async function preparePlugin (plugin) {
let jobName = state.get('name')
let action = 'While initializing plugin ' + plugin.get('plugin')
let runtime = plugin.get('runtime')
if (R.isNil(runtime)) {
throw err('While initializing plugin ' + plugin.get('plugin') + ': Plugin hasn\'t been loaded yet!')
}
let validate = ajv.compile(runtime.sanity)
let sane = validate(R.defaultTo({}, plugin.get('configuration')))
if (!sane) {
throw err(action, new Error(ajv.errorsText(validate.errors, { dataVar: 'configuration' })))
}
let api = {}
if (requires('identity', plugin) && plugin.exists('configuration', 'identity')) {
let id = plugin.select('configuration', 'identity').get()
let credentials
let action = 'While loading identity for plugin ' + plugin.get('plugin')
try {
if (R.is(String, id)) {
credentials = await auth.load(id)
} else {
credentials = id
}
} catch (e) {
throw err(action, e)
}
plugin.select('configuration', 'credentials').set(credentials)
api.client = await plugin.get('runtime').getClient(credentials)
}
if (requires('cache', plugin)) {
api.cache = cache
}
if (requires('storage', plugin)) {
api.storage = {
exists: storage.exists(jobName),
record: storage.record(jobName)
}
}
if (requires('multiqueue', plugin)) {
api.queue = queue
} else {
api.enqueue = queue.enqueue(runtime.limits)
}
api.announce = (metadata) => state.get('announce')(plugin.get('tags'), metadata)
plugin.set('api', api)
plugin.set('outbox', { push: pipePluginLog })
return plugin
}
function schedulePlugin (plugin) {
logSilly(state, 'Setting schedule for plugin ' + plugin.get('plugin') + '...')
let performance = async () => {
try {
let api = plugin.get('api')
let cfg = plugin.get('configuration')
await plugin.get('runtime').act(api, cfg)
} catch (e) {
logError(state, 'While performing a scheduled action', e)
}
}
let schedule = S.scheduleJob(plugin.get('plan'), performance)
let pluginSchedule = plugin.select('schedule')
if (!R.isNil(pluginSchedule.get())) {
pluginSchedule.get().cancel()
}
pluginSchedule.set(schedule)
logSilly(state, 'Scheduled plugin ' + plugin.get('plugin') + ': ' + plugin.get('plan'))
}
async function constructPlugin (plugin) {
logSilly(state, 'Building plugin ' + plugin.get('plugin') + '...')
await loadPlugin(plugin)
await preparePlugin(plugin)
return plugin
}
const act = R.curry(async (plugin, fact) => {
let ftags = R.defaultTo([], fact.apply(R.prop('system:tags')))
let ptags = R.defaultTo([], plugin.get('tags'))
let isTagged
if (R.isEmpty(ptags)) {
isTagged = true
} else {
isTagged = R.reduce(R.or, false, R.map(R.contains(R.__, ftags), ptags))
}
let isNil = fact.isNil()
let graph = R.defaultTo('', fact.apply(R.prop('system:graph')))
let newFact
if (isTagged && !isNil) {
let api = plugin.get('api')
let cfg = R.defaultTo({}, plugin.get('configuration'))
newFact = await plugin.get('runtime').act(api, cfg, fact)
} else {
newFact = fact
}
if (newFact.isNil() && !isNil) {
graph += `-->[NULLED:${plugin.get('plugin')}]`
logInfo(state, graph)
} else if (!newFact.isNil() && isTagged) {
graph += `-->[${plugin.get('plugin')}]`
} else if (!newFact.isNil() && !isTagged) {
graph += `-->[SKIPPED:${plugin.get('plugin')}]`
}
return newFact.map(R.assoc('system:graph', graph))
})
async function construct () {
state.set('prefix', 'JOB ' + state.get('name') + ' |')
logSilly(state, 'Construction in progress...')
let jobName = state.get('name')
let action = 'While building job'
storage.setup(jobName)
let plugin = state.select('chain', 0)
let mutations = []
let outputs = []
while (!R.isNil(plugin) && !R.isNil(plugin.get())) {
try {
await constructPlugin(plugin)
let type = plugin.get('runtime', 'type')
if (type === 'mutator') {
mutations.push(act(plugin))
} else if (type === 'output') {
outputs.push(act(plugin))
} else if (type !== 'input') {
throw new Error('Plugin ' + plugin.get('plugin') + ' is of unknown plugin type!')
}
} catch (e) {
throw err(action, e)
}
plugin = plugin.right()
}
let actions = R.concat(mutations, outputs)
actions.push(async f => {
if (!f.isNil()) {
logInfo(state, f.apply(R.prop('system:graph')))
}
})
let applyChain
if (actions.length === 0) {
applyChain = R.bind(Promise.resolve, Promise)
} else {
applyChain = R.apply(R.pipeP, actions)
}
state.set('announce', R.curry(async (tags, metadata) => {
metadata['system:graph'] = `${metadata.key}: [${metadata.origin}]`
let fact = Knowledge.of(metadata)
if (!R.isNil(tags)) {
fact = fact.map(R.assoc('system:tags', tags))
}
let ttl = Duration.fromObject(R.defaultTo({ seconds: 0 }, state.get('TTL'))).as('seconds')
fact = fact.map(R.assoc('system:ttl', ttl))
try {
await applyChain(fact)
} catch (e) {
logError(state, 'While performing a scheduled action', e)
try {
await storage.erase(jobName, fact)
} catch (e) {
logError(state, 'While attempting to erase a fact from memory', e)
}
}
}))
logSilly(state, 'Ready for action.')
return state
}
function start () {
let plugin = state.select('chain', 0)
while (!R.isNil(plugin) && !R.isNil(plugin.get())) {
if (requires('schedule', plugin)) {
schedulePlugin(plugin)
} else if (requires('activation', plugin)) {
plugin.get('runtime').act(plugin.get('api'), plugin.get('configuration'))
}
plugin = plugin.right()
}
}
function stop () {
let plugin = state.select('chain', 0)
while (!R.isNil(plugin) && !R.isNil(plugin.get())) {
let pluginSchedule = plugin.select('schedule')
if (pluginSchedule.exists()) {
pluginSchedule.get().cancel()
}
plugin = plugin.right()
}
}
async function destruct () {
let action = 'While destroying job'
logSilly(state, 'Killing plugins...')
let savedCreds = {}
let plugin = state.select('chain', 0)
while (!R.isNil(plugin) && !R.isNil(plugin.get())) {
if (requires('identity', plugin)) {
let creds = await plugin.get('runtime').getIdentity(plugin.get('api').client)
if (!R.isNil(creds) && !savedCreds[creds.name]) {
logSilly(state, 'Recording identity ' + creds.name)
try {
await auth.store(creds)
savedCreds[creds.name] = true
} catch (e) {
logError(state, action, e)
}
}
}
try {
await destructPlugin(plugin)
} catch (e) {
logError(state, action, e)
}
plugin = plugin.right()
}
try {
await storage.shrink(state.get('name'))
} catch (e) {
logError(state, action, e)
}
return state
}
return {
construct,
start,
stop,
destruct
}
}