@atomist/automation-client
Version:
Atomist API for software low-level client
518 lines • 23.6 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const cluster = require("cluster");
const stringify = require("json-stringify-safe");
const _ = require("lodash");
const TinyQueue = require("tinyqueue");
const error_1 = require("../../../util/error");
const logger_1 = require("../../../util/logger");
const namespace = require("../../util/cls");
const Deferred_1 = require("../../util/Deferred");
const health_1 = require("../../util/health");
const poll_1 = require("../../util/poll");
const shutdown_1 = require("../../util/shutdown");
const AbstractRequestProcessor_1 = require("../AbstractRequestProcessor");
const RequestProcessor_1 = require("../RequestProcessor");
const WebSocketMessageClient_1 = require("../websocket/WebSocketMessageClient");
const messages_1 = require("./messages");
/* tslint:disable:max-file-line-count */
/**
* A RequestProcessor that delegates to Node.JS Cluster workers to do the actual
* command and event processing.
* @see ClusterWorkerRequestProcessor
*/
class ClusterMasterRequestProcessor extends AbstractRequestProcessor_1.AbstractRequestProcessor {
constructor(automations, configuration, listeners = [], numWorkers = require("os").cpus().length, maxConcurrentPerWorker = 4) {
super(automations, configuration, listeners);
this.automations = automations;
this.configuration = configuration;
this.listeners = listeners;
this.numWorkers = numWorkers;
this.maxConcurrentPerWorker = maxConcurrentPerWorker;
this.commands = new Map();
this.events = new Map();
this.messages = new TinyQueue([], (a, b) => {
if (a.message.type === "atomist:command" && b.message.type !== "atomist:command") {
return -1;
}
else if (a.message.type !== "atomist:command" && b.message.type === "atomist:command") {
return 1;
}
else {
return a.ts - b.ts;
}
});
this.shutdownInitiated = false;
this.replaceWorkers = true;
this.backoffInitiated = false;
this.webSocketLifecycle = configuration.ws.lifecycle;
health_1.registerHealthIndicator(() => {
const cmds = [];
for (const key of this.commands.keys()) {
cmds.push(key);
}
const evts = [];
for (const key of this.events.keys()) {
evts.push(key);
}
if (this.webSocketLifecycle.connected() && this.registration) {
return {
status: health_1.HealthStatus.Up,
detail: {
commands: cmds,
events: evts,
},
};
}
else {
return {
status: health_1.HealthStatus.Down,
detail: {
commands: cmds,
events: evts,
},
};
}
});
shutdown_1.registerShutdownHook(() => __awaiter(this, void 0, void 0, function* () {
if (this.shutdownInitiated) {
return 0;
}
this.shutdownInitiated = true;
const gracePeriod = shutdown_1.terminationGracePeriod(this.configuration);
if (shutdown_1.terminationGraceful(this.configuration)) {
try {
logger_1.logger.debug("Waiting for queue to empty");
yield poll_1.poll(() => this.queueLength() < 1, gracePeriod);
}
catch (e) {
logger_1.logger.warn("Work queue did not empty within grace period");
return 1;
}
}
logger_1.logger.debug("Terminating workers");
yield this.terminateWorkers(gracePeriod);
return 0;
}), 0, "drain work queue and shutdown workers");
this.scheduleQueueLength();
this.scheduleBackoffCheck();
}
onRegistration(registration) {
logger_1.logger.debug("Registration successful: %s", stringify(registration));
this.configuration.ws.session = registration;
this.registration = registration;
messages_1.broadcast({
type: "atomist:registration",
registration: this.registration,
context: undefined,
});
}
onConnect(ws) {
logger_1.logger.debug("WebSocket connection established. Listening for incoming messages");
this.webSocketLifecycle.set(ws);
this.listeners.forEach(l => l.registrationSuccessful(this));
}
onDisconnect() {
this.webSocketLifecycle.reset();
this.registration = undefined;
}
run() {
const ws = () => this.webSocketLifecycle;
const attachEvents = (worker, deferred) => {
worker.on("message", message => {
const msg = message;
// Wait for online message to come in
if (msg.type === "atomist:online") {
deferred.resolve();
return;
}
const ses = namespace.create();
ses.run(() => {
// Only process our messages
if (!msg.type || (msg.type && !msg.type.startsWith("atomist:"))) {
return;
}
namespace.set(msg.context);
logger_1.logger.debug("Received '%s' message from worker '%s': %j", msg.type, worker.id, msg.context);
const invocationId = namespace.get().invocationId;
const ctx = hydrateContext(msg);
if (msg.type === "atomist:message") {
let messageClient;
if (this.commands.has(invocationId)) {
messageClient = this.commands.get(invocationId).dispatched.context.messageClient;
}
else if (this.events.has(invocationId)) {
messageClient = this.events.get(invocationId).dispatched.context.messageClient;
}
else {
logger_1.logger.error("Can't handle message from worker due to missing messageClient");
this.clearNamespace();
return;
}
if (msg.data.destinations && msg.data.destinations.length > 0) {
messageClient.send(msg.data.message, msg.data.destinations, msg.data.options)
.then(this.clearNamespace, this.clearNamespace);
}
else {
messageClient.respond(msg.data.message, msg.data.options)
.then(this.clearNamespace, this.clearNamespace);
}
}
else if (msg.type === "atomist:status") {
ws().send(msg.data);
}
else if (msg.type === "atomist:command_success") {
this.listeners.map(l => () => l.commandSuccessful(msg.event, ctx, msg.data))
.reduce((p, f) => p.then(f), Promise.resolve())
.then(() => {
if (this.commands.has(invocationId)) {
this.commands.get(invocationId).dispatched.result.resolve(msg.data);
this.commands.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
})
.catch(e => {
logger_1.logger.warn(`Failed to run listeners: ${e.message}`);
error_1.printError(e);
if (this.commands.has(invocationId)) {
this.commands.get(invocationId).dispatched.result.resolve(msg.data);
this.commands.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
});
}
else if (msg.type === "atomist:command_failure") {
this.listeners.map(l => () => l.commandFailed(msg.event, ctx, msg.data))
.reduce((p, f) => p.then(f), Promise.resolve())
.then(() => {
if (this.commands.has(invocationId)) {
this.commands.get(invocationId).dispatched.result.reject(msg.data);
this.commands.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
})
.catch(e => {
logger_1.logger.warn(`Failed to run listeners: ${e.message}`);
error_1.printError(e);
if (this.commands.has(invocationId)) {
this.commands.get(invocationId).dispatched.result.reject(msg.data);
this.commands.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
});
}
else if (msg.type === "atomist:event_success") {
this.listeners.map(l => () => l.eventSuccessful(msg.event, ctx, msg.data))
.reduce((p, f) => p.then(f), Promise.resolve())
.then(() => {
if (this.events.has(invocationId)) {
this.events.get(invocationId).dispatched.result.resolve(msg.data);
this.events.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
})
.catch(e => {
logger_1.logger.warn(`Failed to run listeners: ${e.message}`);
error_1.printError(e);
if (this.events.has(invocationId)) {
this.events.get(invocationId).dispatched.result.resolve(msg.data);
this.events.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
});
}
else if (msg.type === "atomist:event_failure") {
this.listeners.map(l => () => l.eventFailed(msg.event, ctx, msg.data))
.reduce((p, f) => p.then(f), Promise.resolve())
.then(() => {
if (this.events.has(invocationId)) {
this.events.get(invocationId).dispatched.result.reject(msg.data);
this.events.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
})
.catch(e => {
logger_1.logger.warn(`Failed to run listeners: ${e.message}`);
error_1.printError(e);
if (this.events.has(invocationId)) {
this.events.get(invocationId).dispatched.result.reject(msg.data);
this.events.delete(invocationId);
}
this.clearNamespace();
this.startMessage();
});
}
else if (msg.type === "atomist:shutdown") {
logger_1.logger.info(`Immediate shutdown requested from worker`);
this.configuration.ws.termination.graceful = false;
shutdown_1.safeExit(msg.data);
}
});
});
};
attachEvents.bind(this);
const promises = [];
for (let i = 0; i < this.numWorkers; i++) {
const worker = cluster.fork();
const deferred = new Deferred_1.Deferred();
promises.push(deferred.promise);
attachEvents(worker, deferred);
}
cluster.on("disconnect", worker => {
logger_1.logger.warn(`Worker '${worker.id}' disconnected`);
});
cluster.on("online", worker => {
logger_1.logger.debug(`Worker '${worker.id}' connected`);
this.startMessage();
});
cluster.on("exit", (worker, code, signal) => {
const workerId = worker.id;
// Remove all inflight work for died worker
for (const [k, v] of this.events.entries()) {
if (v.worker === workerId) {
this.events.delete(k);
}
}
for (const [k, v] of this.commands.entries()) {
if (v.worker === workerId) {
this.commands.delete(k);
}
}
if (this.replaceWorkers) {
logger_1.logger.warn(`Worker '${worker.id}' exited with status '${code}' and signal '${signal}', replacing...`);
attachEvents(cluster.fork(), new Deferred_1.Deferred());
}
else {
logger_1.logger.debug(`Worker '${worker.id}' shut down with status '${code}' and signal '${signal}'`);
}
});
return Promise.all(promises);
}
invokeCommand(ci, ctx, command, callback) {
const message = {
type: "atomist:command",
registration: this.registration,
context: ctx.context,
data: command,
};
logger_1.logger.debug(`Queuing incoming command handler request '${command.command}'`);
const dispatched = new Dispatched(new Deferred_1.Deferred(), ctx);
this.messages.push({ message, dispatched, ts: Date.now() });
callback(dispatched.result.promise);
this.startMessage();
}
invokeEvent(ef, ctx, event, callback) {
const message = {
type: "atomist:event",
registration: this.registration,
context: ctx.context,
data: event,
};
logger_1.logger.debug(`Queuing incoming event subscription '${event.extensions.operationName}'`);
const dispatched = new Dispatched(new Deferred_1.Deferred(), ctx);
this.messages.push({ message, dispatched, ts: Date.now() });
callback(dispatched.result.promise);
this.startMessage();
}
sendStatusMessage(payload, ctx) {
return __awaiter(this, void 0, void 0, function* () {
return this.webSocketLifecycle.send(payload);
});
}
createGraphClient(event) {
return undefined;
}
createMessageClient(event) {
if (RequestProcessor_1.isCommandIncoming(event)) {
return new WebSocketMessageClient_1.WebSocketCommandMessageClient(event, this.webSocketLifecycle, this.configuration);
}
else if (RequestProcessor_1.isEventIncoming(event)) {
return new WebSocketMessageClient_1.WebSocketEventMessageClient(event, this.webSocketLifecycle, this.configuration);
}
}
assignWorker() {
let workers = [];
for (const id in cluster.workers) {
if (cluster.workers.hasOwnProperty(id)) {
const worker = cluster.workers[id];
if (worker.isConnected) {
workers.push({ worker, messages: 0 });
}
}
}
const deadEvents = [];
this.events.forEach((e, k) => {
const worker = workers.find(w => w.worker.id === e.worker);
if (!!worker) {
worker.messages = worker.messages + 1;
}
else {
deadEvents.push(k);
}
});
deadEvents.forEach(de => this.events.delete(de));
const deadCommands = [];
this.commands.forEach((c, k) => {
const worker = workers.find(w => w.worker.id === c.worker);
if (!!worker) {
worker.messages = worker.messages + 1;
}
else {
deadCommands.push(k);
}
});
deadCommands.forEach(dc => this.commands.delete(dc));
workers = workers.filter(w => w.messages < this.maxConcurrentPerWorker);
if (workers.length === 0) {
return undefined;
}
return workers[Math.floor(Math.random() * workers.length)].worker;
}
startMessage() {
if (this.messages.length > 0) {
const worker = this.assignWorker();
if (!!worker) {
const message = this.messages.pop();
namespace.set(message.message.context);
if (message.message.type === "atomist:command") {
this.commands.set(message.message.context.invocationId, {
dispatched: message.dispatched,
worker: worker.id,
});
logger_1.logger.debug("Incoming command handler request '%s' dispatching to worker '%s'", message.message.data.command, worker.id);
}
else if (message.message.type === "atomist:event") {
this.events.set(message.message.context.invocationId, {
dispatched: message.dispatched,
worker: worker.id,
});
logger_1.logger.debug("Incoming event handler subscription '%s' dispatching to worker '%s'", message.message.data.extensions.operationName, worker.id);
}
worker.send(message.message);
}
}
this.reportQueueLength();
}
queueLength() {
return this.messages.length + this.events.size + this.commands.size;
}
scheduleQueueLength() {
if (this.configuration.statsd.enabled) {
setInterval(() => {
this.reportQueueLength();
}, 1000).unref();
}
}
scheduleBackoffCheck() {
const workers = this.numWorkers;
const maxConcurrent = this.maxConcurrentPerWorker;
const threshold = _.get(this.configuration, "ws.backoff.threshold") || (workers * maxConcurrent);
const interval = _.get(this.configuration, "ws.backoff.interval") || 2500;
const duration = _.get(this.configuration, "ws.backoff.duration") || 5000;
let factor = _.get(this.configuration, "ws.backoff.factor") || 0.5;
if (factor > 1) {
factor = 0.5;
}
else if (factor < 0) {
factor = 0;
}
setInterval(() => {
const messageCount = this.messages.length;
const statsd = this.configuration.statsd.__instance;
if ((!this.backoffInitiated && messageCount >= threshold) ||
(this.backoffInitiated && messageCount >= (threshold * factor)) ||
this.shutdownInitiated) {
WebSocketMessageClient_1.sendMessage({
control: {
name: "backoff",
params: {
millis: duration,
},
},
}, this.webSocketLifecycle.get(), false);
if (!this.backoffInitiated) {
logger_1.logger.debug(`Initiated incoming messages backoff. queue size: ${messageCount}, threshold: ${threshold}`);
}
this.backoffInitiated = true;
if (!!statsd) {
statsd.gauge("work_queue.backoff", 1, [], () => {
});
}
}
else {
if (this.backoffInitiated) {
logger_1.logger.debug(`Stopped incoming messages backoff. queue size: ${messageCount}, threshold: ${threshold}`);
}
this.backoffInitiated = false;
if (!!statsd) {
statsd.gauge("work_queue.backoff", 0, [], () => {
});
}
}
}, interval).unref();
}
reportQueueLength() {
if (this.configuration.statsd.enabled) {
const statsd = this.configuration.statsd.__instance;
if (!!statsd) {
statsd.gauge("work_queue.pending", this.messages.length, 1, [], () => {
});
statsd.gauge("work_queue.events", this.events.size, 1, [], () => {
});
statsd.gauge("work_queue.commands", this.commands.size, 1, [], () => {
});
}
}
}
terminateWorkers(gracePeriod) {
return __awaiter(this, void 0, void 0, function* () {
if (!this.replaceWorkers) {
return;
}
this.replaceWorkers = false;
const workerIds = Object.keys(cluster.workers).map(id => cluster.workers[id].id);
logger_1.logger.debug("Sending workers the shutdown message");
workerIds.forEach(id => cluster.workers[id].send({ type: "atomist:shutdown" }));
try {
logger_1.logger.debug("Waiting for workers to exit");
yield poll_1.poll(() => Object.keys(cluster.workers).length < 1, gracePeriod);
logger_1.logger.debug("All workers have exited");
}
catch (e) {
logger_1.logger.warn(`Not all workers exited in allotted time: ${Object.keys(cluster.workers).join(",")}`);
}
});
}
}
exports.ClusterMasterRequestProcessor = ClusterMasterRequestProcessor;
class Dispatched {
constructor(result, context) {
this.result = result;
this.context = context;
}
}
function hydrateContext(msg) {
return {
invocationId: msg.context.invocationId,
correlationId: msg.context.correlationId,
workspaceId: msg.context.workspaceId,
context: msg.context,
};
}
//# sourceMappingURL=ClusterMasterRequestProcessor.js.map