reshuffle-aws-connectors
Version:
A set of Reshuffle connectors for AWS services
641 lines • 31.4 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
exports.__esModule = true;
exports.AWSLambdaConnector = exports.Folder = void 0;
var crypto_1 = __importDefault(require("crypto"));
var fs_1 = require("fs");
var Folder_1 = require("./Folder");
exports.Folder = Folder_1.Folder;
var BaseAWSConnector_1 = require("./BaseAWSConnector");
var COMMAND_TAG_NAME = 'deployed-by';
var COMMAND_TAG_VALUE = 'reshuffle-aws-lambda-connector';
var QUEUE_CONCURRENCY_LIMIT = 100;
function createQueue(functionName, payloads, maxConcurrent) {
return {
functionName: functionName,
payloads: payloads,
id: crypto_1["default"].randomBytes(8).toString('hex'),
active: 0,
complete: 0,
statuses: payloads.map(function () { return null; }),
resolutions: new Array(payloads.length),
maxConcurrent: Math.min(payloads.length, Math.round(maxConcurrent), QUEUE_CONCURRENCY_LIMIT)
};
}
function getNextJobFromQueue(queue) {
if (queueIsComplete(queue) || !queueHasPendingJobs(queue)) {
return;
}
var index = queue.statuses.findIndex(function (st) { return st === null; });
if (index < 0) {
return;
}
queue.statuses[index] = 'running';
queue.active++;
return {
functionName: queue.functionName,
payload: queue.payloads[index],
queue: queue,
index: index
};
}
function queueHasPendingJobs(queue) {
return queue.active < queue.maxConcurrent;
}
function queueIsComplete(queue) {
return queue.complete === queue.payloads.length;
}
function onQueueJobDone(queue, job, resolution) {
var index = job.index;
if (queue.statuses[index] !== 'running') {
return;
}
queue.statuses[index] = 'complete';
queue.resolutions[index] = resolution;
queue.complete++;
queue.active--;
}
var AWSLambdaConnector = /** @class */ (function (_super) {
__extends(AWSLambdaConnector, _super);
function AWSLambdaConnector(app, options, id) {
var _this = _super.call(this, app, options, id) || this;
if (!_this.options.region) {
throw new Error('No region');
}
_this.lambda = _this.account.getClient('Lambda');
return _this;
}
// Events /////////////////////////////////////////////////////////
AWSLambdaConnector.prototype.on = function (options, handler, eventId) {
if (options.type !== 'QueueComplete') {
throw new Error("Invalid event type: " + options.type);
}
var eid = eventId || { account: this.account, options: options };
return this.eventManager.addEvent(options, handler, eid);
};
// Actions ////////////////////////////////////////////////////////
AWSLambdaConnector.prototype.command = function (functionName, executable, command, files, options) {
if (files === void 0) { files = []; }
if (options === void 0) { options = {}; }
return __awaiter(this, void 0, void 0, function () {
var urls, _i, urls_1, url;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (typeof command !== 'string' || command.trim().length === 0) {
throw new Error("Invalid commmand: " + command);
}
if (typeof executable === 'string' && executable.startsWith('http')) {
BaseAWSConnector_1.validateURL(executable);
}
else if (typeof executable === 'string' && executable.startsWith('s3')) {
BaseAWSConnector_1.validateS3URL(executable);
}
else {
throw new Error("Invalid executable: " + executable);
}
urls = Array.isArray(files) ? files : [files];
for (_i = 0, urls_1 = urls; _i < urls_1.length; _i++) {
url = urls_1[_i];
BaseAWSConnector_1.validateS3URL(url);
}
return [4 /*yield*/, this.commandCreateFunction(functionName, executable, options)];
case 1:
_a.sent();
return [2 /*return*/, this.invoke(functionName, { command: command, urls: urls })];
}
});
});
};
AWSLambdaConnector.prototype.commandCreateFunction = function (functionName, executable, options) {
return __awaiter(this, void 0, void 0, function () {
function folderHandler(folder) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, folder.copy('index.js', __dirname + "/lambda-command.js")];
case 1:
_a.sent();
return [4 /*yield*/, folder.copy('Folder.js', __dirname + "/Folder.js")];
case 2:
_a.sent();
return [4 /*yield*/, folder.write('package.json', JSON.stringify({
main: 'index.js',
dependencies: {
'aws-sdk': '^2.741.0',
jszip: '^3.5.0',
'node-fetch': '^2.6.0',
rimraf: '^3.0.2'
}
}))];
case 3:
_a.sent();
return [4 /*yield*/, folder.exec('npm install')];
case 4:
_a.sent();
return [2 /*return*/];
}
});
});
}
var info;
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!!options.force) return [3 /*break*/, 2];
return [4 /*yield*/, this.getFunctionInfo(functionName)];
case 1:
info = _b.sent();
if (info) {
if (info.Tags[COMMAND_TAG_NAME] !== COMMAND_TAG_VALUE) {
throw new Error("Lambda exists but is not a command function: " + functionName);
}
console.log(functionName + ": Using existing Lambda function");
return [2 /*return*/];
}
_b.label = 2;
case 2: return [4 /*yield*/, this.createInFolder(functionName, folderHandler, __assign(__assign({}, options), { env: __assign(__assign({}, (options.env || {})), { EXECUTABLE: executable }), tags: __assign(__assign({}, (options.tags || {})), (_a = {}, _a[COMMAND_TAG_NAME] = COMMAND_TAG_VALUE, _a)), timeout: options.timeout || 300 }))];
case 3:
_b.sent();
return [2 /*return*/];
}
});
});
};
AWSLambdaConnector.prototype.create = function (functionName, payload, options) {
if (options === void 0) { options = {}; }
return __awaiter(this, void 0, void 0, function () {
var buffer, role;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.validateFunctionName(functionName);
return [4 /*yield*/, this.makeBuffer(payload)];
case 1:
buffer = _a.sent();
return [4 /*yield*/, this.identity.getOrCreateServiceRole(options.roleName || 'reshuffle_AWSLambdaConnector', 'lambda.amazonaws.com', this.identity.createSimplePolicy('arn:aws:logs:*:*:*', [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents',
]))];
case 2:
role = _a.sent();
return [2 /*return*/, this.lambda
.createFunction({
Code: { ZipFile: buffer },
Environment: options.env ? { Variables: options.env } : {},
FunctionName: functionName,
Handler: 'index.handler',
MemorySize: options.memorySize || 256,
Publish: true,
Runtime: options.runtime || 'nodejs12.x',
Role: role.Arn,
Tags: options.tags || {},
Timeout: options.timeout || 3
})
.promise()];
}
});
});
};
AWSLambdaConnector.prototype.makeBuffer = function (payload) {
return __awaiter(this, void 0, void 0, function () {
var _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (payload.buffer instanceof Buffer) {
return [2 /*return*/, payload.buffer];
}
if (!(typeof payload.filename === 'string')) return [3 /*break*/, 2];
_a = payload;
return [4 /*yield*/, fs_1.promises.readFile(payload.filename, 'utf-8')];
case 1:
_a.code = _b.sent();
_b.label = 2;
case 2:
if (typeof payload.code !== 'string' || payload.code.length === 0) {
throw new Error("Invalid code for lambda function: " + payload.code);
}
return [2 /*return*/, Folder_1.zipOne('index.js', payload.code)];
}
});
});
};
AWSLambdaConnector.prototype.createFromBuffer = function (functionName, buffer, options) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.create(functionName, { buffer: buffer }, options)];
});
});
};
AWSLambdaConnector.prototype.createFromCode = function (functionName, code, options) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.create(functionName, { code: code }, options)];
});
});
};
AWSLambdaConnector.prototype.createFromFile = function (functionName, filename, options) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.create(functionName, { filename: filename }, options)];
});
});
};
AWSLambdaConnector.prototype.createInFolder = function (functionName, folderHandler, options) {
if (options === void 0) { options = {}; }
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
return [2 /*return*/, this.inFolder(functionName, folderHandler, function (buffer) {
return _this.createFromBuffer(functionName, buffer, options);
})];
});
});
};
AWSLambdaConnector.prototype["delete"] = function (functionName) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.validateFunctionName(functionName);
return [4 /*yield*/, this.lambda
.deleteFunction({
FunctionName: functionName
})
.promise()];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
AWSLambdaConnector.prototype.enqueue = function (functionName, payload, maxConcurrent) {
if (maxConcurrent === void 0) { maxConcurrent = QUEUE_CONCURRENCY_LIMIT; }
return __awaiter(this, void 0, void 0, function () {
var payloads, queue;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.validateFunctionName(functionName);
payloads = Array.isArray(payload) ? payload : [payload];
if (typeof maxConcurrent !== 'number' || maxConcurrent < 1) {
throw new Error("Invalid max concurrent: " + maxConcurrent);
}
queue = createQueue(functionName, payloads, maxConcurrent);
console.log('Create queue:', queue.id, queue.functionName);
return [4 /*yield*/, this.store.update('queues', function (queues) {
if (queues === void 0) { queues = {}; }
return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
queues[queue.id] = queue;
return [2 /*return*/, queues];
});
});
})];
case 1:
_a.sent();
setTimeout(function () { return _this.onQueueReady(); }, 0);
return [2 /*return*/, queue.id];
}
});
});
};
AWSLambdaConnector.prototype.onQueueReady = function () {
return __awaiter(this, void 0, void 0, function () {
var jobs, complete, startJob, _i, jobs_1, job;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
jobs = [];
complete = [];
return [4 /*yield*/, this.store.update('queues', function (queues) { return __awaiter(_this, void 0, void 0, function () {
var _i, _a, queue, job;
return __generator(this, function (_b) {
for (_i = 0, _a = Object.values(queues); _i < _a.length; _i++) {
queue = _a[_i];
if (queueIsComplete(queue)) {
console.log('Qeueue complete:', queue.id);
complete.push(queue);
delete queues[queue.id];
continue;
}
for (;;) {
job = getNextJobFromQueue(queue);
if (!job) {
break;
}
jobs.push(job);
}
}
return [2 /*return*/, queues];
});
}); })];
case 1:
_a.sent();
startJob = function (job) { return __awaiter(_this, void 0, void 0, function () {
var res, resolution;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
console.log('Starting job:', job.functionName, job.payload);
return [4 /*yield*/, this.lambda
.invoke({
FunctionName: job.functionName,
Payload: JSON.stringify(job.payload)
})
.promise()];
case 1:
res = _a.sent();
resolution = this.parseResponse(res, job.functionName);
return [4 /*yield*/, this.store.update('queues', function (queues) { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
onQueueJobDone(queues[job.queue.id], job, resolution);
console.log('Job done:', job.functionName, job.payload);
return [2 /*return*/, queues];
});
}); })];
case 2:
_a.sent();
void this.onQueueReady();
return [2 /*return*/];
}
});
}); };
for (_i = 0, jobs_1 = jobs; _i < jobs_1.length; _i++) {
job = jobs_1[_i];
void startJob(job);
}
if (!(0 < complete.length)) return [3 /*break*/, 3];
return [4 /*yield*/, this.eventManager.fire(function (ec) { return ec.options.type === 'QueueComplete'; }, complete.map(function (queue) { return ({
qid: queue.id,
payloads: queue.payloads,
resolutions: queue.resolutions
}); }))];
case 2:
_a.sent();
_a.label = 3;
case 3: return [2 /*return*/];
}
});
});
};
AWSLambdaConnector.prototype.getFunctionInfo = function (functionName) {
return __awaiter(this, void 0, void 0, function () {
var info, e_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, this.lambda
.getFunction({
FunctionName: functionName
})
.promise()]; // must explicitly await for catch
case 1:
info = _a.sent() // must explicitly await for catch
;
return [2 /*return*/, info];
case 2:
e_1 = _a.sent();
if (e_1.code === 'ResourceNotFoundException') {
return [2 /*return*/];
}
throw e_1;
case 3: return [2 /*return*/];
}
});
});
};
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
AWSLambdaConnector.prototype.invoke = function (functionName, payload) {
return __awaiter(this, void 0, void 0, function () {
var res, response;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.validateFunctionName(functionName);
return [4 /*yield*/, this.lambda
.invoke({
FunctionName: functionName,
Payload: JSON.stringify(payload)
})
.promise()];
case 1:
res = _a.sent();
response = this.parseResponse(res, functionName);
if (response instanceof Error) {
throw response;
}
return [2 /*return*/, response];
}
});
});
};
AWSLambdaConnector.prototype.parseResponse = function (res, functionName) {
function err(code) {
var fn = functionName ? ": " + functionName : '';
return new Error("Error " + code + " invoking lambda function" + fn);
}
if (res.StatusCode !== 200) {
return err(res.StatusCode);
}
var payload = JSON.parse(res.Payload);
if (payload === null) {
return err('unknown');
}
if (typeof payload.statusCode !== 'number') {
return payload;
}
if (payload.statusCode < 200 || 300 <= payload.statusCode) {
return err(payload.statusCode || payload.errorMessage || res.FunctionError);
}
try {
return JSON.parse(payload.body);
}
catch (_a) {
return payload.body;
}
};
AWSLambdaConnector.prototype.validateFunctionName = function (functionName) {
if (typeof functionName !== 'string') {
throw new Error("Lambda function name not a string: " + functionName);
}
if (!/^[a-zA-z0-9\-]{1,64}$/.test(functionName)) {
throw new Error("Invalid lambda function name: " + functionName);
}
};
AWSLambdaConnector.prototype.listFunctions = function () {
return __awaiter(this, void 0, void 0, function () {
var res;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.lambda.listFunctions({}).promise()];
case 1:
res = _a.sent();
return [2 /*return*/, res.Functions];
}
});
});
};
AWSLambdaConnector.prototype.updateCode = function (functionName, payload) {
return __awaiter(this, void 0, void 0, function () {
var buffer;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.validateFunctionName(functionName);
return [4 /*yield*/, this.makeBuffer(payload)];
case 1:
buffer = _a.sent();
return [2 /*return*/, this.lambda
.updateFunctionCode({
ZipFile: buffer,
FunctionName: functionName
})
.promise()];
}
});
});
};
AWSLambdaConnector.prototype.updateCodeFromBuffer = function (functionName, buffer) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.updateCode(functionName, { buffer: buffer })];
});
});
};
AWSLambdaConnector.prototype.updateCodeFromCode = function (functionName, code) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.updateCode(functionName, { code: code })];
});
});
};
AWSLambdaConnector.prototype.updateCodeFromFile = function (functionName, filename) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, this.updateCode(functionName, { filename: filename })];
});
});
};
AWSLambdaConnector.prototype.updateCodeInFolder = function (functionName, folderHandler) {
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
return [2 /*return*/, this.inFolder(functionName, folderHandler, function (buffer) {
return _this.updateCodeFromBuffer(functionName, buffer);
})];
});
});
};
AWSLambdaConnector.prototype.inFolder = function (functionName, folderHandler, action) {
return __awaiter(this, void 0, void 0, function () {
var folder, buffer, res;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
folder = new Folder_1.Folder("reshuffle-awslambdaconnector-" + crypto_1["default"].randomBytes(8).toString('hex'));
_a.label = 1;
case 1:
_a.trys.push([1, , 5, 7]);
console.debug(functionName + ": Building code folder");
return [4 /*yield*/, folderHandler(folder)];
case 2:
_a.sent();
console.debug(functionName + ": Packaging folder");
return [4 /*yield*/, folder.zip()];
case 3:
buffer = _a.sent();
console.debug(functionName + ": Package size " + buffer.length + " bytes");
console.debug(functionName + ": Deploying to Lambda");
return [4 /*yield*/, action(buffer)]; // must explicitly await for finally
case 4:
res = _a.sent() // must explicitly await for finally
;
return [2 /*return*/, res];
case 5: return [4 /*yield*/, folder.destroy()];
case 6:
_a.sent();
return [7 /*endfinally*/];
case 7: return [2 /*return*/];
}
});
});
};
// SDK ////////////////////////////////////////////////////////////
AWSLambdaConnector.prototype.sdk = function (options) {
return this.account.getClient('Lambda', options);
};
return AWSLambdaConnector;
}(BaseAWSConnector_1.BaseAWSConnector));
exports.AWSLambdaConnector = AWSLambdaConnector;
//# sourceMappingURL=AWSLambdaConnector.js.map