redioactive
Version:
Reactive streams for chaining overlapping promises.
682 lines (681 loc) • 30.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.httpSource = void 0;
const redio_1 = require("./redio");
const http_1 = __importStar(require("http"));
const https_1 = __importStar(require("https"));
const util_1 = require("util");
const url_1 = require("url");
const http_common_1 = require("./http-common");
const dns_1 = require("dns");
/* Code for sending values over HTTP/S. */
const servers = {};
const serversS = {};
const streamIDs = {};
function isPull(c) {
return c.type === 'pull';
}
function isPush(c) {
return c.type === 'push';
}
// function wait(t: number): Promise<void> {
// return new Promise((resolve) => setTimeout(resolve, t))
// }
function noMatch(req, res) {
if (res.writableEnded)
return;
const message = {};
if (req.url && req.method && req.method === 'GET') {
const url = new url_1.URL(req.url);
if (!Object.keys(streamIDs).find((x) => url.pathname.startsWith(x))) {
message.status = 404;
message.message = `Redioactive: HTTP/S source: No stream available for pathname "${url.pathname}".`;
}
}
else {
message.status = req.method ? 405 : 500;
message.message = req.method
? `Redioactive: HTTP/S source: Method ${req.method} not allowed for resource`
: `Redioactive: HTTP/S source: Cannot determine method type`;
}
if (message.status) {
res.statusCode = message.status;
const result = JSON.stringify(message, null, 2);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', Buffer.byteLength(result, 'utf8'));
res.end(result, 'utf8');
}
}
function httpSource(uri, options) {
if (!options)
throw new Error('HTTP options must be specified - for now.');
const tChest = new Map();
let info;
const url = new url_1.URL(uri, `http://localhost:${options.httpPort || options.httpsPort}`);
url.pathname = url.pathname.replace(/\/+/g, '/');
let protocol;
let agent;
if (url.pathname.endsWith('/')) {
url.pathname = url.pathname.slice(0, -1);
}
if (uri.toLowerCase().startsWith('http')) {
info = (0, redio_1.literal)({
type: 'push',
protocol: uri.toLowerCase().startsWith('https') ? http_common_1.ProtocolType.https : http_common_1.ProtocolType.http,
body: http_common_1.BodyType.primitive,
idType: http_common_1.IdType.counter,
delta: http_common_1.DeltaType.one,
manifest: {},
root: url.pathname
});
protocol = info.protocol === http_common_1.ProtocolType.http ? http_1.default : https_1.default;
agent = new protocol.Agent(Object.assign({ keepAlive: true, host: url.hostname }, (options && options.requestOptions) || {}));
}
else {
let server = undefined;
let serverS = undefined;
const root = url.pathname;
if (options.httpPort) {
if (options && !options.extraStreamRoot) {
// Set with first element
streamIDs[root] = { httpPort: options.httpPort, httpsPort: options.httpsPort };
}
server = servers[options.httpPort];
if (!server) {
server = options.serverOptions ? (0, http_1.createServer)(options.serverOptions) : (0, http_1.createServer)();
server.keepAliveTimeout = (options && options.keepAliveTimeout) || 5000;
servers[options.httpPort] = server;
server.listen(options.httpPort, () => {
console.log(`Redioactive: HTTP/S source: HTTP pull server for stream ${root} listening on ${options.httpPort}`);
});
}
server.on('request', pullRequest);
server.on('error', (err) => {
// TODO interrupt and push error?
console.error(err);
});
}
if (options.httpsPort) {
if (options && !options.extraStreamRoot) {
streamIDs[root] = { httpPort: options.httpPort, httpsPort: options.httpsPort };
}
serverS = serversS[options.httpsPort];
if (!serverS) {
serverS = options.serverOptions ? (0, https_1.createServer)(options.serverOptions) : (0, https_1.createServer)();
serverS.keepAliveTimeout = (options && options.keepAliveTimeout) || 5000;
serversS[options.httpsPort] = serverS;
serverS.listen(options.httpsPort, () => {
console.log(`Redioactive: HTTP/S source: HTTPS server for stream ${root} listening on ${options.httpsPort}`);
});
}
serverS.on('request', pullRequest);
serverS.on('error', (err) => {
// TODO interrupt and push error?
console.error(err);
});
}
info = (0, redio_1.literal)({
type: 'pull',
protocol: server && serverS ? http_common_1.ProtocolType.both : serverS ? http_common_1.ProtocolType.https : http_common_1.ProtocolType.http,
body: http_common_1.BodyType.primitive,
idType: http_common_1.IdType.counter,
delta: http_common_1.DeltaType.one,
manifest: {},
httpPort: options.httpPort,
httpsPort: options.httpsPort,
server,
serverS,
root
});
}
let fuzzyGap = (options && options.fuzzy) || 0.0;
const fuzzFactor = (options && options.fuzzy) || 0.0;
function fuzzyMatch(id) {
if (tChest.size === 0) {
return undefined;
}
const exact = tChest.get(id);
if (exact || fuzzFactor === 0.0) {
return exact;
}
else {
const key = fuzzyIDMatch(id, tChest.keys());
return key ? tChest.get(key) : undefined;
}
}
function fuzzyIDMatch(id, keys) {
if (info.idType !== http_common_1.IdType.string) {
const gap = fuzzyGap * fuzzFactor;
const idn = +id;
const [min, max] = [idn - gap, idn + gap];
for (const key of keys) {
const keyn = +key;
if (keyn >= min && keyn <= max) {
return key;
}
}
return undefined;
}
else {
// IdType === string
for (const key of keys) {
let score = id.length > key.length ? id.length - key.length : key.length - id.length;
for (let x = id.length - 1; x >= 0 && score / id.length <= fuzzFactor; x--) {
if (x < key.length) {
score += key[x] === id[x] ? 0 : 1;
}
}
if (score / id.length <= fuzzFactor) {
return key;
}
}
return undefined;
}
}
const blobContentType = (options && options.contentType) || 'application/octet-stream';
const pendings = [];
let nextId;
function pullRequest(req, res) {
var _a;
if (res.writableEnded)
return;
if (req.url && isPull(info) && req.method === 'GET') {
let path = req.url.replace(/\/+/g, '/');
if (path.endsWith('/')) {
path = path.slice(0, -1);
}
if (path.startsWith(info.root)) {
const id = path.slice(info.root.length + 1);
if (id === 'debug.json') {
return debug(res);
}
if (id === 'manifest.json') {
const maniStr = JSON.stringify(info.manifest);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', `${Buffer.byteLength(maniStr, 'utf8')}`);
res.end(maniStr, 'utf8');
return;
}
if (id === 'end') {
return endStream(req, res);
}
if (id.match(/start|latest/)) {
res.statusCode = 302;
const isSSL = Object.prototype.hasOwnProperty.call(req.socket, 'encrypted');
res.setHeader('Location', `${isSSL ? 'https' : 'http'}://${req.headers['host']}${info.root}/${id === 'start' ? lowWaterMark : highWaterMark}`);
res.setHeader('Redioactive-BodyType', info.body);
res.setHeader('Redioactive-IdType', info.idType);
res.setHeader('Redioactive-DeltaType', info.delta);
res.setHeader('Redioactive-BufferSize', `${bufferSize}`);
if (options && options.cadence) {
res.setHeader('Redioactive-Cadence', `${options.cadence}`);
}
res.end();
return;
}
const value = fuzzyMatch(id);
if (value) {
res.setHeader('Redioactive-Id', value.id);
res.setHeader('Redioactive-NextId', value.nextId);
res.setHeader('Redioactive-PrevId', value.prevId);
// TODO parts and parallel
if (info.body !== http_common_1.BodyType.blob) {
const json = JSON.stringify(value.value);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', `${Buffer.byteLength(json, 'utf8')}`);
res.end(json, 'utf8');
}
else {
res.setHeader('Content-Type', blobContentType);
res.setHeader('Content-Length', `${(value.blob && value.blob.length) || 0}`);
// Assuming that receiver us happy with UTF-8 in headers
res.setHeader('Redioactive-Details', JSON.stringify(value.value));
res.end(value.blob || Buffer.alloc(0));
}
//res.on('finish', () => { - commented out to go parrallel - might work?
// Relying on undocument features of promises that you can safely resolve twice
setImmediate(value.nextFn);
//})
return;
}
else {
// for (const k of [nextId.toString()][Symbol.iterator]()) {
// console.log('**** About to fuzzy match with nextId', k)
// }
if (fuzzyIDMatch(id, [nextId.toString()][Symbol.iterator]())) {
// console.log('*** Fuzzy match with next', id)
const pending = new Promise((resolve) => {
const clearer = setTimeout(resolve, (options && options.timeout) || 5000);
pendings.push(() => {
clearTimeout(clearer);
resolve();
});
});
pending.then(() => {
pullRequest(req, res);
});
(_a = tChest.get(highWaterMark.toString())) === null || _a === void 0 ? void 0 : _a.nextFn();
return;
}
let [status, message] = [404, ''];
switch (info.idType) {
case http_common_1.IdType.counter:
case http_common_1.IdType.number:
if (+id < lowWaterMark) {
if (+id < lowestOfTheLow) {
status = 405;
message = `Request for a value with a sequence identifier "${id}" that is before the start of a stream "${lowestOfTheLow}".`;
}
else {
status = 410; // Gone
message = `Request for value with sequence identifier "${id}" that is before the current low water mark of "${lowWaterMark}".`;
}
}
else if (+id > highWaterMark) {
if (!ended) {
message = `Request for value with sequence identifier "${id}" that is beyond the current high water mark of "${highWaterMark}".`;
}
else {
status = 405; // METHOD NOT ALLOWED - I understand your request, but never for this resource pal!
message = `Request for a value with a sequence identifier "${id}" that is beyond the end of a finished stream.`;
}
}
else {
message = `Unmatched in-range request for a value with a sequence identifier "${id}".`;
}
break;
case http_common_1.IdType.string:
message = `Unmatched string sequence identifier "${id}".`;
break;
}
const json = JSON.stringify({
status,
statusMessage: http_1.STATUS_CODES[status],
message
}, null, 2);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', `${Buffer.byteLength(json, 'utf8')}`);
res.statusCode = status;
res.end(json, 'utf8');
return;
}
}
}
noMatch(req, res);
}
function debug(res) {
const keys = [];
for (const key of tChest.keys()) {
keys.push(key);
}
const debugInfo = {
info,
tChestSize: tChest.size,
bufferSize,
fuzzFactor,
fuzzyGap,
uri,
url,
streamIDs,
options,
ended,
lowestOfTheLow,
lowWaterMark,
highWaterMark,
nextId,
keys
};
const debugString = JSON.stringify(debugInfo, null, 2);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', `${Buffer.byteLength(debugString, 'utf8')}`);
res.end(debugString, 'utf8');
}
function endStream(req, res) {
const isSSL = Object.prototype.hasOwnProperty.call(req.socket, 'encrypted');
const port = req.socket.localPort;
if (isPull(info)) {
try {
info.server &&
info.server.close(() => {
isPull(info) && delete streamIDs[info.root];
console.log(`Redioactive: HTTP/S source: ${isSSL ? 'HTTPS' : 'HTTP'} server for stream ${(isPull(info) && info.root) || 'unknown'} on port ${port} closed.`);
});
info.serverS &&
info.serverS.close(() => {
isPull(info) && delete streamIDs[info.root];
console.log(`Redioactive: HTTP/S source: ${isSSL ? 'HTTPS' : 'HTTP'} server for stream ${(isPull(info) && info.root) || 'unknown'} on port ${port} closed.`);
});
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', 2);
res.end('OK', 'utf8');
delete streamIDs[info.root];
if (!Object.values(streamIDs).some((x) => isPull(info) &&
((x.httpPort && x.httpPort === info.httpPort) ||
(x.httpsPort && x.httpsPort === info.httpsPort)))) {
isPull(info) && info.httpPort && delete servers[info.httpPort];
isPull(info) && info.httpsPort && delete serversS[info.httpsPort];
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
}
catch (err) {
console.error(`Redioactive: HTTP source: error closing ${info.protocol} ${info.type} stream: ${err.message}`);
}
}
}
function checkObject(t) {
if (!options)
return;
if (options.seqId || options.extraStreamRoot || options.delta || options.blob) {
if (typeof t !== 'object' || Array.isArray(t)) {
throw new Error('HTTP stream properties from values (seqId, extraStreamRoot, delta, blob) requested but first stream value is not an object.');
}
}
const tr = t;
if (options.seqId &&
typeof tr[options.seqId] !== 'string' &&
typeof tr[options.seqId] !== 'number') {
throw new Error('Sequence identifer property expected but not present - or not a string or number - in first value in the stream.');
}
if (options.extraStreamRoot && typeof [options.extraStreamRoot] !== 'string') {
throw new Error('Extra stream root expected but no string property is present in the first stream value.');
}
if (options.delta &&
typeof tr[options.delta] !== 'string' &&
typeof tr[options.delta] !== 'number') {
throw new Error('Delta value expected but no string or number delta property is present on the first stream value.');
}
if (options.blob && !Buffer.isBuffer(tr[options.blob])) {
throw new Error('Data blob expected but no Buffer is present in the first value of the stream.');
}
if (typeof options.manifest === 'string' && typeof tr[options.manifest] !== 'object') {
throw new Error('Manifest object expected but it is not present in the first value of the stream.');
}
}
function initFromObject(t) {
if (typeof t !== 'object') {
info.body = http_common_1.BodyType.primitive;
}
else if (options && options.blob) {
info.body = http_common_1.BodyType.blob;
}
else {
info.body = http_common_1.BodyType.json;
}
const tr = t;
info.idType = http_common_1.IdType.counter;
if (options && options.seqId) {
info.idType = typeof tr[options.seqId] === 'number' ? http_common_1.IdType.number : http_common_1.IdType.string;
}
if (options && options.extraStreamRoot) {
if (isPull(info)) {
info.root = `${info.root}/${tr[options.extraStreamRoot]}`;
streamIDs[info.root] = { httpPort: options.httpPort, httpsPort: options.httpsPort };
}
// TODO do something for push
}
info.delta = http_common_1.DeltaType.one;
if (options && options.delta) {
if (typeof info.delta === 'number') {
info.delta = http_common_1.DeltaType.fixed;
}
else {
info.delta = typeof tr[options.delta] === 'number' ? http_common_1.DeltaType.variable : http_common_1.DeltaType.string;
}
}
if (options && options.manifest) {
if (typeof options.manifest === 'string') {
info.manifest =
typeof tr[options.manifest] === 'object'
? tr[options.manifest]
: {};
}
else {
info.manifest = options.manifest;
}
}
}
let idCounter = 0;
async function push(currentId) {
// console.log(
// `Pushing ${currentId} with counter ${idCounter} compared to lowest ${lowestOfTheLow}`
// )
const manifestSender = new Promise((resolve, reject) => {
if (idCounter !== lowestOfTheLow) {
resolve();
return;
}
// First time out, send manifest
dns_1.promises
.lookup(url.hostname)
.then((host) => {
url.hostname = host.address;
}, () => {
/* Does not matter - fall back on given hostname and default DNS behaviour */
})
.then(() => {
const req = protocol.request(Object.assign((options && options.requestOptions) || {}, {
hostname: url.hostname,
protocol: url.protocol,
port: url.port,
path: `${info.root}/manifest.json`,
method: 'POST',
headers: {
'Redioactive-BodyType': info.body,
'Redioactive-IdType': info.idType,
'Redioactive-DeltaType': info.delta,
'Redioactive-BufferSize': `${bufferSize}`,
'Redioactive-NextId': currentId // Defines the start
},
agent
}), (res) => {
if (res.statusCode === 200 || res.statusCode === 201) {
resolve();
}
else {
reject(new Error(`After posting manifest, unexptected response code "${res.statusCode}"`));
}
res.on('error', reject);
res.on('error', console.error);
});
if (options && options.cadence) {
req.setHeader('Redioactive-Cadence', `${options.cadence}`);
}
req.on('error', reject);
const maniJSON = JSON.stringify(info.manifest);
req.setHeader('Content-Length', `${Buffer.byteLength(maniJSON, 'utf8')}`);
req.setHeader('Content-Type', 'application/json');
req.end(maniJSON, 'utf8');
});
});
return manifestSender
.then(() => {
return new Promise((resolve, reject) => {
const sendBag = tChest.get(currentId.toString());
if (!sendBag) {
throw new Error('Redioactive: HTTP/S source: Could not find element to push.');
}
const req = protocol.request(Object.assign((options && options.requestOptions) || {}, {
hostname: url.hostname,
protocol: url.protocol,
port: url.port,
path: `${info.root}/${currentId}`,
method: 'POST',
headers: {
'Redioactive-Id': sendBag.id,
'Redioactive-NextId': sendBag.nextId,
'Redioactive-PrevId': sendBag.prevId
},
agent
}), (res) => {
// Received when all data is consumed
if (res.statusCode === 200 || res.statusCode === 201) {
setImmediate(sendBag.nextFn);
resolve();
return;
}
reject(new Error(`Redioactive: HTTP/S source: Received unexpected response of POST request for "${currentId}": ${res.statusCode}`));
});
req.on('error', reject);
let valueStr = '';
switch (info.body) {
case http_common_1.BodyType.primitive:
case http_common_1.BodyType.json:
valueStr = JSON.stringify(sendBag.value);
req.setHeader('Content-Type', 'application/json');
req.setHeader('Content-Length', `${Buffer.byteLength(valueStr, 'utf8')}`);
req.end(valueStr, 'utf8');
break;
case http_common_1.BodyType.blob:
req.setHeader('Content-Type', blobContentType);
req.setHeader('Content-Length', (sendBag.blob && sendBag.blob.length) || 0);
req.setHeader('Redioactive-Details', JSON.stringify(sendBag.value));
req.end(sendBag.blob || Buffer.alloc(0));
break;
}
});
})
.catch((err) => {
const sendBag = tChest.get(currentId.toString());
if (sendBag) {
setImmediate(() => {
sendBag.errorFn(err);
});
}
else {
throw new Error(`Redioactive: HTTP/S source: Unable to forward error for Id "${currentId}": ${err.message}`);
}
});
}
let ended = false;
const bufferSize = (options && options.bufferSizeMax) || 10;
let highWaterMark = 0;
let lowWaterMark = 0;
let lowestOfTheLow = 0;
let pushChain = Promise.resolve();
return async (t) => new Promise((resolve, reject) => {
if ((0, redio_1.isNil)(t) || (0, util_1.isError)(t)) {
return;
}
if (idCounter++ === 0 && !(0, redio_1.isEnd)(t)) {
// Do some first time out checks
checkObject(t);
initFromObject(t);
}
if (info.idType !== http_common_1.IdType.string && tChest.size > 1 && idCounter <= bufferSize) {
const keys = tChest.keys();
let prev = keys.next().value;
let sum = 0;
for (const key of keys) {
sum += +key - +prev;
prev = key;
}
fuzzyGap = sum / (tChest.size - 1);
}
const tr = t;
const currentId = info.idType === http_common_1.IdType.counter ? idCounter : tr[options.seqId];
if (idCounter === 1) {
lowWaterMark = currentId;
highWaterMark = currentId;
lowestOfTheLow = currentId;
}
while (pendings.length > 0) {
const resolvePending = pendings.pop();
resolvePending && setImmediate(resolvePending);
}
if (!(0, redio_1.isEnd)(t)) {
switch (info.delta) {
case http_common_1.DeltaType.one:
nextId = currentId + 1;
break;
case http_common_1.DeltaType.fixed:
nextId = currentId + options.delta;
break;
case http_common_1.DeltaType.variable:
nextId = currentId + tr[options.delta];
break;
case http_common_1.DeltaType.string:
nextId = tr[options.delta];
break;
}
}
else {
nextId = currentId;
ended = true;
}
const value = info.body === http_common_1.BodyType.primitive ? t : Object.assign({}, t);
if (typeof value === 'object') {
options && options.seqId && delete value[options.seqId];
options &&
options.extraStreamRoot &&
delete value[options.extraStreamRoot];
options && options.blob && delete value[options.blob];
options &&
typeof options.delta === 'string' &&
delete value[options.delta];
options &&
typeof options.manifest === 'string' &&
delete value[options.manifest];
}
const blob = (options &&
options.blob &&
Buffer.isBuffer(tr[options.blob]) &&
tr[options.blob]) ||
undefined;
tChest.set(currentId.toString(), (0, redio_1.literal)({
value,
blob,
counter: idCounter,
id: currentId,
nextId: nextId,
prevId: highWaterMark,
nextFn: resolve,
errorFn: reject
}));
highWaterMark = currentId;
if (tChest.size > bufferSize) {
const keys = tChest.keys();
const toRemove = tChest.size - bufferSize;
for (let x = 0; x < toRemove; x++) {
tChest.delete(keys.next().value);
}
lowWaterMark = keys.next().value;
}
if (tChest.size < bufferSize || // build the initial buffer
(options && options.backPressure === false)) {
const timeout = (options && options.cadence) || 0;
if (timeout) {
setTimeout(resolve, timeout);
}
else {
setImmediate(resolve);
}
}
if (isPush(info)) {
pushChain = pushChain.then(() => push(currentId));
}
});
}
exports.httpSource = httpSource;