frolyk
Version:
Stream processing library for Kafka in Node
225 lines • 11 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const highland_1 = __importDefault(require("highland"));
const long_1 = __importDefault(require("long"));
const offsets_1 = require("../offsets");
const processors_1 = require("../processors");
const createContext = function ({ assignment, processors, offsetReset, initialState }) {
return __awaiter(this, void 0, void 0, function* () {
initialState = Object.assign({ lowOffset: 0, messages: [] }, (initialState || {}));
if (!offsetReset)
offsetReset = offsets_1.LogicalOffset.Latest;
let producedOffset = initialState.lowOffset - 1;
let consumedOffset = initialState.lowOffset - 1;
let seekToOffset = -1;
let committedOffset = { offset: '-1', metadata: null };
const stream = highland_1.default();
const committedOffsets = [];
const injectedMessages = [];
const producedMessages = [];
const createMessage = (payload) => {
const value = !payload.value ? null :
Buffer.isBuffer(payload.value) ? payload.value :
Buffer.from(JSON.stringify(payload.value));
const key = !payload.key ? null :
Buffer.isBuffer(payload.key) ? payload.key :
Buffer.from(JSON.stringify(payload.key));
const offset = payload.offset && long_1.default.fromValue(payload.offset) || long_1.default.fromNumber(producedOffset + 1);
if (offset.lte(producedOffset)) {
throw new Error('Offset of injected message must be at or higher than the current highwatermark');
}
producedOffset = offset.toNumber();
return Object.assign(Object.assign({ partition: assignment.partition, timestamp: `${Date.now()}`, topic: assignment.topic }, payload), { value,
key,
offset });
};
const injectMessage = (message) => {
injectedMessages.push(message);
return writeMessageToStream(message);
};
const injectError = (error) => {
return stream.write(error);
};
const writeMessageToStream = (message) => {
stream.write(message);
return message;
};
const highOffset = () => {
const lastMessage = injectedMessages[injectedMessages.length - 1];
return lastMessage ? lastMessage.offset.add(1) : long_1.default.fromNumber(initialState.lowOffset);
};
const lowOffset = () => {
const firstMessage = injectedMessages[0];
return firstMessage ? firstMessage.offset : long_1.default.fromNumber(initialState.lowOffset);
};
const assignmentContext = {
caughtUp(offset) {
return __awaiter(this, void 0, void 0, function* () {
// TODO: deal with logical offsets
return long_1.default.fromValue(offset).add(1) >= highOffset();
});
},
commitOffset(newOffset, metadata = null) {
return __awaiter(this, void 0, void 0, function* () {
newOffset = long_1.default.fromValue(newOffset);
if (newOffset.lte(-1)) {
throw new Error('Offset must be a valid absolute offset to commit it');
}
const offset = { offset: newOffset.toString(), metadata };
committedOffset = offset;
committedOffsets.push(offset);
return Promise.resolve();
});
},
committed() {
return __awaiter(this, void 0, void 0, function* () {
return Promise.resolve(Object.assign({}, committedOffset));
});
},
isEmpty() {
return __awaiter(this, void 0, void 0, function* () {
return Promise.resolve(highOffset().subtract(lowOffset()).lte(0));
});
},
/* istanbul ignore next */
log(tags, payload) { },
seek(soughtOffset) {
return __awaiter(this, void 0, void 0, function* () {
// resolve the requested offset to a message that has been injected
const absoluteOffset = offsets_1.isEarliest(soughtOffset) ? lowOffset() :
offsets_1.isLatest(soughtOffset) ? highOffset() :
long_1.default.fromValue(soughtOffset);
const outOfRange = absoluteOffset < lowOffset() || absoluteOffset >= highOffset();
let closestIndex = injectedMessages.findIndex(({ offset }) => offset.gte(absoluteOffset));
/* istanbul ignore else */
if (!outOfRange && closestIndex > -1) {
let nextMessage = injectedMessages[closestIndex];
// update the offset we're currently consuming or reset to start or end
seekToOffset = long_1.default.fromValue(nextMessage.offset).toNumber();
}
else if (offsets_1.isLatest(soughtOffset) || offsets_1.isLatest(offsetReset)) {
seekToOffset = highOffset().toNumber();
}
else if (offsets_1.isEarliest(soughtOffset) || offsets_1.isEarliest(offsetReset)) {
seekToOffset = lowOffset().toNumber();
}
// replay any messages if necessary
if (consumedOffset >= seekToOffset) {
setTimeout(() => {
injectedMessages.slice(closestIndex).forEach(writeMessageToStream);
});
}
});
},
send(messages) {
return __awaiter(this, void 0, void 0, function* () {
if (!Array.isArray(messages))
messages = [messages];
return messages.map(createMessage).map((message) => {
producedMessages.push(message);
if (message.topic === assignment.topic && message.partition === assignment.partition) {
injectMessage(message);
}
return {
topicName: message.topic,
partition: message.partition,
errorCode: 0,
offset: message.offset.toString(),
timestamp: message.timestamp
};
});
});
},
watermarks() {
return __awaiter(this, void 0, void 0, function* () {
return {
highOffset: highOffset().toString(),
lowOffset: lowOffset().toString()
};
});
},
topic: assignment.topic,
partition: assignment.partition,
// TODO: decide if we want to support prefixes
// prefix: assignment.prefix
group: assignment.group
};
const initialMessages = initialState.messages.map(createMessage).map(injectMessage);
const controlledStream = stream.map((messageOrError) => {
if (messageOrError instanceof Error) {
throw messageOrError;
}
return messageOrError;
}).filter(({ offset }) => {
if (seekToOffset > -1) {
return long_1.default.fromValue(offset).equals(seekToOffset);
}
else {
return true;
}
}).map((message) => {
consumedOffset = long_1.default.fromValue(message.offset).toNumber();
seekToOffset = -1;
return Object.assign(Object.assign({}, message), { offset: message.offset.toString(), highWaterOffset: highOffset().toString() });
});
const [processingPipeline, offsetsStream] = yield processors_1.createPipeline(assignmentContext, processors);
const processedStream = controlledStream.through(processingPipeline);
const processingResults = [];
const processing = processedStream.tap((result) => {
processingResults.push(result);
}).last().toPromise(Promise);
const processedOffsets = [];
offsetsStream.each((offset) => {
processedOffsets.push(offset);
});
function inject(payload) {
if (payload instanceof Error) {
injectError(payload);
return payload;
}
else {
let internal = createMessage(payload);
injectMessage(internal);
return Object.assign(Object.assign({}, internal), { offset: internal.offset.toString(), highWaterOffset: highOffset().toString() });
}
}
return {
inject,
committedOffsets,
caughtUp() {
return __awaiter(this, void 0, void 0, function* () {
yield offsetsStream.observe()
.map((offset) => __awaiter(this, void 0, void 0, function* () { return assignmentContext.caughtUp(`${offset}`); }))
.flatMap((awaiting) => highland_1.default(awaiting))
.find((isCaughtUp) => isCaughtUp)
.toPromise(Promise);
});
},
end() {
return __awaiter(this, void 0, void 0, function* () {
stream.end();
yield processing;
});
},
initialMessages,
processing,
processingResults,
processedOffsets,
producedMessages
};
});
};
exports.default = createContext;
//# sourceMappingURL=local.js.map