@fluent-org/logger
Version:
A node fluent protocol compatible logger
567 lines • 20.7 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.FluentClient = void 0;
const error_1 = require("./error");
const event_time_1 = require("./event_time");
const auth_1 = require("./auth");
const socket_1 = require("./socket");
const modes_1 = require("./modes");
const crypto = require("crypto");
const event_retrier_1 = require("./event_retrier");
const util_1 = require("./util");
const defaultLimit = (limit) => {
return {
size: +Infinity,
length: +Infinity,
...(limit || {}),
};
};
/**
* A Fluent Client. Connects to a FluentD server using the [Forward protocol](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1).
*/
class FluentClient {
/**
* Creates a new FluentClient
*
* @param tag_prefix A prefix to prefix to all tags. For example, passing the prefix "foo" will cause `emit("bar", data)` to emit with `foo.bar`.
* @param options The client options
*/
constructor(tag_prefix = null, options = {}) {
var _a;
this.ackQueue = new Map();
this.emitQueue = new Set();
this.notFlushableLimitTimeoutId = null;
this.nextFlushTimeoutId = null;
this.flushing = false;
this.willFlushNextTick = null;
options = options || {};
this.eventMode = options.eventMode || "PackedForward";
if (this.eventMode === "Message") {
this.sendQueue = new modes_1.MessageQueue();
}
else if (this.eventMode === "Forward") {
this.sendQueue = new modes_1.ForwardQueue();
}
else if (this.eventMode === "PackedForward") {
this.sendQueue = new modes_1.PackedForwardQueue();
}
else if (this.eventMode === "CompressedPackedForward") {
this.sendQueue = new modes_1.CompressedPackedForwardQueue();
}
else {
throw new error_1.ConfigError("Unknown event mode: " + this.eventMode);
}
if (options.eventRetry) {
this.retrier = new event_retrier_1.EventRetrier(options.eventRetry);
}
else {
this.retrier = null;
}
this.tag_prefix = tag_prefix;
this.ackEnabled = !!options.ack;
this.ackOptions = {
ackTimeout: 500,
...(options.ack || {}),
};
this.milliseconds = !!options.milliseconds;
this.flushInterval = options.flushInterval || 0;
this.sendQueueSyncFlushLimit = options.sendQueueSyncFlushLimit
? defaultLimit(options.sendQueueSyncFlushLimit)
: null;
this.sendQueueIntervalFlushLimit = options.sendQueueIntervalFlushLimit
? defaultLimit(options.sendQueueIntervalFlushLimit)
: null;
this.sendQueueMaxLimit = options.sendQueueMaxLimit
? defaultLimit(options.sendQueueMaxLimit)
: null;
this.sendQueueNotFlushableLimit = options.sendQueueNotFlushableLimit
? defaultLimit(options.sendQueueNotFlushableLimit)
: null;
this.sendQueueNotFlushableLimitDelay =
options.sendQueueNotFlushableLimitDelay || 0;
this.disconnectOptions = {
waitForPending: false,
waitForPendingDelay: 0,
socketDisconnectDelay: 0,
...(options.disconnect || {}),
};
this.socket = this.createSocket(options.security, options.socket);
this.socket.on(socket_1.FluentSocketEvent.WRITABLE, () => this.handleWritable());
this.socket.on(socket_1.FluentSocketEvent.ACK, (chunkId) => this.handleAck(chunkId));
// Only connect if we're able to reconnect and user has not disabled auto connect
// Otherwise we expect an explicit connect() which will handle connection errors
const autoConnect = !((_a = options.socket) === null || _a === void 0 ? void 0 : _a.disableReconnect) && !options.disableAutoconnect;
if (autoConnect) {
// Catch errors and noop them, so the constructor doesn't throw unhandled promises
// They can be handled by the socket "error" event handler anyway
this.connect().catch(() => { });
}
}
/**
* Attaches an event listener to the underlying socket
*
* See FluentSocketEvent for more info
*/
socketOn(event, listener // eslint-disable-line @typescript-eslint/no-explicit-any
) {
this.socket.on(event, listener);
}
/**
* Constructs a new socket
*
* @param security The security options, if any
* @param options The socket options, if any
* @returns A new FluentSocket
*/
createSocket(security, options) {
if (security) {
return new auth_1.FluentAuthSocket(security, options);
}
else {
return new socket_1.FluentSocket(options);
}
}
emit(a, b, c) {
let label, data, timestamp;
if (typeof a === "string") {
label = a;
if (typeof b === "object") {
data = b;
}
else {
return Promise.reject(new error_1.DataTypeError("data must be an object"));
}
if (!c ||
typeof c === "number" ||
c instanceof Date ||
c instanceof event_time_1.default) {
timestamp = c || null;
}
else {
return Promise.reject(new error_1.DataTypeError("timestamp was not a valid timestamp"));
}
}
else {
label = null;
if (typeof a === "object") {
data = a;
}
else {
return Promise.reject(new error_1.DataTypeError("data must be an object"));
}
if (!b ||
typeof b === "number" ||
b instanceof Date ||
b instanceof event_time_1.default) {
timestamp = b || null;
}
else {
return Promise.reject(new error_1.DataTypeError("timestamp was not a valid timestamp"));
}
}
const tag = this.makeTag(label);
if (tag === null || tag.length === 0) {
return Promise.reject(new error_1.MissingTagError("tag is missing"));
}
let millisOrEventTime;
if (timestamp === null || timestamp instanceof Date) {
millisOrEventTime = timestamp ? timestamp.getTime() : Date.now();
}
else {
millisOrEventTime = timestamp;
}
let time;
if (typeof millisOrEventTime === "number") {
// Convert timestamp to EventTime or number in second resolution
time = this.milliseconds
? event_time_1.default.fromTimestamp(millisOrEventTime)
: Math.floor(millisOrEventTime / 1000);
}
else {
time = millisOrEventTime;
}
let emitPromise;
if (this.retrier !== null) {
emitPromise = this.retrier.retryPromise(() => this.pushEvent(tag, time, data));
}
else {
emitPromise = this.pushEvent(tag, time, data);
}
if (!this.emitQueue.has(emitPromise)) {
this.emitQueue.add(emitPromise);
emitPromise
.finally(() => this.emitQueue.delete(emitPromise))
.catch(() => { });
}
return emitPromise;
}
/**
* Pushes an event onto the sendQueue
*
* Also drops items from the queue if it is too large (size/length)
*
* @param tag The event tag
* @param time The event timestamp
* @param data The event data
* @returns The promise from the sendQueue
*/
pushEvent(tag, time, data) {
const promise = this.sendQueue.push(tag, time, data);
if (this.sendQueueMaxLimit) {
this.dropLimit(this.sendQueueMaxLimit);
}
this.maybeFlush();
return promise;
}
/**
* Called once the underlying socket is writable
*
* Should attempt a flush
*/
handleWritable() {
this.maybeFlush();
}
/**
* Connects the client. Can happen automatically during construction, but can be called after a `disconnect()` to resume the client.
*/
async connect() {
await this.socket.connect();
}
/**
* Closes the socket, and clears both the ackQueue and the sendQueue, rejecting all pending events.
*
* For use during shutdown events, where we don't plan on reconnecting
*/
async shutdown() {
try {
await this.disconnect();
}
finally {
this.sendQueue.clear();
}
}
/**
* Closes the socket and clears the ackQueue. Keeps pending events, which can be sent via a later .connect()
*/
async disconnect() {
try {
// Flush before awaiting
await this.flush();
if (this.disconnectOptions.waitForPending) {
const flushPromise = this.waitForPending();
if (this.disconnectOptions.waitForPendingDelay > 0) {
await util_1.awaitAtMost(flushPromise, this.disconnectOptions.waitForPendingDelay);
}
else {
await flushPromise;
}
}
}
finally {
if (this.disconnectOptions.socketDisconnectDelay > 0) {
await util_1.awaitTimeout(this.disconnectOptions.socketDisconnectDelay);
}
try {
await this.socket.disconnect();
}
finally {
// We want to client to be in a state where nothing is pending that isn't in the sendQueue, now that the socket is unflushable.
// This means nothing is pending acknowledgemnets, and nothing is pending to retry.
// As a result, we can drop all the pending events, or send them once we're connected again
// Drop the acks first, as they can queue up retries which we need to short circuit
await this.clearAcks();
if (this.retrier) {
// Short circuit all retries, so they requeue immediately
await this.retrier.shortCircuit();
}
}
}
}
/**
* Creates a tag from the passed label and the constructor `tagPrefix`.
*
* @param label The label to create a tag from
* @returns The constructed tag, or `null`.
*/
makeTag(label) {
let tag = null;
if (this.tag_prefix && label) {
tag = `${this.tag_prefix}.${label}`;
}
else if (this.tag_prefix) {
tag = this.tag_prefix;
}
else if (label) {
tag = label;
}
return tag;
}
/**
* Flushes to the socket synchronously
*
* Prefer calling `.flush` which will flush on the next tick, allowing events from this tick to queue up.
*
* @returns true if there are more events in the queue to flush, false otherwise
*/
syncFlush() {
if (this.sendQueue.queueLength === 0) {
return false;
}
if (this.flushing) {
return this.sendQueue.queueLength > 0;
}
if (!this.socket.writable()) {
return this.sendQueue.queueLength > 0;
}
this.flushing = true;
if (this.nextFlushTimeoutId !== null) {
clearTimeout(this.nextFlushTimeoutId);
this.nextFlushTimeoutId = null;
}
let availableEvents = true;
while (availableEvents && this.socket.writable()) {
availableEvents = this.sendNext();
}
this.flushing = false;
return availableEvents;
}
/**
* Flushes the event queue. Queues up the flushes for the next tick, preventing multiple flushes at the same time.
*
* @returns A promise, which resolves with a boolean indicating if there are more events to flush.
*/
flush() {
// Prevent duplicate flushes next tick
if (this.willFlushNextTick === null) {
this.willFlushNextTick = new Promise(resolve => process.nextTick(() => {
this.willFlushNextTick = null;
resolve(this.syncFlush());
}));
}
return this.willFlushNextTick;
}
/**
* Potentially triggers a flush
*
* If we're flushing on an interval, check if the queue (size/length) limits have been reached, and otherwise schedule a new flush
*
* If not, just flush
* @returns
*/
maybeFlush() {
// nothing to flush
if (this.sendQueue.queueLength === 0) {
return;
}
// can't flush
if (!this.socket.writable()) {
if (this.sendQueueNotFlushableLimit &&
this.notFlushableLimitTimeoutId === null) {
if (this.sendQueueNotFlushableLimitDelay > 0) {
this.notFlushableLimitTimeoutId = setTimeout(() => {
this.notFlushableLimitTimeoutId = null;
if (this.sendQueueNotFlushableLimit) {
this.dropLimit(this.sendQueueNotFlushableLimit);
}
}, this.sendQueueNotFlushableLimitDelay);
}
else {
this.dropLimit(this.sendQueueNotFlushableLimit);
}
}
return;
}
else {
// When writable, we want to clear the not flushable limit
if (this.notFlushableLimitTimeoutId !== null) {
clearTimeout(this.notFlushableLimitTimeoutId);
this.notFlushableLimitTimeoutId = null;
}
}
// If we've hit a blocking limit
if (this.sendQueueSyncFlushLimit &&
this.shouldLimit(this.sendQueueSyncFlushLimit)) {
this.syncFlush();
}
else if (this.flushInterval > 0) {
if (this.sendQueueIntervalFlushLimit &&
this.shouldLimit(this.sendQueueIntervalFlushLimit)) {
this.flush();
}
else if (this.nextFlushTimeoutId === null) {
// Otherwise, schedule the next flush interval
this.nextFlushTimeoutId = setTimeout(() => {
this.nextFlushTimeoutId = null;
this.flush();
}, this.flushInterval);
}
}
else {
// If we're not flushing on an interval, then try to flush on every emission
this.flush();
}
}
/**
* Drops events until the send queue is below the specified limits
*
* @param limit The limit to enforce
*/
dropLimit(limit) {
if (this.sendQueue.queueSize !== -1 &&
this.sendQueue.queueSize > limit.size) {
while (this.sendQueue.queueSize > limit.size) {
this.sendQueue.dropEntry();
}
}
if (this.sendQueue.queueLength !== -1 &&
this.sendQueue.queueLength > limit.length) {
while (this.sendQueue.queueLength > limit.length) {
this.sendQueue.dropEntry();
}
}
}
/**
* Checks if the sendQueue hits this limit
* @param limit the limit to check
*/
shouldLimit(limit) {
if (this.sendQueue.queueSize !== -1 &&
this.sendQueue.queueSize >= limit.size) {
// If the queue has hit the memory flush limit
return true;
}
else if (this.sendQueue.queueLength !== -1 &&
this.sendQueue.queueLength >= limit.length) {
// If the queue has hit the length flush limit
return true;
}
return false;
}
/**
* Send the front item of the queue to the socket
* @returns True if there was something to send
*/
sendNext() {
let chunk;
if (this.ackEnabled) {
chunk = crypto.randomBytes(16).toString("base64");
}
const nextPacket = this.sendQueue.nextPacket(chunk);
if (nextPacket === null) {
return false;
}
// Set up the ack before the write, in case of an immediate response
if (this.ackEnabled && chunk) {
this.ackQueue.set(chunk, {
timeoutId: this.setupAckTimeout(chunk, nextPacket.deferred, this.ackOptions.ackTimeout),
deferred: nextPacket.deferred,
});
}
// Not awaiting because we don't need to wait for this chunk to be flushed to the kernel buffer
const writePromise = this.socket.write(nextPacket.packet);
// However, we do still want to catch errors
writePromise.catch(err => {
var _a;
// If the chunk was put in the ack queue, and is still there, the deferred hasn't been resolved
if (chunk && this.ackQueue.has(chunk)) {
const ackTimeoutId = (_a = this.ackQueue.get(chunk)) === null || _a === void 0 ? void 0 : _a.timeoutId;
this.ackQueue.delete(chunk);
if (ackTimeoutId) {
clearTimeout(ackTimeoutId);
}
}
nextPacket.deferred.reject(err);
});
if (!chunk) {
// Wait for the promise to resolve before resolving the deferred
writePromise.then(() => nextPacket.deferred.resolve(), () => { });
}
return true;
}
/**
* Creates an event for how long to wait for the ack
*
* @param chunkId The chunk ID we're waiting to ack
* @param deferred The deferred to reject on timeout
* @param ackTimeout The timeout length
* @returns
*/
setupAckTimeout(chunkId, deferred, ackTimeout) {
return setTimeout(() => {
// If the chunk isn't in the queue, then we must have removed it somewhere, assume that it didn't time out
if (this.ackQueue.has(chunkId)) {
deferred.reject(new error_1.AckTimeoutError("ack response timeout"));
this.ackQueue.delete(chunkId);
}
}, ackTimeout);
}
/**
* Called on an acknowledgement from the socket
*
* @param chunkId The chunk ID the socket has acknowledged
* @returns
*/
handleAck(chunkId) {
if (!this.ackQueue.has(chunkId)) {
// Timed out or socket shut down fully before this event could be processed
return;
}
const ackData = this.ackQueue.get(chunkId);
this.ackQueue.delete(chunkId);
if (ackData) {
clearTimeout(ackData.timeoutId);
ackData.deferred.resolve();
}
}
/**
* Fails all acknowledgements
* Called on shutdown
*
* @returns a Promise which resolves once all the handlers depending on the ack result have resolved
*/
async clearAcks() {
for (const data of this.ackQueue.values()) {
clearTimeout(data.timeoutId);
data.deferred.reject(new error_1.AckShutdownError("ack queue emptied"));
}
this.ackQueue = new Map();
// We want this to resolve on the next tick, once handlers depending on the ack result have fully resolved
// i.e we have emptied PromiseJobs
return util_1.awaitNextTick();
}
/**
* Returns the number of queued events that haven't been sent yet
*
* Useful to react if we're queuing up too many events within a single tick
*/
get sendQueueLength() {
return this.sendQueue.queueLength;
}
/**
* Returns whether or not the socket is writable
*
* Useful to react if we're disconnected for any reason
*/
get writable() {
return this.socket.writable();
}
/**
* Returns the number of events that have been queued, but haven't resolved yet
*
* This includes acknowledgements and retries if enabled.
*/
get queueLength() {
return this.emitQueue.size;
}
/**
* Waits for all currently pending events to successfully resolve or reject
*
* @returns A Promise which resolves once all the pending events have successfully been emitted
*/
async waitForPending() {
// Clone the emitQueue, to ignore emit calls made while waiting
await Promise.allSettled(Array.from(this.emitQueue));
}
}
exports.FluentClient = FluentClient;
//# sourceMappingURL=client.js.map