@connektra/logger-service
Version:
A logger service that sends logs to Google Cloud Tasks
277 lines (276 loc) • 8.85 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LogType = exports.FallbackStrategy = void 0;
const tasks_1 = require("@google-cloud/tasks");
const moment_1 = __importDefault(require("moment"));
const pino_1 = __importDefault(require("pino"));
const exponential_backoff_1 = require("exponential-backoff");
var FallbackStrategy;
(function (FallbackStrategy) {
FallbackStrategy["LOCAL_ONLY"] = "local_only";
FallbackStrategy["BUFFER_AND_FLUSH"] = "buffer_and_flush";
FallbackStrategy["DISCARD"] = "discard";
})(FallbackStrategy || (exports.FallbackStrategy = FallbackStrategy = {}));
var LogType;
(function (LogType) {
LogType["INFO"] = "info";
LogType["ERROR"] = "error";
LogType["EXECUTION"] = "execution";
})(LogType || (exports.LogType = LogType = {}));
const localLogger = (0, pino_1.default)();
let loggerConfig = {
projectId: process.env.GOOGLE_CLOUD_PROJECT || "",
location: process.env.LOGGER_QUEUE_LOCATION || "",
queueName: process.env.LOGGER_QUEUE_NAME || "",
serviceUrl: process.env.LOGGER_SERVICE_URL || "",
retryConfig: {
maxRetries: 3,
initialDelayMs: 100,
maxDelayMs: 5000,
backoffFactor: 2,
enableJitter: true,
},
fallbackStrategy: FallbackStrategy.BUFFER_AND_FLUSH,
};
let client;
let parent;
let isConfigured = false;
let logBuffer = [];
const MAX_BUFFER_SIZE = 1000;
/**
* Initialize the logger with configuration
*/
const initialize = () => {
if (!loggerConfig.projectId ||
!loggerConfig.location ||
!loggerConfig.queueName ||
!loggerConfig.serviceUrl) {
localLogger.warn("Logger not properly configured. Using local logging only.");
return false;
}
try {
client = new tasks_1.CloudTasksClient();
parent = client.queuePath(loggerConfig.projectId, loggerConfig.location, loggerConfig.queueName);
isConfigured = true;
return true;
}
catch (error) {
localLogger.error("Error initializing logger:", error);
return false;
}
};
/**
* Attempt to create a task with retry logic
*/
const createTaskWithRetry = async (task) => {
if (!isConfigured)
return;
const { maxRetries, initialDelayMs, maxDelayMs, backoffFactor, enableJitter, } = loggerConfig.retryConfig || {};
try {
await (0, exponential_backoff_1.backOff)(async () => {
try {
await client.createTask({
parent,
task,
});
return true;
}
catch (error) {
const err = error;
if (err.message?.includes("UNAVAILABLE") ||
err.message?.includes("DEADLINE_EXCEEDED") ||
err.message?.includes("INTERNAL") ||
err.message?.includes("RESOURCE_EXHAUSTED")) {
throw err;
}
localLogger.error("Non-retryable error creating log task:", err);
return false;
}
}, {
numOfAttempts: maxRetries || 3,
startingDelay: initialDelayMs || 100,
timeMultiple: backoffFactor || 2,
maxDelay: maxDelayMs || 5000,
jitter: (enableJitter !== false),
});
}
catch (error) {
handleFailedLog(task);
}
};
/**
* Handle failed log attempts based on fallback strategy
*/
const handleFailedLog = (task) => {
const strategy = loggerConfig.fallbackStrategy || FallbackStrategy.LOCAL_ONLY;
const payload = JSON.parse(Buffer.from(task.httpRequest.body, "base64").toString());
switch (strategy) {
case FallbackStrategy.BUFFER_AND_FLUSH:
if (logBuffer.length < MAX_BUFFER_SIZE) {
logBuffer.push(payload);
localLogger.warn(`Added failed log to buffer. Buffer size: ${logBuffer.length}`);
}
else {
localLogger.warn("Log buffer full, discarding log");
}
break;
case FallbackStrategy.LOCAL_ONLY:
localLogger.warn("Failed to send log to Cloud Tasks after retries, logging locally only");
break;
case FallbackStrategy.DISCARD:
localLogger.warn("Failed to send log to Cloud Tasks after retries, discarding log");
break;
default:
localLogger.warn("Unknown fallback strategy, defaulting to local logging only");
}
};
/**
* Logs a message to Google Cloud Tasks queue with retry and failsafe
* @param payload The log payload
*/
const createLog = (payload) => {
switch (payload.type) {
case LogType.ERROR:
localLogger.error(payload);
break;
case LogType.INFO:
localLogger.info(payload);
break;
case LogType.EXECUTION:
localLogger.info({ execution: true, ...payload });
break;
}
if (!isConfigured)
return;
if (!payload.timestamp) {
payload.timestamp = (0, moment_1.default)().toISOString();
}
const task = {
httpRequest: {
httpMethod: "POST",
url: loggerConfig.serviceUrl,
headers: {
"Content-Type": "application/json",
},
body: Buffer.from(JSON.stringify(payload)).toString("base64"),
},
};
createTaskWithRetry(task).catch((err) => {
localLogger.error("Error in createTaskWithRetry:", err);
});
};
/**
* Attempt to flush buffered logs
*/
const flushBuffer = async () => {
if (logBuffer.length === 0)
return;
localLogger.info(`Flushing log buffer with ${logBuffer.length} items`);
const tempBuffer = [...logBuffer];
logBuffer = [];
const promises = tempBuffer.map((payload) => {
const task = {
httpRequest: {
httpMethod: "POST",
url: loggerConfig.serviceUrl,
headers: {
"Content-Type": "application/json",
},
body: Buffer.from(JSON.stringify(payload)).toString("base64"),
},
};
return createTaskWithRetry(task);
});
try {
await Promise.allSettled(promises);
localLogger.info(`Buffer flush completed. ${tempBuffer.length} logs processed.`);
}
catch (error) {
localLogger.error("Error flushing buffer:", error);
}
};
/**
* Configure the logger
* @param config Configuration for the logger
*/
const configure = (config) => {
if (config) {
loggerConfig = {
...loggerConfig,
...config,
retryConfig: {
...loggerConfig.retryConfig,
...(config.retryConfig || {}),
maxRetries: config.retryConfig?.maxRetries ??
loggerConfig.retryConfig?.maxRetries ??
3,
initialDelayMs: config.retryConfig?.initialDelayMs ??
loggerConfig.retryConfig?.initialDelayMs ??
100,
maxDelayMs: config.retryConfig?.maxDelayMs ??
loggerConfig.retryConfig?.maxDelayMs ??
5000,
backoffFactor: config.retryConfig?.backoffFactor ??
loggerConfig.retryConfig?.backoffFactor ??
2,
enableJitter: config.retryConfig?.enableJitter ??
loggerConfig.retryConfig?.enableJitter ??
true,
},
};
}
initialize();
};
/**
* Method for logging info without blocking
*/
const info = (payload) => {
createLog({
type: LogType.INFO,
log: payload.log,
message: payload.message,
source: payload.source,
});
};
/**
* Method for logging error without blocking
*/
const error = (payload) => {
createLog({
type: LogType.ERROR,
log: payload.log,
message: payload.message,
source: payload.source,
});
};
/**
* Method for logging execution data without blocking
*/
const executionLog = (payload) => {
createLog({
type: LogType.EXECUTION,
log: payload.log,
message: payload.message,
source: payload.source,
});
};
if (typeof process !== "undefined") {
const FLUSH_INTERVAL_MS = 60000;
setInterval(() => {
if (logBuffer.length > 0) {
flushBuffer().catch((err) => {
localLogger.error("Error in auto-flush:", err);
});
}
}, FLUSH_INTERVAL_MS);
}
initialize();
const log = {
info,
error,
executionLog,
};
exports.default = log;