@connektra/logger-service/index.ts

A logger service that sends logs to Google Cloud Tasks

import { CloudTasksClient } from "@google-cloud/tasks";
import moment from "moment";
import pino from "pino";
import { backOff } from "exponential-backoff";
import { JitterType } from "exponential-backoff/dist/options";

export interface LoggerConfig {
  projectId: string;
  location: string;
  queueName: string;
  serviceAccountEmail?: string;
  serviceUrl: string;
  retryConfig?: RetryConfig;
  fallbackStrategy?: FallbackStrategy;
}

export interface RetryConfig {
  maxRetries: number;
  initialDelayMs: number;
  maxDelayMs: number;
  backoffFactor: number;
  enableJitter?: boolean;
}

export enum FallbackStrategy {
  LOCAL_ONLY = "local_only",
  BUFFER_AND_FLUSH = "buffer_and_flush",
  DISCARD = "discard",
}

export enum LogType {
  INFO = "info",
  ERROR = "error",
  EXECUTION = "execution",
}

export interface LogPayload {
  type: LogType;
  log: any;
  message?: string;
  timestamp?: string;
  source?: string;
}

export interface LogBasePayload {
  log: any;
  message?: string;
  source?: string;
}

interface Logger {
  info: (payload: LogBasePayload) => void;
  error: (payload: LogBasePayload) => void;
  executionLog: (payload: LogBasePayload) => void;
}

const localLogger = pino();

let loggerConfig: LoggerConfig = {
  projectId: process.env.GOOGLE_CLOUD_PROJECT || "",
  location: process.env.LOGGER_QUEUE_LOCATION || "",
  queueName: process.env.LOGGER_QUEUE_NAME || "",
  serviceUrl: process.env.LOGGER_SERVICE_URL || "",
  retryConfig: {
    maxRetries: 3,
    initialDelayMs: 100,
    maxDelayMs: 5000,
    backoffFactor: 2,
    enableJitter: true,
  },
  fallbackStrategy: FallbackStrategy.BUFFER_AND_FLUSH,
};

let client: CloudTasksClient;
let parent: string;
let isConfigured = false;
let logBuffer: LogPayload[] = [];
const MAX_BUFFER_SIZE = 1000;

/**
 * Initialize the logger with configuration
 */
const initialize = (): boolean => {
  if (
    !loggerConfig.projectId ||
    !loggerConfig.location ||
    !loggerConfig.queueName ||
    !loggerConfig.serviceUrl
  ) {
    localLogger.warn(
      "Logger not properly configured. Using local logging only."
    );
    return false;
  }

  try {
    client = new CloudTasksClient();
    parent = client.queuePath(
      loggerConfig.projectId,
      loggerConfig.location,
      loggerConfig.queueName
    );
    isConfigured = true;
    return true;
  } catch (error) {
    localLogger.error("Error initializing logger:", error);
    return false;
  }
};

/**
 * Attempt to create a task with retry logic
 */
const createTaskWithRetry = async (task: any): Promise<void> => {
  if (!isConfigured) return;

  const {
    maxRetries,
    initialDelayMs,
    maxDelayMs,
    backoffFactor,
    enableJitter,
  } = loggerConfig.retryConfig || {};

  try {
    await backOff(
      async () => {
        try {
          await client.createTask({
            parent,
            task,
          });
          return true;
        } catch (error) {
          const err = error as Error;

          if (
            err.message?.includes("UNAVAILABLE") ||
            err.message?.includes("DEADLINE_EXCEEDED") ||
            err.message?.includes("INTERNAL") ||
            err.message?.includes("RESOURCE_EXHAUSTED")
          ) {
            throw err;
          }

          localLogger.error("Non-retryable error creating log task:", err);
          return false;
        }
      },
      {
        numOfAttempts: maxRetries || 3,
        startingDelay: initialDelayMs || 100,
        timeMultiple: backoffFactor || 2,
        maxDelay: maxDelayMs || 5000,
        jitter: (enableJitter !== false) as unknown as JitterType,
      }
    );
  } catch (error) {
    handleFailedLog(task);
  }
};

/**
 * Handle failed log attempts based on fallback strategy
 */
const handleFailedLog = (task: any): void => {
  const strategy = loggerConfig.fallbackStrategy || FallbackStrategy.LOCAL_ONLY;
  const payload = JSON.parse(
    Buffer.from(task.httpRequest.body, "base64").toString()
  );

  switch (strategy) {
    case FallbackStrategy.BUFFER_AND_FLUSH:
      if (logBuffer.length < MAX_BUFFER_SIZE) {
        logBuffer.push(payload);
        localLogger.warn(
          `Added failed log to buffer. Buffer size: ${logBuffer.length}`
        );
      } else {
        localLogger.warn("Log buffer full, discarding log");
      }
      break;
    case FallbackStrategy.LOCAL_ONLY:
      localLogger.warn(
        "Failed to send log to Cloud Tasks after retries, logging locally only"
      );
      break;
    case FallbackStrategy.DISCARD:
      localLogger.warn(
        "Failed to send log to Cloud Tasks after retries, discarding log"
      );
      break;
    default:
      localLogger.warn(
        "Unknown fallback strategy, defaulting to local logging only"
      );
  }
};

/**
 * Logs a message to Google Cloud Tasks queue with retry and failsafe
 * @param payload The log payload
 */
const createLog = (payload: LogPayload): void => {
  switch (payload.type) {
    case LogType.ERROR:
      localLogger.error(payload);
      break;
    case LogType.INFO:
      localLogger.info(payload);
      break;
    case LogType.EXECUTION:
      localLogger.info({ execution: true, ...payload });
      break;
  }

  if (!isConfigured) return;

  if (!payload.timestamp) {
    payload.timestamp = moment().toISOString();
  }

  const task = {
    httpRequest: {
      httpMethod: "POST" as "POST",
      url: loggerConfig.serviceUrl,
      headers: {
        "Content-Type": "application/json",
      },
      body: Buffer.from(JSON.stringify(payload)).toString("base64"),
    },
  };

  createTaskWithRetry(task).catch((err) => {
    localLogger.error("Error in createTaskWithRetry:", err);
  });
};

/**
 * Attempt to flush buffered logs
 */
const flushBuffer = async (): Promise<void> => {
  if (logBuffer.length === 0) return;

  localLogger.info(`Flushing log buffer with ${logBuffer.length} items`);

  const tempBuffer = [...logBuffer];
  logBuffer = [];

  const promises = tempBuffer.map((payload) => {
    const task = {
      httpRequest: {
        httpMethod: "POST" as "POST",
        url: loggerConfig.serviceUrl,
        headers: {
          "Content-Type": "application/json",
        },
        body: Buffer.from(JSON.stringify(payload)).toString("base64"),
      },
    };

    return createTaskWithRetry(task);
  });

  try {
    await Promise.allSettled(promises);
    localLogger.info(
      `Buffer flush completed. ${tempBuffer.length} logs processed.`
    );
  } catch (error) {
    localLogger.error("Error flushing buffer:", error);
  }
};

/**
 * Configure the logger
 * @param config Configuration for the logger
 */
const configure = (config?: Partial<LoggerConfig>): void => {
  if (config) {
    loggerConfig = {
      ...loggerConfig,
      ...config,

      retryConfig: {
        ...loggerConfig.retryConfig,
        ...(config.retryConfig || {}),
        maxRetries:
          config.retryConfig?.maxRetries ??
          loggerConfig.retryConfig?.maxRetries ??
          3,
        initialDelayMs:
          config.retryConfig?.initialDelayMs ??
          loggerConfig.retryConfig?.initialDelayMs ??
          100,
        maxDelayMs:
          config.retryConfig?.maxDelayMs ??
          loggerConfig.retryConfig?.maxDelayMs ??
          5000,
        backoffFactor:
          config.retryConfig?.backoffFactor ??
          loggerConfig.retryConfig?.backoffFactor ??
          2,
        enableJitter:
          config.retryConfig?.enableJitter ??
          loggerConfig.retryConfig?.enableJitter ??
          true,
      },
    };
  }
  initialize();
};

/**
 * Method for logging info without blocking
 */
const info = (payload: LogBasePayload): void => {
  createLog({
    type: LogType.INFO,
    log: payload.log,
    message: payload.message,
    source: payload.source,
  });
};

/**
 * Method for logging error without blocking
 */
const error = (payload: LogBasePayload): void => {
  createLog({
    type: LogType.ERROR,
    log: payload.log,
    message: payload.message,
    source: payload.source,
  });
};

/**
 * Method for logging execution data without blocking
 */
const executionLog = (payload: LogBasePayload): void => {
  createLog({
    type: LogType.EXECUTION,
    log: payload.log,
    message: payload.message,
    source: payload.source,
  });
};

if (typeof process !== "undefined") {
  const FLUSH_INTERVAL_MS = 60000;
  setInterval(() => {
    if (logBuffer.length > 0) {
      flushBuffer().catch((err) => {
        localLogger.error("Error in auto-flush:", err);
      });
    }
  }, FLUSH_INTERVAL_MS);
}

initialize();

const log: Logger = {
  info,
  error,
  executionLog,
};

export default log;