pino-transport-rotating-file
Version:
Plugin for pino to transport logs to rotating files
433 lines (428 loc) • 16 kB
JavaScript
;
Object.defineProperty(exports, '__esModule', { value: true });
var node_buffer = require('node:buffer');
var node_fs = require('node:fs');
var promises = require('node:fs/promises');
var node_path = require('node:path');
var node_stream = require('node:stream');
var node_util = require('node:util');
var node_zlib = require('node:zlib');
var build = require('pino-abstract-transport');
var pinoPretty = require('pino-pretty');
var rotatingFileStream = require('rotating-file-stream');
/** @typedef {import('node:stream').TransformCallback} TransformCallback */
/** @typedef {import('node:zlib').ZlibOptions} ZlibOptions */
/** @typedef {import('pino-abstract-transport').OnUnknown} OnUnknown */
/** @typedef {import('rotating-file-stream').RotatingFileStream} RotatingFileStream */
/**
* @typedef {Object} CreateErrorLogger
* @property {(message: string, error?: unknown) => void} log - Logs an error message.
* @property {() => void} destroy - Destroys the error logger.
*/
/**
* @typedef {'B' | 'K' | 'M' | 'G'} StorageUnit
* @typedef {'s' | 'm' | 'h' | 'd' | 'M'} TimeUnit
* @typedef {`${number}${StorageUnit}`} Size
* @typedef {`${number}${TimeUnit}`} Interval
* @typedef {'iso' | 'unix' | 'utc' | 'rfc2822' | 'epoch'} TimestampFormat
*/
/**
* @typedef {Object} PinoTransportOptions
* @property {string} dir - The directory to store the log files.
* @property {string} filename - The base filename for the log files.
* @property {boolean} enabled - Whether the transport is enabled.
* @property {Size} size - The size at which to rotate the log files.
* @property {Interval} interval - The interval at which to rotate the log files.
* @property {boolean} compress - Whether to compress the log files.
* @property {boolean} immutable - Whether to use immutable log files.
* @property {number} [retentionDays=30] - The number of days to retain log files.
* @property {ZlibOptions} [compressionOptions] - The options to use for compression.
* @property {string} [errorLogFile] - The path to the error log file.
* @property {TimestampFormat} [timestampFormat='iso'] - The format to use for the timestamp.
* @property {boolean} [skipPretty=false] - Whether to skip pretty formatting.
* @property {number} [errorFlushIntervalMs=60000] - The interval at which to flush the error log buffer.
*/
/**
* @constant {Function} pipelineAsync
* @description Promisified version of the Node.js pipeline function for handling streams.
*/
const pipelineAsync = node_util.promisify(node_stream.pipeline);
/**
* @constant {Record<StorageUnit, number>} sizeUnits
* @description The conversion factor for storage units.
*/
const sizeUnits = {
B: 1,
K: 1024,
M: 1024 ** 2,
G: 1024 ** 3,
};
/**
* @constant {Record<TimeUnit, number>} timeUnits
* @description The conversion factor for time units.
*/
const timeUnits = {
s: 1000,
m: 60 * 1000,
h: 60 * 60 * 1000,
d: 24 * 60 * 60 * 1000,
M: 30 * 24 * 60 * 60 * 1000,
};
/**
* @function validateSize
* @description Validates the size option for the rotating log file.
*
* @param {Size} size - The size option to validate.
*/
function validateSize(size) {
const match = /^(\d+)([BKMG])$/.exec(size);
if (!match)
throw new Error(`Invalid size format: ${size}. Expected format: <number><B|K|M|G>`);
const [_, num, unit] = match;
if (Number.parseInt(num, 10) <= 0)
throw new Error(`Size must be positive: ${size}`);
if (!sizeUnits[unit])
throw new Error(`Unknown size unit: ${unit}`);
}
/**
* @function validateInterval
* @description Validates the interval option for the rotating log file.
*
* @param {Interval} interval - The interval option to validate.
*/
function validateInterval(interval) {
const match = /^(\d+)([smhdM])$/.exec(interval);
if (!match)
throw new Error(`Invalid interval format: ${interval}. Expected format: <number><s|m|h|d|M>`);
const [_, num, unit] = match;
if (Number.parseInt(num, 10) <= 0)
throw new Error(`Interval must be positive: ${interval}`);
if (!timeUnits[unit])
throw new Error(`Unknown time unit: ${unit}`);
}
/**
* @function validateTimestampFormat
* @description Validates the timestampFormat option for the rotating log file.
*
* @param {TimestampFormat} format - The timestampFormat option to validate.
*/
function validateTimestampFormat(format) {
const validFormats = [
'iso',
'unix',
'utc',
'rfc2822',
'epoch',
];
if (!validFormats.includes(format)) {
throw new Error(`Invalid timestampFormat: ${format}. Expected one of: ${validFormats.join(', ')}`);
}
}
/**
* @function generator
* @description Generates the file path for the rotating log file.
*
* @param {number | Date | null} time - The time to use in the filename.
* @param {string} dir - The directory to store the log files.
* @param {string} filename - The base filename for the log files.
* @param {TimestampFormat} timestampFormat - The format to use for the timestamp.
*
* @returns {string} The generated file path.
*/
function generator(time, dir, filename, timestampFormat = 'iso') {
if (!time)
return node_path.join(dir, `${filename}.log`);
const _date = new Date(time);
let timestamp;
switch (timestampFormat) {
case 'iso':
timestamp = _date.toISOString().replace(/[-:T]/g, '').split('.')[0];
break;
case 'unix':
timestamp = _date.getTime().toString();
break;
case 'utc':
timestamp = _date
.toUTCString()
.replace(/[^a-zA-Z0-9]/g, '-')
.replace(/-+/g, '-')
.trim();
break;
case 'rfc2822':
timestamp = _date
.toString()
.replace(/[^a-zA-Z0-9]/g, '-')
.replace(/-+/g, '-')
.trim();
break;
case 'epoch':
timestamp = (_date.getTime() / 1000).toFixed(0);
break;
default:
timestamp = _date.toISOString().replace(/[-:T]/g, '').split('.')[0];
break;
}
const fullName = `${filename}-${timestamp}.log`;
return node_path.join(dir, fullName.length > 200 ? fullName.slice(0, 200) : fullName);
}
/**
* @function fileExists
* @description Checks if a file exists at the given path.
*
* @param {string} path - The path to the file.
*
* @returns {Promise<boolean>} A promise that resolves to true if the file exists, otherwise false.
*/
async function fileExists(path) {
try {
await promises.access(path);
return true;
}
catch {
return false;
}
}
/**
* @function createErrorLogger
* @description Creates a logger for handling errors during compression and cleanup.
*
* @param {string} errorLogFile - The path to the error log file.
* @param {number} bufferSize - The maximum number of log messages to buffer before flushing.
* @param {number} flushIntervalMs - The interval at which to flush the log buffer.
*
* @returns {CreateErrorLogger} The error logger instance.
*/
const createErrorLogger = (errorLogFile, bufferSize = 100, flushIntervalMs = 60 * 1000) => {
let buffer = [];
let flushInterval = null;
/**
* @function flush
* @description Flushes the log buffer to the error log file.
*
* @returns {void}
*/
const flush = () => {
if (buffer.length === 0)
return;
const logMessage = buffer.join('');
buffer = [];
if (errorLogFile) {
node_fs.createWriteStream(errorLogFile, { flags: 'a' }).write(logMessage);
}
else {
console.error(logMessage);
}
};
/**
* @function startFlushInterval
* @description Starts the interval for flushing the log buffer.
*
* @returns {void}
*/
const startFlushInterval = () => {
flushInterval = setInterval(flush, flushIntervalMs);
};
startFlushInterval();
/**
* @function log
* @description Logs an error message to the error log file.
*
* @param {string} message - The error message to log.
* @param {unknown} error - The error object to log.
*
* @returns {void}
*/
const log = (message, error) => {
const logMessage = `${new Date().toISOString()} - ${message}${error ? `: ${error}` : ''}\n`;
buffer = [...buffer, logMessage];
if (buffer.length >= bufferSize)
flush();
};
const destroy = () => {
flush();
if (flushInterval)
clearInterval(flushInterval);
};
return { log, destroy };
};
/**
* @function compressFile
* @description Compresses a file using gzip and writes the compressed file to the destination.
*
* @param {string} src - The path to the source file.
* @param {string} dest - The path to the destination file.
* @param {ZlibOptions} compressionOptions - The options to use for the compression.
* @param {CreateErrorLogger} errorLogger - The error logger instance.
*
* @returns {Promise<void>} A promise that resolves once the file has been compressed.
*/
async function compressFile(src, dest, compressionOptions, errorLogger) {
try {
const exists = await fileExists(src);
if (!exists) {
errorLogger.log(`Skipping compression, file does not exist: ${src}`);
return;
}
const stats = await promises.stat(src);
if (stats.isDirectory()) {
errorLogger.log(`Skipping compression, source is a directory: ${src}`);
return;
}
await pipelineAsync(node_fs.createReadStream(src), node_zlib.createGzip(compressionOptions), node_fs.createWriteStream(dest));
if (!(await fileExists(dest))) {
throw new Error(`Compression failed: Destination file ${dest} not found`);
}
await promises.unlink(src);
}
catch (error) {
errorLogger.log(`Error compressing file ${src}`, error);
throw error;
}
}
/**
* @function cleanupOldFiles
* @description Cleans up old log files in the directory based on the retention period.
*
* @param {string} dir - The directory to clean up.
* @param {string} filename - The base filename for the log files.
* @param {number} retentionDays - The number of days to retain log files.
* @param {CreateErrorLogger} errorLogger - The error logger instance.
*
* @returns {Promise<void>} A promise that resolves once the old files have been cleaned up.
*/
async function cleanupOldFiles(dir, filename, retentionDays, errorLogger) {
try {
const files = await promises.readdir(dir);
const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
await Promise.all(files
.filter((file) => file.startsWith(filename) &&
(file.endsWith('.log') || file.endsWith('.log.gz')))
.map(async (file) => {
const filePath = node_path.join(dir, file);
const stats = await promises.stat(filePath);
if (stats.mtimeMs < cutoff) {
await promises.unlink(filePath);
errorLogger.log(`Deleted old log file: ${filePath}`);
}
}));
}
catch (error) {
errorLogger.log(`Error during log file cleanup in ${dir}`, error);
}
}
/**
* @function pinoTransportRotatingFile
* @description Creates a Pino transport for rotating log files.
*
* @param {Partial<PinoTransportOptions>} options - The options for the rotating log file transport.
*
* @returns {Promise<Transform & OnUnknown>} A promise that resolves to the transport instance.
*/
async function pinoTransportRotatingFile(options = {
dir: '',
filename: 'app',
enabled: true,
size: '100K',
interval: '1d',
compress: true,
immutable: true,
retentionDays: 30,
compressionOptions: {
level: 6,
strategy: 0,
},
errorLogFile: undefined,
timestampFormat: 'iso',
skipPretty: false,
errorFlushIntervalMs: 60 * 1000,
}) {
const { dir, filename = 'app', enabled = true, size = '100K', interval = '1d', compress = true, immutable = true, retentionDays = 30, compressionOptions = {
level: 6,
strategy: 0,
}, errorLogFile, timestampFormat = 'iso', skipPretty = false, errorFlushIntervalMs = 60 * 1000, } = options;
if (!enabled) {
return build((source) => source, {
parse: 'lines',
expectPinoConfig: true,
// @ts-expect-error
enablePipelining: false,
close() { },
});
}
if (!dir)
throw new Error('Missing required option: dir');
validateSize(size);
validateInterval(interval);
validateTimestampFormat(timestampFormat);
const errorLogger = createErrorLogger(errorLogFile, 100, errorFlushIntervalMs);
const rotatingStream = rotatingFileStream.createStream((time) => generator(time, dir, filename, timestampFormat), { size, interval, immutable });
const compressedFiles = new Map();
const MAX_COMPRESSION_AGE = 24 * 60 * 60 * 1000;
if (compress) {
rotatingStream.on('rotated', async (rotatedFile) => {
try {
if (compressedFiles.has(rotatedFile))
return;
const isFile = (await promises.stat(rotatedFile)).isFile();
if (isFile) {
const compressedFile = `${rotatedFile}.gz`;
await compressFile(rotatedFile, compressedFile, compressionOptions, errorLogger);
compressedFiles.set(rotatedFile, Date.now());
setTimeout(() => compressedFiles.delete(rotatedFile), MAX_COMPRESSION_AGE);
}
else {
errorLogger.log(`Skipping compression, rotated file is a directory: ${rotatedFile}`);
}
}
catch (err) {
errorLogger.log(`Error compressing rotated file ${rotatedFile}`, err);
}
});
}
let cleanupInterval;
if (retentionDays > 0) {
await cleanupOldFiles(dir, filename, retentionDays, errorLogger);
cleanupInterval = setInterval(() => cleanupOldFiles(dir, filename, retentionDays, errorLogger), 24 * 60 * 60 * 1000);
}
return build((source) => {
const prettyStream = skipPretty
? source
: new node_stream.Transform({
objectMode: true,
autoDestroy: true,
transform(chunk, encoding, callback) {
try {
const logMessage = node_buffer.Buffer.isBuffer(chunk)
? chunk.toString(encoding)
: chunk;
const prettyLog = pinoPretty.prettyFactory({ colorize: false })(logMessage);
callback(null, prettyLog);
}
catch (error) {
callback(error instanceof Error
? error
: new Error(String(error) || 'An unknown error occurred'));
}
},
});
node_stream.pipeline(source, prettyStream, rotatingStream, (err) => {
if (err)
errorLogger.log('Failed to write log in transport', err);
});
return prettyStream;
}, {
parse: 'lines',
expectPinoConfig: true,
// @ts-expect-error
enablePipelining: false,
async close() {
errorLogger.destroy();
if (cleanupInterval)
clearInterval(cleanupInterval);
await new Promise((resolve) => rotatingStream.end(() => resolve()));
},
});
}
exports.default = pinoTransportRotatingFile;
exports.pinoTransportRotatingFile = pinoTransportRotatingFile;
//# sourceMappingURL=index.js.map