bandeng-logger
Version:
Logger from bandeng developer team.
598 lines (521 loc) • 21.9 kB
JavaScript
const fs = require('fs')
// Removed global processingPromise - now each bufferSystem has its own
// 全局节流变量，避免高并发下的日志泛滥
let lastQueueWarningTime = 0
let lastBufferWarningTime = 0
/**
* @description 初始化缓冲区系统
* @param {Object} options - 配置选项
* @returns {Object} 缓冲区配置
*/
function initBufferSystem(options, paths) {
// 参数验证
if (!options || typeof options !== 'object') {
throw new Error('initBufferSystem: options must be a valid object')
}
// 从配置中获取缓冲区相关参数，提供合理的默认值
const bufferSize = options.bufferSize || 64 * 1024 // 默认64KB
const flushInterval = options.flushInterval || 1000 // 默认1秒
// 主日志缓冲区
const mainBuffer = {
data: [],
size: 0,
isFlushing: false
}
// 错误日志缓冲区
const errorBuffer = {
data: [],
size: 0,
isFlushing: false
}
// 缓冲区刷新定时器
const flushTimer = null
// 并发控制队列（每个实例独立）
const writeQueue = []
const isWriting = false
// 每个实例独立的Promise链，用于确保顺序处理
const processingPromise = Promise.resolve()
// 根据缓冲区大小动态计算队列大小限制
// 队列最大大小：缓冲区大小的10倍，或者至少1000条，可通过 options 覆盖
const writeQueueMaxSize = options.writeQueueMaxSize || Math.max(1000, Math.floor((bufferSize * 10) / 1024))
// 队列内存使用量限制：缓冲区大小的8倍，或者至少10MB，可通过 options 覆盖
const maxQueueMemoryUsage = options.maxQueueMemoryUsage || Math.max(10 * 1024 * 1024, bufferSize * 8)
// 文件清理控制
const isCleaningFiles = false
// 销毁控制
const isDestroyed = false
// 废弃方法警告控制
const tracePerformanceDeprecationWarned = false
// 统计信息 - 使用BigInt存储大数字避免溢出
const stats = {
totalLogs: 0n, // BigInt for potentially large numbers
flushedLogs: 0n,
failedFlushes: 0,
bufferFlushCount: 0,
lastFlushTime: Date.now(),
slowWrites: 0
}
// 缓冲区统计信息
const bufferStats = {
main: {maxSize: 0, maxCount: 0, totalWrites: 0},
error: {maxSize: 0, maxCount: 0, totalWrites: 0}
}
return {
mainBuffer,
errorBuffer,
flushTimer,
writeQueue,
isWriting,
processingPromise,
writeQueueMaxSize,
queueMemoryUsage: 0, // 初始内存使用量为0
maxQueueMemoryUsage,
isCleaningFiles,
isDestroyed,
tracePerformanceDeprecationWarned,
stats,
bufferStats,
paths, // 保存路径信息供异步操作使用
// 保存配置以供其他函数使用
config: {
bufferSize,
flushInterval
}
}
}
/**
* @description 刷新所有缓冲区
* @param {Object} bufferSystem - 缓冲区系统
* @param {Object} paths - 路径信息
* @returns {Object} 更新后的缓冲区系统
*/
async function flushBuffers(bufferSystem, paths) {
let flushedLogs = 0n // BigInt 类型用于处理大数字避免溢出
// 刷新主日志缓冲区
if (bufferSystem.mainBuffer.data.length > 0 && !bufferSystem.mainBuffer.isFlushing) {
const originalLength = BigInt(bufferSystem.mainBuffer.data.length)
bufferSystem.mainBuffer = await flushSingleBuffer(bufferSystem.mainBuffer, paths.logFile)
flushedLogs += originalLength // 使用刷新前的长度
}
// 刷新错误日志缓冲区
if (bufferSystem.errorBuffer.data.length > 0 && !bufferSystem.errorBuffer.isFlushing) {
const originalLength = BigInt(bufferSystem.errorBuffer.data.length)
bufferSystem.errorBuffer = await flushSingleBuffer(bufferSystem.errorBuffer, paths.errorLogFile)
flushedLogs += originalLength // 使用刷新前的长度
}
// 更新统计信息
bufferSystem.stats = {
...bufferSystem.stats,
flushedLogs: bufferSystem.stats.flushedLogs + flushedLogs,
bufferFlushCount: bufferSystem.stats.bufferFlushCount + 1,
lastFlushTime: Date.now()
}
// 注意：移除自动调度，避免与destroy时的手动刷新冲突
// 缓冲区刷新现在只通过阈值触发和destroy时手动调用
return bufferSystem
}
/**
* @description 刷新单个缓冲区
* @param {Object} buffer - 缓冲区对象
* @param {string} filePath - 文件路径
* @returns {Object} 更新后的缓冲区
*/
async function flushSingleBuffer(buffer, filePath) {
if (buffer.isFlushing || buffer.data.length === 0) {
return buffer
}
const newBuffer = {...buffer, isFlushing: true}
const dataToWrite = buffer.data.join('')
let retryCount = 0
const maxRetries = 3
while (retryCount < maxRetries) {
try {
await fs.promises.appendFile(filePath, dataToWrite)
// 成功时清理缓冲区
return {
...newBuffer,
data: [],
size: 0,
isFlushing: false
}
} catch (error) {
retryCount++
if (retryCount >= maxRetries) {
console.error(`Failed to flush log buffer to ${filePath} after ${maxRetries} retries:`, error.message)
// 保留数据用于后续重试
return {
...newBuffer,
isFlushing: false
}
}
// 指数退避
const delay = Math.min(100 * Math.pow(2, retryCount), 1000)
await new Promise(resolve => setTimeout(resolve, delay))
}
}
}
/**
* @description 强制同步刷新缓冲区
* @param {Object} bufferSystem - 缓冲区系统
* @param {Object} paths - 路径信息
* @param {Object} options - 配置选项
* @param {string} level - 日志级别
* @returns {Object} 更新后的缓冲区系统
*/
async function forceFlushBuffers(bufferSystem, paths, options, level) {
if (options.forceSyncLevels.includes(level)) {
if (bufferSystem.mainBuffer.data.length > 0) {
try {
await fs.promises.appendFile(paths.logFile, bufferSystem.mainBuffer.data.join(''))
bufferSystem.mainBuffer = {
...bufferSystem.mainBuffer,
data: [],
size: 0
}
} catch (error) {
console.error('Failed to force flush main buffer:', error.message)
}
}
if (bufferSystem.errorBuffer.data.length > 0) {
try {
await fs.promises.appendFile(paths.errorLogFile, bufferSystem.errorBuffer.data.join(''))
bufferSystem.errorBuffer = {
...bufferSystem.errorBuffer,
data: [],
size: 0
}
} catch (error) {
console.error('Failed to force flush error buffer:', error.message)
}
}
}
return bufferSystem
}
/**
* @description 添加日志到缓冲区（使用队列确保并发安全）
* @param {Object} bufferSystem - 缓冲区系统
* @param {string} message - 日志消息
* @param {string} level - 日志级别
* @returns {Object} 更新后的缓冲区系统
*/
function addToBuffer(bufferSystem, message, level, paths, options) {
// 更新统计信息
const newStats = {
...bufferSystem.stats,
totalLogs: bufferSystem.stats.totalLogs + 1n // BigInt 类型用于处理大数字避免溢出
}
// 计算新条目的内存使用量（使用因子估算额外开销）
const entrySize = Buffer.byteLength(message + '\n', 'utf8') * 1.5 // 使用因子估算额外开销
// 检查队列大小和内存限制
if (
bufferSystem.writeQueue.length >= bufferSystem.writeQueueMaxSize ||
bufferSystem.queueMemoryUsage + entrySize > bufferSystem.maxQueueMemoryUsage
) {
// 节流警告输出，避免高并发下的日志泛滥（每5秒最多输出一次）
const now = Date.now()
const timeSinceLastWarning = now - lastQueueWarningTime
const WARNING_THROTTLE_MS = 5000 // 5秒节流
if (timeSinceLastWarning >= WARNING_THROTTLE_MS) {
console.warn('[LOGGER_WARNING] Write queue is full or memory limit reached, dropping log entry')
console.warn(
`[LOGGER_WARNING] Queue size: ${bufferSystem.writeQueue.length}/${bufferSystem.writeQueueMaxSize}, Memory: ${(
bufferSystem.queueMemoryUsage /
1024 /
1024
).toFixed(2)}MB/${(bufferSystem.maxQueueMemoryUsage / 1024 / 1024).toFixed(2)}MB`
)
// 更新全局最后警告时间
lastQueueWarningTime = now
}
return {
...bufferSystem,
stats: newStats
// 注意：内存使用量不累积，因为消息被丢弃
}
}
// 将写入操作加入队列
const newWriteQueue = [
...bufferSystem.writeQueue,
{
message: message + '\n',
level,
timestamp: Date.now(),
size: entrySize
}
]
// 更新内存使用量
const newQueueMemoryUsage = bufferSystem.queueMemoryUsage + entrySize // 更新内存使用量
const newBufferSystem = {
...bufferSystem,
writeQueue: newWriteQueue,
queueMemoryUsage: newQueueMemoryUsage,
stats: newStats
}
// 链式处理以确保每个实例的顺序执行
newBufferSystem.processingPromise = newBufferSystem.processingPromise.then(() => processWriteQueue(newBufferSystem, paths, options))
// 返回带有队列的缓冲区系统（处理是异步的）
return newBufferSystem
}
/**
* @description 处理写入队列（确保顺序性和并发安全）
* @param {Object} bufferSystem - 缓冲区系统
* @param {Object} paths - 路径信息
* @param {Object} options - 配置选项
* @returns {Object} 更新后的缓冲区系统
*/
async function processWriteQueue(bufferSystem, paths, options) {
if (!bufferSystem.writeQueue || bufferSystem.writeQueue.length === 0) {
return bufferSystem
}
// 顺序处理队列条目
while (bufferSystem.writeQueue.length > 0) {
const entry = bufferSystem.writeQueue.shift()
if (!entry) break
// 记录此条目的内存大小，确保无论如何都要清理
const entryMemorySize = entry.size // 记录此条目的内存大小，确保无论如何都要清理
let retryCount = 0
const maxRetries = 3
let entryProcessed = false
while (retryCount < maxRetries && !entryProcessed) {
try {
// 写入缓冲区
const result = await writeToBuffers(entry.message, entry.level, bufferSystem, paths, options)
// 更新缓冲区系统
Object.assign(bufferSystem, result)
entryProcessed = true
} catch (error) {
retryCount++
// 更新统计信息
bufferSystem.stats = {
...bufferSystem.stats,
failedFlushes: bufferSystem.stats.failedFlushes + 1
}
if (retryCount >= maxRetries) {
// 达到最大重试次数，记录最终错误
console.error('[LOGGER_ERROR] Failed to write log entry after', maxRetries, 'retries:', error.message)
// 将失败的条目记录到错误日志中
try {
const fallbackResult = await writeToBuffers(
`[FAILED_LOG_ENTRY] ${new Date().toISOString()} ${entry.level.toUpperCase()} ${entry.message}`,
'error',
bufferSystem,
paths,
options
)
// 更新缓冲区系统
Object.assign(bufferSystem, fallbackResult)
} catch (fallbackError) {
console.error('[LOGGER_CRITICAL] Even fallback error logging failed:', fallbackError.message)
}
break
} else {
// 等待一段时间后重试
const delay = Math.min(100 * Math.pow(2, retryCount), 1000)
await new Promise(resolve => setTimeout(resolve, delay))
console.warn(`[LOGGER_WARN] Retrying log write (attempt ${retryCount}/${maxRetries}):`, error.message)
}
}
}
// 无论成功与否，都要清理此条目的内存使用量，避免内存泄漏
bufferSystem.queueMemoryUsage = Math.max(0, bufferSystem.queueMemoryUsage - entryMemorySize)
}
return bufferSystem
}
/**
* @description 实际写入缓冲区（优化大小监控）
* @param {string} logEntry - 日志条目
* @param {string} level - 日志级别
* @param {Object} bufferSystem - 缓冲区系统
* @param {Object} paths - 路径信息
* @param {Object} options - 配置选项
* @returns {Object} 更新后的缓冲区系统
*/
async function writeToBuffers(logEntry, level, bufferSystem, paths, options) {
const entrySize = Buffer.byteLength(logEntry, 'utf8')
// 主日志缓冲区检查和写入
const mainResult = await writeToBuffer(bufferSystem.mainBuffer, logEntry, entrySize, 'main', paths.logFile, options, bufferSystem)
bufferSystem.mainBuffer = mainResult.buffer
// 错误日志缓冲区（如果需要分离错误日志）
if (options.separateErrorLog && shouldSeparateErrorLog(level, options)) {
const errorResult = await writeToBuffer(bufferSystem.errorBuffer, logEntry, entrySize, 'error', paths.errorLogFile, options, bufferSystem)
bufferSystem.errorBuffer = errorResult.buffer
}
// 更新bufferStats
bufferSystem.bufferStats = {
...bufferSystem.bufferStats,
main: {
...bufferSystem.bufferStats.main,
maxSize: Math.max(bufferSystem.bufferStats.main.maxSize, bufferSystem.mainBuffer.size),
maxCount: Math.max(bufferSystem.bufferStats.main.maxCount, bufferSystem.mainBuffer.data.length),
totalWrites: bufferSystem.bufferStats.main.totalWrites + 1
}
}
if (options.separateErrorLog && shouldSeparateErrorLog(level, options)) {
bufferSystem.bufferStats.error = {
...bufferSystem.bufferStats.error,
maxSize: Math.max(bufferSystem.bufferStats.error.maxSize, bufferSystem.errorBuffer.size),
maxCount: Math.max(bufferSystem.bufferStats.error.maxCount, bufferSystem.errorBuffer.data.length),
totalWrites: bufferSystem.bufferStats.error.totalWrites + 1
}
}
// 检查是否需要强制同步刷新
await forceFlushBuffers(bufferSystem, paths, options, level)
return bufferSystem
}
/**
* @description 写入单个缓冲区（带智能大小监控）
* @param {Object} buffer - 缓冲区对象
* @param {string} logEntry - 日志条目
* @param {number} entrySize - 条目大小
* @param {string} bufferType - 缓冲区类型
* @param {string} filePath - 文件路径
* @param {Object} options - 配置选项
* @param {Object} bufferSystem - 缓冲区系统（可选，用于获取配置）
* @returns {Object} 写入结果
*/
async function writeToBuffer(buffer, logEntry, entrySize, bufferType, filePath, options, bufferSystem = null) {
// 优先使用bufferSystem中的配置，如果没有则使用传入的options
const bufferSize = (bufferSystem && bufferSystem.config && bufferSystem.config.bufferSize) || options.bufferSize || 64 * 1024
// 检查是否会溢出，如果会则先刷新缓冲区
if (buffer.size + entrySize > bufferSize) {
const flushedBuffer = await flushSingleBuffer(buffer, filePath)
buffer = flushedBuffer
}
// 动态调整：如果写入后缓冲区即将溢出，提前刷新（80%阈值）
const newSize = buffer.size + entrySize
const threshold = bufferSize * 0.8 // 80%阈值
if (newSize > threshold) {
// 节流缓冲区警告，避免高并发下的日志泛滥
const now = Date.now()
const timeSinceLastBufferWarning = now - lastBufferWarningTime
const BUFFER_WARNING_THROTTLE_MS = 2000 // 2秒节流，比队列警告更频繁但仍有节流
if (timeSinceLastBufferWarning >= BUFFER_WARNING_THROTTLE_MS) {
// 使用console.log而不是this.debug()来避免递归调用
console.log(
`[logger] Buffer ${bufferType} approaching capacity (${newSize}/${bufferSize} bytes, ${((newSize / bufferSize) * 100).toFixed(
1
)}%), triggering flush`
)
// 更新全局最后缓冲区警告时间
lastBufferWarningTime = now
}
// 非阻塞刷新：启动异步flush，同时创建一个新缓冲区继续写入
// 这样可以避免阻塞新的日志记录
flushSingleBuffer(buffer, filePath).catch(error => {
// 静默处理flush错误，避免影响主日志流程
console.warn(`[LOGGER_WARN] Background buffer flush failed: ${error.message}`)
})
// 立即创建一个新的空缓冲区，让新日志可以继续写入
buffer = {
data: [],
size: 0,
isFlushing: false,
maxSize: buffer.maxSize,
maxCount: buffer.maxCount,
totalWrites: buffer.totalWrites
}
}
const newBuffer = {
...buffer,
data: [...buffer.data, logEntry],
size: buffer.size + entrySize
}
// 记录缓冲区使用情况统计（不在此处记录debug日志，避免递归）
// 注意：这里我们不更新bufferStats，因为它应该由调用方处理
return {
buffer: newBuffer
}
}
/**
* @description 判断是否需要分离错误日志
* @param {string} level - 日志级别
* @param {Object} options - 配置选项
* @returns {boolean} 是否需要分离
*/
function shouldSeparateErrorLog(level, options) {
// 支持配置哪些级别需要分离
const separateLevels = options.separateErrorLevels || ['error', 'fatal']
return separateLevels.includes(level)
}
/**
* @description 清空缓冲区统计信息
* @param {Object} bufferSystem - 缓冲区系统
* @returns {Object} 更新后的缓冲区系统
*/
function clearBufferStats(bufferSystem) {
return {
...bufferSystem,
stats: {
totalLogs: 0n,
flushedLogs: 0n,
failedFlushes: 0,
bufferFlushCount: 0,
lastFlushTime: Date.now(),
slowWrites: 0
},
bufferStats: {
main: {maxSize: 0, maxCount: 0, totalWrites: 0},
error: {maxSize: 0, maxCount: 0, totalWrites: 0}
},
tracePerformanceDeprecationWarned: false
}
}
/**
* @description 清空缓存
* @param {Object} bufferSystem - 缓冲区系统
* @returns {Object} 更新后的缓冲区系统
*/
function clearBufferCache(bufferSystem) {
return {
...bufferSystem,
writeQueue: [],
queueMemoryUsage: 0,
isWriting: false
}
}
/**
* @description 清理缓冲区系统资源
* @param {Object} bufferSystem - 缓冲区系统
* @returns {Promise<Object>} 更新后的缓冲区系统
*/
async function destroy(bufferSystem) {
// 标记为销毁以防止新的异步操作
bufferSystem.isDestroyed = true
// 清除任何挂起的定时器
if (bufferSystem.flushTimer) {
clearInterval(bufferSystem.flushTimer)
bufferSystem.flushTimer = null
}
// 同步刷新剩余的缓冲区以避免新的异步操作
try {
if (bufferSystem.mainBuffer.data.length > 0 && bufferSystem.paths && bufferSystem.paths.logFile) {
const fs = require('fs').promises
await fs.appendFile(bufferSystem.paths.logFile, bufferSystem.mainBuffer.data.join(''))
}
if (bufferSystem.errorBuffer.data.length > 0 && bufferSystem.paths && bufferSystem.paths.errorLogFile) {
const fs = require('fs').promises
await fs.appendFile(bufferSystem.paths.errorLogFile, bufferSystem.errorBuffer.data.join(''))
}
} catch (error) {
// 忽略销毁期间的刷新错误
}
// 清除队列并重置
return {
...bufferSystem,
mainBuffer: {...bufferSystem.mainBuffer, data: [], size: 0},
errorBuffer: {...bufferSystem.errorBuffer, data: [], size: 0},
writeQueue: [],
queueMemoryUsage: 0,
isWriting: false,
isDestroyed: true
}
}
module.exports = {
initBufferSystem,
flushBuffers,
addToBuffer,
clearBufferStats,
clearBufferCache,
destroy
}