apitally
Version:
Simple API monitoring & analytics for REST APIs built with Express, Fastify, Hono, Koa, and NestJS.
1 lines • 93.6 kB
Source Map (JSON)
{"version":3,"sources":["../../src/express/middleware.ts","../../src/common/client.ts","../../src/common/consumerRegistry.ts","../../src/common/logging.ts","../../src/common/paramValidation.ts","../../src/common/requestCounter.ts","../../src/common/requestLogger.ts","../../src/common/sentry.ts","../../src/common/serverErrorCounter.ts","../../src/common/tempGzipFile.ts","../../src/common/validationErrorCounter.ts","../../src/common/packageVersions.ts","../../src/common/utils.ts","../../src/express/utils.js"],"sourcesContent":["import type { Express, NextFunction, Request, Response, Router } from \"express\";\nimport { performance } from \"perf_hooks\";\n\nimport { ApitallyClient } from \"../common/client.js\";\nimport { consumerFromStringOrObject } from \"../common/consumerRegistry.js\";\nimport { getPackageVersion } from \"../common/packageVersions.js\";\nimport { convertBody, convertHeaders } from \"../common/requestLogger.js\";\nimport {\n ApitallyConfig,\n ApitallyConsumer,\n StartupData,\n ValidationError,\n} from \"../common/types.js\";\nimport { parseContentLength } from \"../common/utils.js\";\nimport { getEndpoints, parseExpressPath } from \"./utils.js\";\n\ndeclare module \"express\" {\n interface Request {\n apitallyConsumer?: ApitallyConsumer | string | null;\n consumerIdentifier?: ApitallyConsumer | string | null; // For backwards compatibility\n }\n}\n\nexport const useApitally = (\n app: Express | Router,\n config: ApitallyConfig & { basePath?: string },\n) => {\n const client = new ApitallyClient(config);\n const middleware = getMiddleware(app, client);\n app.use(middleware);\n setTimeout(() => {\n client.setStartupData(getAppInfo(app, config.basePath, config.appVersion));\n }, 1000);\n};\n\nconst getMiddleware = (app: Express | Router, client: ApitallyClient) => {\n let errorHandlerConfigured = false;\n\n return (req: Request, res: Response, next: NextFunction) => {\n if (!client.isEnabled()) {\n next();\n return;\n }\n if (!errorHandlerConfigured) {\n // Add error handling middleware to the bottom of the stack when handling the first request\n app.use(\n (err: Error, req: Request, res: Response, next: NextFunction): void => {\n res.locals.serverError = err;\n next(err);\n },\n );\n errorHandlerConfigured = true;\n }\n try {\n const startTime = performance.now();\n const originalSend = res.send;\n res.send = (body) => {\n const contentType = res.get(\"content-type\");\n if (client.requestLogger.isSupportedContentType(contentType)) {\n res.locals.body = body;\n }\n return originalSend.call(res, body);\n };\n\n res.on(\"finish\", () => {\n try {\n const responseTime = performance.now() - startTime;\n const path = getRoutePath(req);\n const consumer = getConsumer(req);\n client.consumerRegistry.addOrUpdateConsumer(consumer);\n\n const requestSize = parseContentLength(req.get(\"content-length\"));\n const responseSize = parseContentLength(res.get(\"content-length\"));\n\n if (path) {\n client.requestCounter.addRequest({\n consumer: consumer?.identifier,\n method: req.method,\n path,\n statusCode: res.statusCode,\n responseTime: responseTime,\n requestSize,\n responseSize,\n });\n if (\n (res.statusCode === 400 || res.statusCode === 422) &&\n res.locals.body\n ) {\n let jsonBody: any;\n try {\n jsonBody = JSON.parse(res.locals.body);\n } catch {\n // Ignore\n }\n if (jsonBody) {\n const validationErrors: ValidationError[] = [];\n if (validationErrors.length === 0) {\n validationErrors.push(\n ...extractExpressValidatorErrors(jsonBody),\n );\n }\n if (validationErrors.length === 0) {\n validationErrors.push(...extractCelebrateErrors(jsonBody));\n }\n if (validationErrors.length === 0) {\n validationErrors.push(\n ...extractNestValidationErrors(jsonBody),\n );\n }\n validationErrors.forEach((error) => {\n client.validationErrorCounter.addValidationError({\n consumer: consumer?.identifier,\n method: req.method,\n path: req.route.path,\n ...error,\n });\n });\n }\n }\n if (res.statusCode === 500 && res.locals.serverError) {\n const serverError = res.locals.serverError as Error;\n client.serverErrorCounter.addServerError({\n consumer: consumer?.identifier,\n method: req.method,\n path: req.route.path,\n type: serverError.name,\n msg: serverError.message,\n traceback: serverError.stack || \"\",\n });\n }\n }\n if (client.requestLogger.enabled) {\n client.requestLogger.logRequest(\n {\n timestamp: Date.now() / 1000,\n method: req.method,\n path,\n url: `${req.protocol}://${req.host}${req.originalUrl}`,\n headers: convertHeaders(req.headers),\n size: requestSize,\n consumer: consumer?.identifier,\n body: convertBody(req.body, req.get(\"content-type\")),\n },\n {\n statusCode: res.statusCode,\n responseTime: responseTime / 1000,\n headers: convertHeaders(res.getHeaders()),\n size: responseSize,\n body: convertBody(res.locals.body, res.get(\"content-type\")),\n },\n res.locals.serverError,\n );\n }\n } catch (error) {\n client.logger.error(\n \"Error while logging request in Apitally middleware.\",\n { request: req, response: res, error },\n );\n }\n });\n } catch (error) {\n client.logger.error(\"Error in Apitally middleware.\", {\n request: req,\n response: res,\n error,\n });\n } finally {\n next();\n }\n };\n};\n\nconst getRoutePath = (req: Request) => {\n if (!req.route) {\n return;\n }\n if (req.baseUrl) {\n const routerPath = getRouterPath(req.app._router.stack, req.baseUrl);\n return req.route.path === \"/\" ? routerPath : routerPath + req.route.path;\n }\n return req.route.path;\n};\n\nconst getRouterPath = (stack: any[], baseUrl: string) => {\n const routerPaths: string[] = [];\n while (stack && stack.length > 0) {\n const routerLayer = stack.find(\n (layer) =>\n layer.name === \"router\" && layer.path && layer.regexp?.test(baseUrl),\n );\n if (routerLayer) {\n if (routerLayer.keys.length > 0) {\n const parsedPath = parseExpressPath(\n routerLayer.regexp,\n routerLayer.keys,\n );\n routerPaths.push(\"/\" + parsedPath);\n } else {\n routerPaths.push(routerLayer.path);\n }\n stack = routerLayer.handle?.stack;\n baseUrl = baseUrl.slice(routerLayer.path.length);\n } else {\n break;\n }\n }\n return routerPaths.filter((path) => path !== \"/\").join(\"\");\n};\n\nconst getConsumer = (req: Request) => {\n if (req.apitallyConsumer) {\n return consumerFromStringOrObject(req.apitallyConsumer);\n } else if (req.consumerIdentifier) {\n // For backwards compatibility\n process.emitWarning(\n \"The consumerIdentifier property on the request object is deprecated. Use apitallyConsumer instead.\",\n \"DeprecationWarning\",\n );\n return consumerFromStringOrObject(req.consumerIdentifier);\n }\n return null;\n};\n\nconst extractExpressValidatorErrors = (responseBody: any) => {\n try {\n const errors: ValidationError[] = [];\n if (\n responseBody &&\n responseBody.errors &&\n Array.isArray(responseBody.errors)\n ) {\n responseBody.errors.forEach((error: any) => {\n if (error.location && error.path && error.msg && error.type) {\n errors.push({\n loc: `${error.location}.${error.path}`,\n msg: error.msg,\n type: error.type,\n });\n }\n });\n }\n return errors;\n } catch (error) {\n return [];\n }\n};\n\nconst extractCelebrateErrors = (responseBody: any) => {\n try {\n const errors: ValidationError[] = [];\n if (responseBody && responseBody.validation) {\n Object.values(responseBody.validation).forEach((error: any) => {\n if (\n error.source &&\n error.keys &&\n Array.isArray(error.keys) &&\n error.message\n ) {\n error.keys.forEach((key: string) => {\n errors.push({\n loc: `${error.source}.${key}`,\n msg: subsetJoiMessage(error.message, key),\n type: \"\",\n });\n });\n }\n });\n }\n return errors;\n } catch (error) {\n return [];\n }\n};\n\nconst extractNestValidationErrors = (responseBody: any) => {\n try {\n const errors: ValidationError[] = [];\n if (responseBody && Array.isArray(responseBody.message)) {\n responseBody.message.forEach((message: any) => {\n errors.push({\n loc: \"\",\n msg: message,\n type: \"\",\n });\n });\n }\n return errors;\n } catch (error) {\n return [];\n }\n};\n\nconst subsetJoiMessage = (message: string, key: string) => {\n const messageWithKey = message\n .split(\". \")\n .find((message) => message.includes(`\"${key}\"`));\n return messageWithKey ? messageWithKey : message;\n};\n\nconst getAppInfo = (\n app: Express | Router,\n basePath?: string,\n appVersion?: string,\n): StartupData => {\n const versions: Array<[string, string]> = [\n [\"nodejs\", process.version.replace(/^v/, \"\")],\n ];\n const expressVersion = getPackageVersion(\"express\");\n const nestjsVersion = getPackageVersion(\"@nestjs/core\");\n const apitallyVersion = getPackageVersion(\"../..\");\n if (expressVersion) {\n versions.push([\"express\", expressVersion]);\n }\n if (nestjsVersion) {\n versions.push([\"nestjs\", nestjsVersion]);\n }\n if (apitallyVersion) {\n versions.push([\"apitally\", apitallyVersion]);\n }\n if (appVersion) {\n versions.push([\"app\", appVersion]);\n }\n return {\n paths: getEndpoints(app, basePath || \"\"),\n versions: Object.fromEntries(versions),\n client: \"js:express\",\n };\n};\n","import { randomUUID } from \"crypto\";\nimport fetchRetry from \"fetch-retry\";\n\nimport ConsumerRegistry from \"./consumerRegistry.js\";\nimport { Logger, getLogger } from \"./logging.js\";\nimport { isValidClientId, isValidEnv } from \"./paramValidation.js\";\nimport RequestCounter from \"./requestCounter.js\";\nimport RequestLogger from \"./requestLogger.js\";\nimport ServerErrorCounter from \"./serverErrorCounter.js\";\nimport {\n ApitallyConfig,\n StartupData,\n StartupPayload,\n SyncPayload,\n} from \"./types.js\";\nimport ValidationErrorCounter from \"./validationErrorCounter.js\";\n\nconst SYNC_INTERVAL = 60000; // 60 seconds\nconst INITIAL_SYNC_INTERVAL = 10000; // 10 seconds\nconst INITIAL_SYNC_INTERVAL_DURATION = 3600000; // 1 hour\nconst MAX_QUEUE_TIME = 3.6e6; // 1 hour\n\nclass HTTPError extends Error {\n public response: Response;\n\n constructor(response: Response) {\n const reason = response.status\n ? `status code ${response.status}`\n : \"an unknown error\";\n super(`Request failed with ${reason}`);\n this.response = response;\n }\n}\n\nexport class ApitallyClient {\n private clientId: string;\n private env: string;\n\n private static instance?: ApitallyClient;\n private instanceUuid: string;\n private syncDataQueue: SyncPayload[];\n private syncIntervalId?: NodeJS.Timeout;\n public startupData?: StartupData;\n private startupDataSent: boolean = false;\n private enabled: boolean = true;\n\n public requestCounter: RequestCounter;\n public requestLogger: RequestLogger;\n public validationErrorCounter: ValidationErrorCounter;\n public serverErrorCounter: ServerErrorCounter;\n public consumerRegistry: ConsumerRegistry;\n public logger: Logger;\n\n constructor({\n clientId,\n env = \"dev\",\n requestLoggingConfig,\n logger,\n }: ApitallyConfig) {\n if (ApitallyClient.instance) {\n throw new Error(\"Apitally client is already initialized\");\n }\n if (!isValidClientId(clientId)) {\n throw new Error(\n `Invalid Apitally client ID '${clientId}' (expecting hexadecimal UUID format)`,\n );\n }\n if (!isValidEnv(env)) {\n throw new Error(\n `Invalid env '${env}' (expecting 1-32 alphanumeric lowercase characters and hyphens only)`,\n );\n }\n\n ApitallyClient.instance = this;\n this.clientId = clientId;\n this.env = env;\n this.instanceUuid = randomUUID();\n this.syncDataQueue = [];\n this.requestCounter = new RequestCounter();\n this.requestLogger = new RequestLogger(requestLoggingConfig);\n this.validationErrorCounter = new ValidationErrorCounter();\n this.serverErrorCounter = new ServerErrorCounter();\n this.consumerRegistry = new ConsumerRegistry();\n this.logger = logger || getLogger();\n\n this.startSync();\n this.handleShutdown = this.handleShutdown.bind(this);\n }\n\n public static getInstance() {\n if (!ApitallyClient.instance) {\n throw new Error(\"Apitally client is not initialized\");\n }\n return ApitallyClient.instance;\n }\n\n public isEnabled() {\n return this.enabled;\n }\n\n public static async shutdown() {\n if (ApitallyClient.instance) {\n await ApitallyClient.instance.handleShutdown();\n }\n }\n\n public async handleShutdown() {\n this.enabled = false;\n this.stopSync();\n await this.sendSyncData();\n await this.sendLogData();\n await this.requestLogger.close();\n ApitallyClient.instance = undefined;\n }\n\n private getHubUrlPrefix() {\n const baseURL =\n process.env.APITALLY_HUB_BASE_URL || \"https://hub.apitally.io\";\n const version = \"v2\";\n return `${baseURL}/${version}/${this.clientId}/${this.env}/`;\n }\n\n private async sendData(url: string, payload: any) {\n const fetchWithRetry = fetchRetry(fetch, {\n retries: 3,\n retryDelay: 1000,\n retryOn: [408, 429, 500, 502, 503, 504],\n });\n const response = await fetchWithRetry(this.getHubUrlPrefix() + url, {\n method: \"POST\",\n body: JSON.stringify(payload),\n headers: { \"Content-Type\": \"application/json\" },\n });\n if (!response.ok) {\n throw new HTTPError(response);\n }\n }\n\n private startSync() {\n this.sync();\n this.syncIntervalId = setInterval(() => {\n this.sync();\n }, INITIAL_SYNC_INTERVAL);\n setTimeout(() => {\n clearInterval(this.syncIntervalId);\n this.syncIntervalId = setInterval(() => {\n this.sync();\n }, SYNC_INTERVAL);\n }, INITIAL_SYNC_INTERVAL_DURATION);\n }\n\n private async sync() {\n try {\n const promises = [this.sendSyncData(), this.sendLogData()];\n if (!this.startupDataSent) {\n promises.push(this.sendStartupData());\n }\n await Promise.all(promises);\n } catch (error) {\n this.logger.error(\"Error while syncing with Apitally Hub\", {\n error,\n });\n }\n }\n\n private stopSync() {\n if (this.syncIntervalId) {\n clearInterval(this.syncIntervalId);\n this.syncIntervalId = undefined;\n }\n }\n\n public setStartupData(data: StartupData) {\n this.startupData = data;\n this.startupDataSent = false;\n this.sendStartupData();\n }\n\n private async sendStartupData() {\n if (this.startupData) {\n this.logger.debug(\"Sending startup data to Apitally Hub\");\n const payload: StartupPayload = {\n instance_uuid: this.instanceUuid,\n message_uuid: randomUUID(),\n ...this.startupData,\n };\n try {\n await this.sendData(\"startup\", payload);\n this.startupDataSent = true;\n } catch (error) {\n const handled = this.handleHubError(error);\n if (!handled) {\n this.logger.error((error as Error).message);\n this.logger.debug(\n \"Error while sending startup data to Apitally Hub (will retry)\",\n { error },\n );\n }\n }\n }\n }\n\n private async sendSyncData() {\n this.logger.debug(\"Synchronizing data with Apitally Hub\");\n const newPayload: SyncPayload = {\n timestamp: Date.now() / 1000,\n instance_uuid: this.instanceUuid,\n message_uuid: randomUUID(),\n requests: this.requestCounter.getAndResetRequests(),\n validation_errors:\n this.validationErrorCounter.getAndResetValidationErrors(),\n server_errors: this.serverErrorCounter.getAndResetServerErrors(),\n consumers: this.consumerRegistry.getAndResetUpdatedConsumers(),\n };\n this.syncDataQueue.push(newPayload);\n\n let i = 0;\n while (this.syncDataQueue.length > 0) {\n const payload = this.syncDataQueue.shift();\n if (payload) {\n try {\n if (Date.now() - payload.timestamp * 1000 <= MAX_QUEUE_TIME) {\n if (i > 0) {\n await this.randomDelay();\n }\n await this.sendData(\"sync\", payload);\n i += 1;\n }\n } catch (error) {\n const handled = this.handleHubError(error);\n if (!handled) {\n this.logger.debug(\n \"Error while synchronizing data with Apitally Hub (will retry)\",\n { error },\n );\n this.syncDataQueue.push(payload);\n break;\n }\n }\n }\n }\n }\n\n private async sendLogData() {\n this.logger.debug(\"Sending request log data to Apitally Hub\");\n await this.requestLogger.rotateFile();\n\n const fetchWithRetry = fetchRetry(fetch, {\n retries: 3,\n retryDelay: 1000,\n retryOn: [408, 429, 500, 502, 503, 504],\n });\n\n let i = 0;\n let logFile;\n while ((logFile = this.requestLogger.getFile())) {\n if (i > 0) {\n await this.randomDelay();\n }\n\n try {\n const response = await fetchWithRetry(\n `${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`,\n {\n method: \"POST\",\n body: await logFile.getContent(),\n },\n );\n\n if (response.status === 402 && response.headers.has(\"Retry-After\")) {\n const retryAfter = parseInt(\n response.headers.get(\"Retry-After\") ?? \"0\",\n );\n if (retryAfter > 0) {\n this.requestLogger.suspendUntil = Date.now() + retryAfter * 1000;\n this.requestLogger.clear();\n return;\n }\n }\n\n if (!response.ok) {\n throw new HTTPError(response);\n }\n\n logFile.delete();\n } catch (error) {\n this.requestLogger.retryFileLater(logFile);\n break;\n }\n\n i++;\n if (i >= 10) break;\n }\n }\n\n private handleHubError(error: unknown) {\n if (error instanceof HTTPError) {\n if (error.response.status === 404) {\n this.logger.error(`Invalid Apitally client ID: '${this.clientId}'`);\n this.enabled = false;\n this.stopSync();\n return true;\n }\n if (error.response.status === 422) {\n this.logger.error(\"Received validation error from Apitally Hub\");\n return true;\n }\n }\n return false;\n }\n\n private async randomDelay() {\n const delay = 100 + Math.random() * 400;\n await new Promise((resolve) => setTimeout(resolve, delay));\n }\n}\n","import { ApitallyConsumer } from \"./types.js\";\n\nexport const consumerFromStringOrObject = (\n consumer: ApitallyConsumer | string,\n) => {\n if (typeof consumer === \"string\") {\n consumer = String(consumer).trim().substring(0, 128);\n return consumer ? { identifier: consumer } : null;\n } else {\n consumer.identifier = String(consumer.identifier).trim().substring(0, 128);\n consumer.name = consumer.name?.trim().substring(0, 64);\n consumer.group = consumer.group?.trim().substring(0, 64);\n return consumer.identifier ? consumer : null;\n }\n};\n\nexport default class ConsumerRegistry {\n private consumers: Map<string, ApitallyConsumer>;\n private updated: Set<string>;\n\n constructor() {\n this.consumers = new Map();\n this.updated = new Set();\n }\n\n public addOrUpdateConsumer(consumer?: ApitallyConsumer | null) {\n if (!consumer || (!consumer.name && !consumer.group)) {\n return;\n }\n const existing = this.consumers.get(consumer.identifier);\n if (!existing) {\n this.consumers.set(consumer.identifier, consumer);\n this.updated.add(consumer.identifier);\n } else {\n if (consumer.name && consumer.name !== existing.name) {\n existing.name = consumer.name;\n this.updated.add(consumer.identifier);\n }\n if (consumer.group && consumer.group !== existing.group) {\n existing.group = consumer.group;\n this.updated.add(consumer.identifier);\n }\n }\n }\n\n public getAndResetUpdatedConsumers() {\n const data: Array<ApitallyConsumer> = [];\n this.updated.forEach((identifier) => {\n const consumer = this.consumers.get(identifier);\n if (consumer) {\n data.push(consumer);\n }\n });\n this.updated.clear();\n return data;\n }\n}\n","import { createLogger, format, transports } from \"winston\";\n\nexport interface Logger {\n debug: (message: string, meta?: object) => void;\n info: (message: string, meta?: object) => void;\n warn: (message: string, meta?: object) => void;\n error: (message: string, meta?: object) => void;\n}\n\nexport const getLogger = () => {\n return createLogger({\n level: process.env.APITALLY_DEBUG ? \"debug\" : \"warn\",\n format: format.combine(\n format.colorize(),\n format.timestamp(),\n format.printf(\n (info) => `${info.timestamp} ${info.level}: ${info.message}`,\n ),\n ),\n transports: [new transports.Console()],\n });\n};\n","export function isValidClientId(clientId: string): boolean {\n const regexExp =\n /^[0-9a-f]{8}-[0-9a-f]{4}-[4][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;\n return regexExp.test(clientId);\n}\n\nexport function isValidEnv(env: string): boolean {\n const regexExp = /^[\\w-]{1,32}$/;\n return regexExp.test(env);\n}\n","import { RequestInfo, RequestsItem } from \"./types.js\";\n\nexport default class RequestCounter {\n private requestCounts: Map<string, number>;\n private requestSizeSums: Map<string, number>;\n private responseSizeSums: Map<string, number>;\n private responseTimes: Map<string, Map<number, number>>;\n private requestSizes: Map<string, Map<number, number>>;\n private responseSizes: Map<string, Map<number, number>>;\n\n constructor() {\n this.requestCounts = new Map<string, number>();\n this.requestSizeSums = new Map<string, number>();\n this.responseSizeSums = new Map<string, number>();\n this.responseTimes = new Map<string, Map<number, number>>();\n this.requestSizes = new Map<string, Map<number, number>>();\n this.responseSizes = new Map<string, Map<number, number>>();\n }\n\n private getKey(requestInfo: RequestInfo) {\n return [\n requestInfo.consumer || \"\",\n requestInfo.method.toUpperCase(),\n requestInfo.path,\n requestInfo.statusCode,\n ].join(\"|\");\n }\n\n addRequest(requestInfo: RequestInfo) {\n const key = this.getKey(requestInfo);\n\n // Increment request count\n this.requestCounts.set(key, (this.requestCounts.get(key) || 0) + 1);\n\n // Add response time\n if (!this.responseTimes.has(key)) {\n this.responseTimes.set(key, new Map<number, number>());\n }\n const responseTimeMap = this.responseTimes.get(key)!;\n const responseTimeMsBin = Math.floor(requestInfo.responseTime / 10) * 10; // Rounded to nearest 10ms\n responseTimeMap.set(\n responseTimeMsBin,\n (responseTimeMap.get(responseTimeMsBin) || 0) + 1,\n );\n\n // Add request size\n if (requestInfo.requestSize !== undefined) {\n requestInfo.requestSize = Number(requestInfo.requestSize);\n this.requestSizeSums.set(\n key,\n (this.requestSizeSums.get(key) || 0) + requestInfo.requestSize,\n );\n if (!this.requestSizes.has(key)) {\n this.requestSizes.set(key, new Map<number, number>());\n }\n const requestSizeMap = this.requestSizes.get(key)!;\n const requestSizeKbBin = Math.floor(requestInfo.requestSize / 1000); // Rounded down to nearest KB\n requestSizeMap.set(\n requestSizeKbBin,\n (requestSizeMap.get(requestSizeKbBin) || 0) + 1,\n );\n }\n\n // Add response size\n if (requestInfo.responseSize !== undefined) {\n requestInfo.responseSize = Number(requestInfo.responseSize);\n this.responseSizeSums.set(\n key,\n (this.responseSizeSums.get(key) || 0) + requestInfo.responseSize,\n );\n if (!this.responseSizes.has(key)) {\n this.responseSizes.set(key, new Map<number, number>());\n }\n const responseSizeMap = this.responseSizes.get(key)!;\n const responseSizeKbBin = Math.floor(requestInfo.responseSize / 1000); // Rounded down to nearest KB\n responseSizeMap.set(\n responseSizeKbBin,\n (responseSizeMap.get(responseSizeKbBin) || 0) + 1,\n );\n }\n }\n\n getAndResetRequests() {\n const data: Array<RequestsItem> = [];\n this.requestCounts.forEach((count, key) => {\n const [consumer, method, path, statusCodeStr] = key.split(\"|\");\n const responseTimes =\n this.responseTimes.get(key) || new Map<number, number>();\n const requestSizes =\n this.requestSizes.get(key) || new Map<number, number>();\n const responseSizes =\n this.responseSizes.get(key) || new Map<number, number>();\n data.push({\n consumer: consumer || null,\n method,\n path,\n status_code: parseInt(statusCodeStr),\n request_count: count,\n request_size_sum: this.requestSizeSums.get(key) || 0,\n response_size_sum: this.responseSizeSums.get(key) || 0,\n response_times: Object.fromEntries(responseTimes),\n request_sizes: Object.fromEntries(requestSizes),\n response_sizes: Object.fromEntries(responseSizes),\n });\n });\n\n // Reset the counts and times\n this.requestCounts.clear();\n this.requestSizeSums.clear();\n this.responseSizeSums.clear();\n this.responseTimes.clear();\n this.requestSizes.clear();\n this.responseSizes.clear();\n\n return data;\n }\n}\n","import AsyncLock from \"async-lock\";\nimport { Buffer } from \"buffer\";\nimport { randomUUID } from \"crypto\";\nimport { unlinkSync, writeFileSync } from \"fs\";\nimport { IncomingHttpHeaders, OutgoingHttpHeaders } from \"http\";\nimport { tmpdir } from \"os\";\nimport { join } from \"path\";\n\nimport { getSentryEventId } from \"./sentry.js\";\nimport {\n truncateExceptionMessage,\n truncateExceptionStackTrace,\n} from \"./serverErrorCounter.js\";\nimport TempGzipFile from \"./tempGzipFile.js\";\n\nconst MAX_BODY_SIZE = 50_000; // 50 KB (uncompressed)\nconst MAX_FILE_SIZE = 1_000_000; // 1 MB (compressed)\nconst MAX_FILES = 50;\nconst MAX_PENDING_WRITES = 100;\nconst BODY_TOO_LARGE = Buffer.from(\"<body too large>\");\nconst BODY_MASKED = Buffer.from(\"<masked>\");\nconst MASKED = \"******\";\nconst ALLOWED_CONTENT_TYPES = [\"application/json\", \"text/plain\"];\nconst EXCLUDE_PATH_PATTERNS = [\n /\\/_?healthz?$/i,\n /\\/_?health[_-]?checks?$/i,\n /\\/_?heart[_-]?beats?$/i,\n /\\/ping$/i,\n /\\/ready$/i,\n /\\/live$/i,\n];\nconst EXCLUDE_USER_AGENT_PATTERNS = [\n /health[-_ ]?check/i,\n /microsoft-azure-application-lb/i,\n /googlehc/i,\n /kube-probe/i,\n];\nconst MASK_QUERY_PARAM_PATTERNS = [\n /auth/i,\n /api-?key/i,\n /secret/i,\n /token/i,\n /password/i,\n /pwd/i,\n];\nconst MASK_HEADER_PATTERNS = [\n /auth/i,\n /api-?key/i,\n /secret/i,\n /token/i,\n /cookie/i,\n];\n\nexport type Request = {\n timestamp: number;\n method: string;\n path?: string;\n url: string;\n headers: [string, string][];\n size?: number;\n consumer?: string;\n body?: Buffer;\n};\n\nexport type Response = {\n statusCode: number;\n responseTime: number;\n headers: [string, string][];\n size?: number;\n body?: Buffer;\n};\n\nexport type RequestLoggingConfig = {\n enabled: boolean;\n logQueryParams: boolean;\n logRequestHeaders: boolean;\n logRequestBody: boolean;\n logResponseHeaders: boolean;\n logResponseBody: boolean;\n logException: boolean;\n maskQueryParams: RegExp[];\n maskHeaders: RegExp[];\n maskRequestBodyCallback?: (request: Request) => Buffer | null | undefined;\n maskResponseBodyCallback?: (\n request: Request,\n response: Response,\n ) => Buffer | null | undefined;\n excludePaths: RegExp[];\n excludeCallback?: (request: Request, response: Response) => boolean;\n};\n\nconst DEFAULT_CONFIG: RequestLoggingConfig = {\n enabled: false,\n logQueryParams: true,\n logRequestHeaders: false,\n logRequestBody: false,\n logResponseHeaders: true,\n logResponseBody: false,\n logException: true,\n maskQueryParams: [],\n maskHeaders: [],\n excludePaths: [],\n};\n\nexport default class RequestLogger {\n public config: RequestLoggingConfig;\n public enabled: boolean;\n public suspendUntil: number | null = null;\n private pendingWrites: string[] = [];\n private currentFile: TempGzipFile | null = null;\n private files: TempGzipFile[] = [];\n private maintainIntervalId?: NodeJS.Timeout;\n private lock = new AsyncLock();\n\n constructor(config?: Partial<RequestLoggingConfig>) {\n this.config = { ...DEFAULT_CONFIG, ...config };\n this.enabled = this.config.enabled && checkWritableFs();\n\n if (this.enabled) {\n this.maintainIntervalId = setInterval(() => {\n this.maintain();\n }, 1000);\n }\n }\n\n private shouldExcludePath(urlPath: string) {\n const patterns = [...this.config.excludePaths, ...EXCLUDE_PATH_PATTERNS];\n return matchPatterns(urlPath, patterns);\n }\n\n private shouldExcludeUserAgent(userAgent?: string) {\n return userAgent\n ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS)\n : false;\n }\n\n private shouldMaskQueryParam(name: string) {\n const patterns = [\n ...this.config.maskQueryParams,\n ...MASK_QUERY_PARAM_PATTERNS,\n ];\n return matchPatterns(name, patterns);\n }\n\n private shouldMaskHeader(name: string) {\n const patterns = [...this.config.maskHeaders, ...MASK_HEADER_PATTERNS];\n return matchPatterns(name, patterns);\n }\n\n private hasSupportedContentType(headers: [string, string][]) {\n const contentType = headers.find(\n ([k]) => k.toLowerCase() === \"content-type\",\n )?.[1];\n return this.isSupportedContentType(contentType);\n }\n\n public isSupportedContentType(contentType?: string | null) {\n return (\n typeof contentType === \"string\" &&\n ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t))\n );\n }\n\n private maskQueryParams(search: string) {\n const params = new URLSearchParams(search);\n for (const [key] of params) {\n if (this.shouldMaskQueryParam(key)) {\n params.set(key, MASKED);\n }\n }\n return params.toString();\n }\n\n private maskHeaders(headers: [string, string][]): [string, string][] {\n return headers.map(([k, v]) => [k, this.shouldMaskHeader(k) ? MASKED : v]);\n }\n\n logRequest(request: Request, response: Response, error?: Error) {\n if (!this.enabled || this.suspendUntil !== null) return;\n\n const url = new URL(request.url);\n const path = request.path ?? url.pathname;\n const userAgent = request.headers.find(\n ([k]) => k.toLowerCase() === \"user-agent\",\n )?.[1];\n\n if (\n this.shouldExcludePath(path) ||\n this.shouldExcludeUserAgent(userAgent) ||\n (this.config.excludeCallback?.(request, response) ?? false)\n ) {\n return;\n }\n\n // Process query params\n url.search = this.config.logQueryParams\n ? this.maskQueryParams(url.search)\n : \"\";\n request.url = url.toString();\n\n // Process request body\n if (\n !this.config.logRequestBody ||\n !this.hasSupportedContentType(request.headers)\n ) {\n request.body = undefined;\n } else if (request.body) {\n if (request.body.length > MAX_BODY_SIZE) {\n request.body = BODY_TOO_LARGE;\n } else if (this.config.maskRequestBodyCallback) {\n try {\n request.body =\n this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;\n if (request.body.length > MAX_BODY_SIZE) {\n request.body = BODY_TOO_LARGE;\n }\n } catch {\n request.body = undefined;\n }\n }\n }\n\n // Process response body\n if (\n !this.config.logResponseBody ||\n !this.hasSupportedContentType(response.headers)\n ) {\n response.body = undefined;\n } else if (response.body) {\n if (response.body.length > MAX_BODY_SIZE) {\n response.body = BODY_TOO_LARGE;\n } else if (this.config.maskResponseBodyCallback) {\n try {\n response.body =\n this.config.maskResponseBodyCallback(request, response) ??\n BODY_MASKED;\n if (response.body.length > MAX_BODY_SIZE) {\n response.body = BODY_TOO_LARGE;\n }\n } catch {\n response.body = undefined;\n }\n }\n }\n\n // Process headers\n request.headers = this.config.logRequestHeaders\n ? this.maskHeaders(request.headers)\n : [];\n response.headers = this.config.logResponseHeaders\n ? this.maskHeaders(response.headers)\n : [];\n\n const item = {\n uuid: randomUUID(),\n request: skipEmptyValues(request),\n response: skipEmptyValues(response),\n exception:\n error && this.config.logException\n ? {\n type: error.name,\n message: truncateExceptionMessage(error.message),\n stacktrace: truncateExceptionStackTrace(error.stack || \"\"),\n sentryEventId: getSentryEventId(),\n }\n : null,\n };\n [item.request.body, item.response.body].forEach((body) => {\n if (body) {\n // @ts-expect-error Different return type\n body.toJSON = function () {\n return this.toString(\"base64\");\n };\n }\n });\n this.pendingWrites.push(JSON.stringify(item));\n\n if (this.pendingWrites.length > MAX_PENDING_WRITES) {\n this.pendingWrites.shift();\n }\n }\n\n async writeToFile() {\n if (!this.enabled || this.pendingWrites.length === 0) {\n return;\n }\n return this.lock.acquire(\"file\", async () => {\n if (!this.currentFile) {\n this.currentFile = new TempGzipFile();\n }\n while (this.pendingWrites.length > 0) {\n const item = this.pendingWrites.shift();\n if (item) {\n await this.currentFile.writeLine(Buffer.from(item));\n }\n }\n });\n }\n\n getFile() {\n return this.files.shift();\n }\n\n retryFileLater(file: TempGzipFile) {\n this.files.unshift(file);\n }\n\n async rotateFile() {\n return this.lock.acquire(\"file\", async () => {\n if (this.currentFile) {\n await this.currentFile.close();\n this.files.push(this.currentFile);\n this.currentFile = null;\n }\n });\n }\n\n async maintain() {\n await this.writeToFile();\n if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {\n await this.rotateFile();\n }\n while (this.files.length > MAX_FILES) {\n const file = this.files.shift();\n file?.delete();\n }\n if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {\n this.suspendUntil = null;\n }\n }\n\n async clear() {\n this.pendingWrites = [];\n await this.rotateFile();\n this.files.forEach((file) => {\n file.delete();\n });\n this.files = [];\n }\n\n async close() {\n this.enabled = false;\n await this.clear();\n if (this.maintainIntervalId) {\n clearInterval(this.maintainIntervalId);\n }\n }\n}\n\nexport function convertHeaders(\n headers:\n | Headers\n | IncomingHttpHeaders\n | OutgoingHttpHeaders\n | Record<string, string | string[] | number | undefined>,\n) {\n if (headers instanceof Headers) {\n return Array.from(headers.entries());\n }\n return Object.entries(headers).flatMap(([key, value]) => {\n if (value === undefined) {\n return [];\n }\n if (Array.isArray(value)) {\n return value.map((v) => [key, v]);\n }\n return [[key, value.toString()]];\n }) as [string, string][];\n}\n\nexport function convertBody(body: any, contentType?: string | null) {\n if (!body || !contentType) {\n return;\n }\n try {\n if (contentType.startsWith(\"application/json\")) {\n if (isValidJsonString(body)) {\n return Buffer.from(body);\n } else {\n return Buffer.from(JSON.stringify(body));\n }\n }\n if (contentType.startsWith(\"text/\") && typeof body === \"string\") {\n return Buffer.from(body);\n }\n } catch (error) {\n return;\n }\n}\n\nfunction isValidJsonString(body: any) {\n if (typeof body !== \"string\") {\n return false;\n }\n try {\n JSON.parse(body);\n return true;\n } catch {\n return false;\n }\n}\n\nfunction matchPatterns(value: string, patterns: RegExp[]) {\n return patterns.some((pattern) => {\n return pattern.test(value);\n });\n}\n\nfunction skipEmptyValues<T extends Record<string, any>>(data: T) {\n return Object.fromEntries(\n Object.entries(data).filter(([_, v]) => {\n if (v == null || Number.isNaN(v)) return false;\n if (Array.isArray(v) || Buffer.isBuffer(v) || typeof v === \"string\") {\n return v.length > 0;\n }\n return true;\n }),\n ) as Partial<T>;\n}\n\nfunction checkWritableFs() {\n try {\n const testPath = join(tmpdir(), `apitally-${randomUUID()}`);\n writeFileSync(testPath, \"test\");\n unlinkSync(testPath);\n return true;\n } catch (error) {\n return false;\n }\n}\n","import type * as Sentry from \"@sentry/node\";\n\nlet sentry: typeof Sentry | undefined;\n\n// Initialize Sentry when the module is loaded\n(async () => {\n try {\n sentry = await import(\"@sentry/node\");\n } catch (e) {\n // Sentry SDK is not installed, ignore\n }\n})();\n\n/**\n * Returns the last Sentry event ID if available\n */\nexport function getSentryEventId(): string | undefined {\n if (sentry && sentry.lastEventId) {\n return sentry.lastEventId();\n }\n return undefined;\n}\n","import { createHash } from \"crypto\";\n\nimport { getSentryEventId } from \"./sentry.js\";\nimport { ConsumerMethodPath, ServerError, ServerErrorsItem } from \"./types.js\";\n\nconst MAX_MSG_LENGTH = 2048;\nconst MAX_STACKTRACE_LENGTH = 65536;\n\nexport default class ServerErrorCounter {\n private errorCounts: Map<string, number>;\n private errorDetails: Map<string, ConsumerMethodPath & ServerError>;\n private sentryEventIds: Map<string, string>;\n\n constructor() {\n this.errorCounts = new Map();\n this.errorDetails = new Map();\n this.sentryEventIds = new Map();\n }\n\n public addServerError(serverError: ConsumerMethodPath & ServerError) {\n const key = this.getKey(serverError);\n if (!this.errorDetails.has(key)) {\n this.errorDetails.set(key, serverError);\n }\n this.errorCounts.set(key, (this.errorCounts.get(key) || 0) + 1);\n\n const sentryEventId = getSentryEventId();\n if (sentryEventId) {\n this.sentryEventIds.set(key, sentryEventId);\n }\n }\n\n public getAndResetServerErrors() {\n const data: Array<ServerErrorsItem> = [];\n this.errorCounts.forEach((count, key) => {\n const serverError = this.errorDetails.get(key);\n if (serverError) {\n data.push({\n consumer: serverError.consumer || null,\n method: serverError.method,\n path: serverError.path,\n type: serverError.type,\n msg: truncateExceptionMessage(serverError.msg),\n traceback: truncateExceptionStackTrace(serverError.traceback),\n sentry_event_id: this.sentryEventIds.get(key) || null,\n error_count: count,\n });\n }\n });\n this.errorCounts.clear();\n this.errorDetails.clear();\n return data;\n }\n\n private getKey(serverError: ConsumerMethodPath & ServerError) {\n const hashInput = [\n serverError.consumer || \"\",\n serverError.method.toUpperCase(),\n serverError.path,\n serverError.type,\n serverError.msg.trim(),\n serverError.traceback.trim(),\n ].join(\"|\");\n return createHash(\"md5\").update(hashInput).digest(\"hex\");\n }\n}\n\nexport function truncateExceptionMessage(msg: string) {\n if (msg.length <= MAX_MSG_LENGTH) {\n return msg;\n }\n const suffix = \"... (truncated)\";\n const cutoff = MAX_MSG_LENGTH - suffix.length;\n return msg.substring(0, cutoff) + suffix;\n}\n\nexport function truncateExceptionStackTrace(stack: string) {\n const suffix = \"... (truncated) ...\";\n const cutoff = MAX_STACKTRACE_LENGTH - suffix.length;\n const lines = stack.trim().split(\"\\n\");\n const truncatedLines: string[] = [];\n let length = 0;\n for (const line of lines) {\n if (length + line.length + 1 > cutoff) {\n truncatedLines.push(suffix);\n break;\n }\n truncatedLines.push(line);\n length += line.length + 1;\n }\n return truncatedLines.join(\"\\n\");\n}\n","import { Buffer } from \"buffer\";\nimport { randomUUID } from \"crypto\";\nimport { createWriteStream, readFile, unlinkSync, WriteStream } from \"fs\";\nimport { tmpdir } from \"os\";\nimport { join } from \"path\";\nimport { createGzip, Gzip } from \"zlib\";\n\nexport default class TempGzipFile {\n public uuid: string;\n private filePath: string;\n private gzip: Gzip;\n private writeStream: WriteStream;\n private readyPromise: Promise<void>;\n private closedPromise: Promise<void>;\n\n constructor() {\n this.uuid = randomUUID();\n this.filePath = join(tmpdir(), `apitally-${this.uuid}.gz`);\n this.writeStream = createWriteStream(this.filePath);\n this.readyPromise = new Promise<void>((resolve, reject) => {\n this.writeStream.once(\"ready\", resolve);\n this.writeStream.once(\"error\", reject);\n });\n this.closedPromise = new Promise<void>((resolve, reject) => {\n this.writeStream.once(\"close\", resolve);\n this.writeStream.once(\"error\", reject);\n });\n this.gzip = createGzip();\n this.gzip.pipe(this.writeStream);\n }\n\n get size() {\n return this.writeStream.bytesWritten;\n }\n\n async writeLine(data: Buffer) {\n await this.readyPromise;\n return new Promise<void>((resolve, reject) => {\n this.gzip.write(Buffer.concat([data, Buffer.from(\"\\n\")]), (error) => {\n if (error) {\n reject(error);\n } else {\n resolve();\n }\n });\n });\n }\n\n async getContent() {\n return new Promise<Buffer>((resolve, reject) => {\n readFile(this.filePath, (error, data) => {\n if (error) {\n reject(error);\n } else {\n resolve(data);\n }\n });\n });\n }\n\n async close() {\n await new Promise<void>((resolve) => {\n this.gzip.end(() => {\n resolve();\n });\n });\n await this.closedPromise;\n }\n\n async delete() {\n await this.close();\n unlinkSync(this.filePath);\n }\n}\n","import { createHash } from \"crypto\";\n\nimport {\n ConsumerMethodPath,\n ValidationError,\n ValidationErrorsItem,\n} from \"./types.js\";\n\nexport default class ValidationErrorCounter {\n private errorCounts: Map<string, number>;\n private errorDetails: Map<string, ConsumerMethodPath & ValidationError>;\n\n constructor() {\n this.errorCounts = new Map();\n this.errorDetails = new Map();\n }\n\n public addValidationError(\n validationError: ConsumerMethodPath & ValidationError,\n ) {\n const key = this.getKey(validationError);\n if (!this.errorDetails.has(key)) {\n this.errorDetails.set(key, validationError);\n }\n this.errorCounts.set(key, (this.errorCounts.get(key) || 0) + 1);\n }\n\n public getAndResetValidationErrors() {\n const data: Array<ValidationErrorsItem> = [];\n this.errorCounts.forEach((count, key) => {\n const validationError = this.errorDetails.get(key);\n if (validationError) {\n data.push({\n consumer: validationError.consumer || null,\n method: validationError.method,\n path: validationError.path,\n loc: validationError.loc.split(\".\"),\n msg: validationError.msg,\n type: validationError.type,\n error_count: count,\n });\n }\n });\n this.errorCounts.clear();\n this.errorDetails.clear();\n return data;\n }\n\n private getKey(validationError: ConsumerMethodPath & ValidationError) {\n const hashInput = [\n validationError.consumer || \"\",\n validationError.method.toUpperCase(),\n validationError.path,\n validationError.loc,\n validationError.msg.trim(),\n validationError.type,\n ].join(\"|\");\n return createHash(\"md5\").update(hashInput).digest(\"hex\");\n }\n}\n","import { createRequire } from \"module\";\n\nexport function getPackageVersion(name: string): string | null {\n const packageJsonPath = `${name}/package.json`;\n try {\n return require(packageJsonPath).version || null;\n } catch (error) {\n try {\n const _require = createRequire(import.meta.url);\n return _require(packageJsonPath).version || null;\n } catch (error) {\n return null;\n }\n }\n}\n","import { OutgoingHttpHeader } from \"http\";\n\nexport function parseContentLength(\n contentLength: OutgoingHttpHeader | undefined | null,\n): number | undefined {\n if (contentLength === undefined || contentLength === null) {\n return undefined;\n }\n if (typeof contentLength === \"number\") {\n return contentLength;\n }\n if (typeof contentLength === \"string\") {\n const parsed = parseInt(contentLength);\n return isNaN(parsed) ? undefined : parsed;\n }\n if (Array.isArray(contentLength)) {\n return parseContentLength(contentLength[0]);\n }\n return undefined;\n}\n","// Adapted from https://github.com/AlbertoFdzM/express-list-endpoints/blob/305535d43008b46f34e18b01947762e039af6d2d/src/index.js\n// and also incorporated changes from https://github.com/AlbertoFdzM/express-list-endpoints/pull/96\n\n/**\n * @typedef {Object} Route\n * @property {Object} methods\n * @property {string | string[]} path\n * @property {any[]} stack\n *\n * @typedef {Object} Endpoint\n * @property {string} path Path name\n * @property {string[]} methods Methods handled\n * @property {string[]} middlewares Mounted middlewares\n */\n\nconst regExpToParseExpressPathRegExp =\n /^\\/\\^\\\\?\\/?(?:(:?[\\w\\\\.-]*(?:\\\\\\/:?[\\w\\\\.-]*)*)|(\\(\\?:\\\\?\\/?\\([^)]+\\)\\)))\\\\\\/.*/;\nconst regExpToReplaceExpressPathRegExpParams = /\\(\\?:\\\\?\\/?\\([^)]+\\)\\)/;\nconst regexpExpressParamRegexp = /\\(\\?:\\\\?\\\\?\\/?\\([^)]+\\)\\)/g;\nconst regexpExpressPathParamRegexp = /(:[^)]+)\\([^)]+\\)/g;\n\nconst EXPRESS_ROOT_PATH_REGEXP_VALUE = \"/^\\\\/?(?=\\\\/|$)/i\";\nconst STACK_ITEM_VALID_NAMES = [\"router\", \"bound dispatch\", \"mounted_app\"];\n\n/**\n * Returns all the verbs detected for the passed route\n * @param {Route} route\n */\nconst getRouteMethods = function (route) {\n let methods = Object.keys(route.methods);\n\n methods = methods.filter((method) => method !== \"_all\");\n methods = methods.map((method) => method.toUpperCase());\n\n return methods;\n};\n\n/**\n * Returns the names (or anonymous) of all the middlewares attached to the\n * passed route\n * @param {Route} route\n * @returns {string[]}\n */\nconst getRouteMiddlewares = function (route) {\n return route.stack.map((item) => {\n return item.handle.name || \"anonymous\";\n });\n};\n\n/**\n * Returns true if found regexp related with express params\n * @param {string} expressPathRegExp\n * @returns {boolean}\n */\nconst hasParams = function (expressPathRegExp) {\n return regexpExpressParamRegexp.test(expressPathRegExp);\n};\n\n/**\n * @param {Route} route Express route object to be parsed\n * @param {string} basePath The basePath the route is on\n * @return {Endpoint[]} Endpoints info\n */\nconst parseExpressRoute = function (route, basePath) {\n const paths = [];\n\n if (Array.isArray(route.path)) {\n paths.push(...route.path);\n } else {\n paths.push(route.path);\n }\n\n /** @type {Endpoint[]} */\n const endpoints = paths.map((path) => {\n const completePath =\n basePath && path === \"/\" ? basePath : `${basePath}${path}`;\n\n /** @type {Endpoint} */\n const endpoint =