apitally
Version:
Simple API monitoring & analytics for REST APIs built with Express, Fastify, NestJS, AdonisJS, Hono, H3, Elysia, Hapi, and Koa.
463 lines • 13.3 kB
JavaScript
var __defProp = Object.defineProperty;
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
import AsyncLock from "async-lock";
import { Buffer as Buffer2 } from "node:buffer";
import { randomUUID } from "node:crypto";
import { unlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { getSentryEventId } from "./sentry.js";
import { truncateExceptionMessage, truncateExceptionStackTrace } from "./serverErrorCounter.js";
import TempGzipFile from "./tempGzipFile.js";
const MAX_BODY_SIZE = 5e4;
const MAX_FILE_SIZE = 1e6;
const MAX_FILES = 50;
const MAX_PENDING_WRITES = 100;
const MAX_LOG_MSG_LENGTH = 2048;
const BODY_TOO_LARGE = Buffer2.from("<body too large>");
const BODY_MASKED = Buffer2.from("<masked>");
const MASKED = "******";
const ALLOWED_CONTENT_TYPES = [
"application/json",
"application/ld+json",
"application/problem+json",
"application/vnd.api+json",
"application/x-ndjson",
"text/plain",
"text/html"
];
const EXCLUDE_PATH_PATTERNS = [
/\/_?healthz?$/i,
/\/_?health[_-]?checks?$/i,
/\/_?heart[_-]?beats?$/i,
/\/ping$/i,
/\/ready$/i,
/\/live$/i
];
const EXCLUDE_USER_AGENT_PATTERNS = [
/health[-_ ]?check/i,
/microsoft-azure-application-lb/i,
/googlehc/i,
/kube-probe/i
];
const MASK_QUERY_PARAM_PATTERNS = [
/auth/i,
/api-?key/i,
/secret/i,
/token/i,
/password/i,
/pwd/i
];
const MASK_HEADER_PATTERNS = [
/auth/i,
/api-?key/i,
/secret/i,
/token/i,
/cookie/i
];
const MASK_BODY_FIELD_PATTERNS = [
/password/i,
/pwd/i,
/token/i,
/secret/i,
/auth/i,
/card[-_ ]?number/i,
/ccv/i,
/ssn/i
];
const DEFAULT_CONFIG = {
enabled: false,
logQueryParams: true,
logRequestHeaders: false,
logRequestBody: false,
logResponseHeaders: true,
logResponseBody: false,
logException: true,
captureLogs: false,
maskQueryParams: [],
maskHeaders: [],
maskBodyFields: [],
excludePaths: []
};
const _RequestLogger = class _RequestLogger {
config;
enabled;
suspendUntil = null;
pendingWrites = [];
currentFile = null;
files = [];
maintainIntervalId;
lock = new AsyncLock();
constructor(config) {
this.config = {
...DEFAULT_CONFIG,
...config
};
this.enabled = this.config.enabled && checkWritableFs();
if (this.enabled) {
this.maintainIntervalId = setInterval(() => {
this.maintain();
}, 1e3);
}
}
get maxBodySize() {
return MAX_BODY_SIZE;
}
shouldExcludePath(urlPath) {
const patterns = [
...this.config.excludePaths,
...EXCLUDE_PATH_PATTERNS
];
return matchPatterns(urlPath, patterns);
}
shouldExcludeUserAgent(userAgent) {
return userAgent ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS) : false;
}
shouldMaskQueryParam(name) {
const patterns = [
...this.config.maskQueryParams,
...MASK_QUERY_PARAM_PATTERNS
];
return matchPatterns(name, patterns);
}
shouldMaskHeader(name) {
const patterns = [
...this.config.maskHeaders,
...MASK_HEADER_PATTERNS
];
return matchPatterns(name, patterns);
}
shouldMaskBodyField(name) {
const patterns = [
...this.config.maskBodyFields,
...MASK_BODY_FIELD_PATTERNS
];
return matchPatterns(name, patterns);
}
getContentType(headers) {
var _a;
return (_a = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a[1];
}
hasSupportedContentType(headers) {
const contentType = this.getContentType(headers);
return this.isSupportedContentType(contentType);
}
isSupportedContentType(contentType) {
return typeof contentType === "string" && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
}
maskQueryParams(search) {
const params = new URLSearchParams(search);
for (const [key] of params) {
if (this.shouldMaskQueryParam(key)) {
params.set(key, MASKED);
}
}
return params.toString();
}
maskHeaders(headers) {
return headers.map(([k, v]) => [
k,
this.shouldMaskHeader(k) ? MASKED : v
]);
}
maskBody(data) {
if (typeof data === "object" && data !== null && !Array.isArray(data)) {
const result = {};
for (const [key, value] of Object.entries(data)) {
if (typeof value === "string" && this.shouldMaskBodyField(key)) {
result[key] = MASKED;
} else {
result[key] = this.maskBody(value);
}
}
return result;
}
if (Array.isArray(data)) {
return data.map((item) => this.maskBody(item));
}
return data;
}
applyMasking(item) {
if (this.config.maskRequestBodyCallback && item.request.body && item.request.body !== BODY_TOO_LARGE) {
try {
const maskedBody = this.config.maskRequestBodyCallback(item.request);
item.request.body = maskedBody ?? BODY_MASKED;
} catch {
item.request.body = void 0;
}
}
if (this.config.maskResponseBodyCallback && item.response.body && item.response.body !== BODY_TOO_LARGE) {
try {
const maskedBody = this.config.maskResponseBodyCallback(item.request, item.response);
item.response.body = maskedBody ?? BODY_MASKED;
} catch {
item.response.body = void 0;
}
}
if (item.request.body && item.request.body.length > MAX_BODY_SIZE) {
item.request.body = BODY_TOO_LARGE;
}
if (item.response.body && item.response.body.length > MAX_BODY_SIZE) {
item.response.body = BODY_TOO_LARGE;
}
for (const key of [
"request",
"response"
]) {
const bodyData = item[key].body;
if (!bodyData || bodyData === BODY_TOO_LARGE || bodyData === BODY_MASKED) {
continue;
}
try {
const contentType = this.getContentType(item[key].headers);
if (!contentType || /\bjson\b/i.test(contentType)) {
const parsedBody = JSON.parse(bodyData.toString());
const maskedBody = this.maskBody(parsedBody);
item[key].body = Buffer2.from(JSON.stringify(maskedBody));
} else if (/\bndjson\b/i.test(contentType)) {
const lines = bodyData.toString().split("\n").filter((line) => line.trim());
const maskedLines = lines.map((line) => {
try {
const parsed = JSON.parse(line);
const masked = this.maskBody(parsed);
return JSON.stringify(masked);
} catch {
return line;
}
});
item[key].body = Buffer2.from(maskedLines.join("\n"));
}
} catch {
}
}
item.request.headers = this.config.logRequestHeaders ? this.maskHeaders(item.request.headers) : [];
item.response.headers = this.config.logResponseHeaders ? this.maskHeaders(item.response.headers) : [];
const url = new URL(item.request.url);
url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
item.request.url = url.toString();
return item;
}
logRequest(request, response, error, logs) {
var _a, _b, _c;
if (!this.enabled || this.suspendUntil !== null) return;
const url = new URL(request.url);
const path = request.path ?? url.pathname;
const userAgent = (_a = request.headers.find(([k]) => k.toLowerCase() === "user-agent")) == null ? void 0 : _a[1];
if (this.shouldExcludePath(path) || this.shouldExcludeUserAgent(userAgent) || (((_c = (_b = this.config).excludeCallback) == null ? void 0 : _c.call(_b, request, response)) ?? false)) {
return;
}
if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
request.body = void 0;
}
if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
response.body = void 0;
}
if (request.size !== void 0 && request.size < 0) {
request.size = void 0;
}
if (response.size !== void 0 && response.size < 0) {
response.size = void 0;
}
const item = {
uuid: randomUUID(),
request,
response,
exception: error && this.config.logException ? {
type: error.name,
message: truncateExceptionMessage(error.message),
stacktrace: truncateExceptionStackTrace(error.stack || ""),
sentryEventId: getSentryEventId()
} : void 0
};
if (logs && logs.length > 0) {
item.logs = logs.map((log) => ({
timestamp: log.timestamp,
logger: log.logger,
level: log.level,
message: truncateLogMessage(log.message)
}));
}
this.pendingWrites.push(item);
if (this.pendingWrites.length > MAX_PENDING_WRITES) {
this.pendingWrites.shift();
}
}
async writeToFile() {
if (!this.enabled || this.pendingWrites.length === 0) {
return;
}
return this.lock.acquire("file", async () => {
if (!this.currentFile) {
this.currentFile = new TempGzipFile();
}
while (this.pendingWrites.length > 0) {
let item = this.pendingWrites.shift();
if (item) {
item = this.applyMasking(item);
const finalItem = {
uuid: item.uuid,
request: skipEmptyValues(item.request),
response: skipEmptyValues(item.response),
exception: item.exception,
logs: item.logs
};
[
finalItem.request.body,
finalItem.response.body
].forEach((body) => {
if (body) {
body.toJSON = function() {
return this.toString("base64");
};
}
});
await this.currentFile.writeLine(Buffer2.from(JSON.stringify(finalItem)));
}
}
});
}
getFile() {
return this.files.shift();
}
retryFileLater(file) {
this.files.unshift(file);
}
async rotateFile() {
return this.lock.acquire("file", async () => {
if (this.currentFile) {
await this.currentFile.close();
this.files.push(this.currentFile);
this.currentFile = null;
}
});
}
async maintain() {
await this.writeToFile();
if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
await this.rotateFile();
}
while (this.files.length > MAX_FILES) {
const file = this.files.shift();
file == null ? void 0 : file.delete();
}
if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
this.suspendUntil = null;
}
}
async clear() {
this.pendingWrites = [];
await this.rotateFile();
this.files.forEach((file) => {
file.delete();
});
this.files = [];
}
async close() {
this.enabled = false;
await this.clear();
if (this.maintainIntervalId) {
clearInterval(this.maintainIntervalId);
}
}
};
__name(_RequestLogger, "RequestLogger");
let RequestLogger = _RequestLogger;
function convertHeaders(headers) {
if (!headers) {
return [];
}
if (headers instanceof Headers) {
return Array.from(headers.entries());
}
return Object.entries(headers).flatMap(([key, value]) => {
if (value === void 0) {
return [];
}
if (Array.isArray(value)) {
return value.map((v) => [
key,
v
]);
}
return [
[
key,
value.toString()
]
];
});
}
__name(convertHeaders, "convertHeaders");
function convertBody(body, contentType) {
if (!body || !contentType) {
return;
}
try {
if (contentType.startsWith("application/json")) {
if (isValidJsonString(body)) {
return Buffer2.from(body);
} else {
return Buffer2.from(JSON.stringify(body));
}
}
if (contentType.startsWith("text/") && typeof body === "string") {
return Buffer2.from(body);
}
} catch (error) {
return;
}
}
__name(convertBody, "convertBody");
function isValidJsonString(body) {
if (typeof body !== "string") {
return false;
}
try {
JSON.parse(body);
return true;
} catch {
return false;
}
}
__name(isValidJsonString, "isValidJsonString");
function matchPatterns(value, patterns) {
return patterns.some((pattern) => {
return pattern.test(value);
});
}
__name(matchPatterns, "matchPatterns");
function skipEmptyValues(data) {
return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
if (v == null || Number.isNaN(v)) return false;
if (Array.isArray(v) || Buffer2.isBuffer(v) || typeof v === "string") {
return v.length > 0;
}
return true;
}));
}
__name(skipEmptyValues, "skipEmptyValues");
function truncateLogMessage(msg) {
if (msg.length > MAX_LOG_MSG_LENGTH) {
const suffix = "... (truncated)";
return msg.slice(0, MAX_LOG_MSG_LENGTH - suffix.length) + suffix;
}
return msg;
}
__name(truncateLogMessage, "truncateLogMessage");
function checkWritableFs() {
try {
const testPath = join(tmpdir(), `apitally-${randomUUID()}`);
writeFileSync(testPath, "test");
unlinkSync(testPath);
return true;
} catch (error) {
return false;
}
}
__name(checkWritableFs, "checkWritableFs");
export {
convertBody,
convertHeaders,
RequestLogger as default
};
//# sourceMappingURL=requestLogger.js.map