@genkit-ai/telemetry-server
Version:
Genkit AI telemetry server
224 lines • 8.48 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LogIndex = exports.LocalFileLogStore = void 0;
const utils_1 = require("@genkit-ai/tools-common/utils");
const async_mutex_1 = require("async-mutex");
const crypto_1 = __importDefault(require("crypto"));
const fs_1 = __importDefault(require("fs"));
const lockfile_1 = __importDefault(require("lockfile"));
const path_1 = __importDefault(require("path"));
function lockFile(file) {
return `${file}.lock`;
}
class LocalFileLogStore {
storeRoot;
indexRoot;
index;
mutex = new async_mutex_1.Mutex();
constructor(options) {
this.storeRoot = path_1.default.resolve(options.storeRoot, '.genkit/logs');
fs_1.default.mkdirSync(this.storeRoot, { recursive: true });
this.indexRoot = path_1.default.resolve(options.indexRoot, '.genkit/logs_idx');
fs_1.default.mkdirSync(this.indexRoot, { recursive: true });
utils_1.logger.debug(`[Telemetry Server] initialized local file log store at root: ${this.storeRoot}`);
this.index = new LogIndex(this.indexRoot);
}
async init() {
}
getCurrentLogFile() {
const now = new Date();
const pad = (n) => n.toString().padStart(2, '0');
const dateStr = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}-${pad(now.getHours())}`;
return path_1.default.resolve(this.storeRoot, `logs-${dateStr}.jsonl`);
}
async save(logs) {
if (logs.length === 0)
return;
await this.mutex.runExclusive(async () => {
const logFile = this.getCurrentLogFile();
const relativeFileName = path_1.default.basename(logFile);
let currentOffset = 0;
if (fs_1.default.existsSync(logFile)) {
currentOffset = fs_1.default.statSync(logFile).size;
}
const indexEntries = [];
let offsetTracker = currentOffset;
const linesToAppend = logs.map((log) => {
if (!log.logId) {
log.logId = crypto_1.default.randomUUID();
}
const line = JSON.stringify(log) + '\n';
const length = Buffer.byteLength(line, 'utf8');
indexEntries.push({
traceId: log.traceId,
spanId: log.spanId,
timestamp: log.timestamp,
severityText: log.severityText,
severityNumber: log.severityNumber,
file: relativeFileName,
offset: offsetTracker,
length: length,
});
offsetTracker += length;
return line;
});
fs_1.default.appendFileSync(logFile, linesToAppend.join(''));
await this.index.add(indexEntries);
});
}
async list(query) {
const startFromIndex = query?.continuationToken
? Number.parseInt(query.continuationToken)
: 0;
const limit = query?.limit ?? 100;
const searchResult = this.index.search({
limit,
startFromIndex,
traceId: query?.traceId,
spanId: query?.spanId,
});
const logs = [];
const fdCache = new Map();
try {
for (const entry of searchResult.entries) {
const logFile = path_1.default.resolve(this.storeRoot, entry.file);
let fd = fdCache.get(logFile);
if (fd === undefined) {
if (fs_1.default.existsSync(logFile)) {
try {
fd = fs_1.default.openSync(logFile, 'r');
fdCache.set(logFile, fd);
}
catch (e) {
utils_1.logger.error(`Error opening log file ${logFile}: ${e}`);
continue;
}
}
}
if (fd !== undefined) {
const buffer = Buffer.alloc(entry.length);
try {
fs_1.default.readSync(fd, buffer, 0, entry.length, entry.offset);
logs.push(JSON.parse(buffer.toString('utf8').trim()));
}
catch (e) {
utils_1.logger.error(`Error reading or parsing log at ${entry.file}:${entry.offset}: ${e}`);
}
}
}
}
finally {
for (const fd of fdCache.values()) {
try {
fs_1.default.closeSync(fd);
}
catch (e) {
utils_1.logger.error(`Error closing file descriptor for log file: ${e}`);
}
}
}
return {
logs,
continuationToken: searchResult.pageLastIndex
? `${searchResult.pageLastIndex}`
: undefined,
};
}
}
exports.LocalFileLogStore = LocalFileLogStore;
class LogIndex {
indexRoot;
currentIndexFile;
constructor(indexRoot) {
this.indexRoot = indexRoot;
this.currentIndexFile = path_1.default.resolve(this.indexRoot, this.newIndexFileName());
fs_1.default.mkdirSync(this.indexRoot, { recursive: true });
}
newIndexFileName() {
return `idx_${(Date.now() + '').padStart(17, '0')}.jsonl`;
}
listIndexFiles() {
return fs_1.default.readdirSync(this.indexRoot).filter((f) => f.startsWith('idx_'));
}
async add(entries) {
if (entries.length === 0)
return;
try {
await new Promise((resolve, reject) => {
lockfile_1.default.lock(lockFile(this.currentIndexFile), { wait: 1000, retries: 5, retryWait: 100 }, (err) => {
if (err)
reject(err);
else
resolve();
});
});
const lines = entries.map((e) => JSON.stringify(e) + '\n').join('');
fs_1.default.appendFileSync(this.currentIndexFile, lines);
}
catch (err) {
utils_1.logger.error(`Failed to lock log index file ${this.currentIndexFile}: ${err}`);
}
finally {
if (fs_1.default.existsSync(lockFile(this.currentIndexFile))) {
lockfile_1.default.unlockSync(lockFile(this.currentIndexFile));
}
}
}
search(query) {
const indexFiles = this.listIndexFiles().sort().reverse();
let skipped = 0;
const entries = [];
let hasMore = false;
for (const idxFile of indexFiles) {
if (hasMore)
break;
const idxTxt = fs_1.default.readFileSync(path_1.default.resolve(this.indexRoot, idxFile), 'utf8');
const fileData = idxTxt
.split('\n')
.filter((l) => l.trim().length > 0)
.map((l) => {
try {
return JSON.parse(l);
}
catch {
return undefined;
}
})
.filter((d) => {
if (!d)
return false;
if (query.traceId && d.traceId !== query.traceId)
return false;
if (query.spanId && d.spanId !== query.spanId)
return false;
return true;
});
fileData.sort((a, b) => b.timestamp - a.timestamp);
for (const entry of fileData) {
if (skipped < query.startFromIndex) {
skipped++;
continue;
}
if (entries.length < query.limit) {
entries.push(entry);
}
else {
hasMore = true;
break;
}
}
}
const result = {
entries,
};
if (hasMore) {
result.pageLastIndex = query.startFromIndex + query.limit;
}
return result;
}
}
exports.LogIndex = LogIndex;
//# sourceMappingURL=file-log-store.js.map