@ai2070/l0
Version:
L0: The Missing Reliability Substrate for AI
372 lines • 13.1 kB
JavaScript
import { analyzeNetworkError } from "../utils/errors";
export class L0Monitor {
config;
telemetry;
tokenTimestamps = [];
constructor(config = {}) {
this.config = {
enabled: config.enabled ?? false,
sampleRate: config.sampleRate ?? 1.0,
includeNetworkDetails: config.includeNetworkDetails ?? true,
includeTimings: config.includeTimings ?? true,
metadata: config.metadata,
};
this.telemetry = this.createInitialTelemetry();
}
isEnabled() {
if (!this.config.enabled)
return false;
return Math.random() < this.config.sampleRate;
}
createInitialTelemetry() {
return {
sessionId: this.generateSessionId(),
startTime: Date.now(),
metrics: {
totalTokens: 0,
totalRetries: 0,
networkRetryCount: 0,
modelRetryCount: 0,
},
network: {
errorCount: 0,
errorsByType: {},
errors: this.config.includeNetworkDetails ? [] : undefined,
},
metadata: this.config.metadata,
};
}
generateSessionId() {
return `l0_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
start() {
if (!this.isEnabled())
return;
this.telemetry.startTime = Date.now();
}
complete() {
if (!this.isEnabled())
return;
this.telemetry.endTime = Date.now();
this.telemetry.duration = this.telemetry.endTime - this.telemetry.startTime;
if (this.config.includeTimings && this.tokenTimestamps.length > 0) {
this.calculateTimingMetrics();
}
}
recordToken(timestamp) {
if (!this.isEnabled())
return;
const ts = timestamp ?? Date.now();
this.telemetry.metrics.totalTokens++;
if (this.config.includeTimings) {
this.tokenTimestamps.push(ts);
if (this.telemetry.metrics.totalTokens === 1) {
this.telemetry.metrics.timeToFirstToken = ts - this.telemetry.startTime;
}
}
}
recordNetworkError(error, retried, delay) {
if (!this.isEnabled())
return;
const analysis = analyzeNetworkError(error);
const errorType = analysis.type;
this.telemetry.network.errorCount++;
this.telemetry.network.errorsByType[errorType] =
(this.telemetry.network.errorsByType[errorType] || 0) + 1;
if (this.config.includeNetworkDetails && this.telemetry.network.errors) {
this.telemetry.network.errors.push({
type: errorType,
message: error.message,
timestamp: Date.now(),
retried,
delay,
});
}
}
recordRetry(isNetworkError) {
if (!this.isEnabled())
return;
this.telemetry.metrics.totalRetries++;
if (isNetworkError) {
this.telemetry.metrics.networkRetryCount++;
}
else {
this.telemetry.metrics.modelRetryCount++;
}
}
recordGuardrailViolations(violations) {
if (!this.isEnabled())
return;
if (!this.telemetry.guardrails) {
this.telemetry.guardrails = {
violationCount: 0,
violationsByRule: {},
violationsByRuleAndSeverity: {},
violationsBySeverity: {
warning: 0,
error: 0,
fatal: 0,
},
};
}
for (const violation of violations) {
this.telemetry.guardrails.violationCount++;
this.telemetry.guardrails.violationsByRule[violation.rule] =
(this.telemetry.guardrails.violationsByRule[violation.rule] || 0) + 1;
if (!this.telemetry.guardrails.violationsByRuleAndSeverity[violation.rule]) {
this.telemetry.guardrails.violationsByRuleAndSeverity[violation.rule] =
{
warning: 0,
error: 0,
fatal: 0,
};
}
const ruleSeverity = this.telemetry.guardrails.violationsByRuleAndSeverity[violation.rule];
if (ruleSeverity) {
ruleSeverity[violation.severity]++;
}
this.telemetry.guardrails.violationsBySeverity[violation.severity]++;
}
}
recordDrift(detected, types) {
if (!this.isEnabled())
return;
this.telemetry.drift = {
detected,
types,
};
}
recordContinuation(enabled, used, checkpointContent) {
if (!this.isEnabled())
return;
if (!this.telemetry.continuation) {
this.telemetry.continuation = {
enabled,
used: false,
continuationCount: 0,
};
}
this.telemetry.continuation.enabled = enabled;
if (used) {
this.telemetry.continuation.used = true;
this.telemetry.continuation.continuationCount =
(this.telemetry.continuation.continuationCount || 0) + 1;
if (checkpointContent) {
this.telemetry.continuation.checkpointContent = checkpointContent;
this.telemetry.continuation.checkpointLength = checkpointContent.length;
}
else {
this.telemetry.continuation.checkpointContent = undefined;
this.telemetry.continuation.checkpointLength = undefined;
}
}
}
logEvent(event) {
if (!this.isEnabled())
return;
if (!this.telemetry.metadata) {
this.telemetry.metadata = {};
}
if (!this.telemetry.metadata.customEvents) {
this.telemetry.metadata.customEvents = [];
}
this.telemetry.metadata.customEvents.push({
...event,
timestamp: Date.now(),
});
}
calculateTimingMetrics() {
if (this.tokenTimestamps.length < 2)
return;
const interTokenTimes = [];
for (let i = 1; i < this.tokenTimestamps.length; i++) {
interTokenTimes.push(this.tokenTimestamps[i] - this.tokenTimestamps[i - 1]);
}
if (interTokenTimes.length > 0) {
const sum = interTokenTimes.reduce((a, b) => a + b, 0);
this.telemetry.metrics.avgInterTokenTime = sum / interTokenTimes.length;
}
if (this.telemetry.duration && this.telemetry.duration > 0) {
this.telemetry.metrics.tokensPerSecond =
(this.telemetry.metrics.totalTokens / this.telemetry.duration) * 1000;
}
}
getTelemetry() {
if (!this.isEnabled())
return undefined;
return this.telemetry;
}
toJSON() {
if (!this.isEnabled())
return "{}";
return JSON.stringify(this.telemetry, null, 2);
}
export() {
return this.getTelemetry();
}
getSummary() {
if (!this.isEnabled())
return undefined;
return {
sessionId: this.telemetry.sessionId,
duration: this.telemetry.duration ?? 0,
tokens: this.telemetry.metrics.totalTokens,
tokensPerSecond: this.telemetry.metrics.tokensPerSecond ?? 0,
retries: this.telemetry.metrics.totalRetries,
networkErrors: this.telemetry.network.errorCount,
violations: this.telemetry.guardrails?.violationCount ?? 0,
};
}
getNetworkErrorBreakdown() {
if (!this.isEnabled())
return {};
return { ...this.telemetry.network.errorsByType };
}
hasNetworkErrors() {
if (!this.isEnabled())
return false;
return this.telemetry.network.errorCount > 0;
}
hasViolations() {
if (!this.isEnabled())
return false;
return (this.telemetry.guardrails?.violationCount ?? 0) > 0;
}
getMostCommonNetworkError() {
if (!this.isEnabled() || this.telemetry.network.errorCount === 0) {
return null;
}
let maxCount = 0;
let mostCommon = null;
for (const [type, count] of Object.entries(this.telemetry.network.errorsByType)) {
if (count > maxCount) {
maxCount = count;
mostCommon = type;
}
}
return mostCommon;
}
reset() {
this.telemetry = this.createInitialTelemetry();
this.tokenTimestamps = [];
}
}
export function createMonitor(config) {
return new L0Monitor(config);
}
export class TelemetryExporter {
static toJSON(telemetry) {
return JSON.stringify(telemetry, null, 2);
}
static toCSV(telemetry) {
const lines = [];
lines.push("sessionId,duration,tokens,tokensPerSecond,retries,networkErrors,violations");
const duration = telemetry.duration ?? 0;
const tokens = telemetry.metrics.totalTokens;
const tokensPerSecond = telemetry.metrics.tokensPerSecond ?? 0;
const retries = telemetry.metrics.totalRetries;
const networkErrors = telemetry.network.errorCount;
const violations = telemetry.guardrails?.violationCount ?? 0;
lines.push(`${telemetry.sessionId},${duration},${tokens},${tokensPerSecond.toFixed(2)},${retries},${networkErrors},${violations}`);
return lines.join("\n");
}
static toLogFormat(telemetry) {
return {
session_id: telemetry.sessionId,
timestamp: telemetry.startTime,
duration_ms: telemetry.duration,
metrics: {
tokens: telemetry.metrics.totalTokens,
tokens_per_second: telemetry.metrics.tokensPerSecond,
time_to_first_token_ms: telemetry.metrics.timeToFirstToken,
avg_inter_token_time_ms: telemetry.metrics.avgInterTokenTime,
total_retries: telemetry.metrics.totalRetries,
network_retries: telemetry.metrics.networkRetryCount,
model_retries: telemetry.metrics.modelRetryCount,
},
network: {
error_count: telemetry.network.errorCount,
errors_by_type: telemetry.network.errorsByType,
},
guardrails: telemetry.guardrails
? {
violation_count: telemetry.guardrails.violationCount,
violations_by_severity: telemetry.guardrails.violationsBySeverity,
}
: null,
drift: telemetry.drift,
metadata: telemetry.metadata,
};
}
static toMetrics(telemetry) {
const metrics = [];
const timestamp = telemetry.endTime ?? telemetry.startTime;
const tags = telemetry.metadata
? Object.fromEntries(Object.entries(telemetry.metadata).map(([k, v]) => [k, String(v)]))
: undefined;
if (telemetry.duration !== undefined) {
metrics.push({
name: "l0.duration",
value: telemetry.duration,
timestamp,
tags,
});
}
metrics.push({
name: "l0.tokens.total",
value: telemetry.metrics.totalTokens,
timestamp,
tags,
});
if (telemetry.metrics.tokensPerSecond !== undefined) {
metrics.push({
name: "l0.tokens.per_second",
value: telemetry.metrics.tokensPerSecond,
timestamp,
tags,
});
}
if (telemetry.metrics.timeToFirstToken !== undefined) {
metrics.push({
name: "l0.time_to_first_token",
value: telemetry.metrics.timeToFirstToken,
timestamp,
tags,
});
}
metrics.push({
name: "l0.retries.total",
value: telemetry.metrics.totalRetries,
timestamp,
tags,
});
metrics.push({
name: "l0.retries.network",
value: telemetry.metrics.networkRetryCount,
timestamp,
tags,
});
metrics.push({
name: "l0.retries.model",
value: telemetry.metrics.modelRetryCount,
timestamp,
tags,
});
metrics.push({
name: "l0.network.errors",
value: telemetry.network.errorCount,
timestamp,
tags,
});
if (telemetry.guardrails) {
metrics.push({
name: "l0.guardrails.violations",
value: telemetry.guardrails.violationCount,
timestamp,
tags,
});
}
return metrics;
}
}
//# sourceMappingURL=monitoring.js.map