llmverify
Version:
AI Output Verification Toolkit — Local-first LLM safety, hallucination detection, PII redaction, prompt injection defense, and runtime monitoring. Zero telemetry. OWASP LLM Top 10 aligned.
101 lines • 9.63 kB
JavaScript
;
/**
* Token Rate Engine
*
* Monitors tokens-per-second throughput and detects degradation.
* Lower token rates may indicate provider throttling or issues.
*
* WHAT THIS DOES:
* ✅ Calculates tokens per second
* ✅ Compares to baseline throughput
* ✅ Detects throughput degradation
*
* WHAT THIS DOES NOT DO:
* ❌ Predict throughput changes
* ❌ Identify cause of slowdowns
* ❌ Account for response complexity
*
* @module engines/runtime/token-rate
* @author Haiec
* @license MIT
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenRateEngine = TokenRateEngine;
const LIMITATIONS = [
'Baseline-dependent: requires sufficient samples',
'Token count may be estimated if not provided by API',
'Does not account for response complexity',
'Cannot distinguish intentional throttling from issues'
];
/**
* Analyzes token generation rate against baseline.
*
* @param call - The call record to analyze
* @param baseline - Current baseline state
* @param thresholds - Optional custom thresholds
* @returns Engine result with token rate analysis
*
* @example
* const result = TokenRateEngine(callRecord, baseline);
* if (result.status === 'warn') {
* console.log('Token rate below normal');
* }
*/
function TokenRateEngine(call, baseline, thresholds) {
const warnRatio = thresholds?.warnRatio ?? 0.8;
const errorRatio = thresholds?.errorRatio ?? 0.2;
// Calculate tokens per second (avoid division by zero)
const seconds = Math.max(call.latencyMs / 1000, 0.001);
const tps = call.responseTokens / seconds;
// No baseline yet - return healthy
if (!baseline || !baseline.avgTokensPerSecond || baseline.avgTokensPerSecond === 0) {
return {
metric: 'token_rate',
value: 0,
status: 'ok',
details: {
tps: Math.round(tps * 100) / 100,
baselineTps: 0,
ratio: 0,
message: 'Baseline not yet established'
},
limitations: LIMITATIONS
};
}
const ratio = tps / baseline.avgTokensPerSecond;
// Normalize value: 0 at warnRatio, 1 at errorRatio (inverted - lower is worse)
let value;
if (ratio >= warnRatio) {
value = 0;
}
else if (ratio <= errorRatio) {
value = 1;
}
else {
value = (warnRatio - ratio) / (warnRatio - errorRatio);
}
// Determine status
let status;
if (value < 0.3) {
status = 'ok';
}
else if (value < 0.6) {
status = 'warn';
}
else {
status = 'error';
}
return {
metric: 'token_rate',
value,
status,
details: {
tps: Math.round(tps * 100) / 100,
baselineTps: Math.round(baseline.avgTokensPerSecond * 100) / 100,
ratio: Math.round(ratio * 100) / 100,
thresholds: { warnRatio, errorRatio }
},
limitations: LIMITATIONS
};
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidG9rZW4tcmF0ZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9lbmdpbmVzL3J1bnRpbWUvdG9rZW4tcmF0ZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUE7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0FtQkc7O0FBeUJILDBDQThEQztBQW5GRCxNQUFNLFdBQVcsR0FBRztJQUNsQixpREFBaUQ7SUFDakQscURBQXFEO0lBQ3JELDBDQUEwQztJQUMxQyx1REFBdUQ7Q0FDeEQsQ0FBQztBQUVGOzs7Ozs7Ozs7Ozs7O0dBYUc7QUFDSCxTQUFnQixlQUFlLENBQzdCLElBQWdCLEVBQ2hCLFFBQW1ELEVBQ25ELFVBQXdEO0lBRXhELE1BQU0sU0FBUyxHQUFHLFVBQVUsRUFBRSxTQUFTLElBQUksR0FBRyxDQUFDO0lBQy9DLE1BQU0sVUFBVSxHQUFHLFVBQVUsRUFBRSxVQUFVLElBQUksR0FBRyxDQUFDO0lBRWpELHVEQUF1RDtJQUN2RCxNQUFNLE9BQU8sR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBQ3ZELE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxjQUFjLEdBQUcsT0FBTyxDQUFDO0lBRTFDLG1DQUFtQztJQUNuQyxJQUFJLENBQUMsUUFBUSxJQUFJLENBQUMsUUFBUSxDQUFDLGtCQUFrQixJQUFJLFFBQVEsQ0FBQyxrQkFBa0IsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUNuRixPQUFPO1lBQ0wsTUFBTSxFQUFFLFlBQVk7WUFDcEIsS0FBSyxFQUFFLENBQUM7WUFDUixNQUFNLEVBQUUsSUFBSTtZQUNaLE9BQU8sRUFBRTtnQkFDUCxHQUFHLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDLEdBQUcsR0FBRztnQkFDaEMsV0FBVyxFQUFFLENBQUM7Z0JBQ2QsS0FBSyxFQUFFLENBQUM7Z0JBQ1IsT0FBTyxFQUFFLDhCQUE4QjthQUN4QztZQUNELFdBQVcsRUFBRSxXQUFXO1NBQ3pCLENBQUM7SUFDSixDQUFDO0lBRUQsTUFBTSxLQUFLLEdBQUcsR0FBRyxHQUFHLFFBQVEsQ0FBQyxrQkFBa0IsQ0FBQztJQUVoRCwrRUFBK0U7SUFDL0UsSUFBSSxLQUFhLENBQUM7SUFDbEIsSUFBSSxLQUFLLElBQUksU0FBUyxFQUFFLENBQUM7UUFDdkIsS0FBSyxHQUFHLENBQUMsQ0FBQztJQUNaLENBQUM7U0FBTSxJQUFJLEtBQUssSUFBSSxVQUFVLEVBQUUsQ0FBQztRQUMvQixLQUFLLEdBQUcsQ0FBQyxDQUFDO0lBQ1osQ0FBQztTQUFNLENBQUM7UUFDTixLQUFLLEdBQUcsQ0FBQyxTQUFTLEdBQUcsS0FBSyxDQUFDLEdBQUcsQ0FBQyxTQUFTLEdBQUcsVUFBVSxDQUFDLENBQUM7SUFDekQsQ0FBQztJQUVELG1CQUFtQjtJQUNuQixJQUFJLE1BQStCLENBQUM7SUFDcEMsSUFBSSxLQUFLLEdBQUcsR0FBRyxFQUFFLENBQUM7UUFDaEIsTUFBTSxHQUFHLElBQUksQ0FBQztJQUNoQixDQUFDO1NBQU0sSUFBSSxLQUFLLEdBQUcsR0FBRyxFQUFFLENBQUM7UUFDdkIsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUNsQixDQUFDO1NBQU0sQ0FBQztRQUNOLE1BQU0sR0FBRyxPQUFPLENBQUM7SUFDbkIsQ0FBQztJQUVELE9BQU87UUFDTCxNQUFNLEVBQUUsWUFBWTtRQUNwQixLQUFLO1FBQ0wsTUFBTTtRQUNOLE9BQU8sRUFBRTtZQUNQLEdBQUcsRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsR0FBRyxHQUFHLENBQUMsR0FBRyxHQUFHO1lBQ2hDLFdBQVcsRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxrQkFBa0IsR0FBRyxHQUFHLENBQUMsR0FBRyxHQUFHO1lBQ2hFLEtBQUssRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssR0FBRyxHQUFHLENBQUMsR0FBRyxHQUFHO1lBQ3BDLFVBQVUsRUFBRSxFQUFFLFNBQVMsRUFBRSxVQUFVLEVBQUU7U0FDdEM7UUFDRCxXQUFXLEVBQUUsV0FBVztLQUN6QixDQUFDO0FBQ0osQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8qKlxuICogVG9rZW4gUmF0ZSBFbmdpbmVcbiAqIFxuICogTW9uaXRvcnMgdG9rZW5zLXBlci1zZWNvbmQgdGhyb3VnaHB1dCBhbmQgZGV0ZWN0cyBkZWdyYWRhdGlvbi5cbiAqIExvd2VyIHRva2VuIHJhdGVzIG1heSBpbmRpY2F0ZSBwcm92aWRlciB0aHJvdHRsaW5nIG9yIGlzc3Vlcy5cbiAqIFxuICogV0hBVCBUSElTIERPRVM6XG4gKiDinIUgQ2FsY3VsYXRlcyB0b2tlbnMgcGVyIHNlY29uZFxuICog4pyFIENvbXBhcmVzIHRvIGJhc2VsaW5lIHRocm91Z2hwdXRcbiAqIOKchSBEZXRlY3RzIHRocm91Z2hwdXQgZGVncmFkYXRpb25cbiAqIFxuICogV0hBVCBUSElTIERPRVMgTk9UIERPOlxuICog4p2MIFByZWRpY3QgdGhyb3VnaHB1dCBjaGFuZ2VzXG4gKiDinYwgSWRlbnRpZnkgY2F1c2Ugb2Ygc2xvd2Rvd25zXG4gKiDinYwgQWNjb3VudCBmb3IgcmVzcG9uc2UgY29tcGxleGl0eVxuICogXG4gKiBAbW9kdWxlIGVuZ2luZXMvcnVudGltZS90b2tlbi1yYXRlXG4gKiBAYXV0aG9yIEhhaWVjXG4gKiBAbGljZW5zZSBNSVRcbiAqL1xuXG5pbXBvcnQgeyBDYWxsUmVjb3JkLCBFbmdpbmVSZXN1bHQsIEJhc2VsaW5lU3RhdGUgfSBmcm9tICcuLi8uLi90eXBlcy9ydW50aW1lJztcblxuY29uc3QgTElNSVRBVElPTlMgPSBbXG4gICdCYXNlbGluZS1kZXBlbmRlbnQ6IHJlcXVpcmVzIHN1ZmZpY2llbnQgc2FtcGxlcycsXG4gICdUb2tlbiBjb3VudCBtYXkgYmUgZXN0aW1hdGVkIGlmIG5vdCBwcm92aWRlZCBieSBBUEknLFxuICAnRG9lcyBub3QgYWNjb3VudCBmb3IgcmVzcG9uc2UgY29tcGxleGl0eScsXG4gICdDYW5ub3QgZGlzdGluZ3Vpc2ggaW50ZW50aW9uYWwgdGhyb3R0bGluZyBmcm9tIGlzc3Vlcydcbl07XG5cbi8qKlxuICogQW5hbHl6ZXMgdG9rZW4gZ2VuZXJhdGlvbiByYXRlIGFnYWluc3QgYmFzZWxpbmUuXG4gKiBcbiAqIEBwYXJhbSBjYWxsIC0gVGhlIGNhbGwgcmVjb3JkIHRvIGFuYWx5emVcbiAqIEBwYXJhbSBiYXNlbGluZSAtIEN1cnJlbnQgYmFzZWxpbmUgc3RhdGVcbiAqIEBwYXJhbSB0aHJlc2hvbGRzIC0gT3B0aW9uYWwgY3VzdG9tIHRocmVzaG9sZHNcbiAqIEByZXR1cm5zIEVuZ2luZSByZXN1bHQgd2l0aCB0b2tlbiByYXRlIGFuYWx5c2lzXG4gKiBcbiAqIEBleGFtcGxlXG4gKiBjb25zdCByZXN1bHQgPSBUb2tlblJhdGVFbmdpbmUoY2FsbFJlY29yZCwgYmFzZWxpbmUpO1xuICogaWYgKHJlc3VsdC5zdGF0dXMgPT09ICd3YXJuJykge1xuICogICBjb25zb2xlLmxvZygnVG9rZW4gcmF0ZSBiZWxvdyBub3JtYWwnKTtcbiAqIH1cbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIFRva2VuUmF0ZUVuZ2luZShcbiAgY2FsbDogQ2FsbFJlY29yZCxcbiAgYmFzZWxpbmU6IFBpY2s8QmFzZWxpbmVTdGF0ZSwgJ2F2Z1Rva2Vuc1BlclNlY29uZCc+LFxuICB0aHJlc2hvbGRzPzogeyB3YXJuUmF0aW8/OiBudW1iZXI7IGVycm9yUmF0aW8/OiBudW1iZXIgfVxuKTogRW5naW5lUmVzdWx0IHtcbiAgY29uc3Qgd2FyblJhdGlvID0gdGhyZXNob2xkcz8ud2FyblJhdGlvID8/IDAuODtcbiAgY29uc3QgZXJyb3JSYXRpbyA9IHRocmVzaG9sZHM/LmVycm9yUmF0aW8gPz8gMC4yO1xuXG4gIC8vIENhbGN1bGF0ZSB0b2tlbnMgcGVyIHNlY29uZCAoYXZvaWQgZGl2aXNpb24gYnkgemVybylcbiAgY29uc3Qgc2Vjb25kcyA9IE1hdGgubWF4KGNhbGwubGF0ZW5jeU1zIC8gMTAwMCwgMC4wMDEpO1xuICBjb25zdCB0cHMgPSBjYWxsLnJlc3BvbnNlVG9rZW5zIC8gc2Vjb25kcztcblxuICAvLyBObyBiYXNlbGluZSB5ZXQgLSByZXR1cm4gaGVhbHRoeVxuICBpZiAoIWJhc2VsaW5lIHx8ICFiYXNlbGluZS5hdmdUb2tlbnNQZXJTZWNvbmQgfHwgYmFzZWxpbmUuYXZnVG9rZW5zUGVyU2Vjb25kID09PSAwKSB7XG4gICAgcmV0dXJuIHtcbiAgICAgIG1ldHJpYzogJ3Rva2VuX3JhdGUnLFxuICAgICAgdmFsdWU6IDAsXG4gICAgICBzdGF0dXM6ICdvaycsXG4gICAgICBkZXRhaWxzOiB7XG4gICAgICAgIHRwczogTWF0aC5yb3VuZCh0cHMgKiAxMDApIC8gMTAwLFxuICAgICAgICBiYXNlbGluZVRwczogMCxcbiAgICAgICAgcmF0aW86IDAsXG4gICAgICAgIG1lc3NhZ2U6ICdCYXNlbGluZSBub3QgeWV0IGVzdGFibGlzaGVkJ1xuICAgICAgfSxcbiAgICAgIGxpbWl0YXRpb25zOiBMSU1JVEFUSU9OU1xuICAgIH07XG4gIH1cblxuICBjb25zdCByYXRpbyA9IHRwcyAvIGJhc2VsaW5lLmF2Z1Rva2Vuc1BlclNlY29uZDtcblxuICAvLyBOb3JtYWxpemUgdmFsdWU6IDAgYXQgd2FyblJhdGlvLCAxIGF0IGVycm9yUmF0aW8gKGludmVydGVkIC0gbG93ZXIgaXMgd29yc2UpXG4gIGxldCB2YWx1ZTogbnVtYmVyO1xuICBpZiAocmF0aW8gPj0gd2FyblJhdGlvKSB7XG4gICAgdmFsdWUgPSAwO1xuICB9IGVsc2UgaWYgKHJhdGlvIDw9IGVycm9yUmF0aW8pIHtcbiAgICB2YWx1ZSA9IDE7XG4gIH0gZWxzZSB7XG4gICAgdmFsdWUgPSAod2FyblJhdGlvIC0gcmF0aW8pIC8gKHdhcm5SYXRpbyAtIGVycm9yUmF0aW8pO1xuICB9XG5cbiAgLy8gRGV0ZXJtaW5lIHN0YXR1c1xuICBsZXQgc3RhdHVzOiAnb2snIHwgJ3dhcm4nIHwgJ2Vycm9yJztcbiAgaWYgKHZhbHVlIDwgMC4zKSB7XG4gICAgc3RhdHVzID0gJ29rJztcbiAgfSBlbHNlIGlmICh2YWx1ZSA8IDAuNikge1xuICAgIHN0YXR1cyA9ICd3YXJuJztcbiAgfSBlbHNlIHtcbiAgICBzdGF0dXMgPSAnZXJyb3InO1xuICB9XG5cbiAgcmV0dXJuIHtcbiAgICBtZXRyaWM6ICd0b2tlbl9yYXRlJyxcbiAgICB2YWx1ZSxcbiAgICBzdGF0dXMsXG4gICAgZGV0YWlsczoge1xuICAgICAgdHBzOiBNYXRoLnJvdW5kKHRwcyAqIDEwMCkgLyAxMDAsXG4gICAgICBiYXNlbGluZVRwczogTWF0aC5yb3VuZChiYXNlbGluZS5hdmdUb2tlbnNQZXJTZWNvbmQgKiAxMDApIC8gMTAwLFxuICAgICAgcmF0aW86IE1hdGgucm91bmQocmF0aW8gKiAxMDApIC8gMTAwLFxuICAgICAgdGhyZXNob2xkczogeyB3YXJuUmF0aW8sIGVycm9yUmF0aW8gfVxuICAgIH0sXG4gICAgbGltaXRhdGlvbnM6IExJTUlUQVRJT05TXG4gIH07XG59XG4iXX0=