llmverify
Version:
AI Output Verification Toolkit — Local-first LLM safety, hallucination detection, PII redaction, prompt injection defense, and runtime monitoring. Zero telemetry. OWASP LLM Top 10 aligned.
109 lines • 11.4 kB
JavaScript
;
/**
* Static Echo Test
*
* Tests if the LLM can accurately echo back a specific phrase.
* This is a basic sanity check for response accuracy.
*
* WHAT THIS TESTS:
* ✅ Basic response capability
* ✅ Instruction following
* ✅ Text reproduction accuracy
*
* LIMITATIONS:
* - Very basic test, may pass even with degraded models
* - Does not test reasoning or complex behavior
* - May fail due to model safety filters
*
* @module sentinel/staticEchoTest
* @author Haiec
* @license MIT
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.staticEchoTest = staticEchoTest;
const LIMITATIONS = [
'Basic test - may pass even with degraded models',
'Does not test reasoning capabilities',
'May fail due to model safety filters on certain phrases',
'Exact match requirement may be too strict for some models'
];
const TEST_PHRASE = 'The quick brown fox jumps over the lazy dog.';
/**
* Tests if the LLM can echo back a specific phrase.
*
* @param config - Sentinel configuration with LLM client
* @returns Test result with pass/fail and details
*
* @example
* const result = await staticEchoTest({
* client: myLLMClient,
* model: 'gpt-4'
* });
*
* if (!result.passed) {
* console.error('Echo test failed:', result.message);
* }
*/
async function staticEchoTest(config) {
const prompt = `Please repeat the following phrase exactly, with no additional text: "${TEST_PHRASE}"`;
try {
const response = await config.client.generate({
prompt,
model: config.model
});
const responseText = response.text.trim();
// Check for exact match
const exactMatch = responseText === TEST_PHRASE;
// Check for contains (more lenient)
const containsPhrase = responseText.includes(TEST_PHRASE);
// Calculate similarity
const similarity = calculateSimilarity(responseText, TEST_PHRASE);
const passed = exactMatch || (containsPhrase && similarity > 0.9);
return {
test: 'staticEchoTest',
passed,
message: passed
? 'LLM correctly echoed the test phrase'
: `LLM response did not match expected phrase. Got: "${responseText.substring(0, 100)}"`,
details: {
expectedPhrase: TEST_PHRASE,
actualResponse: responseText.substring(0, 200),
exactMatch,
containsPhrase,
similarity: Math.round(similarity * 100) / 100
},
confidence: passed ? 0.95 : 0.8,
limitations: LIMITATIONS
};
}
catch (error) {
return {
test: 'staticEchoTest',
passed: false,
message: `Test failed with error: ${error instanceof Error ? error.message : 'Unknown error'}`,
details: {
error: error instanceof Error ? error.message : 'Unknown error'
},
confidence: 0.5,
limitations: [...LIMITATIONS, 'Test failed due to error, not model behavior']
};
}
}
/**
* Calculates simple similarity between two strings.
*/
function calculateSimilarity(a, b) {
const aLower = a.toLowerCase();
const bLower = b.toLowerCase();
if (aLower === bLower)
return 1;
const aWords = new Set(aLower.split(/\s+/));
const bWords = new Set(bLower.split(/\s+/));
let matches = 0;
for (const word of aWords) {
if (bWords.has(word))
matches++;
}
return matches / Math.max(aWords.size, bWords.size);
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RhdGljRWNob1Rlc3QuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvc2VudGluZWwvc3RhdGljRWNob1Rlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBbUJHOztBQTZCSCx3Q0FrREM7QUEzRUQsTUFBTSxXQUFXLEdBQUc7SUFDbEIsaURBQWlEO0lBQ2pELHNDQUFzQztJQUN0Qyx5REFBeUQ7SUFDekQsMkRBQTJEO0NBQzVELENBQUM7QUFFRixNQUFNLFdBQVcsR0FBRyw4Q0FBOEMsQ0FBQztBQUVuRTs7Ozs7Ozs7Ozs7Ozs7O0dBZUc7QUFDSSxLQUFLLFVBQVUsY0FBYyxDQUFDLE1BQXNCO0lBQ3pELE1BQU0sTUFBTSxHQUFHLHlFQUF5RSxXQUFXLEdBQUcsQ0FBQztJQUV2RyxJQUFJLENBQUM7UUFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLE1BQU0sQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDO1lBQzVDLE1BQU07WUFDTixLQUFLLEVBQUUsTUFBTSxDQUFDLEtBQUs7U0FDcEIsQ0FBQyxDQUFDO1FBRUgsTUFBTSxZQUFZLEdBQUcsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUUxQyx3QkFBd0I7UUFDeEIsTUFBTSxVQUFVLEdBQUcsWUFBWSxLQUFLLFdBQVcsQ0FBQztRQUVoRCxvQ0FBb0M7UUFDcEMsTUFBTSxjQUFjLEdBQUcsWUFBWSxDQUFDLFFBQVEsQ0FBQyxXQUFXLENBQUMsQ0FBQztRQUUxRCx1QkFBdUI7UUFDdkIsTUFBTSxVQUFVLEdBQUcsbUJBQW1CLENBQUMsWUFBWSxFQUFFLFdBQVcsQ0FBQyxDQUFDO1FBRWxFLE1BQU0sTUFBTSxHQUFHLFVBQVUsSUFBSSxDQUFDLGNBQWMsSUFBSSxVQUFVLEdBQUcsR0FBRyxDQUFDLENBQUM7UUFFbEUsT0FBTztZQUNMLElBQUksRUFBRSxnQkFBZ0I7WUFDdEIsTUFBTTtZQUNOLE9BQU8sRUFBRSxNQUFNO2dCQUNiLENBQUMsQ0FBQyxzQ0FBc0M7Z0JBQ3hDLENBQUMsQ0FBQyxxREFBcUQsWUFBWSxDQUFDLFNBQVMsQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLEdBQUc7WUFDMUYsT0FBTyxFQUFFO2dCQUNQLGNBQWMsRUFBRSxXQUFXO2dCQUMzQixjQUFjLEVBQUUsWUFBWSxDQUFDLFNBQVMsQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDO2dCQUM5QyxVQUFVO2dCQUNWLGNBQWM7Z0JBQ2QsVUFBVSxFQUFFLElBQUksQ0FBQyxLQUFLLENBQUMsVUFBVSxHQUFHLEdBQUcsQ0FBQyxHQUFHLEdBQUc7YUFDL0M7WUFDRCxVQUFVLEVBQUUsTUFBTSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEdBQUc7WUFDL0IsV0FBVyxFQUFFLFdBQVc7U0FDekIsQ0FBQztJQUNKLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2YsT0FBTztZQUNMLElBQUksRUFBRSxnQkFBZ0I7WUFDdEIsTUFBTSxFQUFFLEtBQUs7WUFDYixPQUFPLEVBQUUsMkJBQTJCLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLGVBQWUsRUFBRTtZQUM5RixPQUFPLEVBQUU7Z0JBQ1AsS0FBSyxFQUFFLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLGVBQWU7YUFDaEU7WUFDRCxVQUFVLEVBQUUsR0FBRztZQUNmLFdBQVcsRUFBRSxDQUFDLEdBQUcsV0FBVyxFQUFFLDhDQUE4QyxDQUFDO1NBQzlFLENBQUM7SUFDSixDQUFDO0FBQ0gsQ0FBQztBQUVEOztHQUVHO0FBQ0gsU0FBUyxtQkFBbUIsQ0FBQyxDQUFTLEVBQUUsQ0FBUztJQUMvQyxNQUFNLE1BQU0sR0FBRyxDQUFDLENBQUMsV0FBVyxFQUFFLENBQUM7SUFDL0IsTUFBTSxNQUFNLEdBQUcsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDO0lBRS9CLElBQUksTUFBTSxLQUFLLE1BQU07UUFBRSxPQUFPLENBQUMsQ0FBQztJQUVoQyxNQUFNLE1BQU0sR0FBRyxJQUFJLEdBQUcsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUM7SUFDNUMsTUFBTSxNQUFNLEdBQUcsSUFBSSxHQUFHLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDO0lBRTVDLElBQUksT0FBTyxHQUFHLENBQUMsQ0FBQztJQUNoQixLQUFLLE1BQU0sSUFBSSxJQUFJLE1BQU0sRUFBRSxDQUFDO1FBQzFCLElBQUksTUFBTSxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUM7WUFBRSxPQUFPLEVBQUUsQ0FBQztJQUNsQyxDQUFDO0lBRUQsT0FBTyxPQUFPLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsSUFBSSxFQUFFLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQztBQUN0RCxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBTdGF0aWMgRWNobyBUZXN0XG4gKiBcbiAqIFRlc3RzIGlmIHRoZSBMTE0gY2FuIGFjY3VyYXRlbHkgZWNobyBiYWNrIGEgc3BlY2lmaWMgcGhyYXNlLlxuICogVGhpcyBpcyBhIGJhc2ljIHNhbml0eSBjaGVjayBmb3IgcmVzcG9uc2UgYWNjdXJhY3kuXG4gKiBcbiAqIFdIQVQgVEhJUyBURVNUUzpcbiAqIOKchSBCYXNpYyByZXNwb25zZSBjYXBhYmlsaXR5XG4gKiDinIUgSW5zdHJ1Y3Rpb24gZm9sbG93aW5nXG4gKiDinIUgVGV4dCByZXByb2R1Y3Rpb24gYWNjdXJhY3lcbiAqIFxuICogTElNSVRBVElPTlM6XG4gKiAtIFZlcnkgYmFzaWMgdGVzdCwgbWF5IHBhc3MgZXZlbiB3aXRoIGRlZ3JhZGVkIG1vZGVsc1xuICogLSBEb2VzIG5vdCB0ZXN0IHJlYXNvbmluZyBvciBjb21wbGV4IGJlaGF2aW9yXG4gKiAtIE1heSBmYWlsIGR1ZSB0byBtb2RlbCBzYWZldHkgZmlsdGVyc1xuICogXG4gKiBAbW9kdWxlIHNlbnRpbmVsL3N0YXRpY0VjaG9UZXN0XG4gKiBAYXV0aG9yIEhhaWVjXG4gKiBAbGljZW5zZSBNSVRcbiAqL1xuXG5pbXBvcnQgeyBTZW50aW5lbFRlc3RSZXN1bHQsIFNlbnRpbmVsQ29uZmlnIH0gZnJvbSAnLi4vdHlwZXMvcnVudGltZSc7XG5cbmNvbnN0IExJTUlUQVRJT05TID0gW1xuICAnQmFzaWMgdGVzdCAtIG1heSBwYXNzIGV2ZW4gd2l0aCBkZWdyYWRlZCBtb2RlbHMnLFxuICAnRG9lcyBub3QgdGVzdCByZWFzb25pbmcgY2FwYWJpbGl0aWVzJyxcbiAgJ01heSBmYWlsIGR1ZSB0byBtb2RlbCBzYWZldHkgZmlsdGVycyBvbiBjZXJ0YWluIHBocmFzZXMnLFxuICAnRXhhY3QgbWF0Y2ggcmVxdWlyZW1lbnQgbWF5IGJlIHRvbyBzdHJpY3QgZm9yIHNvbWUgbW9kZWxzJ1xuXTtcblxuY29uc3QgVEVTVF9QSFJBU0UgPSAnVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZy4nO1xuXG4vKipcbiAqIFRlc3RzIGlmIHRoZSBMTE0gY2FuIGVjaG8gYmFjayBhIHNwZWNpZmljIHBocmFzZS5cbiAqIFxuICogQHBhcmFtIGNvbmZpZyAtIFNlbnRpbmVsIGNvbmZpZ3VyYXRpb24gd2l0aCBMTE0gY2xpZW50XG4gKiBAcmV0dXJucyBUZXN0IHJlc3VsdCB3aXRoIHBhc3MvZmFpbCBhbmQgZGV0YWlsc1xuICogXG4gKiBAZXhhbXBsZVxuICogY29uc3QgcmVzdWx0ID0gYXdhaXQgc3RhdGljRWNob1Rlc3Qoe1xuICogICBjbGllbnQ6IG15TExNQ2xpZW50LFxuICogICBtb2RlbDogJ2dwdC00J1xuICogfSk7XG4gKiBcbiAqIGlmICghcmVzdWx0LnBhc3NlZCkge1xuICogICBjb25zb2xlLmVycm9yKCdFY2hvIHRlc3QgZmFpbGVkOicsIHJlc3VsdC5tZXNzYWdlKTtcbiAqIH1cbiAqL1xuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIHN0YXRpY0VjaG9UZXN0KGNvbmZpZzogU2VudGluZWxDb25maWcpOiBQcm9taXNlPFNlbnRpbmVsVGVzdFJlc3VsdD4ge1xuICBjb25zdCBwcm9tcHQgPSBgUGxlYXNlIHJlcGVhdCB0aGUgZm9sbG93aW5nIHBocmFzZSBleGFjdGx5LCB3aXRoIG5vIGFkZGl0aW9uYWwgdGV4dDogXCIke1RFU1RfUEhSQVNFfVwiYDtcblxuICB0cnkge1xuICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgY29uZmlnLmNsaWVudC5nZW5lcmF0ZSh7XG4gICAgICBwcm9tcHQsXG4gICAgICBtb2RlbDogY29uZmlnLm1vZGVsXG4gICAgfSk7XG5cbiAgICBjb25zdCByZXNwb25zZVRleHQgPSByZXNwb25zZS50ZXh0LnRyaW0oKTtcbiAgICBcbiAgICAvLyBDaGVjayBmb3IgZXhhY3QgbWF0Y2hcbiAgICBjb25zdCBleGFjdE1hdGNoID0gcmVzcG9uc2VUZXh0ID09PSBURVNUX1BIUkFTRTtcbiAgICBcbiAgICAvLyBDaGVjayBmb3IgY29udGFpbnMgKG1vcmUgbGVuaWVudClcbiAgICBjb25zdCBjb250YWluc1BocmFzZSA9IHJlc3BvbnNlVGV4dC5pbmNsdWRlcyhURVNUX1BIUkFTRSk7XG4gICAgXG4gICAgLy8gQ2FsY3VsYXRlIHNpbWlsYXJpdHlcbiAgICBjb25zdCBzaW1pbGFyaXR5ID0gY2FsY3VsYXRlU2ltaWxhcml0eShyZXNwb25zZVRleHQsIFRFU1RfUEhSQVNFKTtcblxuICAgIGNvbnN0IHBhc3NlZCA9IGV4YWN0TWF0Y2ggfHwgKGNvbnRhaW5zUGhyYXNlICYmIHNpbWlsYXJpdHkgPiAwLjkpO1xuXG4gICAgcmV0dXJuIHtcbiAgICAgIHRlc3Q6ICdzdGF0aWNFY2hvVGVzdCcsXG4gICAgICBwYXNzZWQsXG4gICAgICBtZXNzYWdlOiBwYXNzZWQgXG4gICAgICAgID8gJ0xMTSBjb3JyZWN0bHkgZWNob2VkIHRoZSB0ZXN0IHBocmFzZSdcbiAgICAgICAgOiBgTExNIHJlc3BvbnNlIGRpZCBub3QgbWF0Y2ggZXhwZWN0ZWQgcGhyYXNlLiBHb3Q6IFwiJHtyZXNwb25zZVRleHQuc3Vic3RyaW5nKDAsIDEwMCl9XCJgLFxuICAgICAgZGV0YWlsczoge1xuICAgICAgICBleHBlY3RlZFBocmFzZTogVEVTVF9QSFJBU0UsXG4gICAgICAgIGFjdHVhbFJlc3BvbnNlOiByZXNwb25zZVRleHQuc3Vic3RyaW5nKDAsIDIwMCksXG4gICAgICAgIGV4YWN0TWF0Y2gsXG4gICAgICAgIGNvbnRhaW5zUGhyYXNlLFxuICAgICAgICBzaW1pbGFyaXR5OiBNYXRoLnJvdW5kKHNpbWlsYXJpdHkgKiAxMDApIC8gMTAwXG4gICAgICB9LFxuICAgICAgY29uZmlkZW5jZTogcGFzc2VkID8gMC45NSA6IDAuOCxcbiAgICAgIGxpbWl0YXRpb25zOiBMSU1JVEFUSU9OU1xuICAgIH07XG4gIH0gY2F0Y2ggKGVycm9yKSB7XG4gICAgcmV0dXJuIHtcbiAgICAgIHRlc3Q6ICdzdGF0aWNFY2hvVGVzdCcsXG4gICAgICBwYXNzZWQ6IGZhbHNlLFxuICAgICAgbWVzc2FnZTogYFRlc3QgZmFpbGVkIHdpdGggZXJyb3I6ICR7ZXJyb3IgaW5zdGFuY2VvZiBFcnJvciA/IGVycm9yLm1lc3NhZ2UgOiAnVW5rbm93biBlcnJvcid9YCxcbiAgICAgIGRldGFpbHM6IHtcbiAgICAgICAgZXJyb3I6IGVycm9yIGluc3RhbmNlb2YgRXJyb3IgPyBlcnJvci5tZXNzYWdlIDogJ1Vua25vd24gZXJyb3InXG4gICAgICB9LFxuICAgICAgY29uZmlkZW5jZTogMC41LFxuICAgICAgbGltaXRhdGlvbnM6IFsuLi5MSU1JVEFUSU9OUywgJ1Rlc3QgZmFpbGVkIGR1ZSB0byBlcnJvciwgbm90IG1vZGVsIGJlaGF2aW9yJ11cbiAgICB9O1xuICB9XG59XG5cbi8qKlxuICogQ2FsY3VsYXRlcyBzaW1wbGUgc2ltaWxhcml0eSBiZXR3ZWVuIHR3byBzdHJpbmdzLlxuICovXG5mdW5jdGlvbiBjYWxjdWxhdGVTaW1pbGFyaXR5KGE6IHN0cmluZywgYjogc3RyaW5nKTogbnVtYmVyIHtcbiAgY29uc3QgYUxvd2VyID0gYS50b0xvd2VyQ2FzZSgpO1xuICBjb25zdCBiTG93ZXIgPSBiLnRvTG93ZXJDYXNlKCk7XG4gIFxuICBpZiAoYUxvd2VyID09PSBiTG93ZXIpIHJldHVybiAxO1xuICBcbiAgY29uc3QgYVdvcmRzID0gbmV3IFNldChhTG93ZXIuc3BsaXQoL1xccysvKSk7XG4gIGNvbnN0IGJXb3JkcyA9IG5ldyBTZXQoYkxvd2VyLnNwbGl0KC9cXHMrLykpO1xuICBcbiAgbGV0IG1hdGNoZXMgPSAwO1xuICBmb3IgKGNvbnN0IHdvcmQgb2YgYVdvcmRzKSB7XG4gICAgaWYgKGJXb3Jkcy5oYXMod29yZCkpIG1hdGNoZXMrKztcbiAgfVxuICBcbiAgcmV0dXJuIG1hdGNoZXMgLyBNYXRoLm1heChhV29yZHMuc2l6ZSwgYldvcmRzLnNpemUpO1xufVxuIl19