llmverify
Version:
AI Output Verification Toolkit — Local-first LLM safety, hallucination detection, PII redaction, prompt injection defense, and runtime monitoring. Zero telemetry. OWASP LLM Top 10 aligned.
93 lines • 11.4 kB
JavaScript
;
/**
* Sentinel Test Suite
*
* Runs all sentinel tests and aggregates results.
*
* @module sentinel/suite
* @author Haiec
* @license MIT
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.runAllSentinelTests = runAllSentinelTests;
const staticEchoTest_1 = require("./staticEchoTest");
const duplicateQueryTest_1 = require("./duplicateQueryTest");
const structuredListTest_1 = require("./structuredListTest");
const shortReasoningTest_1 = require("./shortReasoningTest");
/**
* Runs all sentinel tests and returns aggregated results.
*
* @param config - Sentinel configuration with LLM client
* @param options - Optional configuration for which tests to run
* @returns Aggregated test suite results
*
* @example
* const suite = await runAllSentinelTests({
* client: myLLMClient,
* model: 'gpt-4'
* });
*
* console.log(`Passed ${suite.passedCount}/${suite.totalCount} tests`);
*
* if (!suite.passed) {
* suite.results.filter(r => !r.passed).forEach(r => {
* console.error(`Failed: ${r.test} - ${r.message}`);
* });
* }
*/
async function runAllSentinelTests(config, options) {
const start = Date.now();
const results = [];
const skipTests = new Set(options?.skipTests || []);
// Define test runners
const tests = [
{ name: 'staticEchoTest', run: () => (0, staticEchoTest_1.staticEchoTest)(config) },
{ name: 'duplicateQueryTest', run: () => (0, duplicateQueryTest_1.duplicateQueryTest)(config) },
{ name: 'structuredListTest', run: () => (0, structuredListTest_1.structuredListTest)(config) },
{ name: 'shortReasoningTest', run: () => (0, shortReasoningTest_1.shortReasoningTest)(config) }
];
// Run tests sequentially
for (const test of tests) {
if (skipTests.has(test.name))
continue;
try {
const result = await test.run();
results.push(result);
if (options?.onTestComplete) {
options.onTestComplete(result);
}
}
catch (error) {
const errorResult = {
test: test.name,
passed: false,
message: `Test threw error: ${error instanceof Error ? error.message : 'Unknown'}`,
details: { error: error instanceof Error ? error.message : 'Unknown' },
confidence: 0,
limitations: ['Test failed due to unhandled error']
};
results.push(errorResult);
if (options?.onTestComplete) {
options.onTestComplete(errorResult);
}
}
}
const end = Date.now();
const passedCount = results.filter(r => r.passed).length;
const totalCount = results.length;
const passRate = totalCount > 0 ? passedCount / totalCount : 0;
const passed = passRate >= 0.75; // Pass if 75%+ tests pass
return {
passed,
passedCount,
totalCount,
passRate: Math.round(passRate * 100) / 100,
results,
timestamp: start,
durationMs: end - start,
summary: passed
? `All critical tests passed (${passedCount}/${totalCount})`
: `Some tests failed (${passedCount}/${totalCount} passed)`
};
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3VpdGUuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvc2VudGluZWwvc3VpdGUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7Ozs7OztHQVFHOztBQW1ESCxrREFpRUM7QUFqSEQscURBQWtEO0FBQ2xELDZEQUEwRDtBQUMxRCw2REFBMEQ7QUFDMUQsNkRBQTBEO0FBd0IxRDs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7R0FvQkc7QUFDSSxLQUFLLFVBQVUsbUJBQW1CLENBQ3ZDLE1BQXNCLEVBQ3RCLE9BR0M7SUFFRCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7SUFDekIsTUFBTSxPQUFPLEdBQXlCLEVBQUUsQ0FBQztJQUN6QyxNQUFNLFNBQVMsR0FBRyxJQUFJLEdBQUcsQ0FBQyxPQUFPLEVBQUUsU0FBUyxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBRXBELHNCQUFzQjtJQUN0QixNQUFNLEtBQUssR0FBb0U7UUFDN0UsRUFBRSxJQUFJLEVBQUUsZ0JBQWdCLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBRSxDQUFDLElBQUEsK0JBQWMsRUFBQyxNQUFNLENBQUMsRUFBRTtRQUM3RCxFQUFFLElBQUksRUFBRSxvQkFBb0IsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUMsSUFBQSx1Q0FBa0IsRUFBQyxNQUFNLENBQUMsRUFBRTtRQUNyRSxFQUFFLElBQUksRUFBRSxvQkFBb0IsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUMsSUFBQSx1Q0FBa0IsRUFBQyxNQUFNLENBQUMsRUFBRTtRQUNyRSxFQUFFLElBQUksRUFBRSxvQkFBb0IsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUMsSUFBQSx1Q0FBa0IsRUFBQyxNQUFNLENBQUMsRUFBRTtLQUN0RSxDQUFDO0lBRUYseUJBQXlCO0lBQ3pCLEtBQUssTUFBTSxJQUFJLElBQUksS0FBSyxFQUFFLENBQUM7UUFDekIsSUFBSSxTQUFTLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUM7WUFBRSxTQUFTO1FBRXZDLElBQUksQ0FBQztZQUNILE1BQU0sTUFBTSxHQUFHLE1BQU0sSUFBSSxDQUFDLEdBQUcsRUFBRSxDQUFDO1lBQ2hDLE9BQU8sQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLENBQUM7WUFFckIsSUFBSSxPQUFPLEVBQUUsY0FBYyxFQUFFLENBQUM7Z0JBQzVCLE9BQU8sQ0FBQyxjQUFjLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDakMsQ0FBQztRQUNILENBQUM7UUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1lBQ2YsTUFBTSxXQUFXLEdBQXVCO2dCQUN0QyxJQUFJLEVBQUUsSUFBSSxDQUFDLElBQUk7Z0JBQ2YsTUFBTSxFQUFFLEtBQUs7Z0JBQ2IsT0FBTyxFQUFFLHFCQUFxQixLQUFLLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUU7Z0JBQ2xGLE9BQU8sRUFBRSxFQUFFLEtBQUssRUFBRSxLQUFLLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUU7Z0JBQ3RFLFVBQVUsRUFBRSxDQUFDO2dCQUNiLFdBQVcsRUFBRSxDQUFDLG9DQUFvQyxDQUFDO2FBQ3BELENBQUM7WUFDRixPQUFPLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDO1lBRTFCLElBQUksT0FBTyxFQUFFLGNBQWMsRUFBRSxDQUFDO2dCQUM1QixPQUFPLENBQUMsY0FBYyxDQUFDLFdBQVcsQ0FBQyxDQUFDO1lBQ3RDLENBQUM7UUFDSCxDQUFDO0lBQ0gsQ0FBQztJQUVELE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztJQUN2QixNQUFNLFdBQVcsR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxDQUFDLE1BQU0sQ0FBQztJQUN6RCxNQUFNLFVBQVUsR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDO0lBQ2xDLE1BQU0sUUFBUSxHQUFHLFVBQVUsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLFdBQVcsR0FBRyxVQUFVLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMvRCxNQUFNLE1BQU0sR0FBRyxRQUFRLElBQUksSUFBSSxDQUFDLENBQUMsMEJBQTBCO0lBRTNELE9BQU87UUFDTCxNQUFNO1FBQ04sV0FBVztRQUNYLFVBQVU7UUFDVixRQUFRLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxRQUFRLEdBQUcsR0FBRyxDQUFDLEdBQUcsR0FBRztRQUMxQyxPQUFPO1FBQ1AsU0FBUyxFQUFFLEtBQUs7UUFDaEIsVUFBVSxFQUFFLEdBQUcsR0FBRyxLQUFLO1FBQ3ZCLE9BQU8sRUFBRSxNQUFNO1lBQ2IsQ0FBQyxDQUFDLDhCQUE4QixXQUFXLElBQUksVUFBVSxHQUFHO1lBQzVELENBQUMsQ0FBQyxzQkFBc0IsV0FBVyxJQUFJLFVBQVUsVUFBVTtLQUM5RCxDQUFDO0FBQ0osQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8qKlxuICogU2VudGluZWwgVGVzdCBTdWl0ZVxuICogXG4gKiBSdW5zIGFsbCBzZW50aW5lbCB0ZXN0cyBhbmQgYWdncmVnYXRlcyByZXN1bHRzLlxuICogXG4gKiBAbW9kdWxlIHNlbnRpbmVsL3N1aXRlXG4gKiBAYXV0aG9yIEhhaWVjXG4gKiBAbGljZW5zZSBNSVRcbiAqL1xuXG5pbXBvcnQgeyBTZW50aW5lbFRlc3RSZXN1bHQsIFNlbnRpbmVsQ29uZmlnIH0gZnJvbSAnLi4vdHlwZXMvcnVudGltZSc7XG5pbXBvcnQgeyBzdGF0aWNFY2hvVGVzdCB9IGZyb20gJy4vc3RhdGljRWNob1Rlc3QnO1xuaW1wb3J0IHsgZHVwbGljYXRlUXVlcnlUZXN0IH0gZnJvbSAnLi9kdXBsaWNhdGVRdWVyeVRlc3QnO1xuaW1wb3J0IHsgc3RydWN0dXJlZExpc3RUZXN0IH0gZnJvbSAnLi9zdHJ1Y3R1cmVkTGlzdFRlc3QnO1xuaW1wb3J0IHsgc2hvcnRSZWFzb25pbmdUZXN0IH0gZnJvbSAnLi9zaG9ydFJlYXNvbmluZ1Rlc3QnO1xuXG4vKipcbiAqIEFnZ3JlZ2F0ZWQgc2VudGluZWwgdGVzdCBzdWl0ZSByZXN1bHRzLlxuICovXG5leHBvcnQgaW50ZXJmYWNlIFNlbnRpbmVsU3VpdGUge1xuICAvKiogT3ZlcmFsbCBwYXNzL2ZhaWwgc3RhdHVzICovXG4gIHBhc3NlZDogYm9vbGVhbjtcbiAgLyoqIE51bWJlciBvZiB0ZXN0cyBwYXNzZWQgKi9cbiAgcGFzc2VkQ291bnQ6IG51bWJlcjtcbiAgLyoqIFRvdGFsIG51bWJlciBvZiB0ZXN0cyAqL1xuICB0b3RhbENvdW50OiBudW1iZXI7XG4gIC8qKiBQYXNzIHJhdGUgKDAtMSkgKi9cbiAgcGFzc1JhdGU6IG51bWJlcjtcbiAgLyoqIEluZGl2aWR1YWwgdGVzdCByZXN1bHRzICovXG4gIHJlc3VsdHM6IFNlbnRpbmVsVGVzdFJlc3VsdFtdO1xuICAvKiogVGltZXN0YW1wIG9mIHRlc3QgcnVuICovXG4gIHRpbWVzdGFtcDogbnVtYmVyO1xuICAvKiogRHVyYXRpb24gaW4gbWlsbGlzZWNvbmRzICovXG4gIGR1cmF0aW9uTXM6IG51bWJlcjtcbiAgLyoqIFN1bW1hcnkgbWVzc2FnZSAqL1xuICBzdW1tYXJ5OiBzdHJpbmc7XG59XG5cbi8qKlxuICogUnVucyBhbGwgc2VudGluZWwgdGVzdHMgYW5kIHJldHVybnMgYWdncmVnYXRlZCByZXN1bHRzLlxuICogXG4gKiBAcGFyYW0gY29uZmlnIC0gU2VudGluZWwgY29uZmlndXJhdGlvbiB3aXRoIExMTSBjbGllbnRcbiAqIEBwYXJhbSBvcHRpb25zIC0gT3B0aW9uYWwgY29uZmlndXJhdGlvbiBmb3Igd2hpY2ggdGVzdHMgdG8gcnVuXG4gKiBAcmV0dXJucyBBZ2dyZWdhdGVkIHRlc3Qgc3VpdGUgcmVzdWx0c1xuICogXG4gKiBAZXhhbXBsZVxuICogY29uc3Qgc3VpdGUgPSBhd2FpdCBydW5BbGxTZW50aW5lbFRlc3RzKHtcbiAqICAgY2xpZW50OiBteUxMTUNsaWVudCxcbiAqICAgbW9kZWw6ICdncHQtNCdcbiAqIH0pO1xuICogXG4gKiBjb25zb2xlLmxvZyhgUGFzc2VkICR7c3VpdGUucGFzc2VkQ291bnR9LyR7c3VpdGUudG90YWxDb3VudH0gdGVzdHNgKTtcbiAqIFxuICogaWYgKCFzdWl0ZS5wYXNzZWQpIHtcbiAqICAgc3VpdGUucmVzdWx0cy5maWx0ZXIociA9PiAhci5wYXNzZWQpLmZvckVhY2gociA9PiB7XG4gKiAgICAgY29uc29sZS5lcnJvcihgRmFpbGVkOiAke3IudGVzdH0gLSAke3IubWVzc2FnZX1gKTtcbiAqICAgfSk7XG4gKiB9XG4gKi9cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBydW5BbGxTZW50aW5lbFRlc3RzKFxuICBjb25maWc6IFNlbnRpbmVsQ29uZmlnLFxuICBvcHRpb25zPzoge1xuICAgIHNraXBUZXN0cz86IHN0cmluZ1tdO1xuICAgIG9uVGVzdENvbXBsZXRlPzogKHJlc3VsdDogU2VudGluZWxUZXN0UmVzdWx0KSA9PiB2b2lkO1xuICB9XG4pOiBQcm9taXNlPFNlbnRpbmVsU3VpdGU+IHtcbiAgY29uc3Qgc3RhcnQgPSBEYXRlLm5vdygpO1xuICBjb25zdCByZXN1bHRzOiBTZW50aW5lbFRlc3RSZXN1bHRbXSA9IFtdO1xuICBjb25zdCBza2lwVGVzdHMgPSBuZXcgU2V0KG9wdGlvbnM/LnNraXBUZXN0cyB8fCBbXSk7XG5cbiAgLy8gRGVmaW5lIHRlc3QgcnVubmVyc1xuICBjb25zdCB0ZXN0czogQXJyYXk8eyBuYW1lOiBzdHJpbmc7IHJ1bjogKCkgPT4gUHJvbWlzZTxTZW50aW5lbFRlc3RSZXN1bHQ+IH0+ID0gW1xuICAgIHsgbmFtZTogJ3N0YXRpY0VjaG9UZXN0JywgcnVuOiAoKSA9PiBzdGF0aWNFY2hvVGVzdChjb25maWcpIH0sXG4gICAgeyBuYW1lOiAnZHVwbGljYXRlUXVlcnlUZXN0JywgcnVuOiAoKSA9PiBkdXBsaWNhdGVRdWVyeVRlc3QoY29uZmlnKSB9LFxuICAgIHsgbmFtZTogJ3N0cnVjdHVyZWRMaXN0VGVzdCcsIHJ1bjogKCkgPT4gc3RydWN0dXJlZExpc3RUZXN0KGNvbmZpZykgfSxcbiAgICB7IG5hbWU6ICdzaG9ydFJlYXNvbmluZ1Rlc3QnLCBydW46ICgpID0+IHNob3J0UmVhc29uaW5nVGVzdChjb25maWcpIH1cbiAgXTtcblxuICAvLyBSdW4gdGVzdHMgc2VxdWVudGlhbGx5XG4gIGZvciAoY29uc3QgdGVzdCBvZiB0ZXN0cykge1xuICAgIGlmIChza2lwVGVzdHMuaGFzKHRlc3QubmFtZSkpIGNvbnRpbnVlO1xuXG4gICAgdHJ5IHtcbiAgICAgIGNvbnN0IHJlc3VsdCA9IGF3YWl0IHRlc3QucnVuKCk7XG4gICAgICByZXN1bHRzLnB1c2gocmVzdWx0KTtcbiAgICAgIFxuICAgICAgaWYgKG9wdGlvbnM/Lm9uVGVzdENvbXBsZXRlKSB7XG4gICAgICAgIG9wdGlvbnMub25UZXN0Q29tcGxldGUocmVzdWx0KTtcbiAgICAgIH1cbiAgICB9IGNhdGNoIChlcnJvcikge1xuICAgICAgY29uc3QgZXJyb3JSZXN1bHQ6IFNlbnRpbmVsVGVzdFJlc3VsdCA9IHtcbiAgICAgICAgdGVzdDogdGVzdC5uYW1lLFxuICAgICAgICBwYXNzZWQ6IGZhbHNlLFxuICAgICAgICBtZXNzYWdlOiBgVGVzdCB0aHJldyBlcnJvcjogJHtlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6ICdVbmtub3duJ31gLFxuICAgICAgICBkZXRhaWxzOiB7IGVycm9yOiBlcnJvciBpbnN0YW5jZW9mIEVycm9yID8gZXJyb3IubWVzc2FnZSA6ICdVbmtub3duJyB9LFxuICAgICAgICBjb25maWRlbmNlOiAwLFxuICAgICAgICBsaW1pdGF0aW9uczogWydUZXN0IGZhaWxlZCBkdWUgdG8gdW5oYW5kbGVkIGVycm9yJ11cbiAgICAgIH07XG4gICAgICByZXN1bHRzLnB1c2goZXJyb3JSZXN1bHQpO1xuICAgICAgXG4gICAgICBpZiAob3B0aW9ucz8ub25UZXN0Q29tcGxldGUpIHtcbiAgICAgICAgb3B0aW9ucy5vblRlc3RDb21wbGV0ZShlcnJvclJlc3VsdCk7XG4gICAgICB9XG4gICAgfVxuICB9XG5cbiAgY29uc3QgZW5kID0gRGF0ZS5ub3coKTtcbiAgY29uc3QgcGFzc2VkQ291bnQgPSByZXN1bHRzLmZpbHRlcihyID0+IHIucGFzc2VkKS5sZW5ndGg7XG4gIGNvbnN0IHRvdGFsQ291bnQgPSByZXN1bHRzLmxlbmd0aDtcbiAgY29uc3QgcGFzc1JhdGUgPSB0b3RhbENvdW50ID4gMCA/IHBhc3NlZENvdW50IC8gdG90YWxDb3VudCA6IDA7XG4gIGNvbnN0IHBhc3NlZCA9IHBhc3NSYXRlID49IDAuNzU7IC8vIFBhc3MgaWYgNzUlKyB0ZXN0cyBwYXNzXG5cbiAgcmV0dXJuIHtcbiAgICBwYXNzZWQsXG4gICAgcGFzc2VkQ291bnQsXG4gICAgdG90YWxDb3VudCxcbiAgICBwYXNzUmF0ZTogTWF0aC5yb3VuZChwYXNzUmF0ZSAqIDEwMCkgLyAxMDAsXG4gICAgcmVzdWx0cyxcbiAgICB0aW1lc3RhbXA6IHN0YXJ0LFxuICAgIGR1cmF0aW9uTXM6IGVuZCAtIHN0YXJ0LFxuICAgIHN1bW1hcnk6IHBhc3NlZFxuICAgICAgPyBgQWxsIGNyaXRpY2FsIHRlc3RzIHBhc3NlZCAoJHtwYXNzZWRDb3VudH0vJHt0b3RhbENvdW50fSlgXG4gICAgICA6IGBTb21lIHRlc3RzIGZhaWxlZCAoJHtwYXNzZWRDb3VudH0vJHt0b3RhbENvdW50fSBwYXNzZWQpYFxuICB9O1xufVxuIl19