llmverify
Version:
AI Output Verification Toolkit — Local-first LLM safety, hallucination detection, PII redaction, prompt injection defense, and runtime monitoring. Zero telemetry. OWASP LLM Top 10 aligned.
98 lines • 9.27 kB
JavaScript
;
/**
* Text Processing Utilities
*
* @module utils/text
* @author Haiec
* @license MIT
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.splitSentences = splitSentences;
exports.splitParagraphs = splitParagraphs;
exports.truncate = truncate;
exports.extractContext = extractContext;
exports.normalizeWhitespace = normalizeWhitespace;
exports.countWords = countWords;
exports.containsCodeBlocks = containsCodeBlocks;
exports.removeCodeBlocks = removeCodeBlocks;
exports.isLikelyQuoted = isLikelyQuoted;
/**
* Split text into sentences
*/
function splitSentences(text) {
const sentences = text
.replace(/([.!?])\s+/g, '$1|')
.split('|')
.map(s => s.trim())
.filter(s => s.length > 0);
return sentences;
}
/**
* Split text into paragraphs
*/
function splitParagraphs(text) {
return text
.split(/\n\s*\n/)
.map(p => p.trim())
.filter(p => p.length > 0);
}
/**
* Truncate text to max length
*/
function truncate(text, maxLen) {
if (text.length <= maxLen)
return text;
return text.substring(0, maxLen) + '...';
}
/**
* Extract surrounding context
*/
function extractContext(text, index, windowSize) {
const start = Math.max(0, index - windowSize);
const end = Math.min(text.length, index + windowSize);
let context = text.substring(start, end);
if (start > 0)
context = '...' + context;
if (end < text.length)
context = context + '...';
return context;
}
/**
* Normalize whitespace
*/
function normalizeWhitespace(text) {
return text.replace(/\s+/g, ' ').trim();
}
/**
* Count words in text
*/
function countWords(text) {
return text.split(/\s+/).filter(w => w.length > 0).length;
}
/**
* Check if text contains code blocks
*/
function containsCodeBlocks(text) {
return /```[\s\S]*?```|`[^`]+`/.test(text);
}
/**
* Remove code blocks from text
*/
function removeCodeBlocks(text) {
return text
.replace(/```[\s\S]*?```/g, '')
.replace(/`[^`]+`/g, '');
}
/**
* Check if text is likely quoted/example content
*/
function isLikelyQuoted(text, match) {
const matchIndex = text.indexOf(match);
if (matchIndex === -1)
return false;
const before = text.substring(Math.max(0, matchIndex - 20), matchIndex);
const after = text.substring(matchIndex + match.length, matchIndex + match.length + 20);
const quoteChars = ['"', "'", '`', '\u201C', '\u201D', '\u2018', '\u2019'];
return quoteChars.some(q => before.includes(q) || after.includes(q));
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGV4dC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy91dGlscy90ZXh0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7Ozs7O0dBTUc7O0FBS0gsd0NBUUM7QUFLRCwwQ0FLQztBQUtELDRCQUdDO0FBS0Qsd0NBY0M7QUFLRCxrREFFQztBQUtELGdDQUVDO0FBS0QsZ0RBRUM7QUFLRCw0Q0FJQztBQUtELHdDQWFDO0FBaEdEOztHQUVHO0FBQ0gsU0FBZ0IsY0FBYyxDQUFDLElBQVk7SUFDekMsTUFBTSxTQUFTLEdBQUcsSUFBSTtTQUNuQixPQUFPLENBQUMsYUFBYSxFQUFFLEtBQUssQ0FBQztTQUM3QixLQUFLLENBQUMsR0FBRyxDQUFDO1NBQ1YsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO1NBQ2xCLE1BQU0sQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUM7SUFFN0IsT0FBTyxTQUFTLENBQUM7QUFDbkIsQ0FBQztBQUVEOztHQUVHO0FBQ0gsU0FBZ0IsZUFBZSxDQUFDLElBQVk7SUFDMUMsT0FBTyxJQUFJO1NBQ1IsS0FBSyxDQUFDLFNBQVMsQ0FBQztTQUNoQixHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUM7U0FDbEIsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQztBQUMvQixDQUFDO0FBRUQ7O0dBRUc7QUFDSCxTQUFnQixRQUFRLENBQUMsSUFBWSxFQUFFLE1BQWM7SUFDbkQsSUFBSSxJQUFJLENBQUMsTUFBTSxJQUFJLE1BQU07UUFBRSxPQUFPLElBQUksQ0FBQztJQUN2QyxPQUFPLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxHQUFHLEtBQUssQ0FBQztBQUMzQyxDQUFDO0FBRUQ7O0dBRUc7QUFDSCxTQUFnQixjQUFjLENBQzVCLElBQVksRUFDWixLQUFhLEVBQ2IsVUFBa0I7SUFFbEIsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLEVBQUUsS0FBSyxHQUFHLFVBQVUsQ0FBQyxDQUFDO0lBQzlDLE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sRUFBRSxLQUFLLEdBQUcsVUFBVSxDQUFDLENBQUM7SUFFdEQsSUFBSSxPQUFPLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxLQUFLLEVBQUUsR0FBRyxDQUFDLENBQUM7SUFFekMsSUFBSSxLQUFLLEdBQUcsQ0FBQztRQUFFLE9BQU8sR0FBRyxLQUFLLEdBQUcsT0FBTyxDQUFDO0lBQ3pDLElBQUksR0FBRyxHQUFHLElBQUksQ0FBQyxNQUFNO1FBQUUsT0FBTyxHQUFHLE9BQU8sR0FBRyxLQUFLLENBQUM7SUFFakQsT0FBTyxPQUFPLENBQUM7QUFDakIsQ0FBQztBQUVEOztHQUVHO0FBQ0gsU0FBZ0IsbUJBQW1CLENBQUMsSUFBWTtJQUM5QyxPQUFPLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxFQUFFLEdBQUcsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO0FBQzFDLENBQUM7QUFFRDs7R0FFRztBQUNILFNBQWdCLFVBQVUsQ0FBQyxJQUFZO0lBQ3JDLE9BQU8sSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQztBQUM1RCxDQUFDO0FBRUQ7O0dBRUc7QUFDSCxTQUFnQixrQkFBa0IsQ0FBQyxJQUFZO0lBQzdDLE9BQU8sd0JBQXdCLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0FBQzdDLENBQUM7QUFFRDs7R0FFRztBQUNILFNBQWdCLGdCQUFnQixDQUFDLElBQVk7SUFDM0MsT0FBTyxJQUFJO1NBQ1IsT0FBTyxDQUFDLGlCQUFpQixFQUFFLEVBQUUsQ0FBQztTQUM5QixPQUFPLENBQUMsVUFBVSxFQUFFLEVBQUUsQ0FBQyxDQUFDO0FBQzdCLENBQUM7QUFFRDs7R0FFRztBQUNILFNBQWdCLGNBQWMsQ0FBQyxJQUFZLEVBQUUsS0FBYTtJQUN4RCxNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3ZDLElBQUksVUFBVSxLQUFLLENBQUMsQ0FBQztRQUFFLE9BQU8sS0FBSyxDQUFDO0lBRXBDLE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLEVBQUUsVUFBVSxHQUFHLEVBQUUsQ0FBQyxFQUFFLFVBQVUsQ0FBQyxDQUFDO0lBQ3hFLE1BQU0sS0FBSyxHQUFHLElBQUksQ0FBQyxTQUFTLENBQzFCLFVBQVUsR0FBRyxLQUFLLENBQUMsTUFBTSxFQUN6QixVQUFVLEdBQUcsS0FBSyxDQUFDLE1BQU0sR0FBRyxFQUFFLENBQy9CLENBQUM7SUFFRixNQUFNLFVBQVUsR0FBRyxDQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLFFBQVEsRUFBRSxRQUFRLEVBQUUsUUFBUSxFQUFFLFFBQVEsQ0FBQyxDQUFDO0lBRTNFLE9BQU8sVUFBVSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLE1BQU0sQ0FBQyxRQUFRLENBQUMsQ0FBQyxDQUFDLElBQUksS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ3ZFLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFRleHQgUHJvY2Vzc2luZyBVdGlsaXRpZXNcbiAqIFxuICogQG1vZHVsZSB1dGlscy90ZXh0XG4gKiBAYXV0aG9yIEhhaWVjXG4gKiBAbGljZW5zZSBNSVRcbiAqL1xuXG4vKipcbiAqIFNwbGl0IHRleHQgaW50byBzZW50ZW5jZXNcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIHNwbGl0U2VudGVuY2VzKHRleHQ6IHN0cmluZyk6IHN0cmluZ1tdIHtcbiAgY29uc3Qgc2VudGVuY2VzID0gdGV4dFxuICAgIC5yZXBsYWNlKC8oWy4hP10pXFxzKy9nLCAnJDF8JylcbiAgICAuc3BsaXQoJ3wnKVxuICAgIC5tYXAocyA9PiBzLnRyaW0oKSlcbiAgICAuZmlsdGVyKHMgPT4gcy5sZW5ndGggPiAwKTtcbiAgXG4gIHJldHVybiBzZW50ZW5jZXM7XG59XG5cbi8qKlxuICogU3BsaXQgdGV4dCBpbnRvIHBhcmFncmFwaHNcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIHNwbGl0UGFyYWdyYXBocyh0ZXh0OiBzdHJpbmcpOiBzdHJpbmdbXSB7XG4gIHJldHVybiB0ZXh0XG4gICAgLnNwbGl0KC9cXG5cXHMqXFxuLylcbiAgICAubWFwKHAgPT4gcC50cmltKCkpXG4gICAgLmZpbHRlcihwID0+IHAubGVuZ3RoID4gMCk7XG59XG5cbi8qKlxuICogVHJ1bmNhdGUgdGV4dCB0byBtYXggbGVuZ3RoXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiB0cnVuY2F0ZSh0ZXh0OiBzdHJpbmcsIG1heExlbjogbnVtYmVyKTogc3RyaW5nIHtcbiAgaWYgKHRleHQubGVuZ3RoIDw9IG1heExlbikgcmV0dXJuIHRleHQ7XG4gIHJldHVybiB0ZXh0LnN1YnN0cmluZygwLCBtYXhMZW4pICsgJy4uLic7XG59XG5cbi8qKlxuICogRXh0cmFjdCBzdXJyb3VuZGluZyBjb250ZXh0XG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBleHRyYWN0Q29udGV4dChcbiAgdGV4dDogc3RyaW5nLCBcbiAgaW5kZXg6IG51bWJlciwgXG4gIHdpbmRvd1NpemU6IG51bWJlclxuKTogc3RyaW5nIHtcbiAgY29uc3Qgc3RhcnQgPSBNYXRoLm1heCgwLCBpbmRleCAtIHdpbmRvd1NpemUpO1xuICBjb25zdCBlbmQgPSBNYXRoLm1pbih0ZXh0Lmxlbmd0aCwgaW5kZXggKyB3aW5kb3dTaXplKTtcbiAgXG4gIGxldCBjb250ZXh0ID0gdGV4dC5zdWJzdHJpbmcoc3RhcnQsIGVuZCk7XG4gIFxuICBpZiAoc3RhcnQgPiAwKSBjb250ZXh0ID0gJy4uLicgKyBjb250ZXh0O1xuICBpZiAoZW5kIDwgdGV4dC5sZW5ndGgpIGNvbnRleHQgPSBjb250ZXh0ICsgJy4uLic7XG4gIFxuICByZXR1cm4gY29udGV4dDtcbn1cblxuLyoqXG4gKiBOb3JtYWxpemUgd2hpdGVzcGFjZVxuICovXG5leHBvcnQgZnVuY3Rpb24gbm9ybWFsaXplV2hpdGVzcGFjZSh0ZXh0OiBzdHJpbmcpOiBzdHJpbmcge1xuICByZXR1cm4gdGV4dC5yZXBsYWNlKC9cXHMrL2csICcgJykudHJpbSgpO1xufVxuXG4vKipcbiAqIENvdW50IHdvcmRzIGluIHRleHRcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGNvdW50V29yZHModGV4dDogc3RyaW5nKTogbnVtYmVyIHtcbiAgcmV0dXJuIHRleHQuc3BsaXQoL1xccysvKS5maWx0ZXIodyA9PiB3Lmxlbmd0aCA+IDApLmxlbmd0aDtcbn1cblxuLyoqXG4gKiBDaGVjayBpZiB0ZXh0IGNvbnRhaW5zIGNvZGUgYmxvY2tzXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBjb250YWluc0NvZGVCbG9ja3ModGV4dDogc3RyaW5nKTogYm9vbGVhbiB7XG4gIHJldHVybiAvYGBgW1xcc1xcU10qP2BgYHxgW15gXStgLy50ZXN0KHRleHQpO1xufVxuXG4vKipcbiAqIFJlbW92ZSBjb2RlIGJsb2NrcyBmcm9tIHRleHRcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIHJlbW92ZUNvZGVCbG9ja3ModGV4dDogc3RyaW5nKTogc3RyaW5nIHtcbiAgcmV0dXJuIHRleHRcbiAgICAucmVwbGFjZSgvYGBgW1xcc1xcU10qP2BgYC9nLCAnJylcbiAgICAucmVwbGFjZSgvYFteYF0rYC9nLCAnJyk7XG59XG5cbi8qKlxuICogQ2hlY2sgaWYgdGV4dCBpcyBsaWtlbHkgcXVvdGVkL2V4YW1wbGUgY29udGVudFxuICovXG5leHBvcnQgZnVuY3Rpb24gaXNMaWtlbHlRdW90ZWQodGV4dDogc3RyaW5nLCBtYXRjaDogc3RyaW5nKTogYm9vbGVhbiB7XG4gIGNvbnN0IG1hdGNoSW5kZXggPSB0ZXh0LmluZGV4T2YobWF0Y2gpO1xuICBpZiAobWF0Y2hJbmRleCA9PT0gLTEpIHJldHVybiBmYWxzZTtcbiAgXG4gIGNvbnN0IGJlZm9yZSA9IHRleHQuc3Vic3RyaW5nKE1hdGgubWF4KDAsIG1hdGNoSW5kZXggLSAyMCksIG1hdGNoSW5kZXgpO1xuICBjb25zdCBhZnRlciA9IHRleHQuc3Vic3RyaW5nKFxuICAgIG1hdGNoSW5kZXggKyBtYXRjaC5sZW5ndGgsIFxuICAgIG1hdGNoSW5kZXggKyBtYXRjaC5sZW5ndGggKyAyMFxuICApO1xuICBcbiAgY29uc3QgcXVvdGVDaGFycyA9IFsnXCInLCBcIidcIiwgJ2AnLCAnXFx1MjAxQycsICdcXHUyMDFEJywgJ1xcdTIwMTgnLCAnXFx1MjAxOSddO1xuICBcbiAgcmV0dXJuIHF1b3RlQ2hhcnMuc29tZShxID0+IGJlZm9yZS5pbmNsdWRlcyhxKSB8fCBhZnRlci5pbmNsdWRlcyhxKSk7XG59XG4iXX0=