@sema-agent/core/dist/tools/fs/encoding.js

Stateless, task-oriented AI agent core

export function detectFileEncoding(bytes) {
    if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe)
        return { encoding: "utf16le", hadBom: true };
    if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
        return { encoding: "utf8", hadBom: true };
    return { encoding: "utf8", hadBom: false };
}
function detectLineEndings(raw) {
    let crlf = 0;
    let lf = 0;
    for (let i = 0; i < raw.length; i++) {
        if (raw.charCodeAt(i) === 0x0a) {
            if (i > 0 && raw.charCodeAt(i - 1) === 0x0d)
                crlf++;
            else
                lf++;
        }
    }
    return crlf > lf ? "CRLF" : "LF";
}
export function decodeTextBytes(bytes) {
    const encoding = detectFileEncoding(bytes);
    const body = encoding.hadBom ? bytes.subarray(encoding.encoding === "utf16le" ? 2 : 3) : bytes;
    const raw = Buffer.from(body).toString(encoding.encoding);
    const decoded = { text: raw.replaceAll("\r\n", "\n"), encoding, endings: detectLineEndings(raw) };
    if (encoding.encoding === "utf16le" && body.length % 2 !== 0)
        decoded.malformed = true;
    return decoded;
}
export function encodeTextForFile(text, encoding, endings) {
    const out = endings === "CRLF" ? text.replaceAll("\n", "\r\n") : text;
    if (encoding.encoding === "utf16le") {
        const body = Buffer.from(out, "utf16le");
        return Buffer.concat([Buffer.from([0xff, 0xfe]), body]);
    }
    if (encoding.hadBom) {
        return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(out, "utf8")]);
    }
    return out;
}
export function normalizeEditText(s) {
    return s.replaceAll("\r\n", "\n");
}
export function splitLeadingBom(text) {
    return text.charCodeAt(0) === 0xfeff ? { hadBom: true, text: text.slice(1) } : { hadBom: false, text };
}
export function normalizeFileText(s) {
    return normalizeEditText(splitLeadingBom(s).text);
}