scai/dist/pipeline/modules/cleanupModule.js

> **AI-powered CLI for local code analysis, commit message suggestions, and natural-language queries.** > **100% local • No token cost • Private by design • GDPR-friendly** — made in Denmark/EU with ❤️.

// File: src/modules/cleanupModule.ts
import chalk from "chalk";
/** --- Helper: detect top/bottom fluff/noise --- */
function isTopOrBottomNoise(line) {
    const trimmed = line.trim();
    if (!trimmed)
        return true;
    if (/^```(?:\w+)?$/.test(trimmed))
        return true;
    if (/^<!--.*-->$/.test(trimmed))
        return true;
    const lower = trimmed.toLowerCase();
    return [
        /^i\s/i,
        /^here/iu,
        /^this/iu,
        /^the following/iu,
        /^below/iu,
        /^in this/iu,
        /^we have/iu,
        /the code above/iu,
        /ensures that/iu,
        /it handles/iu,
        /used to/iu,
        /note that/iu,
        /example/iu,
        /summary/iu,
        /added comments/iu,
    ].some((pattern) => pattern.test(lower));
}
/** --- Extract first JSON object or array slice --- */
function extractJsonSlice(text) {
    const objStart = text.indexOf("{");
    const objEnd = text.lastIndexOf("}");
    if (objStart !== -1 && objEnd > objStart)
        return text.slice(objStart, objEnd + 1);
    const arrStart = text.indexOf("[");
    const arrEnd = text.lastIndexOf("]");
    if (arrStart !== -1 && arrEnd > arrStart)
        return text.slice(arrStart, arrEnd + 1);
    return null;
}
/** --- Try parsing JSON with multiple fallbacks --- */
function parseJsonWithFallback(content) {
    try {
        return JSON.parse(content);
    }
    catch { }
    const slice = extractJsonSlice(content);
    if (slice) {
        try {
            return JSON.parse(slice);
        }
        catch { }
    }
    return null;
}
/** --- Module --- */
export const cleanupModule = {
    name: "cleanup",
    description: "Cleans up AI output, removes noise, and extracts valid JSON if possible.",
    groups: ["transform"],
    async run(input) {
        // --- Normalize input ---
        let content = typeof input.content === "string"
            ? input.content
            : JSON.stringify(input.content ?? "");
        content = content.replace(/\r\n/g, "\n");
        // --- Trim top/bottom noise ---
        let lines = content.split("\n");
        while (lines.length && isTopOrBottomNoise(lines[0]))
            lines.shift();
        while (lines.length && isTopOrBottomNoise(lines[lines.length - 1]))
            lines.pop();
        content = lines.join("\n").trim();
        // --- Remove fences, HTML comments, thinking tags ---
        content = content
            .replace(/```(?:json)?/gi, "")
            .replace(/```/g, "")
            .replace(/<!--.*?-->/gs, "")
            .replace(/<think>[\s\S]*?<\/think>/gi, "")
            .trim();
        // --- Attempt parsing ---
        let parsed = null;
        if (content.includes("{") || content.includes("[")) {
            parsed = parseJsonWithFallback(content);
        }
        if (parsed !== null) {
            return { query: input.query, content, data: parsed };
        }
        // --- Fallback: return raw cleaned text ---
        // Use console.debug instead of warn so logs are quieter
        process.stdout.write("\r\x1b[K");
        console.debug(chalk.yellow(` - [cleanupModule] Could not parse JSON, returning raw content.`));
        return { query: input.query, content, data: content };
    },
};