rag-crawler/dist/preset.js

Crawl a website to generate knowledge file for RAG

const PRESET_LIST = [
    {
        name: "github-repo",
        test: "github.com/([^/]+)/([^/]+)/tree/([^/]+)",
        options: {
            exclude: ["changelog", "changes", "license"],
        },
    },
    {
        name: "github-wiki",
        test: "github.com/([^/]+)/([^/]+)/wiki",
        options: {
            exclude: ["_history"],
            extract: "#wiki-body",
        },
    },
];
export default PRESET_LIST;