mongodb-rag-core
Version:
Common elements used by MongoDB Chatbot Framework components.
52 lines • 1.97 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.makeGitHubDataSource = void 0;
const GitDataSource_1 = require("./GitDataSource");
/**
Loads an arbitrary GitHub repo and converts its contents into pages.
*/
const makeGitHubDataSource = ({ name, repoUrl, filter, repoLoaderOptions, handleDocumentInRepo, }) => {
const repoOptions = {};
const { branch, recursive } = repoLoaderOptions ?? {};
if (branch) {
repoOptions["--branch"] = branch;
}
if (recursive) {
repoOptions["--recursive"] = `${recursive}`;
}
// Langchain GitHubRepoLoader consumes a lot of API requests and quickly hits
// the limit. This is now using makeGitDataSource as a drop-in replacement.
return (0, GitDataSource_1.makeGitDataSource)({
name,
repoUri: repoUrl,
repoOptions,
async handlePage(path, pageContent) {
return handleDocumentInRepo({ pageContent, metadata: { source: path } });
},
filter: filter ??
function (path) {
if (repoLoaderOptions === undefined) {
return true;
}
const { ignorePaths, ignoreFiles } = repoLoaderOptions;
for (const ignorePath of ignorePaths ?? []) {
if (path === ignorePath) {
return false;
}
}
for (const ignoreFile of ignoreFiles ?? []) {
if (typeof ignoreFile === "string") {
if (path === ignoreFile) {
return false;
}
}
else if (ignoreFile.test(path)) {
return false;
}
}
return true;
},
});
};
exports.makeGitHubDataSource = makeGitHubDataSource;
//# sourceMappingURL=GitHubDataSource.js.map