mongodb-rag-core
Version:
Common elements used by MongoDB Chatbot Framework components.
90 lines • 3.87 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getAcquitTestsFromGithubRepo = exports.makeAcquitRequireMdOnGithubDataSource = void 0;
const GitHubDataSource_1 = require("./GitHubDataSource");
const acquit_require_1 = __importDefault(require("acquit-require"));
const acquit_1 = __importDefault(require("acquit"));
const removeMarkdownImagesAndLinks_1 = require("./removeMarkdownImagesAndLinks");
const extractMarkdownH1_1 = require("./extractMarkdownH1");
const logger_1 = require("../logger");
/**
Loads an MD/Acquit docs site from a GitHub repo.
[Acquit](https://www.npmjs.com/package/acquit) is a tool for writing tests in comments,
and then extracting them into a test suite.
This function loads the tests from the repo, and then transforms the document content
to include tests from the test suite in the document.
Acquit is used in the [Mongoose ODM](https://mongoosejs.com/docs) documentation.
This data source assumes that the test files are in the same repo as the docs.
*/
const makeAcquitRequireMdOnGithubDataSource = async ({ name, repoUrl, repoLoaderOptions, pathToPageUrl, testFileLoaderOptions, sourceType, metadata, acquitCodeBlockLanguageReplacement = "", }) => {
let testsPromise;
const getLazyLoadedTests = async () => {
if (!testsPromise) {
logger_1.logger.info("Loading acquit tests from GitHub repo");
testsPromise = getAcquitTestsFromGithubRepo(repoUrl, testFileLoaderOptions);
}
return testsPromise;
};
return (0, GitHubDataSource_1.makeGitHubDataSource)({
name,
repoUrl,
repoLoaderOptions: {
...(repoLoaderOptions ?? {}),
ignoreFiles: [
/^(?!.*\.md$).*/i, // Anything BUT the .md extension
...(repoLoaderOptions?.ignoreFiles ?? []),
],
},
async handleDocumentInRepo(document) {
const tests = await getLazyLoadedTests();
const { source } = document.metadata;
const url = pathToPageUrl(source);
const body = (0, removeMarkdownImagesAndLinks_1.removeMarkdownImagesAndLinks)((0, acquit_require_1.default)(document.pageContent, tests)).replaceAll("```acquit\n", `\`\`\`${acquitCodeBlockLanguageReplacement ?? ""}\n`);
const page = {
body: body,
format: "md",
url,
metadata,
sourceType,
sourceName: name,
};
const h1 = (0, extractMarkdownH1_1.extractMarkdownH1)(page.body);
if (h1) {
page.title = h1;
}
return page;
},
});
};
exports.makeAcquitRequireMdOnGithubDataSource = makeAcquitRequireMdOnGithubDataSource;
async function getAcquitTestsFromGithubRepo(repoUrl, repoLoaderOptions) {
const testFileSource = (0, GitHubDataSource_1.makeGitHubDataSource)({
name: "acquit-tests",
repoUrl,
repoLoaderOptions,
async handleDocumentInRepo(document) {
return {
body: document.pageContent,
url: document.metadata.source,
};
},
});
const pages = await testFileSource.fetchPages();
const tests = pages
.map(({ body, url }) => {
try {
return acquit_1.default.parse(body);
}
catch (error) {
logger_1.logger.warn(`Error parsing acquit tests for file ${url}: ${error.message}`);
return [];
}
})
.flat();
return tests;
}
exports.getAcquitTestsFromGithubRepo = getAcquitTestsFromGithubRepo;
//# sourceMappingURL=AcquitRequireMdOnGithubDataSource.js.map