mongodb-rag-ingest
Version:
MongoDB Ingest CLI for the MongoDB Chatbot Framework.
48 lines • 2.21 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.doAllCommand = void 0;
const mongodb_rag_core_1 = require("mongodb-rag-core");
const pages_1 = require("./pages");
const embed_1 = require("./embed");
const withConfig_1 = require("../withConfig");
const commandModule = {
command: "all",
builder(args) {
return (0, withConfig_1.withConfigOptions)(args).option("permanentDeletePages", {
boolean: true,
description: "If true, permanently deletes the pages associated with a data source that is not on the list of valid data sources. If false or unspecified, marks the pages as deleted without removing them from the collection.",
});
},
async handler(args) {
return (0, withConfig_1.withConfig)(exports.doAllCommand, {
...args,
doUpdatePagesCommand: pages_1.doUpdatePagesCommand,
permanentlyDeletePages: args.permanentlyDeletePages,
});
},
describe: "Run 'pages' and 'embed' since last successful run",
};
exports.default = commandModule;
const doAllCommand = async (config, { doUpdatePagesCommand, permanentlyDeletePages, }) => {
const { ingestMetaStore } = config;
const lastSuccessfulRunDate = await ingestMetaStore.loadLastSuccessfulRunDate();
mongodb_rag_core_1.logger.info(`Last successful run date: ${lastSuccessfulRunDate}`);
await doUpdatePagesCommand(config, {});
await (0, embed_1.doUpdateEmbedCommand)(config, {
since: lastSuccessfulRunDate ?? new Date("2023-01-01"),
});
// cleanup - delete pages and embedded content that are no longer in the data sources
await config.pageStore.deletePages({
dataSources: config.dataSources.map(({ name }) => name),
permanent: !!permanentlyDeletePages,
inverse: true,
});
await config.embeddedContentStore.deleteEmbeddedContent({
dataSources: config.dataSources.map(({ name }) => name),
inverseDataSources: true,
});
mongodb_rag_core_1.logger.info(`Updating last successful run date`);
await ingestMetaStore.updateLastSuccessfulRunDate();
};
exports.doAllCommand = doAllCommand;
//# sourceMappingURL=all.js.map