@lg-tools/crawler/dist/cli.js

Crawler for MongoDB documentation and other sites

!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t(require("commander"),require("chalk"),require("mongodb"),require("dotenv"),require("lodash/trimEnd"),require("lodash/trimStart"),require("langchain/text_splitter"),require("node:crypto"),require("@langchain/community/document_loaders/web/cheerio"),require("@langchain/core/documents")):"function"==typeof define&&define.amd?define(["commander","chalk","mongodb","dotenv","lodash/trimEnd","lodash/trimStart","langchain/text_splitter","node:crypto","@langchain/community/document_loaders/web/cheerio","@langchain/core/documents"],t):(e="undefined"!=typeof globalThis?globalThis:e||self)["@lg-tools/crawler"]=t(e.commander,e.chalk,e.mongodb,e.dotenv,e["/trimEnd"],e["/trimStart"],e.text_splitter,e.crypto,e.cheerio,e.documents)}(this,(function(e,t,r,n,o,a,c,i,l,u){"use strict";function s(e){return e&&"object"==typeof e&&"default"in e?e:{default:e}}var f=s(t),d=s(n),p=s(o),h=s(a),g=s(i);function v(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function y(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?v(Object(r),!0).forEach((function(t){k(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):v(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function m(){m=function(){return t};var e,t={},r=Object.prototype,n=r.hasOwnProperty,o=Object.defineProperty||function(e,t,r){e[t]=r.value},a="function"==typeof Symbol?Symbol:{},c=a.iterator||"@@iterator",i=a.asyncIterator||"@@asyncIterator",l=a.toStringTag||"@@toStringTag";function u(e,t,r){return Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}),e[t]}try{u({},"")}catch(e){u=function(e,t,r){return e[t]=r}}function s(e,t,r,n){var a=t&&t.prototype instanceof y?t:y,c=Object.create(a.prototype),i=new P(n||[]);return o(c,"_invoke",{value:C(e,r,i)}),c}function f(e,t,r){try{return{type:"normal",arg:e.call(t,r)}}catch(e){return{type:"throw",arg:e}}}t.wrap=s;var d="suspendedStart",p="suspendedYield",h="executing",g="completed",v={};function y(){}function b(){}function w(){}var x={};u(x,c,(function(){return this}));var k=Object.getPrototypeOf,L=k&&k(k(_([])));L&&L!==r&&n.call(L,c)&&(x=L);var O=w.prototype=y.prototype=Object.create(x);function E(e){["next","throw","return"].forEach((function(t){u(e,t,(function(e){return this._invoke(t,e)}))}))}function R(e,t){function r(o,a,c,i){var l=f(e[o],e,a);if("throw"!==l.type){var u=l.arg,s=u.value;return s&&"object"==typeof s&&n.call(s,"__await")?t.resolve(s.__await).then((function(e){r("next",e,c,i)}),(function(e){r("throw",e,c,i)})):t.resolve(s).then((function(e){u.value=e,c(u)}),(function(e){return r("throw",e,c,i)}))}i(l.arg)}var a;o(this,"_invoke",{value:function(e,n){function o(){return new t((function(t,o){r(e,n,t,o)}))}return a=a?a.then(o,o):o()}})}function C(t,r,n){var o=d;return function(a,c){if(o===h)throw Error("Generator is already running");if(o===g){if("throw"===a)throw c;return{value:e,done:!0}}for(n.method=a,n.arg=c;;){var i=n.delegate;if(i){var l=D(i,n);if(l){if(l===v)continue;return l}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(o===d)throw o=g,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);o=h;var u=f(t,r,n);if("normal"===u.type){if(o=n.done?g:p,u.arg===v)continue;return{value:u.arg,done:n.done}}"throw"===u.type&&(o=g,n.method="throw",n.arg=u.arg)}}}function D(t,r){var n=r.method,o=t.iterator[n];if(o===e)return r.delegate=null,"throw"===n&&t.iterator.return&&(r.method="return",r.arg=e,D(t,r),"throw"===r.method)||"return"!==n&&(r.method="throw",r.arg=new TypeError("The iterator does not provide a '"+n+"' method")),v;var a=f(o,t.iterator,r.arg);if("throw"===a.type)return r.method="throw",r.arg=a.arg,r.delegate=null,v;var c=a.arg;return c?c.done?(r[t.resultName]=c.value,r.next=t.nextLoc,"return"!==r.method&&(r.method="next",r.arg=e),r.delegate=null,v):c:(r.method="throw",r.arg=new TypeError("iterator result is not an object"),r.delegate=null,v)}function j(e){var t={tryLoc:e[0]};1 in e&&(t.catchLoc=e[1]),2 in e&&(t.finallyLoc=e[2],t.afterLoc=e[3]),this.tryEntries.push(t)}function S(e){var t=e.completion||{};t.type="normal",delete t.arg,e.completion=t}function P(e){this.tryEntries=[{tryLoc:"root"}],e.forEach(j,this),this.reset(!0)}function _(t){if(t||""===t){var r=t[c];if(r)return r.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var o=-1,a=function r(){for(;++o<t.length;)if(n.call(t,o))return r.value=t[o],r.done=!1,r;return r.value=e,r.done=!0,r};return a.next=a}}throw new TypeError(typeof t+" is not iterable")}return b.prototype=w,o(O,"constructor",{value:w,configurable:!0}),o(w,"constructor",{value:b,configurable:!0}),b.displayName=u(w,l,"GeneratorFunction"),t.isGeneratorFunction=function(e){var t="function"==typeof e&&e.constructor;return!!t&&(t===b||"GeneratorFunction"===(t.displayName||t.name))},t.mark=function(e){return Object.setPrototypeOf?Object.setPrototypeOf(e,w):(e.__proto__=w,u(e,l,"GeneratorFunction")),e.prototype=Object.create(O),e},t.awrap=function(e){return{__await:e}},E(R.prototype),u(R.prototype,i,(function(){return this})),t.AsyncIterator=R,t.async=function(e,r,n,o,a){void 0===a&&(a=Promise);var c=new R(s(e,r,n,o),a);return t.isGeneratorFunction(r)?c:c.next().then((function(e){return e.done?e.value:c.next()}))},E(O),u(O,l,"Generator"),u(O,c,(function(){return this})),u(O,"toString",(function(){return"[object Generator]"})),t.keys=function(e){var t=Object(e),r=[];for(var n in t)r.push(n);return r.reverse(),function e(){for(;r.length;){var n=r.pop();if(n in t)return e.value=n,e.done=!1,e}return e.done=!0,e}},t.values=_,P.prototype={constructor:P,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=e,this.done=!1,this.delegate=null,this.method="next",this.arg=e,this.tryEntries.forEach(S),!t)for(var r in this)"t"===r.charAt(0)&&n.call(this,r)&&!isNaN(+r.slice(1))&&(this[r]=e)},stop:function(){this.done=!0;var e=this.tryEntries[0].completion;if("throw"===e.type)throw e.arg;return this.rval},dispatchException:function(t){if(this.done)throw t;var r=this;function o(n,o){return i.type="throw",i.arg=t,r.next=n,o&&(r.method="next",r.arg=e),!!o}for(var a=this.tryEntries.length-1;a>=0;--a){var c=this.tryEntries[a],i=c.completion;if("root"===c.tryLoc)return o("end");if(c.tryLoc<=this.prev){var l=n.call(c,"catchLoc"),u=n.call(c,"finallyLoc");if(l&&u){if(this.prev<c.catchLoc)return o(c.catchLoc,!0);if(this.prev<c.finallyLoc)return o(c.finallyLoc)}else if(l){if(this.prev<c.catchLoc)return o(c.catchLoc,!0)}else{if(!u)throw Error("try statement without catch or finally");if(this.prev<c.finallyLoc)return o(c.finallyLoc)}}}},abrupt:function(e,t){for(var r=this.tryEntries.length-1;r>=0;--r){var o=this.tryEntries[r];if(o.tryLoc<=this.prev&&n.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var a=o;break}}a&&("break"===e||"continue"===e)&&a.tryLoc<=t&&t<=a.finallyLoc&&(a=null);var c=a?a.completion:{};return c.type=e,c.arg=t,a?(this.method="next",this.next=a.finallyLoc,v):this.complete(c)},complete:function(e,t){if("throw"===e.type)throw e.arg;return"break"===e.type||"continue"===e.type?this.next=e.arg:"return"===e.type?(this.rval=this.arg=e.arg,this.method="return",this.next="end"):"normal"===e.type&&t&&(this.next=t),v},finish:function(e){for(var t=this.tryEntries.length-1;t>=0;--t){var r=this.tryEntries[t];if(r.finallyLoc===e)return this.complete(r.completion,r.afterLoc),S(r),v}},catch:function(e){for(var t=this.tryEntries.length-1;t>=0;--t){var r=this.tryEntries[t];if(r.tryLoc===e){var n=r.completion;if("throw"===n.type){var o=n.arg;S(r)}return o}}throw Error("illegal catch attempt")},delegateYield:function(t,r,n){return this.delegate={iterator:_(t),resultName:r,nextLoc:n},"next"===this.method&&(this.arg=e),v}},t}function b(e){var t=function(e,t){if("object"!=typeof e||!e)return e;var r=e[Symbol.toPrimitive];if(void 0!==r){var n=r.call(e,t);if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return String(e)}(e,"string");return"symbol"==typeof t?t:t+""}function w(e,t,r,n,o,a,c){try{var i=e[a](c),l=i.value}catch(e){return void r(e)}i.done?t(l):Promise.resolve(l).then(n,o)}function x(e){return function(){var t=this,r=arguments;return new Promise((function(n,o){var a=e.apply(t,r);function c(e){w(a,n,o,c,i,"next",e)}function i(e){w(a,n,o,c,i,"throw",e)}c(void 0)}))}}function k(e,t,r){return(t=b(t))in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function L(e,t){t=e.length;for(var r=0,n=new Array(t);r<t;r++)n[r]=e[r];return n}function O(e,t){var r="undefined"!=typeof Symbol&&e[Symbol.iterator]||e["@@iterator"];if(!r){if(Array.isArray(e)||(r=function(e,t){if(e){if("string"==typeof e)return L(e,t);var r=Object.prototype.toString.call(e).slice(8,-1);return"Object"===r&&e.constructor&&(r=e.constructor.name),"Map"===r||"Set"===r?Array.from(e):"Arguments"===r||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(r)?L(e,t):void 0}}(e))||t){r&&(e=r);var n=0,o=function(){};return{s:o,n:function(){return n>=e.length?{done:!0}:{done:!1,value:e[n++]}},e:function(e){throw e},f:o}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,c=!0,i=!1;return{s:function(){r=r.call(e)},n:function(){var e=r.next();return c=e.done,e},e:function(e){i=!0,a=e},f:function(){try{c||null==r.return||r.return()}finally{if(i)throw a}}}}d.default.config();var E=process.env,R=E.MONGODB_USER,C=E.MONGODB_PASSWORD,D=E.MONGODB_PROJECT_URL,j=E.MONGODB_APP_NAME,S="mongodb+srv://".concat(R,":").concat(C,"@").concat(D,"/?retryWrites=true&w=majority&appName=").concat(j),P="rag-sources",_=[{url:"https://mongodb.design",collection:"mongodb-dot-design"},{url:"https://react.dev/reference/react",collection:"react-dev"},{url:"https://developer.mozilla.org/en-US/docs/Web",collection:"mdn"},{url:"https://css-tricks.com/category/articles/",collection:"css-tricks"},{url:"https://www.nngroup.com/articles/",collection:"nn-group"},{url:"https://www.w3.org/WAI/standards-guidelines/wcag/",collection:"wcag"},{url:"https://atomicdesign.bradfrost.com/table-of-contents/",collection:"atomic-design"}],N=["https://www.mongodb.com","https://github.com"];function U(){return M.apply(this,arguments)}function M(){return(M=x(m().mark((function e(){var t,n;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.prev=0,t=S,n=new r.MongoClient(t),e.next=5,n.connect();case 5:return console.log(f.default.green("Connected to MongoDB Atlas")),e.abrupt("return",n);case 9:throw e.prev=9,e.t0=e.catch(0),console.error(f.default.red("Error connecting to MongoDB Atlas:"),e.t0),e.t0;case 13:case"end":return e.stop()}}),e,null,[[0,9]])})))).apply(this,arguments)}var A=function(e){return new URL(e).hostname.replace(/^www\./,"").replace(/\.[^.]+$/,"").replace(/\./g,"-")},B=function(e){try{return new URL(e)}catch(t){console.error(f.default.red("Invalid URL: ".concat(e)),t),process.exit(1)}},I=new Map,G=new Map,W=function(){var e=x(m().mark((function e(t,r,n){var o,a,c,i,l;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(!I.has(t)){e.next=8;break}return o=I.get(t),a=T(o,t,r),n&&console.log(f.default.gray("robots.txt already fetched for ".concat(t))),n&&console.log(f.default.gray("Crawling ".concat(r," on ").concat(t," is ").concat(a?f.default.green("allowed"):f.default.red("disallowed")," by robots.txt"))),e.abrupt("return",a);case 8:return c="https://".concat(t,"/robots.txt"),n&&console.log(f.default.gray("Fetching robots.txt for ".concat(c,"..."))),e.next=12,fetch(c).then((function(e){return e.text()})).catch((function(e){return n&&console.log(f.default.red("Error fetching robots.txt: ".concat(e))),""}));case 12:return i=e.sent,l=T(i,t,r),n&&console.log(f.default.gray("Crawling ".concat(r," on ").concat(t," is ").concat(l?f.default.green("allowed"):f.default.red("disallowed")," by robots.txt"))),e.abrupt("return",l);case 16:case"end":return e.stop()}}),e)})));return function(t,r,n){return e.apply(this,arguments)}}(),T=function(e,t,r){var n,o,a;if(G.has(t)&&null!==(n=G.get(t))&&void 0!==n&&n.has(r))return null===(a=G.get(t))||void 0===a?void 0:a.get(r);var c=!e.split("\n").filter((function(e){return e.startsWith("Disallow:")})).map((function(e){return e.replace("Disallow:","").trim()})).some((function(e){return r.startsWith(e)}));return G.has(t)||G.set(t,new Map),null===(o=G.get(t))||void 0===o||o.set(r,c),c},q=function(e,t){var r="".concat(e,"|").concat(t);return g.default.createHash("md5").update(r).digest("hex")},F=function(){var e=x(m().mark((function e(t){var r,n,o,a,c,i,s;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return r=new l.CheerioWebBaseLoader(t,{selector:"body"}),e.next=3,r.scrape();case 3:return(n=e.sent)("script").remove(),n("style").remove(),o=n("a").get().map((function(e){return e.attribs.href})),a=n("title").text()||n("h1").text()||t.replace(/^(https?:\/\/)/,""),c=n(r.selector).text(),i={source:r.webPath,title:a},s=new u.Document({pageContent:c,metadata:i}),e.abrupt("return",{doc:s,title:a,links:o});case 12:case"end":return e.stop()}}),e)})));return function(t){return e.apply(this,arguments)}}(),Y=function(e){var t=e.title,r=e.href,n=e.checksum;return function(e,o){return{Name:t||"".concat(r," - Chunk ").concat(o),Body:e.pageContent,Link:r,checksum:n,metadata:y(y({},e.metadata),{},{chunk_id:o,source_url:r,crawled_at:new Date})}}};function z(e){return $.apply(this,arguments)}function $(){return $=x(m().mark((function e(t){var r,n,o,a,i,l,u,s,d,p,h,g,v,y,b,w,x,k,L,O,E,R;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(r=t.href,n=t.collectionName,o=t.mongoClient,a=t.verbose,i=void 0!==a&&a,l=t.dryRun,u=void 0!==l&&l,e.prev=1,i&&console.log(f.default.gray("Processing URL:"),f.default.blue(r)),s=new URL(r),d=s.hostname,p=s.pathname,W(d,p,i)){e.next=8;break}return i&&console.log(f.default.red("Crawling disallowed by robots.txt for ".concat(d,". Skipping..."))),e.abrupt("return",{docCount:0,links:[]});case 8:return e.next=10,F(r);case 10:return h=e.sent,g=h.doc,v=h.title,y=h.links,i&&console.log(f.default.gray("Loaded document",'"'.concat(f.default.bold(v),'"'),"from",f.default.blue(r))),b=new c.RecursiveCharacterTextSplitter({chunkSize:1e3,chunkOverlap:200}),e.next=18,b.splitDocuments([g]);case 18:if(w=e.sent,i&&console.log(f.default.gray("Split into ".concat(w.length," chunks"))),x=q(r,g.pageContent),k=w.map(Y({title:v,href:r,checksum:x})),!(w.length<=0)){e.next=25;break}return i&&console.log(f.default.gray("No content to process for URL: ".concat(r))),e.abrupt("return",{docCount:0,links:y,documents:[]});case 25:if(L=0,!u){e.next=31;break}i&&console.log(f.default.yellow("[DRY RUN] Would insert ".concat(k.length,' documents into collection: "').concat(n,'"'))),L=k.length,e.next=43;break;case 31:if(O=o.db(P),E=O.collection(n),i&&console.log(f.default.green("Inserting ".concat(k.length,' documents into collection: "').concat(n,'"'))),!(k.length>0)){e.next=42;break}return e.next=37,E.insertMany(k);case 37:R=e.sent,L=R.insertedCount,i&&console.log(f.default.gray("Successfully inserted ".concat(R.insertedCount," documents into collection ").concat(n))),e.next=43;break;case 42:i&&console.log(f.default.gray("No documents to insert for URL: ".concat(r)));case 43:return e.abrupt("return",{docCount:L,links:y,documents:k});case 46:return e.prev=46,e.t0=e.catch(1),console.error(f.default.red("Error processing URL ".concat(r,":")),f.default.gray(e.t0)),e.abrupt("return",{docCount:0,links:[]});case 50:case"end":return e.stop()}}),e,null,[[1,46]])}))),$.apply(this,arguments)}function H(e){return J.apply(this,arguments)}function J(){return(J=x(m().mark((function e(t){var r,n,o,a,c,i,l,u,s,d,g,v,y,b,w,x,k,L,E,R,C,D,j,S;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(r=t.baseUrl,n=t.collectionName,o=t.mongoClient,a=t.relativePath,c=t.maxDepth,i=void 0===c?3:c,l=t.currentDepth,u=void 0===l?0:l,s=t.visited,d=void 0===s?new Set:s,g=t.verbose,v=void 0!==g&&g,y=t.dryRun,b=void 0!==y&&y,w=t.enableRecursion,x=void 0===w||w,0===u&&v&&console.log(f.default.green("\nStarting crawl of "+r)),k=(null==a?void 0:a.length)&&(null==a?void 0:a.startsWith("/")),L="".concat(p.default(r,"/")).concat(k?"/"+h.default(a,"/"):""),E=B(L),!(u>i||d.has(E.href))){e.next=8;break}return v&&console.log(f.default.gray("Already visited ".concat(E," or reached max depth (").concat(u,"/").concat(i,")"))),e.abrupt("return");case 8:return v&&console.log("\n",f.default.gray("Crawling ".concat(E," (depth: ").concat(u,"/").concat(i,")"))),d.add(E.href),e.next=12,z({href:E.href,collectionName:n,mongoClient:o,verbose:v,dryRun:b});case 12:if(R=e.sent,C=R.links,u!==i&&x){e.next=17;break}return v&&console.log(f.default.gray("Reached max depth, or not allowed to crawl further links from ".concat(E))),e.abrupt("return");case 17:e.prev=17,D=O(C),e.prev=19,S=m().mark((function e(){var t,a,c,l,s,p,g;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if(t=j.value,a=t.startsWith("/")||t.startsWith("#"),c=B(a?E.origin+"/"+h.default(t,"/"):t),l=d.has(c.href),s=t.startsWith(r),g=(p=!a&&!s)&&N.some((function(e){return t.startsWith(e)})),!l){e.next=10;break}return v&&console.log(f.default.gray("Already visited link: ".concat(t))),e.abrupt("return",0);case 10:if(!p||g){e.next=13;break}return v&&console.log(f.default.gray("Skipping external link: ".concat(t," not in allowed domains"))),e.abrupt("return",0);case 13:return console.log(f.default.gray("Found new link: ".concat(t))),v&&console.log(f.default.gray("Preparing to crawl ".concat(c.origin," with relative path ").concat(c.pathname," ").concat(p?" (external)":""))),e.next=17,H({baseUrl:c.origin,relativePath:c.pathname,collectionName:n,mongoClient:o,maxDepth:i,currentDepth:u+1,visited:d,verbose:v,dryRun:b,enableRecursion:!p});case 17:case"end":return e.stop()}}),e)})),D.s();case 22:if((j=D.n()).done){e.next=29;break}return e.delegateYield(S(),"t0",24);case 24:if(0!==e.t0){e.next=27;break}return e.abrupt("continue",27);case 27:e.next=22;break;case 29:e.next=34;break;case 31:e.prev=31,e.t1=e.catch(19),D.e(e.t1);case 34:return e.prev=34,D.f(),e.finish(34);case 37:e.next=42;break;case 39:e.prev=39,e.t2=e.catch(17),console.error(f.default.red("Error crawling ".concat(r,":")),e.t2);case 42:case"end":return e.stop()}}),e,null,[[17,39],[19,31,34,37]])})))).apply(this,arguments)}function K(){return(K=x(m().mark((function e(t){var r,n,o,a,c,i,l,u,s,d,p,h,g,v;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return r=t.verbose,n=void 0!==r&&r,o=t.depth,a=void 0===o?"3":o,c=t.url,i=t.dryRun,l=void 0!==i&&i,u=parseInt(a,10),e.prev=2,e.next=5,U();case 5:if(s=e.sent,l&&n&&console.log(f.default.yellow("Running in dry-run mode - no data will be inserted into MongoDB")),!c){e.next=21;break}if(d=_.find((function(e){return e.url===c})),!d){e.next=15;break}return n&&console.log(f.default.gray("Crawling URL ".concat(c," with collection name ").concat(d.collection))),e.next=13,H({baseUrl:c,collectionName:d.collection,mongoClient:s,maxDepth:u,verbose:n,dryRun:l});case 13:e.next=19;break;case 15:return p=A(c),n&&console.log(f.default.gray("URL ".concat(c," not found in sources configuration. Creating new collection ").concat(p))),e.next=19,H({baseUrl:c,collectionName:p,mongoClient:s,maxDepth:u,verbose:n,dryRun:l});case 19:e.next=39;break;case 21:n&&console.log(f.default.gray("Crawling all ".concat(_.length," configured sources"))),h=O(_),e.prev=23,h.s();case 25:if((g=h.n()).done){e.next=31;break}return v=g.value,e.next=29,H({baseUrl:v.url,collectionName:v.collection,mongoClient:s,maxDepth:u,verbose:n,dryRun:l});case 29:e.next=25;break;case 31:e.next=36;break;case 33:e.prev=33,e.t0=e.catch(23),h.e(e.t0);case 36:return e.prev=36,h.f(),e.finish(36);case 39:return e.next=41,s.close();case 41:n&&console.log(f.default.gray("Closed MongoDB connection")),e.next=47;break;case 44:e.prev=44,e.t1=e.catch(2),console.error("Error during crawl process:",e.t1);case 47:case"end":return e.stop()}}),e,null,[[2,44],[23,33,36,39]])})))).apply(this,arguments)}function Q(){return(Q=x(m().mark((function e(t){var r,n,o,a,c,i,l,u,s,d,p,h,g,v,y,b;return m().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return r=t.verbose,n=void 0!==r&&r,o=t.dryRun,a=void 0!==o&&o,c=t.days,i=parseInt(void 0===c?"7":c,10),e.prev=2,e.next=5,U();case 5:l=e.sent,u=l.db(P),a&&n&&console.log(f.default.yellow("Running in dry-run mode - no documents will be deleted from MongoDB")),(s=new Date).setDate(s.getDate()-i),n&&console.log(f.default.gray("Pruning documents older than ".concat(i," days (before ").concat(s.toISOString(),")"))),d=O(_),e.prev=12,d.s();case 14:if((p=d.n()).done){e.next=35;break}return h=p.value,g=u.collection(h.collection),v={"metadata.crawled_at":{$lt:s}},e.next=20,g.countDocuments(v);case 20:if(0!==(y=e.sent)){e.next=24;break}return n&&console.log(f.default.gray("No documents to prune in collection ".concat(h.collection))),e.abrupt("continue",33);case 24:if(n&&console.log(f.default.gray("Found ".concat(y," documents to prune in collection ").concat(h.collection))),a){e.next=32;break}return e.next=28,g.deleteMany(v);case 28:b=e.sent,console.log(f.default.green("✂️ Pruned ".concat(b.deletedCount," documents from collection ").concat(h.collection))),e.next=33;break;case 32:console.log(f.default.yellow("[DRY RUN] Would prune ".concat(y," documents from collection ").concat(h.collection)));case 33:e.next=14;break;case 35:e.next=40;break;case 37:e.prev=37,e.t0=e.catch(12),d.e(e.t0);case 40:return e.prev=40,d.f(),e.finish(40);case 43:return e.next=45,l.close();case 45:n&&console.log(f.default.gray("Closed MongoDB connection")),e.next=51;break;case 48:e.prev=48,e.t1=e.catch(2),console.error("Error during prune process:",e.t1);case 51:case"end":return e.stop()}}),e,null,[[2,48],[12,37,40,43]])})))).apply(this,arguments)}var V=new e.Command;return V.name("lg-crawler").description("A CLI tool for crawling and analyzing website content for LeafyGreen AI"),V.command("crawl").description("Run the crawler").option("-v, --verbose","Enable verbose output",!1).option("-d, --depth <number>","Maximum crawl depth","3").option("--url <url>","Specific URL to crawl. If not provided, the crawler will scan all URLs defined in the config.").option("--dry-run","Run crawler without inserting documents into MongoDB",!1).action((function(e){return K.apply(this,arguments)})),V.command("prune").description("Prune old documents from MongoDB collections used by LeafyGreen Crawler").option("-v, --verbose","Enable verbose output",!1).option("--dry-run","Run prune without deleting documents from MongoDB",!1).option("-d, --days <number>","Keep documents newer than this many days","7").action((function(e){return Q.apply(this,arguments)})),V.parse(process.argv),V}));
//# sourceMappingURL=cli.js.map