@lg-tools/crawler/dist/esm/index.js

Crawler for MongoDB documentation and other sites

import t from"dotenv";import e from"chalk";import{RecursiveCharacterTextSplitter as r}from"langchain/text_splitter";import n from"node:crypto";import{CheerioWebBaseLoader as o}from"@langchain/community/document_loaders/web/cheerio";import{Document as a}from"@langchain/core/documents";import c from"lodash/trimEnd";import i from"lodash/trimStart";t.config();var u=process.env;u.MONGODB_USER,u.MONGODB_PASSWORD,u.MONGODB_PROJECT_URL,u.MONGODB_APP_NAME;var l="rag-sources",s=[{url:"https://mongodb.design",collection:"mongodb-dot-design"},{url:"https://react.dev/reference/react",collection:"react-dev"},{url:"https://developer.mozilla.org/en-US/docs/Web",collection:"mdn"},{url:"https://css-tricks.com/category/articles/",collection:"css-tricks"},{url:"https://www.nngroup.com/articles/",collection:"nn-group"},{url:"https://www.w3.org/WAI/standards-guidelines/wcag/",collection:"wcag"},{url:"https://atomicdesign.bradfrost.com/table-of-contents/",collection:"atomic-design"}],f=["https://www.mongodb.com","https://github.com"];function h(t,e){var r=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),r.push.apply(r,n)}return r}function p(t){for(var e=1;e<arguments.length;e++){var r=null!=arguments[e]?arguments[e]:{};e%2?h(Object(r),!0).forEach((function(e){m(t,e,r[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(r)):h(Object(r)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(r,e))}))}return t}function d(){d=function(){return e};var t,e={},r=Object.prototype,n=r.hasOwnProperty,o=Object.defineProperty||function(t,e,r){t[e]=r.value},a="function"==typeof Symbol?Symbol:{},c=a.iterator||"@@iterator",i=a.asyncIterator||"@@asyncIterator",u=a.toStringTag||"@@toStringTag";function l(t,e,r){return Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}),t[e]}try{l({},"")}catch(t){l=function(t,e,r){return t[e]=r}}function s(t,e,r,n){var a=e&&e.prototype instanceof m?e:m,c=Object.create(a.prototype),i=new C(n||[]);return o(c,"_invoke",{value:P(t,r,i)}),c}function f(t,e,r){try{return{type:"normal",arg:t.call(e,r)}}catch(t){return{type:"throw",arg:t}}}e.wrap=s;var h="suspendedStart",p="suspendedYield",g="executing",v="completed",y={};function m(){}function b(){}function w(){}var x={};l(x,c,(function(){return this}));var k=Object.getPrototypeOf,O=k&&k(k(R([])));O&&O!==r&&n.call(O,c)&&(x=O);var L=w.prototype=m.prototype=Object.create(x);function E(t){["next","throw","return"].forEach((function(e){l(t,e,(function(t){return this._invoke(e,t)}))}))}function j(t,e){function r(o,a,c,i){var u=f(t[o],t,a);if("throw"!==u.type){var l=u.arg,s=l.value;return s&&"object"==typeof s&&n.call(s,"__await")?e.resolve(s.__await).then((function(t){r("next",t,c,i)}),(function(t){r("throw",t,c,i)})):e.resolve(s).then((function(t){l.value=t,c(l)}),(function(t){return r("throw",t,c,i)}))}i(u.arg)}var a;o(this,"_invoke",{value:function(t,n){function o(){return new e((function(e,o){r(t,n,e,o)}))}return a=a?a.then(o,o):o()}})}function P(e,r,n){var o=h;return function(a,c){if(o===g)throw Error("Generator is already running");if(o===v){if("throw"===a)throw c;return{value:t,done:!0}}for(n.method=a,n.arg=c;;){var i=n.delegate;if(i){var u=S(i,n);if(u){if(u===y)continue;return u}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(o===h)throw o=v,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);o=g;var l=f(e,r,n);if("normal"===l.type){if(o=n.done?v:p,l.arg===y)continue;return{value:l.arg,done:n.done}}"throw"===l.type&&(o=v,n.method="throw",n.arg=l.arg)}}}function S(e,r){var n=r.method,o=e.iterator[n];if(o===t)return r.delegate=null,"throw"===n&&e.iterator.return&&(r.method="return",r.arg=t,S(e,r),"throw"===r.method)||"return"!==n&&(r.method="throw",r.arg=new TypeError("The iterator does not provide a '"+n+"' method")),y;var a=f(o,e.iterator,r.arg);if("throw"===a.type)return r.method="throw",r.arg=a.arg,r.delegate=null,y;var c=a.arg;return c?c.done?(r[e.resultName]=c.value,r.next=e.nextLoc,"return"!==r.method&&(r.method="next",r.arg=t),r.delegate=null,y):c:(r.method="throw",r.arg=new TypeError("iterator result is not an object"),r.delegate=null,y)}function _(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function N(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function C(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(_,this),this.reset(!0)}function R(e){if(e||""===e){var r=e[c];if(r)return r.call(e);if("function"==typeof e.next)return e;if(!isNaN(e.length)){var o=-1,a=function r(){for(;++o<e.length;)if(n.call(e,o))return r.value=e[o],r.done=!1,r;return r.value=t,r.done=!0,r};return a.next=a}}throw new TypeError(typeof e+" is not iterable")}return b.prototype=w,o(L,"constructor",{value:w,configurable:!0}),o(w,"constructor",{value:b,configurable:!0}),b.displayName=l(w,u,"GeneratorFunction"),e.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===b||"GeneratorFunction"===(e.displayName||e.name))},e.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,w):(t.__proto__=w,l(t,u,"GeneratorFunction")),t.prototype=Object.create(L),t},e.awrap=function(t){return{__await:t}},E(j.prototype),l(j.prototype,i,(function(){return this})),e.AsyncIterator=j,e.async=function(t,r,n,o,a){void 0===a&&(a=Promise);var c=new j(s(t,r,n,o),a);return e.isGeneratorFunction(r)?c:c.next().then((function(t){return t.done?t.value:c.next()}))},E(L),l(L,u,"Generator"),l(L,c,(function(){return this})),l(L,"toString",(function(){return"[object Generator]"})),e.keys=function(t){var e=Object(t),r=[];for(var n in e)r.push(n);return r.reverse(),function t(){for(;r.length;){var n=r.pop();if(n in e)return t.value=n,t.done=!1,t}return t.done=!0,t}},e.values=R,C.prototype={constructor:C,reset:function(e){if(this.prev=0,this.next=0,this.sent=this._sent=t,this.done=!1,this.delegate=null,this.method="next",this.arg=t,this.tryEntries.forEach(N),!e)for(var r in this)"t"===r.charAt(0)&&n.call(this,r)&&!isNaN(+r.slice(1))&&(this[r]=t)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(e){if(this.done)throw e;var r=this;function o(n,o){return i.type="throw",i.arg=e,r.next=n,o&&(r.method="next",r.arg=t),!!o}for(var a=this.tryEntries.length-1;a>=0;--a){var c=this.tryEntries[a],i=c.completion;if("root"===c.tryLoc)return o("end");if(c.tryLoc<=this.prev){var u=n.call(c,"catchLoc"),l=n.call(c,"finallyLoc");if(u&&l){if(this.prev<c.catchLoc)return o(c.catchLoc,!0);if(this.prev<c.finallyLoc)return o(c.finallyLoc)}else if(u){if(this.prev<c.catchLoc)return o(c.catchLoc,!0)}else{if(!l)throw Error("try statement without catch or finally");if(this.prev<c.finallyLoc)return o(c.finallyLoc)}}}},abrupt:function(t,e){for(var r=this.tryEntries.length-1;r>=0;--r){var o=this.tryEntries[r];if(o.tryLoc<=this.prev&&n.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var a=o;break}}a&&("break"===t||"continue"===t)&&a.tryLoc<=e&&e<=a.finallyLoc&&(a=null);var c=a?a.completion:{};return c.type=t,c.arg=e,a?(this.method="next",this.next=a.finallyLoc,y):this.complete(c)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),y},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.finallyLoc===t)return this.complete(r.completion,r.afterLoc),N(r),y}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.tryLoc===t){var n=r.completion;if("throw"===n.type){var o=n.arg;N(r)}return o}}throw Error("illegal catch attempt")},delegateYield:function(e,r,n){return this.delegate={iterator:R(e),resultName:r,nextLoc:n},"next"===this.method&&(this.arg=t),y}},e}function g(t){var e=function(t,e){if("object"!=typeof t||!t)return t;var r=t[Symbol.toPrimitive];if(void 0!==r){var n=r.call(t,e);if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return String(t)}(t,"string");return"symbol"==typeof e?e:e+""}function v(t,e,r,n,o,a,c){try{var i=t[a](c),u=i.value}catch(t){return void r(t)}i.done?e(u):Promise.resolve(u).then(n,o)}function y(t){return function(){var e=this,r=arguments;return new Promise((function(n,o){var a=t.apply(e,r);function c(t){v(a,n,o,c,i,"next",t)}function i(t){v(a,n,o,c,i,"throw",t)}c(void 0)}))}}function m(t,e,r){return(e=g(e))in t?Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}):t[e]=r,t}function b(t,e){e=t.length;for(var r=0,n=new Array(e);r<e;r++)n[r]=t[r];return n}function w(t,e){var r="undefined"!=typeof Symbol&&t[Symbol.iterator]||t["@@iterator"];if(!r){if(Array.isArray(t)||(r=function(t,e){if(t){if("string"==typeof t)return b(t,e);var r=Object.prototype.toString.call(t).slice(8,-1);return"Object"===r&&t.constructor&&(r=t.constructor.name),"Map"===r||"Set"===r?Array.from(t):"Arguments"===r||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(r)?b(t,e):void 0}}(t))||e){r&&(t=r);var n=0,o=function(){};return{s:o,n:function(){return n>=t.length?{done:!0}:{done:!1,value:t[n++]}},e:function(t){throw t},f:o}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,c=!0,i=!1;return{s:function(){r=r.call(t)},n:function(){var t=r.next();return c=t.done,t},e:function(t){i=!0,a=t},f:function(){try{c||null==r.return||r.return()}finally{if(i)throw a}}}}var x=new Map,k=new Map,O=function(){var t=y(d().mark((function t(r,n,o){var a,c,i,u,l;return d().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(!x.has(r)){t.next=8;break}return a=x.get(r),c=L(a,r,n),o&&console.log(e.gray("robots.txt already fetched for ".concat(r))),o&&console.log(e.gray("Crawling ".concat(n," on ").concat(r," is ").concat(c?e.green("allowed"):e.red("disallowed")," by robots.txt"))),t.abrupt("return",c);case 8:return i="https://".concat(r,"/robots.txt"),o&&console.log(e.gray("Fetching robots.txt for ".concat(i,"..."))),t.next=12,fetch(i).then((function(t){return t.text()})).catch((function(t){return o&&console.log(e.red("Error fetching robots.txt: ".concat(t))),""}));case 12:return u=t.sent,l=L(u,r,n),o&&console.log(e.gray("Crawling ".concat(n," on ").concat(r," is ").concat(l?e.green("allowed"):e.red("disallowed")," by robots.txt"))),t.abrupt("return",l);case 16:case"end":return t.stop()}}),t)})));return function(e,r,n){return t.apply(this,arguments)}}(),L=function(t,e,r){var n,o,a;if(k.has(e)&&null!==(n=k.get(e))&&void 0!==n&&n.has(r))return null===(a=k.get(e))||void 0===a?void 0:a.get(r);var c=!t.split("\n").filter((function(t){return t.startsWith("Disallow:")})).map((function(t){return t.replace("Disallow:","").trim()})).some((function(t){return r.startsWith(t)}));return k.has(e)||k.set(e,new Map),null===(o=k.get(e))||void 0===o||o.set(r,c),c},E=function(t,e){var r="".concat(t,"|").concat(e);return n.createHash("md5").update(r).digest("hex")},j=function(){var t=y(d().mark((function t(e){var r,n,c,i,u,l,s;return d().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:return r=new o(e,{selector:"body"}),t.next=3,r.scrape();case 3:return(n=t.sent)("script").remove(),n("style").remove(),c=n("a").get().map((function(t){return t.attribs.href})),i=n("title").text()||n("h1").text()||e.replace(/^(https?:\/\/)/,""),u=n(r.selector).text(),l={source:r.webPath,title:i},s=new a({pageContent:u,metadata:l}),t.abrupt("return",{doc:s,title:i,links:c});case 12:case"end":return t.stop()}}),t)})));return function(e){return t.apply(this,arguments)}}(),P=function(t){var e=t.title,r=t.href,n=t.checksum;return function(t,o){return{Name:e||"".concat(r," - Chunk ").concat(o),Body:t.pageContent,Link:r,checksum:n,metadata:p(p({},t.metadata),{},{chunk_id:o,source_url:r,crawled_at:new Date})}}};function S(t){return _.apply(this,arguments)}function _(){return(_=y(d().mark((function t(n){var o,a,c,i,u,s,f,h,p,g,v,y,m,b,w,x,k,L,S,_,N,C;return d().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(o=n.href,a=n.collectionName,c=n.mongoClient,i=n.verbose,u=void 0!==i&&i,s=n.dryRun,f=void 0!==s&&s,t.prev=1,u&&console.log(e.gray("Processing URL:"),e.blue(o)),h=new URL(o),p=h.hostname,g=h.pathname,O(p,g,u)){t.next=8;break}return u&&console.log(e.red("Crawling disallowed by robots.txt for ".concat(p,". Skipping..."))),t.abrupt("return",{docCount:0,links:[]});case 8:return t.next=10,j(o);case 10:return v=t.sent,y=v.doc,m=v.title,b=v.links,u&&console.log(e.gray("Loaded document",'"'.concat(e.bold(m),'"'),"from",e.blue(o))),w=new r({chunkSize:1e3,chunkOverlap:200}),t.next=18,w.splitDocuments([y]);case 18:if(x=t.sent,u&&console.log(e.gray("Split into ".concat(x.length," chunks"))),k=E(o,y.pageContent),L=x.map(P({title:m,href:o,checksum:k})),!(x.length<=0)){t.next=25;break}return u&&console.log(e.gray("No content to process for URL: ".concat(o))),t.abrupt("return",{docCount:0,links:b,documents:[]});case 25:if(S=0,!f){t.next=31;break}u&&console.log(e.yellow("[DRY RUN] Would insert ".concat(L.length,' documents into collection: "').concat(a,'"'))),S=L.length,t.next=43;break;case 31:if(_=c.db(l),N=_.collection(a),u&&console.log(e.green("Inserting ".concat(L.length,' documents into collection: "').concat(a,'"'))),!(L.length>0)){t.next=42;break}return t.next=37,N.insertMany(L);case 37:C=t.sent,S=C.insertedCount,u&&console.log(e.gray("Successfully inserted ".concat(C.insertedCount," documents into collection ").concat(a))),t.next=43;break;case 42:u&&console.log(e.gray("No documents to insert for URL: ".concat(o)));case 43:return t.abrupt("return",{docCount:S,links:b,documents:L});case 46:return t.prev=46,t.t0=t.catch(1),console.error(e.red("Error processing URL ".concat(o,":")),e.gray(t.t0)),t.abrupt("return",{docCount:0,links:[]});case 50:case"end":return t.stop()}}),t,null,[[1,46]])})))).apply(this,arguments)}var N=function(t){try{return new URL(t)}catch(r){console.error(e.red("Invalid URL: ".concat(t)),r),process.exit(1)}};function C(t){return R.apply(this,arguments)}function R(){return(R=y(d().mark((function t(r){var n,o,a,u,l,s,h,p,g,v,y,m,b,x,k,O,L,E,j,P,_,R,D,A;return d().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(n=r.baseUrl,o=r.collectionName,a=r.mongoClient,u=r.relativePath,l=r.maxDepth,s=void 0===l?3:l,h=r.currentDepth,p=void 0===h?0:h,g=r.visited,v=void 0===g?new Set:g,y=r.verbose,m=void 0!==y&&y,b=r.dryRun,x=void 0!==b&&b,k=r.enableRecursion,O=void 0===k||k,0===p&&m&&console.log(e.green("\nStarting crawl of "+n)),L=(null==u?void 0:u.length)&&(null==u?void 0:u.startsWith("/")),E="".concat(c(n,"/")).concat(L?"/"+i(u,"/"):""),j=N(E),!(p>s||v.has(j.href))){t.next=8;break}return m&&console.log(e.gray("Already visited ".concat(j," or reached max depth (").concat(p,"/").concat(s,")"))),t.abrupt("return");case 8:return m&&console.log("\n",e.gray("Crawling ".concat(j," (depth: ").concat(p,"/").concat(s,")"))),v.add(j.href),t.next=12,S({href:j.href,collectionName:o,mongoClient:a,verbose:m,dryRun:x});case 12:if(P=t.sent,_=P.links,p!==s&&O){t.next=17;break}return m&&console.log(e.gray("Reached max depth, or not allowed to crawl further links from ".concat(j))),t.abrupt("return");case 17:t.prev=17,R=w(_),t.prev=19,A=d().mark((function t(){var r,c,u,l,h,g,y;return d().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(r=D.value,c=r.startsWith("/")||r.startsWith("#"),u=N(c?j.origin+"/"+i(r,"/"):r),l=v.has(u.href),h=r.startsWith(n),y=(g=!c&&!h)&&f.some((function(t){return r.startsWith(t)})),!l){t.next=10;break}return m&&console.log(e.gray("Already visited link: ".concat(r))),t.abrupt("return",0);case 10:if(!g||y){t.next=13;break}return m&&console.log(e.gray("Skipping external link: ".concat(r," not in allowed domains"))),t.abrupt("return",0);case 13:return console.log(e.gray("Found new link: ".concat(r))),m&&console.log(e.gray("Preparing to crawl ".concat(u.origin," with relative path ").concat(u.pathname," ").concat(g?" (external)":""))),t.next=17,C({baseUrl:u.origin,relativePath:u.pathname,collectionName:o,mongoClient:a,maxDepth:s,currentDepth:p+1,visited:v,verbose:m,dryRun:x,enableRecursion:!g});case 17:case"end":return t.stop()}}),t)})),R.s();case 22:if((D=R.n()).done){t.next=29;break}return t.delegateYield(A(),"t0",24);case 24:if(0!==t.t0){t.next=27;break}return t.abrupt("continue",27);case 27:t.next=22;break;case 29:t.next=34;break;case 31:t.prev=31,t.t1=t.catch(19),R.e(t.t1);case 34:return t.prev=34,R.f(),t.finish(34);case 37:t.next=42;break;case 39:t.prev=39,t.t2=t.catch(17),console.error(e.red("Error crawling ".concat(n,":")),t.t2);case 42:case"end":return t.stop()}}),t,null,[[17,39],[19,31,34,37]])})))).apply(this,arguments)}export{l as MDB_DB,s as SOURCES,f as allowedDomains,S as processSingleUrl,C as recursiveCrawlFromBaseURL};
//# sourceMappingURL=index.js.map