@lg-tools/crawler/dist/index.js

Crawler for MongoDB documentation and other sites

!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("dotenv"),require("chalk"),require("langchain/text_splitter"),require("node:crypto"),require("@langchain/community/document_loaders/web/cheerio"),require("@langchain/core/documents"),require("lodash/trimEnd"),require("lodash/trimStart")):"function"==typeof define&&define.amd?define(["exports","dotenv","chalk","langchain/text_splitter","node:crypto","@langchain/community/document_loaders/web/cheerio","@langchain/core/documents","lodash/trimEnd","lodash/trimStart"],e):e((t="undefined"!=typeof globalThis?globalThis:t||self)["@lg-tools/crawler"]={},t.dotenv,t.chalk,t.text_splitter,t.crypto,t.cheerio,t.documents,t["/trimEnd"],t["/trimStart"])}(this,(function(t,e,r,n,o,a,c,i,u){"use strict";function l(t){return t&&"object"==typeof t&&"default"in t?t:{default:t}}var s=l(e),f=l(r),h=l(o),d=l(i),p=l(u);s.default.config();var g=process.env;g.MONGODB_USER,g.MONGODB_PASSWORD,g.MONGODB_PROJECT_URL,g.MONGODB_APP_NAME;var v="rag-sources",y=["https://www.mongodb.com","https://github.com"];function m(t,e){var r=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),r.push.apply(r,n)}return r}function b(t){for(var e=1;e<arguments.length;e++){var r=null!=arguments[e]?arguments[e]:{};e%2?m(Object(r),!0).forEach((function(e){L(t,e,r[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(r)):m(Object(r)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(r,e))}))}return t}function w(){w=function(){return e};var t,e={},r=Object.prototype,n=r.hasOwnProperty,o=Object.defineProperty||function(t,e,r){t[e]=r.value},a="function"==typeof Symbol?Symbol:{},c=a.iterator||"@@iterator",i=a.asyncIterator||"@@asyncIterator",u=a.toStringTag||"@@toStringTag";function l(t,e,r){return Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}),t[e]}try{l({},"")}catch(t){l=function(t,e,r){return t[e]=r}}function s(t,e,r,n){var a=e&&e.prototype instanceof y?e:y,c=Object.create(a.prototype),i=new R(n||[]);return o(c,"_invoke",{value:j(t,r,i)}),c}function f(t,e,r){try{return{type:"normal",arg:t.call(e,r)}}catch(t){return{type:"throw",arg:t}}}e.wrap=s;var h="suspendedStart",d="suspendedYield",p="executing",g="completed",v={};function y(){}function m(){}function b(){}var x={};l(x,c,(function(){return this}));var k=Object.getPrototypeOf,O=k&&k(k(D([])));O&&O!==r&&n.call(O,c)&&(x=O);var L=b.prototype=y.prototype=Object.create(x);function E(t){["next","throw","return"].forEach((function(e){l(t,e,(function(t){return this._invoke(e,t)}))}))}function S(t,e){function r(o,a,c,i){var u=f(t[o],t,a);if("throw"!==u.type){var l=u.arg,s=l.value;return s&&"object"==typeof s&&n.call(s,"__await")?e.resolve(s.__await).then((function(t){r("next",t,c,i)}),(function(t){r("throw",t,c,i)})):e.resolve(s).then((function(t){l.value=t,c(l)}),(function(t){return r("throw",t,c,i)}))}i(u.arg)}var a;o(this,"_invoke",{value:function(t,n){function o(){return new e((function(e,o){r(t,n,e,o)}))}return a=a?a.then(o,o):o()}})}function j(e,r,n){var o=h;return function(a,c){if(o===p)throw Error("Generator is already running");if(o===g){if("throw"===a)throw c;return{value:t,done:!0}}for(n.method=a,n.arg=c;;){var i=n.delegate;if(i){var u=_(i,n);if(u){if(u===v)continue;return u}}if("next"===n.method)n.sent=n._sent=n.arg;else if("throw"===n.method){if(o===h)throw o=g,n.arg;n.dispatchException(n.arg)}else"return"===n.method&&n.abrupt("return",n.arg);o=p;var l=f(e,r,n);if("normal"===l.type){if(o=n.done?g:d,l.arg===v)continue;return{value:l.arg,done:n.done}}"throw"===l.type&&(o=g,n.method="throw",n.arg=l.arg)}}}function _(e,r){var n=r.method,o=e.iterator[n];if(o===t)return r.delegate=null,"throw"===n&&e.iterator.return&&(r.method="return",r.arg=t,_(e,r),"throw"===r.method)||"return"!==n&&(r.method="throw",r.arg=new TypeError("The iterator does not provide a '"+n+"' method")),v;var a=f(o,e.iterator,r.arg);if("throw"===a.type)return r.method="throw",r.arg=a.arg,r.delegate=null,v;var c=a.arg;return c?c.done?(r[e.resultName]=c.value,r.next=e.nextLoc,"return"!==r.method&&(r.method="next",r.arg=t),r.delegate=null,v):c:(r.method="throw",r.arg=new TypeError("iterator result is not an object"),r.delegate=null,v)}function P(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function C(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function R(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(P,this),this.reset(!0)}function D(e){if(e||""===e){var r=e[c];if(r)return r.call(e);if("function"==typeof e.next)return e;if(!isNaN(e.length)){var o=-1,a=function r(){for(;++o<e.length;)if(n.call(e,o))return r.value=e[o],r.done=!1,r;return r.value=t,r.done=!0,r};return a.next=a}}throw new TypeError(typeof e+" is not iterable")}return m.prototype=b,o(L,"constructor",{value:b,configurable:!0}),o(b,"constructor",{value:m,configurable:!0}),m.displayName=l(b,u,"GeneratorFunction"),e.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===m||"GeneratorFunction"===(e.displayName||e.name))},e.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,b):(t.__proto__=b,l(t,u,"GeneratorFunction")),t.prototype=Object.create(L),t},e.awrap=function(t){return{__await:t}},E(S.prototype),l(S.prototype,i,(function(){return this})),e.AsyncIterator=S,e.async=function(t,r,n,o,a){void 0===a&&(a=Promise);var c=new S(s(t,r,n,o),a);return e.isGeneratorFunction(r)?c:c.next().then((function(t){return t.done?t.value:c.next()}))},E(L),l(L,u,"Generator"),l(L,c,(function(){return this})),l(L,"toString",(function(){return"[object Generator]"})),e.keys=function(t){var e=Object(t),r=[];for(var n in e)r.push(n);return r.reverse(),function t(){for(;r.length;){var n=r.pop();if(n in e)return t.value=n,t.done=!1,t}return t.done=!0,t}},e.values=D,R.prototype={constructor:R,reset:function(e){if(this.prev=0,this.next=0,this.sent=this._sent=t,this.done=!1,this.delegate=null,this.method="next",this.arg=t,this.tryEntries.forEach(C),!e)for(var r in this)"t"===r.charAt(0)&&n.call(this,r)&&!isNaN(+r.slice(1))&&(this[r]=t)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(e){if(this.done)throw e;var r=this;function o(n,o){return i.type="throw",i.arg=e,r.next=n,o&&(r.method="next",r.arg=t),!!o}for(var a=this.tryEntries.length-1;a>=0;--a){var c=this.tryEntries[a],i=c.completion;if("root"===c.tryLoc)return o("end");if(c.tryLoc<=this.prev){var u=n.call(c,"catchLoc"),l=n.call(c,"finallyLoc");if(u&&l){if(this.prev<c.catchLoc)return o(c.catchLoc,!0);if(this.prev<c.finallyLoc)return o(c.finallyLoc)}else if(u){if(this.prev<c.catchLoc)return o(c.catchLoc,!0)}else{if(!l)throw Error("try statement without catch or finally");if(this.prev<c.finallyLoc)return o(c.finallyLoc)}}}},abrupt:function(t,e){for(var r=this.tryEntries.length-1;r>=0;--r){var o=this.tryEntries[r];if(o.tryLoc<=this.prev&&n.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var a=o;break}}a&&("break"===t||"continue"===t)&&a.tryLoc<=e&&e<=a.finallyLoc&&(a=null);var c=a?a.completion:{};return c.type=t,c.arg=e,a?(this.method="next",this.next=a.finallyLoc,v):this.complete(c)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),v},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.finallyLoc===t)return this.complete(r.completion,r.afterLoc),C(r),v}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var r=this.tryEntries[e];if(r.tryLoc===t){var n=r.completion;if("throw"===n.type){var o=n.arg;C(r)}return o}}throw Error("illegal catch attempt")},delegateYield:function(e,r,n){return this.delegate={iterator:D(e),resultName:r,nextLoc:n},"next"===this.method&&(this.arg=t),v}},e}function x(t){var e=function(t,e){if("object"!=typeof t||!t)return t;var r=t[Symbol.toPrimitive];if(void 0!==r){var n=r.call(t,e);if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return String(t)}(t,"string");return"symbol"==typeof e?e:e+""}function k(t,e,r,n,o,a,c){try{var i=t[a](c),u=i.value}catch(t){return void r(t)}i.done?e(u):Promise.resolve(u).then(n,o)}function O(t){return function(){var e=this,r=arguments;return new Promise((function(n,o){var a=t.apply(e,r);function c(t){k(a,n,o,c,i,"next",t)}function i(t){k(a,n,o,c,i,"throw",t)}c(void 0)}))}}function L(t,e,r){return(e=x(e))in t?Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}):t[e]=r,t}function E(t,e){e=t.length;for(var r=0,n=new Array(e);r<e;r++)n[r]=t[r];return n}function S(t,e){var r="undefined"!=typeof Symbol&&t[Symbol.iterator]||t["@@iterator"];if(!r){if(Array.isArray(t)||(r=function(t,e){if(t){if("string"==typeof t)return E(t,e);var r=Object.prototype.toString.call(t).slice(8,-1);return"Object"===r&&t.constructor&&(r=t.constructor.name),"Map"===r||"Set"===r?Array.from(t):"Arguments"===r||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(r)?E(t,e):void 0}}(t))||e){r&&(t=r);var n=0,o=function(){};return{s:o,n:function(){return n>=t.length?{done:!0}:{done:!1,value:t[n++]}},e:function(t){throw t},f:o}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,c=!0,i=!1;return{s:function(){r=r.call(t)},n:function(){var t=r.next();return c=t.done,t},e:function(t){i=!0,a=t},f:function(){try{c||null==r.return||r.return()}finally{if(i)throw a}}}}var j=new Map,_=new Map,P=function(){var t=O(w().mark((function t(e,r,n){var o,a,c,i,u;return w().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(!j.has(e)){t.next=8;break}return o=j.get(e),a=C(o,e,r),n&&console.log(f.default.gray("robots.txt already fetched for ".concat(e))),n&&console.log(f.default.gray("Crawling ".concat(r," on ").concat(e," is ").concat(a?f.default.green("allowed"):f.default.red("disallowed")," by robots.txt"))),t.abrupt("return",a);case 8:return c="https://".concat(e,"/robots.txt"),n&&console.log(f.default.gray("Fetching robots.txt for ".concat(c,"..."))),t.next=12,fetch(c).then((function(t){return t.text()})).catch((function(t){return n&&console.log(f.default.red("Error fetching robots.txt: ".concat(t))),""}));case 12:return i=t.sent,u=C(i,e,r),n&&console.log(f.default.gray("Crawling ".concat(r," on ").concat(e," is ").concat(u?f.default.green("allowed"):f.default.red("disallowed")," by robots.txt"))),t.abrupt("return",u);case 16:case"end":return t.stop()}}),t)})));return function(e,r,n){return t.apply(this,arguments)}}(),C=function(t,e,r){var n,o,a;if(_.has(e)&&null!==(n=_.get(e))&&void 0!==n&&n.has(r))return null===(a=_.get(e))||void 0===a?void 0:a.get(r);var c=!t.split("\n").filter((function(t){return t.startsWith("Disallow:")})).map((function(t){return t.replace("Disallow:","").trim()})).some((function(t){return r.startsWith(t)}));return _.has(e)||_.set(e,new Map),null===(o=_.get(e))||void 0===o||o.set(r,c),c},R=function(t,e){var r="".concat(t,"|").concat(e);return h.default.createHash("md5").update(r).digest("hex")},D=function(){var t=O(w().mark((function t(e){var r,n,o,i,u,l,s;return w().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:return r=new a.CheerioWebBaseLoader(e,{selector:"body"}),t.next=3,r.scrape();case 3:return(n=t.sent)("script").remove(),n("style").remove(),o=n("a").get().map((function(t){return t.attribs.href})),i=n("title").text()||n("h1").text()||e.replace(/^(https?:\/\/)/,""),u=n(r.selector).text(),l={source:r.webPath,title:i},s=new c.Document({pageContent:u,metadata:l}),t.abrupt("return",{doc:s,title:i,links:o});case 12:case"end":return t.stop()}}),t)})));return function(e){return t.apply(this,arguments)}}(),N=function(t){var e=t.title,r=t.href,n=t.checksum;return function(t,o){return{Name:e||"".concat(r," - Chunk ").concat(o),Body:t.pageContent,Link:r,checksum:n,metadata:b(b({},t.metadata),{},{chunk_id:o,source_url:r,crawled_at:new Date})}}};function U(t){return A.apply(this,arguments)}function A(){return A=O(w().mark((function t(e){var r,o,a,c,i,u,l,s,h,d,p,g,y,m,b,x,k,O,L,E,S,j;return w().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(r=e.href,o=e.collectionName,a=e.mongoClient,c=e.verbose,i=void 0!==c&&c,u=e.dryRun,l=void 0!==u&&u,t.prev=1,i&&console.log(f.default.gray("Processing URL:"),f.default.blue(r)),s=new URL(r),h=s.hostname,d=s.pathname,P(h,d,i)){t.next=8;break}return i&&console.log(f.default.red("Crawling disallowed by robots.txt for ".concat(h,". Skipping..."))),t.abrupt("return",{docCount:0,links:[]});case 8:return t.next=10,D(r);case 10:return p=t.sent,g=p.doc,y=p.title,m=p.links,i&&console.log(f.default.gray("Loaded document",'"'.concat(f.default.bold(y),'"'),"from",f.default.blue(r))),b=new n.RecursiveCharacterTextSplitter({chunkSize:1e3,chunkOverlap:200}),t.next=18,b.splitDocuments([g]);case 18:if(x=t.sent,i&&console.log(f.default.gray("Split into ".concat(x.length," chunks"))),k=R(r,g.pageContent),O=x.map(N({title:y,href:r,checksum:k})),!(x.length<=0)){t.next=25;break}return i&&console.log(f.default.gray("No content to process for URL: ".concat(r))),t.abrupt("return",{docCount:0,links:m,documents:[]});case 25:if(L=0,!l){t.next=31;break}i&&console.log(f.default.yellow("[DRY RUN] Would insert ".concat(O.length,' documents into collection: "').concat(o,'"'))),L=O.length,t.next=43;break;case 31:if(E=a.db(v),S=E.collection(o),i&&console.log(f.default.green("Inserting ".concat(O.length,' documents into collection: "').concat(o,'"'))),!(O.length>0)){t.next=42;break}return t.next=37,S.insertMany(O);case 37:j=t.sent,L=j.insertedCount,i&&console.log(f.default.gray("Successfully inserted ".concat(j.insertedCount," documents into collection ").concat(o))),t.next=43;break;case 42:i&&console.log(f.default.gray("No documents to insert for URL: ".concat(r)));case 43:return t.abrupt("return",{docCount:L,links:m,documents:O});case 46:return t.prev=46,t.t0=t.catch(1),console.error(f.default.red("Error processing URL ".concat(r,":")),f.default.gray(t.t0)),t.abrupt("return",{docCount:0,links:[]});case 50:case"end":return t.stop()}}),t,null,[[1,46]])}))),A.apply(this,arguments)}var G=function(t){try{return new URL(t)}catch(e){console.error(f.default.red("Invalid URL: ".concat(t)),e),process.exit(1)}};function T(t){return W.apply(this,arguments)}function W(){return(W=O(w().mark((function t(e){var r,n,o,a,c,i,u,l,s,h,g,v,m,b,x,k,O,L,E,j,_,P,C,R;return w().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(r=e.baseUrl,n=e.collectionName,o=e.mongoClient,a=e.relativePath,c=e.maxDepth,i=void 0===c?3:c,u=e.currentDepth,l=void 0===u?0:u,s=e.visited,h=void 0===s?new Set:s,g=e.verbose,v=void 0!==g&&g,m=e.dryRun,b=void 0!==m&&m,x=e.enableRecursion,k=void 0===x||x,0===l&&v&&console.log(f.default.green("\nStarting crawl of "+r)),O=(null==a?void 0:a.length)&&(null==a?void 0:a.startsWith("/")),L="".concat(d.default(r,"/")).concat(O?"/"+p.default(a,"/"):""),E=G(L),!(l>i||h.has(E.href))){t.next=8;break}return v&&console.log(f.default.gray("Already visited ".concat(E," or reached max depth (").concat(l,"/").concat(i,")"))),t.abrupt("return");case 8:return v&&console.log("\n",f.default.gray("Crawling ".concat(E," (depth: ").concat(l,"/").concat(i,")"))),h.add(E.href),t.next=12,U({href:E.href,collectionName:n,mongoClient:o,verbose:v,dryRun:b});case 12:if(j=t.sent,_=j.links,l!==i&&k){t.next=17;break}return v&&console.log(f.default.gray("Reached max depth, or not allowed to crawl further links from ".concat(E))),t.abrupt("return");case 17:t.prev=17,P=S(_),t.prev=19,R=w().mark((function t(){var e,a,c,u,s,d,g;return w().wrap((function(t){for(;;)switch(t.prev=t.next){case 0:if(e=C.value,a=e.startsWith("/")||e.startsWith("#"),c=G(a?E.origin+"/"+p.default(e,"/"):e),u=h.has(c.href),s=e.startsWith(r),g=(d=!a&&!s)&&y.some((function(t){return e.startsWith(t)})),!u){t.next=10;break}return v&&console.log(f.default.gray("Already visited link: ".concat(e))),t.abrupt("return",0);case 10:if(!d||g){t.next=13;break}return v&&console.log(f.default.gray("Skipping external link: ".concat(e," not in allowed domains"))),t.abrupt("return",0);case 13:return console.log(f.default.gray("Found new link: ".concat(e))),v&&console.log(f.default.gray("Preparing to crawl ".concat(c.origin," with relative path ").concat(c.pathname," ").concat(d?" (external)":""))),t.next=17,T({baseUrl:c.origin,relativePath:c.pathname,collectionName:n,mongoClient:o,maxDepth:i,currentDepth:l+1,visited:h,verbose:v,dryRun:b,enableRecursion:!d});case 17:case"end":return t.stop()}}),t)})),P.s();case 22:if((C=P.n()).done){t.next=29;break}return t.delegateYield(R(),"t0",24);case 24:if(0!==t.t0){t.next=27;break}return t.abrupt("continue",27);case 27:t.next=22;break;case 29:t.next=34;break;case 31:t.prev=31,t.t1=t.catch(19),P.e(t.t1);case 34:return t.prev=34,P.f(),t.finish(34);case 37:t.next=42;break;case 39:t.prev=39,t.t2=t.catch(17),console.error(f.default.red("Error crawling ".concat(r,":")),t.t2);case 42:case"end":return t.stop()}}),t,null,[[17,39],[19,31,34,37]])})))).apply(this,arguments)}t.MDB_DB=v,t.SOURCES=[{url:"https://mongodb.design",collection:"mongodb-dot-design"},{url:"https://react.dev/reference/react",collection:"react-dev"},{url:"https://developer.mozilla.org/en-US/docs/Web",collection:"mdn"},{url:"https://css-tricks.com/category/articles/",collection:"css-tricks"},{url:"https://www.nngroup.com/articles/",collection:"nn-group"},{url:"https://www.w3.org/WAI/standards-guidelines/wcag/",collection:"wcag"},{url:"https://atomicdesign.bradfrost.com/table-of-contents/",collection:"atomic-design"}],t.allowedDomains=y,t.processSingleUrl=U,t.recursiveCrawlFromBaseURL=T}));
//# sourceMappingURL=index.js.map