@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 5.52 kB
Source Map (JSON)
{"version":3,"file":"gitbook.cjs","names":["CheerioWebBaseLoader","webPath: string","params: GitbookLoaderParams","$: CheerioAPI","url?: string","Document","documents: Document[]"],"sources":["../../../src/document_loaders/web/gitbook.ts"],"sourcesContent":["import type { CheerioAPI } from \"cheerio\";\nimport { Document } from \"@langchain/core/documents\";\nimport { CheerioWebBaseLoader } from \"./cheerio.js\";\n\n/**\n * Interface representing the parameters for configuring the\n * GitbookLoader. It has an optional property shouldLoadAllPaths, which\n * indicates whether all paths should be loaded.\n */\ninterface GitbookLoaderParams {\n shouldLoadAllPaths?: boolean;\n}\n\n/**\n * Class representing a document loader specifically designed for loading\n * documents from Gitbook. It extends the CheerioWebBaseLoader.\n */\nexport class GitbookLoader extends CheerioWebBaseLoader {\n shouldLoadAllPaths = false;\n\n private readonly baseUrl: string;\n\n constructor(\n public webPath: string,\n params: GitbookLoaderParams = {}\n ) {\n const path =\n params.shouldLoadAllPaths === true ? `${webPath}/sitemap.xml` : webPath;\n super(path);\n\n this.baseUrl = webPath;\n this.webPath = path;\n\n this.shouldLoadAllPaths =\n params.shouldLoadAllPaths ?? this.shouldLoadAllPaths;\n }\n\n /**\n * Method that scrapes the web document using Cheerio and loads the\n * content based on the value of shouldLoadAllPaths. If shouldLoadAllPaths\n * is true, it calls the loadAllPaths() method to load all paths.\n * Otherwise, it calls the loadPath() method to load a single path.\n * @returns Promise resolving to an array of Document instances.\n */\n public async load(): Promise<Document[]> {\n const $ = await this.scrape();\n\n if (this.shouldLoadAllPaths === true) {\n return this.loadAllPaths($);\n }\n return this.loadPath($);\n }\n\n /**\n * Private method that loads the content of a single path from the Gitbook\n * web document. It extracts the page content by selecting all elements\n * inside the \"main\" element, filters out empty text nodes, and joins the\n * remaining text nodes with line breaks. It extracts the title by\n * selecting the first \"h1\" element inside the \"main\" element. It creates\n * a Document instance with the extracted page content and metadata\n * containing the source URL and title.\n * @param $ CheerioAPI instance representing the loaded web document.\n * @param url Optional string representing the URL of the web document.\n * @returns Array of Document instances.\n */\n private loadPath($: CheerioAPI, url?: string): Document[] {\n const pageContent = $(\"main *\")\n .contents()\n .toArray()\n .map((element) =>\n element.type === \"text\" ? $(element).text().trim() : null\n )\n .filter((text) => text)\n .join(\"\\n\");\n\n const title = $(\"main h1\").first().text().trim();\n\n return [\n new Document({\n pageContent,\n metadata: { source: url ?? this.webPath, title },\n }),\n ];\n }\n\n /**\n * Private method that loads the content of all paths from the Gitbook web\n * document. It extracts the URLs of all paths from the \"loc\" elements in\n * the sitemap.xml. It iterates over each URL, scrapes the web document\n * using the _scrape() method, and calls the loadPath() method to load the\n * content of each path. It collects all the loaded documents and returns\n * them as an array.\n * @param $ CheerioAPI instance representing the loaded web document.\n * @returns Promise resolving to an array of Document instances.\n */\n private async loadAllPaths($: CheerioAPI): Promise<Document[]> {\n const urls = $(\"loc\")\n .toArray()\n .map((element) => $(element).text());\n\n const documents: Document[] = [];\n for (const url of urls) {\n const buildUrl = url.includes(this.baseUrl) ? url : this.baseUrl + url;\n console.log(`Fetching text from ${buildUrl}`);\n const html = await GitbookLoader._scrape(\n buildUrl,\n this.caller,\n this.timeout\n );\n documents.push(...this.loadPath(html, buildUrl));\n }\n console.log(`Fetched ${documents.length} documents.`);\n return documents;\n }\n}\n"],"mappings":";;;;;;;;;;;AAiBA,IAAa,gBAAb,MAAa,sBAAsBA,0DAAqB;CACtD,qBAAqB;CAErB,AAAiB;CAEjB,YACSC,SACPC,SAA8B,CAAE,GAChC;EACA,MAAM,OACJ,OAAO,uBAAuB,OAAO,GAAG,QAAQ,YAAY,CAAC,GAAG;EAClE,MAAM,KAAK;EALJ;EAOP,KAAK,UAAU;EACf,KAAK,UAAU;EAEf,KAAK,qBACH,OAAO,sBAAsB,KAAK;CACrC;;;;;;;;CASD,MAAa,OAA4B;EACvC,MAAM,IAAI,MAAM,KAAK,QAAQ;AAE7B,MAAI,KAAK,uBAAuB,KAC9B,QAAO,KAAK,aAAa,EAAE;AAE7B,SAAO,KAAK,SAAS,EAAE;CACxB;;;;;;;;;;;;;CAcD,AAAQ,SAASC,GAAeC,KAA0B;EACxD,MAAM,cAAc,EAAE,SAAS,CAC5B,UAAU,CACV,SAAS,CACT,IAAI,CAAC,YACJ,QAAQ,SAAS,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,KACtD,CACA,OAAO,CAAC,SAAS,KAAK,CACtB,KAAK,KAAK;EAEb,MAAM,QAAQ,EAAE,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM;AAEhD,SAAO,CACL,IAAIC,oCAAS;GACX;GACA,UAAU;IAAE,QAAQ,OAAO,KAAK;IAAS;GAAO;EACjD,EACF;CACF;;;;;;;;;;;CAYD,MAAc,aAAaF,GAAoC;EAC7D,MAAM,OAAO,EAAE,MAAM,CAClB,SAAS,CACT,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC;EAEtC,MAAMG,YAAwB,CAAE;AAChC,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,WAAW,IAAI,SAAS,KAAK,QAAQ,GAAG,MAAM,KAAK,UAAU;GACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,UAAU,CAAC;GAC7C,MAAM,OAAO,MAAM,cAAc,QAC/B,UACA,KAAK,QACL,KAAK,QACN;GACD,UAAU,KAAK,GAAG,KAAK,SAAS,MAAM,SAAS,CAAC;EACjD;EACD,QAAQ,IAAI,CAAC,QAAQ,EAAE,UAAU,OAAO,WAAW,CAAC,CAAC;AACrD,SAAO;CACR;AACF"}