@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 4.44 kB
Source Map (JSON)
{"version":3,"file":"spider.cjs","names":["BaseDocumentLoader","loaderParams: SpiderLoaderParameters","Spider","spiderDocs: SpiderDocument[]","Document"],"sources":["../../../src/document_loaders/web/spider.ts"],"sourcesContent":["import { Spider } from \"@spider-cloud/spider-client\";\nimport { Document, type DocumentInterface } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\n/**\n * Interface representing the parameters for the Spider loader. It\n * includes properties such as the URL to scrape or crawl and the API key.\n */\ninterface SpiderLoaderParameters {\n /**\n * URL to scrape or crawl\n */\n url: string;\n\n /**\n * API key for Spider. If not provided, the default value is the value of the SPIDER_API_KEY environment variable.\n */\n apiKey?: string;\n\n /**\n * Mode of operation. Can be either \"crawl\" or \"scrape\". If not provided, the default value is \"scrape\".\n */\n mode?: \"crawl\" | \"scrape\";\n params?: Record<string, unknown>;\n}\ninterface SpiderDocument {\n content: string;\n metadata: Record<string, unknown>;\n}\n\n/**\n * Class representing a document loader for loading data from\n * Spider (spider.cloud). It extends the BaseDocumentLoader class.\n * @example\n * ```typescript\n * const loader = new SpiderLoader({\n * url: \"{url}\",\n * apiKey: \"{apiKey}\",\n * mode: \"crawl\"\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class SpiderLoader extends BaseDocumentLoader {\n private apiKey: string;\n\n private url: string;\n\n private mode: \"crawl\" | \"scrape\";\n\n private params?: Record<string, unknown>;\n\n constructor(loaderParams: SpiderLoaderParameters) {\n super();\n const {\n apiKey = getEnvironmentVariable(\"SPIDER_API_KEY\"),\n url,\n mode = \"scrape\",\n params,\n } = loaderParams;\n if (!apiKey) {\n throw new Error(\n \"Spider API key not set. You can set it as SPIDER_API_KEY in your .env file, or pass it to Spider.\"\n );\n }\n\n this.apiKey = apiKey;\n this.url = url;\n this.mode = mode;\n this.params = params || { metadata: true, return_format: \"markdown\" };\n }\n\n /**\n * Loads the data from the Spider.\n * @returns An array of Documents representing the retrieved data.\n * @throws An error if the data could not be loaded.\n */\n public async load(): Promise<DocumentInterface[]> {\n const app = new Spider({ apiKey: this.apiKey });\n let spiderDocs: SpiderDocument[];\n\n if (this.mode === \"scrape\") {\n const response = await app.scrapeUrl(this.url, this.params);\n if (response.error) {\n throw new Error(\n `Spider: Failed to scrape URL. Error: ${response.error}`\n );\n }\n spiderDocs = response as SpiderDocument[];\n } else if (this.mode === \"crawl\") {\n const response = await app.crawlUrl(this.url, this.params);\n if (response.error) {\n throw new Error(\n `Spider: Failed to crawl URL. Error: ${response.error}`\n );\n }\n spiderDocs = response as SpiderDocument[];\n } else {\n throw new Error(\n `Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`\n );\n }\n\n return spiderDocs.map(\n (doc) =>\n new Document({\n pageContent: doc.content || \"\",\n metadata: doc.metadata || {},\n })\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4CA,IAAa,eAAb,cAAkCA,0DAAmB;CACnD,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YAAYC,cAAsC;EAChD,OAAO;EACP,MAAM,EACJ,gEAAgC,iBAAiB,EACjD,KACA,OAAO,UACP,QACD,GAAG;AACJ,MAAI,CAAC,OACH,OAAM,IAAI,MACR;EAIJ,KAAK,SAAS;EACd,KAAK,MAAM;EACX,KAAK,OAAO;EACZ,KAAK,SAAS,UAAU;GAAE,UAAU;GAAM,eAAe;EAAY;CACtE;;;;;;CAOD,MAAa,OAAqC;EAChD,MAAM,MAAM,IAAIC,oCAAO,EAAE,QAAQ,KAAK,OAAQ;EAC9C,IAAIC;AAEJ,MAAI,KAAK,SAAS,UAAU;GAC1B,MAAM,WAAW,MAAM,IAAI,UAAU,KAAK,KAAK,KAAK,OAAO;AAC3D,OAAI,SAAS,MACX,OAAM,IAAI,MACR,CAAC,qCAAqC,EAAE,SAAS,OAAO;GAG5D,aAAa;EACd,WAAU,KAAK,SAAS,SAAS;GAChC,MAAM,WAAW,MAAM,IAAI,SAAS,KAAK,KAAK,KAAK,OAAO;AAC1D,OAAI,SAAS,MACX,OAAM,IAAI,MACR,CAAC,oCAAoC,EAAE,SAAS,OAAO;GAG3D,aAAa;EACd,MACC,OAAM,IAAI,MACR,CAAC,mBAAmB,EAAE,KAAK,KAAK,qCAAqC,CAAC;AAI1E,SAAO,WAAW,IAChB,CAAC,QACC,IAAIC,oCAAS;GACX,aAAa,IAAI,WAAW;GAC5B,UAAU,IAAI,YAAY,CAAE;EAC7B,GACJ;CACF;AACF"}