@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 5.83 kB
Source Map (JSON)
{"version":3,"file":"firecrawl.cjs","names":["BaseDocumentLoader","loaderParams: FirecrawlLoaderParameters","params: ConstructorParameters<typeof FirecrawlApp>[0]","FirecrawlApp","firecrawlDocs: FirecrawlDocument[]","Document"],"sources":["../../../src/document_loaders/web/firecrawl.ts"],"sourcesContent":["import FirecrawlApp from \"@mendable/firecrawl-js\";\nimport { Document, type DocumentInterface } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\n/**\n * Interface representing the parameters for the Firecrawl loader. It\n * includes properties such as the URL to scrape or crawl and the API key.\n */\ninterface FirecrawlLoaderParameters {\n /**\n * URL to scrape or crawl\n */\n url: string;\n\n /**\n * API key for Firecrawl. If not provided, the default value is the value of the FIRECRAWL_API_KEY environment variable.\n */\n apiKey?: string;\n\n /**\n * API URL for Firecrawl.\n */\n apiUrl?: string;\n /**\n * Mode of operation. Can be \"crawl\", \"scrape\", or \"map\". If not provided, the default value is \"crawl\".\n */\n mode?: \"crawl\" | \"scrape\" | \"map\";\n params?: Record<string, unknown>;\n}\n\ninterface FirecrawlDocument {\n markdown?: string;\n html?: string;\n rawHtml?: string;\n metadata?: Record<string, unknown>;\n}\n\n/**\n * Class representing a document loader for loading data from\n * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.\n * @example\n * ```typescript\n * const loader = new FireCrawlLoader({\n * url: \"{url}\",\n * apiKey: \"{apiKey}\",\n * mode: \"crawl\"\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class FireCrawlLoader extends BaseDocumentLoader {\n private apiKey: string;\n\n private apiUrl?: string;\n\n private url: string;\n\n private mode: \"crawl\" | \"scrape\" | \"map\";\n\n private params?: Record<string, unknown>;\n\n constructor(loaderParams: FirecrawlLoaderParameters) {\n super();\n const {\n apiKey = getEnvironmentVariable(\"FIRECRAWL_API_KEY\"),\n apiUrl,\n url,\n mode = \"crawl\",\n params,\n } = loaderParams;\n if (!apiKey) {\n throw new Error(\n \"Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.\"\n );\n }\n\n this.apiKey = apiKey;\n this.apiUrl = apiUrl;\n this.url = url;\n this.mode = mode;\n this.params = params;\n }\n\n /**\n * Loads data from Firecrawl.\n * @returns An array of Documents representing the retrieved data.\n * @throws An error if the data could not be loaded.\n */\n public async load(): Promise<DocumentInterface[]> {\n const params: ConstructorParameters<typeof FirecrawlApp>[0] = {\n apiKey: this.apiKey,\n };\n if (this.apiUrl !== undefined) {\n params.apiUrl = this.apiUrl;\n }\n const app = new FirecrawlApp(params);\n let firecrawlDocs: FirecrawlDocument[];\n\n if (this.mode === \"scrape\") {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const response = await app.scrapeUrl(this.url, this.params as any);\n if (!response.success) {\n throw new Error(\n `Firecrawl: Failed to scrape URL. Error: ${response.error}`\n );\n }\n firecrawlDocs = [response] as FirecrawlDocument[];\n } else if (this.mode === \"crawl\") {\n const response = await app.crawlUrl(this.url, this.params);\n if (!response.success) {\n throw new Error(\n `Firecrawl: Failed to crawl URL. Error: ${response.error}`\n );\n }\n firecrawlDocs = response.data as FirecrawlDocument[];\n } else if (this.mode === \"map\") {\n const response = await app.mapUrl(this.url, this.params);\n if (!response.success) {\n throw new Error(\n `Firecrawl: Failed to map URL. Error: ${response.error}`\n );\n }\n firecrawlDocs = response.links as FirecrawlDocument[];\n\n return firecrawlDocs.map(\n (doc) =>\n new Document({\n pageContent: JSON.stringify(doc),\n })\n );\n } else {\n throw new Error(\n `Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`\n );\n }\n\n return firecrawlDocs.map(\n (doc) =>\n new Document({\n pageContent: doc.markdown || doc.html || doc.rawHtml || \"\",\n metadata: doc.metadata || {},\n })\n );\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAa,kBAAb,cAAqCA,0DAAmB;CACtD,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YAAYC,cAAyC;EACnD,OAAO;EACP,MAAM,EACJ,gEAAgC,oBAAoB,EACpD,QACA,KACA,OAAO,SACP,QACD,GAAG;AACJ,MAAI,CAAC,OACH,OAAM,IAAI,MACR;EAIJ,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,MAAM;EACX,KAAK,OAAO;EACZ,KAAK,SAAS;CACf;;;;;;CAOD,MAAa,OAAqC;EAChD,MAAMC,SAAwD,EAC5D,QAAQ,KAAK,OACd;AACD,MAAI,KAAK,WAAW,QAClB,OAAO,SAAS,KAAK;EAEvB,MAAM,MAAM,IAAIC,gCAAa;EAC7B,IAAIC;AAEJ,MAAI,KAAK,SAAS,UAAU;GAE1B,MAAM,WAAW,MAAM,IAAI,UAAU,KAAK,KAAK,KAAK,OAAc;AAClE,OAAI,CAAC,SAAS,QACZ,OAAM,IAAI,MACR,CAAC,wCAAwC,EAAE,SAAS,OAAO;GAG/D,gBAAgB,CAAC,QAAS;EAC3B,WAAU,KAAK,SAAS,SAAS;GAChC,MAAM,WAAW,MAAM,IAAI,SAAS,KAAK,KAAK,KAAK,OAAO;AAC1D,OAAI,CAAC,SAAS,QACZ,OAAM,IAAI,MACR,CAAC,uCAAuC,EAAE,SAAS,OAAO;GAG9D,gBAAgB,SAAS;EAC1B,WAAU,KAAK,SAAS,OAAO;GAC9B,MAAM,WAAW,MAAM,IAAI,OAAO,KAAK,KAAK,KAAK,OAAO;AACxD,OAAI,CAAC,SAAS,QACZ,OAAM,IAAI,MACR,CAAC,qCAAqC,EAAE,SAAS,OAAO;GAG5D,gBAAgB,SAAS;AAEzB,UAAO,cAAc,IACnB,CAAC,QACC,IAAIC,oCAAS,EACX,aAAa,KAAK,UAAU,IAAI,CACjC,GACJ;EACF,MACC,OAAM,IAAI,MACR,CAAC,mBAAmB,EAAE,KAAK,KAAK,qCAAqC,CAAC;AAI1E,SAAO,cAAc,IACnB,CAAC,QACC,IAAIA,oCAAS;GACX,aAAa,IAAI,YAAY,IAAI,QAAQ,IAAI,WAAW;GACxD,UAAU,IAAI,YAAY,CAAE;EAC7B,GACJ;CACF;AACF"}