@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 7.59 kB
Source Map (JSON)
{"version":3,"file":"searchapi.cjs","names":["BaseDocumentLoader","params: SearchApiParameters","preparedParams: [string, string][]","output: unknown","responseType: string","documents: Document[]","Document","data: Record<string, unknown>","url: string"],"sources":["../../../src/document_loaders/web/searchapi.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\ntype JSONPrimitive = string | number | boolean | null;\ntype JSONValue = JSONPrimitive | JSONObject | JSONArray;\ninterface JSONObject {\n [key: string]: JSONValue;\n}\ninterface JSONArray extends Array<JSONValue> {}\n\n/**\n * SearchApiParameters Type Definition.\n *\n * For more parameters and supported search engines, refer specific engine documentation:\n * Google - https://www.searchapi.io/docs/google\n * Google News - https://www.searchapi.io/docs/google-news\n * Google Scholar - https://www.searchapi.io/docs/google-scholar\n * YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts\n * and others.\n *\n */\ntype SearchApiParameters = {\n [key: string]: JSONValue;\n};\n\n/**\n * Class representing a document loader for loading search results from\n * the SearchApi. It extends the BaseDocumentLoader class.\n * @example\n * ```typescript\n * const loader = new SearchApiLoader({\n * q: \"{query}\",\n * apiKey: \"{apiKey}\",\n * engine: \"google\",\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class SearchApiLoader extends BaseDocumentLoader {\n private apiKey: string;\n\n private parameters: SearchApiParameters;\n\n constructor(params: SearchApiParameters) {\n super();\n\n const { apiKey = getEnvironmentVariable(\"SEARCHAPI_API_KEY\") } = params;\n\n if (typeof apiKey !== \"string\") {\n throw new Error(\"Invalid type for apiKey. Expected string.\");\n }\n\n if (!apiKey) {\n throw new Error(\n \"SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.\"\n );\n }\n\n this.apiKey = apiKey;\n this.parameters = { ...params };\n }\n\n /**\n * Builds the URL for the SearchApi search request.\n * @returns The URL for the search request.\n */\n public buildUrl(): string {\n this.parameters = {\n engine: \"google\",\n api_key: this.apiKey,\n ...this.parameters,\n };\n\n const preparedParams: [string, string][] = Object.entries(this.parameters)\n .filter(\n ([key, value]) =>\n value !== undefined && value !== null && key !== \"apiKey\"\n )\n .map(([key, value]) => [key, `${value}`]);\n\n const searchParams = new URLSearchParams(preparedParams);\n return `https://www.searchapi.io/api/v1/search?${searchParams}`;\n }\n\n /**\n * Extracts documents from the provided output.\n * @param output - The output to extract documents from.\n * @param responseType - The type of the response to extract documents from.\n * @returns An array of Documents.\n */\n private extractDocuments(output: unknown, responseType: string): Document[] {\n const documents: Document[] = [];\n const results = Array.isArray(output) ? output : [output];\n if (responseType === \"transcripts\") {\n const pageContent = results.map((result) => result.text).join(\"\\n\");\n const metadata = {\n source: \"SearchApi\",\n responseType,\n };\n documents.push(new Document({ pageContent, metadata }));\n } else {\n for (const result of results) {\n const pageContent = JSON.stringify(result);\n const metadata = {\n source: \"SearchApi\",\n responseType,\n };\n documents.push(new Document({ pageContent, metadata }));\n }\n }\n\n return documents;\n }\n\n /**\n * Processes the response data from the SearchApi search request and converts it into an array of Documents.\n * @param data - The response data from the SearchApi search request.\n * @returns An array of Documents.\n */\n public processResponseData(data: Record<string, unknown>): Document[] {\n const documents: Document[] = [];\n const responseTypes = [\n \"answer_box\",\n \"shopping_results\",\n \"knowledge_graph\",\n \"organic_results\",\n \"transcripts\",\n ];\n for (const responseType of responseTypes) {\n if (responseType in data) {\n documents.push(\n ...this.extractDocuments(data[responseType], responseType)\n );\n }\n }\n return documents;\n }\n\n /**\n * Fetches the data from the provided URL and returns it as a JSON object.\n * If an error occurs during the fetch operation, an exception is thrown with the error message.\n * @param url - The URL to fetch data from.\n * @returns A promise that resolves to the fetched data as a JSON object.\n * @throws An error if the fetch operation fails.\n */\n private async fetchData(url: string): Promise<Record<string, unknown>> {\n const response = await fetch(url);\n const data = await response.json();\n\n if (data.error) {\n throw new Error(\n `Failed to load search results from SearchApi due to: ${data.error}`\n );\n }\n\n return data;\n }\n\n /**\n * Loads the search results from the SearchApi.\n * @returns An array of Documents representing the search results.\n * @throws An error if the search results could not be loaded.\n */\n public async load(): Promise<Document[]> {\n const url = this.buildUrl();\n\n const data = await this.fetchData(url);\n try {\n return this.processResponseData(data);\n } catch (error) {\n console.error(error);\n throw new Error(\n `Failed to process search results from SearchApi: ${error}`\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,kBAAb,cAAqCA,0DAAmB;CACtD,AAAQ;CAER,AAAQ;CAER,YAAYC,QAA6B;EACvC,OAAO;EAEP,MAAM,EAAE,gEAAgC,oBAAoB,EAAE,GAAG;AAEjE,MAAI,OAAO,WAAW,SACpB,OAAM,IAAI,MAAM;AAGlB,MAAI,CAAC,OACH,OAAM,IAAI,MACR;EAIJ,KAAK,SAAS;EACd,KAAK,aAAa,EAAE,GAAG,OAAQ;CAChC;;;;;CAMD,AAAO,WAAmB;EACxB,KAAK,aAAa;GAChB,QAAQ;GACR,SAAS,KAAK;GACd,GAAG,KAAK;EACT;EAED,MAAMC,iBAAqC,OAAO,QAAQ,KAAK,WAAW,CACvE,OACC,CAAC,CAAC,KAAK,MAAM,KACX,UAAU,UAAa,UAAU,QAAQ,QAAQ,SACpD,CACA,IAAI,CAAC,CAAC,KAAK,MAAM,KAAK,CAAC,KAAK,GAAG,OAAO,AAAC,EAAC;EAE3C,MAAM,eAAe,IAAI,gBAAgB;AACzC,SAAO,CAAC,uCAAuC,EAAE,cAAc;CAChE;;;;;;;CAQD,AAAQ,iBAAiBC,QAAiBC,cAAkC;EAC1E,MAAMC,YAAwB,CAAE;EAChC,MAAM,UAAU,MAAM,QAAQ,OAAO,GAAG,SAAS,CAAC,MAAO;AACzD,MAAI,iBAAiB,eAAe;GAClC,MAAM,cAAc,QAAQ,IAAI,CAAC,WAAW,OAAO,KAAK,CAAC,KAAK,KAAK;GACnE,MAAM,WAAW;IACf,QAAQ;IACR;GACD;GACD,UAAU,KAAK,IAAIC,oCAAS;IAAE;IAAa;GAAU,GAAE;EACxD,MACC,MAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,cAAc,KAAK,UAAU,OAAO;GAC1C,MAAM,WAAW;IACf,QAAQ;IACR;GACD;GACD,UAAU,KAAK,IAAIA,oCAAS;IAAE;IAAa;GAAU,GAAE;EACxD;AAGH,SAAO;CACR;;;;;;CAOD,AAAO,oBAAoBC,MAA2C;EACpE,MAAMF,YAAwB,CAAE;EAChC,MAAM,gBAAgB;GACpB;GACA;GACA;GACA;GACA;EACD;AACD,OAAK,MAAM,gBAAgB,cACzB,KAAI,gBAAgB,MAClB,UAAU,KACR,GAAG,KAAK,iBAAiB,KAAK,eAAe,aAAa,CAC3D;AAGL,SAAO;CACR;;;;;;;;CASD,MAAc,UAAUG,KAA+C;EACrE,MAAM,WAAW,MAAM,MAAM,IAAI;EACjC,MAAM,OAAO,MAAM,SAAS,MAAM;AAElC,MAAI,KAAK,MACP,OAAM,IAAI,MACR,CAAC,qDAAqD,EAAE,KAAK,OAAO;AAIxE,SAAO;CACR;;;;;;CAOD,MAAa,OAA4B;EACvC,MAAM,MAAM,KAAK,UAAU;EAE3B,MAAM,OAAO,MAAM,KAAK,UAAU,IAAI;AACtC,MAAI;AACF,UAAO,KAAK,oBAAoB,KAAK;EACtC,SAAQ,OAAO;GACd,QAAQ,MAAM,MAAM;AACpB,SAAM,IAAI,MACR,CAAC,iDAAiD,EAAE,OAAO;EAE9D;CACF;AACF"}