@langchain/community/dist/document_loaders/web/searchapi.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"searchapi.cjs","names":["BaseDocumentLoader","Document"],"sources":["../../../src/document_loaders/web/searchapi.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\ntype JSONPrimitive = string | number | boolean | null;\ntype JSONValue = JSONPrimitive | JSONObject | JSONArray;\ninterface JSONObject {\n  [key: string]: JSONValue;\n}\ninterface JSONArray extends Array<JSONValue> {}\n\n/**\n * SearchApiParameters Type Definition.\n *\n * For more parameters and supported search engines, refer specific engine documentation:\n * Google - https://www.searchapi.io/docs/google\n * Google News - https://www.searchapi.io/docs/google-news\n * Google Scholar - https://www.searchapi.io/docs/google-scholar\n * YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts\n * and others.\n *\n */\ntype SearchApiParameters = {\n  [key: string]: JSONValue;\n};\n\n/**\n * Class representing a document loader for loading search results from\n * the SearchApi. It extends the BaseDocumentLoader class.\n * @example\n * ```typescript\n * const loader = new SearchApiLoader({\n *   q: \"{query}\",\n *   apiKey: \"{apiKey}\",\n *   engine: \"google\",\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class SearchApiLoader extends BaseDocumentLoader {\n  private apiKey: string;\n\n  private parameters: SearchApiParameters;\n\n  constructor(params: SearchApiParameters) {\n    super();\n\n    const { apiKey = getEnvironmentVariable(\"SEARCHAPI_API_KEY\") } = params;\n\n    if (typeof apiKey !== \"string\") {\n      throw new Error(\"Invalid type for apiKey. Expected string.\");\n    }\n\n    if (!apiKey) {\n      throw new Error(\n        \"SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.\"\n      );\n    }\n\n    this.apiKey = apiKey;\n    this.parameters = { ...params };\n  }\n\n  /**\n   * Builds the URL for the SearchApi search request.\n   * @returns The URL for the search request.\n   */\n  public buildUrl(): string {\n    this.parameters = {\n      engine: \"google\",\n      api_key: this.apiKey,\n      ...this.parameters,\n    };\n\n    const preparedParams: [string, string][] = Object.entries(this.parameters)\n      .filter(\n        ([key, value]) =>\n          value !== undefined && value !== null && key !== \"apiKey\"\n      )\n      .map(([key, value]) => [key, `${value}`]);\n\n    const searchParams = new URLSearchParams(preparedParams);\n    return `https://www.searchapi.io/api/v1/search?${searchParams}`;\n  }\n\n  /**\n   * Extracts documents from the provided output.\n   * @param output - The output to extract documents from.\n   * @param responseType - The type of the response to extract documents from.\n   * @returns An array of Documents.\n   */\n  private extractDocuments(output: unknown, responseType: string): Document[] {\n    const documents: Document[] = [];\n    const results = Array.isArray(output) ? output : [output];\n    if (responseType === \"transcripts\") {\n      const pageContent = results.map((result) => result.text).join(\"\\n\");\n      const metadata = {\n        source: \"SearchApi\",\n        responseType,\n      };\n      documents.push(new Document({ pageContent, metadata }));\n    } else {\n      for (const result of results) {\n        const pageContent = JSON.stringify(result);\n        const metadata = {\n          source: \"SearchApi\",\n          responseType,\n        };\n        documents.push(new Document({ pageContent, metadata }));\n      }\n    }\n\n    return documents;\n  }\n\n  /**\n   * Processes the response data from the SearchApi search request and converts it into an array of Documents.\n   * @param data - The response data from the SearchApi search request.\n   * @returns An array of Documents.\n   */\n  public processResponseData(data: Record<string, unknown>): Document[] {\n    const documents: Document[] = [];\n    const responseTypes = [\n      \"answer_box\",\n      \"shopping_results\",\n      \"knowledge_graph\",\n      \"organic_results\",\n      \"transcripts\",\n    ];\n    for (const responseType of responseTypes) {\n      if (responseType in data) {\n        documents.push(\n          ...this.extractDocuments(data[responseType], responseType)\n        );\n      }\n    }\n    return documents;\n  }\n\n  /**\n   * Fetches the data from the provided URL and returns it as a JSON object.\n   * If an error occurs during the fetch operation, an exception is thrown with the error message.\n   * @param url - The URL to fetch data from.\n   * @returns A promise that resolves to the fetched data as a JSON object.\n   * @throws An error if the fetch operation fails.\n   */\n  private async fetchData(url: string): Promise<Record<string, unknown>> {\n    const response = await fetch(url);\n    const data = await response.json();\n\n    if (data.error) {\n      throw new Error(\n        `Failed to load search results from SearchApi due to: ${data.error}`\n      );\n    }\n\n    return data;\n  }\n\n  /**\n   * Loads the search results from the SearchApi.\n   * @returns An array of Documents representing the search results.\n   * @throws An error if the search results could not be loaded.\n   */\n  public async load(): Promise<Document[]> {\n    const url = this.buildUrl();\n\n    const data = await this.fetchData(url);\n    try {\n      return this.processResponseData(data);\n    } catch (error) {\n      console.error(error);\n      throw new Error(\n        `Failed to process search results from SearchApi: ${error}`\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,kBAAb,cAAqCA,sCAAAA,mBAAmB;CACtD;CAEA;CAEA,YAAY,QAA6B;AACvC,SAAO;EAEP,MAAM,EAAE,UAAA,GAAA,0BAAA,wBAAgC,oBAAoB,KAAK;AAEjE,MAAI,OAAO,WAAW,SACpB,OAAM,IAAI,MAAM,4CAA4C;AAG9D,MAAI,CAAC,OACH,OAAM,IAAI,MACR,6GACD;AAGH,OAAK,SAAS;AACd,OAAK,aAAa,EAAE,GAAG,QAAQ;;;;;;CAOjC,WAA0B;AACxB,OAAK,aAAa;GAChB,QAAQ;GACR,SAAS,KAAK;GACd,GAAG,KAAK;GACT;EAED,MAAM,iBAAqC,OAAO,QAAQ,KAAK,WAAW,CACvE,QACE,CAAC,KAAK,WACL,UAAU,KAAA,KAAa,UAAU,QAAQ,QAAQ,SACpD,CACA,KAAK,CAAC,KAAK,WAAW,CAAC,KAAK,GAAG,QAAQ,CAAC;AAG3C,SAAO,0CADc,IAAI,gBAAgB,eAAe;;;;;;;;CAU1D,iBAAyB,QAAiB,cAAkC;EAC1E,MAAM,YAAwB,EAAE;EAChC,MAAM,UAAU,MAAM,QAAQ,OAAO,GAAG,SAAS,CAAC,OAAO;AACzD,MAAI,iBAAiB,eAAe;GAClC,MAAM,cAAc,QAAQ,KAAK,WAAW,OAAO,KAAK,CAAC,KAAK,KAAK;GACnE,MAAM,WAAW;IACf,QAAQ;IACR;IACD;AACD,aAAU,KAAK,IAAIC,0BAAAA,SAAS;IAAE;IAAa;IAAU,CAAC,CAAC;QAEvD,MAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,cAAc,KAAK,UAAU,OAAO;GAC1C,MAAM,WAAW;IACf,QAAQ;IACR;IACD;AACD,aAAU,KAAK,IAAIA,0BAAAA,SAAS;IAAE;IAAa;IAAU,CAAC,CAAC;;AAI3D,SAAO;;;;;;;CAQT,oBAA2B,MAA2C;EACpE,MAAM,YAAwB,EAAE;AAQhC,OAAK,MAAM,gBAPW;GACpB;GACA;GACA;GACA;GACA;GACD,CAEC,KAAI,gBAAgB,KAClB,WAAU,KACR,GAAG,KAAK,iBAAiB,KAAK,eAAe,aAAa,CAC3D;AAGL,SAAO;;;;;;;;;CAUT,MAAc,UAAU,KAA+C;EAErE,MAAM,OAAO,OADI,MAAM,MAAM,IAAI,EACL,MAAM;AAElC,MAAI,KAAK,MACP,OAAM,IAAI,MACR,wDAAwD,KAAK,QAC9D;AAGH,SAAO;;;;;;;CAQT,MAAa,OAA4B;EACvC,MAAM,MAAM,KAAK,UAAU;EAE3B,MAAM,OAAO,MAAM,KAAK,UAAU,IAAI;AACtC,MAAI;AACF,UAAO,KAAK,oBAAoB,KAAK;WAC9B,OAAO;AACd,WAAQ,MAAM,MAAM;AACpB,SAAM,IAAI,MACR,oDAAoD,QACrD"}