@langchain/community/dist/document_loaders/fs/unstructured.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"unstructured.cjs","names":["BaseDocumentLoader","filepathOrBufferOptions: string | UnstructuredMemoryLoaderOptions","unstructuredOptions: UnstructuredLoaderOptions | string","documents: Document[]","Document","DirectoryLoader","directoryPathOrLegacyApiUrl: string","optionsOrLegacyDirectoryPath: UnstructuredDirectoryLoaderOptions | string","legacyOptionUnknown: UnknownHandling","UnknownHandling","options: UnstructuredDirectoryLoaderOptions","p: string","loadersObject: LoadersMapping","filetype: string"],"sources":["../../../src/document_loaders/fs/unstructured.ts"],"sourcesContent":["import type { basename as BasenameT } from \"node:path\";\nimport type { readFile as ReadFileT } from \"node:fs/promises\";\nimport { Document } from \"@langchain/core/documents\";\nimport { getEnv, getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { StringWithAutocomplete } from \"@langchain/core/utils/types\";\nimport {\n  DirectoryLoader,\n  UnknownHandling,\n  LoadersMapping,\n} from \"@langchain/classic/document_loaders/fs/directory\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\nexport const UNSTRUCTURED_API_FILETYPES = [\n  \".txt\",\n  \".text\",\n  \".pdf\",\n  \".docx\",\n  \".doc\",\n  \".jpg\",\n  \".jpeg\",\n  \".eml\",\n  \".html\",\n  \".htm\",\n  \".md\",\n  \".pptx\",\n  \".ppt\",\n  \".msg\",\n  \".rtf\",\n  \".xlsx\",\n  \".xls\",\n  \".odt\",\n  \".epub\",\n];\n\n/**\n * Represents an element returned by the Unstructured API. It has\n * properties for the element type, text content, and metadata.\n */\ntype Element = {\n  type: string;\n  text: string;\n  // this is purposefully loosely typed\n  metadata: {\n    [key: string]: unknown;\n  };\n};\n\n/**\n * Represents the available strategies for the UnstructuredLoader. It can\n * be one of \"hi_res\", \"fast\", \"ocr_only\", or \"auto\".\n */\nexport type UnstructuredLoaderStrategy =\n  | \"hi_res\"\n  | \"fast\"\n  | \"ocr_only\"\n  | \"auto\";\n\n/**\n * Represents the available hi-res models for the UnstructuredLoader. It can\n * be one of \"chipper\".\n */\nexport type HiResModelName = \"chipper\";\n\n/**\n * To enable or disable table extraction for file types other than PDF, set\n * the skipInferTableTypes property in the UnstructuredLoaderOptions object.\n * The skipInferTableTypes property is an array of file types for which table\n * extraction is disabled. For example, to disable table extraction for .docx\n * and .doc files, set the skipInferTableTypes property to [\"docx\", \"doc\"].\n * You can also disable table extraction for all file types other than PDF by\n * setting the skipInferTableTypes property to [].\n */\nexport type SkipInferTableTypes =\n  | \"txt\"\n  | \"text\"\n  | \"pdf\"\n  | \"docx\"\n  | \"doc\"\n  | \"jpg\"\n  | \"jpeg\"\n  | \"eml\"\n  | \"html\"\n  | \"htm\"\n  | \"md\"\n  | \"pptx\"\n  | \"ppt\"\n  | \"msg\"\n  | \"rtf\"\n  | \"xlsx\"\n  | \"xls\"\n  | \"odt\"\n  | \"epub\";\n\n/**\n * Set the chunking_strategy to chunk text into larger or smaller elements. Defaults to None with optional arg of by_title\n */\nexport type ChunkingStrategy = \"None\" | \"by_title\";\n\nexport type UnstructuredLoaderOptions = {\n  apiKey?: string;\n  apiUrl?: string;\n  strategy?: StringWithAutocomplete<UnstructuredLoaderStrategy>;\n  encoding?: string;\n  ocrLanguages?: Array<string>;\n  coordinates?: boolean;\n  pdfInferTableStructure?: boolean;\n  xmlKeepTags?: boolean;\n  skipInferTableTypes?: Array<StringWithAutocomplete<SkipInferTableTypes>>;\n  hiResModelName?: StringWithAutocomplete<HiResModelName>;\n  includePageBreaks?: boolean;\n  chunkingStrategy?: StringWithAutocomplete<ChunkingStrategy>;\n  multiPageSections?: boolean;\n  combineUnderNChars?: number;\n  newAfterNChars?: number;\n  maxCharacters?: number;\n  extractImageBlockTypes?: string[];\n  overlap?: number;\n  overlapAll?: boolean;\n};\n\nexport type UnstructuredDirectoryLoaderOptions = UnstructuredLoaderOptions & {\n  recursive?: boolean;\n  unknown?: UnknownHandling;\n};\n\nexport type UnstructuredMemoryLoaderOptions = {\n  buffer: Buffer;\n  fileName: string;\n};\n\n/**\n * A document loader that uses the Unstructured API to load unstructured\n * documents. It supports both the new syntax with options object and the\n * legacy syntax for backward compatibility. The load() method sends a\n * partitioning request to the Unstructured API and retrieves the\n * partitioned elements. It creates a Document instance for each element\n * and returns an array of Document instances.\n *\n * It accepts either a filepath or an object containing a buffer and a filename\n * as input.\n */\nexport class UnstructuredLoader extends BaseDocumentLoader {\n  public filePath: string;\n\n  private buffer?: Buffer;\n\n  private fileName?: string;\n\n  private apiUrl = \"https://api.unstructured.io/general/v0/general\";\n\n  private apiKey?: string;\n\n  private strategy: StringWithAutocomplete<UnstructuredLoaderStrategy> =\n    \"hi_res\";\n\n  private encoding?: string;\n\n  private ocrLanguages: Array<string> = [];\n\n  private coordinates?: boolean;\n\n  private pdfInferTableStructure?: boolean;\n\n  private xmlKeepTags?: boolean;\n\n  private skipInferTableTypes?: Array<\n    StringWithAutocomplete<SkipInferTableTypes>\n  >;\n\n  private hiResModelName?: StringWithAutocomplete<HiResModelName>;\n\n  private includePageBreaks?: boolean;\n\n  private chunkingStrategy?: StringWithAutocomplete<ChunkingStrategy>;\n\n  private multiPageSections?: boolean;\n\n  private combineUnderNChars?: number;\n\n  private newAfterNChars?: number;\n\n  private maxCharacters?: number;\n\n  private extractImageBlockTypes?: string[];\n\n  private overlap?: number;\n\n  private overlapAll?: boolean;\n\n  constructor(\n    filepathOrBufferOptions: string | UnstructuredMemoryLoaderOptions,\n    unstructuredOptions: UnstructuredLoaderOptions | string = {}\n  ) {\n    super();\n\n    // Temporary shim to avoid breaking existing users\n    // Remove when API keys are enforced by Unstructured and existing code will break anyway\n    const isLegacySyntax = typeof unstructuredOptions === \"string\";\n    const isMemorySyntax = typeof filepathOrBufferOptions === \"object\";\n\n    if (isMemorySyntax) {\n      this.buffer = filepathOrBufferOptions.buffer;\n      this.fileName = filepathOrBufferOptions.fileName;\n    } else if (isLegacySyntax) {\n      this.filePath = unstructuredOptions;\n      this.apiUrl = filepathOrBufferOptions;\n    } else {\n      this.filePath = filepathOrBufferOptions;\n    }\n\n    if (!isLegacySyntax) {\n      const options = unstructuredOptions;\n      this.apiKey =\n        options.apiKey ?? getEnvironmentVariable(\"UNSTRUCTURED_API_KEY\");\n      this.apiUrl =\n        options.apiUrl ??\n        getEnvironmentVariable(\"UNSTRUCTURED_API_URL\") ??\n        this.apiUrl;\n      this.strategy = options.strategy ?? this.strategy;\n      this.encoding = options.encoding;\n      this.ocrLanguages = options.ocrLanguages ?? this.ocrLanguages;\n      this.coordinates = options.coordinates;\n      this.pdfInferTableStructure = options.pdfInferTableStructure;\n      this.xmlKeepTags = options.xmlKeepTags;\n      this.skipInferTableTypes = options.skipInferTableTypes;\n      this.hiResModelName = options.hiResModelName;\n      this.includePageBreaks = options.includePageBreaks;\n      this.chunkingStrategy = options.chunkingStrategy;\n      this.multiPageSections = options.multiPageSections;\n      this.combineUnderNChars = options.combineUnderNChars;\n      this.newAfterNChars = options.newAfterNChars;\n      this.maxCharacters = options.maxCharacters;\n      this.extractImageBlockTypes = options.extractImageBlockTypes;\n      this.overlap = options.overlap;\n      this.overlapAll = options.overlapAll ?? false;\n    }\n  }\n\n  async _partition() {\n    let buffer = this.buffer;\n    let fileName = this.fileName;\n\n    if (!buffer) {\n      const { readFile, basename } = await this.imports();\n\n      buffer = await readFile(this.filePath);\n      fileName = basename(this.filePath);\n\n      // I'm aware this reads the file into memory first, but we have lots of work\n      // to do on then consuming Documents in a streaming fashion anyway, so not\n      // worried about this for now.\n    }\n\n    const formData = new FormData();\n    formData.append(\"files\", new Blob([buffer]), fileName);\n    formData.append(\"strategy\", this.strategy);\n    this.ocrLanguages.forEach((language) => {\n      formData.append(\"ocr_languages\", language);\n    });\n    if (this.encoding) {\n      formData.append(\"encoding\", this.encoding);\n    }\n    if (this.coordinates === true) {\n      formData.append(\"coordinates\", \"true\");\n    }\n    if (this.pdfInferTableStructure === true) {\n      formData.append(\"pdf_infer_table_structure\", \"true\");\n    }\n    if (this.xmlKeepTags === true) {\n      formData.append(\"xml_keep_tags\", \"true\");\n    }\n    if (this.skipInferTableTypes) {\n      formData.append(\n        \"skip_infer_table_types\",\n        JSON.stringify(this.skipInferTableTypes)\n      );\n    }\n    if (this.hiResModelName) {\n      formData.append(\"hi_res_model_name\", this.hiResModelName);\n    }\n    if (this.includePageBreaks) {\n      formData.append(\"include_page_breaks\", \"true\");\n    }\n    if (this.chunkingStrategy) {\n      formData.append(\"chunking_strategy\", this.chunkingStrategy);\n    }\n    if (this.multiPageSections !== undefined) {\n      formData.append(\n        \"multipage_sections\",\n        this.multiPageSections ? \"true\" : \"false\"\n      );\n    }\n    if (this.combineUnderNChars !== undefined) {\n      formData.append(\"combine_under_n_chars\", String(this.combineUnderNChars));\n    }\n    if (this.newAfterNChars !== undefined) {\n      formData.append(\"new_after_n_chars\", String(this.newAfterNChars));\n    }\n    if (this.maxCharacters !== undefined) {\n      formData.append(\"max_characters\", String(this.maxCharacters));\n    }\n\n    if (this.extractImageBlockTypes !== undefined) {\n      formData.append(\n        \"extract_image_block_types\",\n        JSON.stringify(this.extractImageBlockTypes)\n      );\n    }\n\n    if (this.overlap !== undefined) {\n      formData.append(\"overlap\", String(this.overlap));\n    }\n\n    if (this.overlapAll === true) {\n      formData.append(\"overlap_all\", \"true\");\n    }\n\n    const headers = {\n      \"UNSTRUCTURED-API-KEY\": this.apiKey ?? \"\",\n    };\n\n    const response = await fetch(this.apiUrl, {\n      method: \"POST\",\n      body: formData,\n      headers,\n    });\n\n    if (!response.ok) {\n      throw new Error(\n        `Failed to partition file ${this.filePath} with error ${\n          response.status\n        } and message ${await response.text()}`\n      );\n    }\n\n    const elements = await response.json();\n    if (!Array.isArray(elements)) {\n      throw new Error(\n        `Expected partitioning request to return an array, but got ${elements}`\n      );\n    }\n    return elements.filter((el) => typeof el.text === \"string\") as Element[];\n  }\n\n  async load(): Promise<Document[]> {\n    const elements = await this._partition();\n\n    const documents: Document[] = [];\n    for (const element of elements) {\n      const { metadata, text } = element;\n      if (typeof text === \"string\" && text !== \"\") {\n        documents.push(\n          new Document({\n            pageContent: text,\n            metadata: {\n              ...metadata,\n              category: element.type,\n            },\n          })\n        );\n      }\n    }\n\n    return documents;\n  }\n\n  async imports(): Promise<{\n    readFile: typeof ReadFileT;\n    basename: typeof BasenameT;\n  }> {\n    try {\n      const { readFile } = await import(\"node:fs/promises\");\n      const { basename } = await import(\"node:path\");\n      return { readFile, basename };\n    } catch (e) {\n      console.error(e);\n      throw new Error(\n        `Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`\n      );\n    }\n  }\n}\n\n/**\n * A document loader that loads unstructured documents from a directory\n * using the UnstructuredLoader. It creates a UnstructuredLoader instance\n * for each supported file type and passes it to the DirectoryLoader\n * constructor.\n * @example\n * ```typescript\n * const loader = new UnstructuredDirectoryLoader(\"path/to/directory\", {\n *   apiKey: \"MY_API_KEY\",\n * });\n * const docs = await loader.load();\n * ```\n */\nexport class UnstructuredDirectoryLoader extends DirectoryLoader {\n  constructor(\n    directoryPathOrLegacyApiUrl: string,\n    optionsOrLegacyDirectoryPath: UnstructuredDirectoryLoaderOptions | string,\n    legacyOptionRecursive = true,\n    legacyOptionUnknown: UnknownHandling = UnknownHandling.Warn\n  ) {\n    let directoryPath;\n    let options: UnstructuredDirectoryLoaderOptions;\n    // Temporary shim to avoid breaking existing users\n    // Remove when API keys are enforced by Unstructured and existing code will break anyway\n    const isLegacySyntax = typeof optionsOrLegacyDirectoryPath === \"string\";\n    if (isLegacySyntax) {\n      directoryPath = optionsOrLegacyDirectoryPath;\n      options = {\n        apiUrl: directoryPathOrLegacyApiUrl,\n        recursive: legacyOptionRecursive,\n        unknown: legacyOptionUnknown,\n      };\n    } else {\n      directoryPath = directoryPathOrLegacyApiUrl;\n      options = optionsOrLegacyDirectoryPath;\n    }\n    const loader = (p: string) => new UnstructuredLoader(p, options);\n    const loaders = UNSTRUCTURED_API_FILETYPES.reduce(\n      (loadersObject: LoadersMapping, filetype: string) => {\n        loadersObject[filetype] = loader;\n        return loadersObject;\n      },\n      {}\n    );\n    super(directoryPath, loaders, options.recursive, options.unknown);\n  }\n}\n\nexport { UnknownHandling };\n"],"mappings":";;;;;;;;;;;;;;AAYA,MAAa,6BAA6B;CACxC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACD;;;;;;;;;;;;AA6GD,IAAa,qBAAb,cAAwCA,0DAAmB;CACzD,AAAO;CAEP,AAAQ;CAER,AAAQ;CAER,AAAQ,SAAS;CAEjB,AAAQ;CAER,AAAQ,WACN;CAEF,AAAQ;CAER,AAAQ,eAA8B,CAAE;CAExC,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAIR,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YACEC,yBACAC,sBAA0D,CAAE,GAC5D;EACA,OAAO;EAIP,MAAM,iBAAiB,OAAO,wBAAwB;EACtD,MAAM,iBAAiB,OAAO,4BAA4B;AAE1D,MAAI,gBAAgB;GAClB,KAAK,SAAS,wBAAwB;GACtC,KAAK,WAAW,wBAAwB;EACzC,WAAU,gBAAgB;GACzB,KAAK,WAAW;GAChB,KAAK,SAAS;EACf,OACC,KAAK,WAAW;AAGlB,MAAI,CAAC,gBAAgB;GACnB,MAAM,UAAU;GAChB,KAAK,SACH,QAAQ,iEAAiC,uBAAuB;GAClE,KAAK,SACH,QAAQ,iEACe,uBAAuB,IAC9C,KAAK;GACP,KAAK,WAAW,QAAQ,YAAY,KAAK;GACzC,KAAK,WAAW,QAAQ;GACxB,KAAK,eAAe,QAAQ,gBAAgB,KAAK;GACjD,KAAK,cAAc,QAAQ;GAC3B,KAAK,yBAAyB,QAAQ;GACtC,KAAK,cAAc,QAAQ;GAC3B,KAAK,sBAAsB,QAAQ;GACnC,KAAK,iBAAiB,QAAQ;GAC9B,KAAK,oBAAoB,QAAQ;GACjC,KAAK,mBAAmB,QAAQ;GAChC,KAAK,oBAAoB,QAAQ;GACjC,KAAK,qBAAqB,QAAQ;GAClC,KAAK,iBAAiB,QAAQ;GAC9B,KAAK,gBAAgB,QAAQ;GAC7B,KAAK,yBAAyB,QAAQ;GACtC,KAAK,UAAU,QAAQ;GACvB,KAAK,aAAa,QAAQ,cAAc;EACzC;CACF;CAED,MAAM,aAAa;EACjB,IAAI,SAAS,KAAK;EAClB,IAAI,WAAW,KAAK;AAEpB,MAAI,CAAC,QAAQ;GACX,MAAM,EAAE,UAAU,UAAU,GAAG,MAAM,KAAK,SAAS;GAEnD,SAAS,MAAM,SAAS,KAAK,SAAS;GACtC,WAAW,SAAS,KAAK,SAAS;EAKnC;EAED,MAAM,WAAW,IAAI;EACrB,SAAS,OAAO,SAAS,IAAI,KAAK,CAAC,MAAO,IAAG,SAAS;EACtD,SAAS,OAAO,YAAY,KAAK,SAAS;EAC1C,KAAK,aAAa,QAAQ,CAAC,aAAa;GACtC,SAAS,OAAO,iBAAiB,SAAS;EAC3C,EAAC;AACF,MAAI,KAAK,UACP,SAAS,OAAO,YAAY,KAAK,SAAS;AAE5C,MAAI,KAAK,gBAAgB,MACvB,SAAS,OAAO,eAAe,OAAO;AAExC,MAAI,KAAK,2BAA2B,MAClC,SAAS,OAAO,6BAA6B,OAAO;AAEtD,MAAI,KAAK,gBAAgB,MACvB,SAAS,OAAO,iBAAiB,OAAO;AAE1C,MAAI,KAAK,qBACP,SAAS,OACP,0BACA,KAAK,UAAU,KAAK,oBAAoB,CACzC;AAEH,MAAI,KAAK,gBACP,SAAS,OAAO,qBAAqB,KAAK,eAAe;AAE3D,MAAI,KAAK,mBACP,SAAS,OAAO,uBAAuB,OAAO;AAEhD,MAAI,KAAK,kBACP,SAAS,OAAO,qBAAqB,KAAK,iBAAiB;AAE7D,MAAI,KAAK,sBAAsB,QAC7B,SAAS,OACP,sBACA,KAAK,oBAAoB,SAAS,QACnC;AAEH,MAAI,KAAK,uBAAuB,QAC9B,SAAS,OAAO,yBAAyB,OAAO,KAAK,mBAAmB,CAAC;AAE3E,MAAI,KAAK,mBAAmB,QAC1B,SAAS,OAAO,qBAAqB,OAAO,KAAK,eAAe,CAAC;AAEnE,MAAI,KAAK,kBAAkB,QACzB,SAAS,OAAO,kBAAkB,OAAO,KAAK,cAAc,CAAC;AAG/D,MAAI,KAAK,2BAA2B,QAClC,SAAS,OACP,6BACA,KAAK,UAAU,KAAK,uBAAuB,CAC5C;AAGH,MAAI,KAAK,YAAY,QACnB,SAAS,OAAO,WAAW,OAAO,KAAK,QAAQ,CAAC;AAGlD,MAAI,KAAK,eAAe,MACtB,SAAS,OAAO,eAAe,OAAO;EAGxC,MAAM,UAAU,EACd,wBAAwB,KAAK,UAAU,GACxC;EAED,MAAM,WAAW,MAAM,MAAM,KAAK,QAAQ;GACxC,QAAQ;GACR,MAAM;GACN;EACD,EAAC;AAEF,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MACR,CAAC,yBAAyB,EAAE,KAAK,SAAS,YAAY,EACpD,SAAS,OACV,aAAa,EAAE,MAAM,SAAS,MAAM,EAAE;EAI3C,MAAM,WAAW,MAAM,SAAS,MAAM;AACtC,MAAI,CAAC,MAAM,QAAQ,SAAS,CAC1B,OAAM,IAAI,MACR,CAAC,0DAA0D,EAAE,UAAU;AAG3E,SAAO,SAAS,OAAO,CAAC,OAAO,OAAO,GAAG,SAAS,SAAS;CAC5D;CAED,MAAM,OAA4B;EAChC,MAAM,WAAW,MAAM,KAAK,YAAY;EAExC,MAAMC,YAAwB,CAAE;AAChC,OAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,EAAE,UAAU,MAAM,GAAG;AAC3B,OAAI,OAAO,SAAS,YAAY,SAAS,IACvC,UAAU,KACR,IAAIC,oCAAS;IACX,aAAa;IACb,UAAU;KACR,GAAG;KACH,UAAU,QAAQ;IACnB;GACF,GACF;EAEJ;AAED,SAAO;CACR;CAED,MAAM,UAGH;AACD,MAAI;GACF,MAAM,EAAE,UAAU,GAAG,MAAM,OAAO;GAClC,MAAM,EAAE,UAAU,GAAG,MAAM,OAAO;AAClC,UAAO;IAAE;IAAU;GAAU;EAC9B,SAAQ,GAAG;GACV,QAAQ,MAAM,EAAE;AAChB,SAAM,IAAI,MACR,CAAC,qHAAqH,0CAAU,CAAC,+CAA+C,CAAC;EAEpL;CACF;AACF;;;;;;;;;;;;;;AAeD,IAAa,8BAAb,cAAiDC,kEAAgB;CAC/D,YACEC,6BACAC,8BACA,wBAAwB,MACxBC,sBAAuCC,kEAAgB,MACvD;EACA,IAAI;EACJ,IAAIC;EAGJ,MAAM,iBAAiB,OAAO,iCAAiC;AAC/D,MAAI,gBAAgB;GAClB,gBAAgB;GAChB,UAAU;IACR,QAAQ;IACR,WAAW;IACX,SAAS;GACV;EACF,OAAM;GACL,gBAAgB;GAChB,UAAU;EACX;EACD,MAAM,SAAS,CAACC,MAAc,IAAI,mBAAmB,GAAG;EACxD,MAAM,UAAU,2BAA2B,OACzC,CAACC,eAA+BC,aAAqB;GACnD,cAAc,YAAY;AAC1B,UAAO;EACR,GACD,CAAE,EACH;EACD,MAAM,eAAe,SAAS,QAAQ,WAAW,QAAQ,QAAQ;CAClE;AACF"}