langchain/dist/storage/file_system.cjs.map

Typescript bindings for langchain

{"version":3,"file":"file_system.cjs","names":["BaseStore","fs","path"],"sources":["../../src/storage/file_system.ts"],"sourcesContent":["import * as fs from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\n/**\n * File system implementation of the BaseStore using a dictionary. Used for\n * storing key-value pairs in the file system.\n * @example\n * ```typescript\n * const store = await LocalFileStore.fromPath(\"./messages\");\n * await store.mset(\n *   Array.from({ length: 5 }).map((_, index) => [\n *     `message:id:${index}`,\n *     new TextEncoder().encode(\n *       JSON.stringify(\n *         index % 2 === 0\n *           ? new AIMessage(\"ai stuff...\")\n *           : new HumanMessage(\"human stuff...\"),\n *       ),\n *     ),\n *   ]),\n * );\n * const retrievedMessages = await store.mget([\"message:id:0\", \"message:id:1\"]);\n * console.log(retrievedMessages.map((v) => new TextDecoder().decode(v)));\n * for await (const key of store.yieldKeys(\"message:id:\")) {\n *   await store.mdelete([key]);\n * }\n * ```\n *\n * @security **Security Notice** This file store\n * can alter any text file in the provided directory and any subfolders.\n * Make sure that the path you specify when initializing the store is free\n * of other files.\n */\nexport class LocalFileStore extends BaseStore<string, Uint8Array> {\n  lc_namespace = [\"langchain\", \"storage\"];\n\n  rootPath: string;\n\n  private keyLocks: Map<string, Promise<void>> = new Map();\n\n  constructor(fields: { rootPath: string }) {\n    super(fields);\n    this.rootPath = fields.rootPath;\n  }\n\n  /**\n   * Read and parse the file at the given path.\n   * @param key The key to read the file for.\n   * @returns Promise that resolves to the parsed file content.\n   */\n  private async getParsedFile(key: string): Promise<Uint8Array | undefined> {\n    // Validate the key to prevent path traversal\n    if (!/^[a-zA-Z0-9_\\-:.]+$/.test(key)) {\n      throw new Error(\n        \"Invalid key. Only alphanumeric characters, underscores, hyphens, colons, and periods are allowed.\"\n      );\n    }\n    try {\n      const fileContent = await fs.readFile(this.getFullPath(key));\n      if (!fileContent) {\n        return undefined;\n      }\n      return fileContent;\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    } catch (e: any) {\n      // File does not exist yet.\n      if (\"code\" in e && e.code === \"ENOENT\") {\n        return undefined;\n      }\n      throw new Error(\n        `Error reading and parsing file at path: ${\n          this.rootPath\n        }.\\nError: ${JSON.stringify(e)}`\n      );\n    }\n  }\n\n  /**\n   * Writes the given key-value pairs to the file at the given path.\n   * @param fileContent An object with the key-value pairs to be written to the file.\n   */\n  private async setFileContent(content: Uint8Array, key: string) {\n    await this.withKeyLock(key, async () => {\n      const fullPath = this.getFullPath(key);\n      try {\n        await this.writeFileAtomically(content, fullPath);\n      } catch (error) {\n        throw new Error(\n          `Error writing file at path: ${fullPath}.\\nError: ${JSON.stringify(\n            error\n          )}`\n        );\n      }\n    });\n  }\n\n  /**\n   * Returns the full path of the file where the value of the given key is stored.\n   * @param key the key to get the full path for\n   */\n  private getFullPath(key: string): string {\n    try {\n      const keyAsTxtFile = `${key}.txt`;\n\n      // Validate the key to prevent path traversal\n      if (!/^[a-zA-Z0-9_.\\-/]+$/.test(key)) {\n        throw new Error(`Invalid characters in key: ${key}`);\n      }\n\n      const rootPath = path.resolve(this.rootPath);\n      const fullPath = path.resolve(rootPath, keyAsTxtFile);\n      const relativePath = path.relative(rootPath, fullPath);\n\n      if (\n        relativePath === \"..\" ||\n        relativePath.startsWith(`..${path.sep}`) ||\n        path.isAbsolute(relativePath)\n      ) {\n        throw new Error(\n          `Invalid key: ${key}. Key should be relative to the root path. ` +\n            `Root path: ${this.rootPath}, Full path: ${fullPath}`\n        );\n      }\n\n      return fullPath;\n    } catch (e) {\n      throw new Error(\n        `Error getting full path for key: ${key}.\\nError: ${String(e)}`\n      );\n    }\n  }\n\n  /**\n   * Retrieves the values associated with the given keys from the store.\n   * @param keys Keys to retrieve values for.\n   * @returns Array of values associated with the given keys.\n   */\n  async mget(keys: string[]) {\n    const values: (Uint8Array | undefined)[] = [];\n    for (const key of keys) {\n      const fileContent = await this.getParsedFile(key);\n      values.push(fileContent);\n    }\n    return values;\n  }\n\n  /**\n   * Sets the values for the given keys in the store.\n   * The last value for duplicate keys will be used.\n   * @param keyValuePairs Array of key-value pairs to set in the store.\n   * @returns Promise that resolves when all key-value pairs have been set.\n   */\n  async mset(keyValuePairs: [string, Uint8Array][]): Promise<void> {\n    const deduped = new Map<string, Uint8Array>();\n    for (const [key, value] of keyValuePairs) {\n      deduped.set(key, value);\n    }\n\n    await Promise.all(\n      Array.from(deduped.entries(), ([key, value]) =>\n        this.setFileContent(value, key)\n      )\n    );\n  }\n\n  /**\n   * Deletes the given keys and their associated values from the store.\n   * @param keys Keys to delete from the store.\n   * @returns Promise that resolves when all keys have been deleted.\n   */\n  async mdelete(keys: string[]): Promise<void> {\n    await Promise.all(\n      keys.map((key) =>\n        this.withKeyLock(key, async () => {\n          try {\n            await fs.unlink(this.getFullPath(key));\n          } catch (error) {\n            // Ignore missing files so deletes remain idempotent.\n            if (!error || (error as { code?: string }).code !== \"ENOENT\") {\n              throw error;\n            }\n          }\n        })\n      )\n    );\n  }\n\n  /**\n   * Asynchronous generator that yields keys from the store. If a prefix is\n   * provided, it only yields keys that start with the prefix.\n   * @param prefix Optional prefix to filter keys.\n   * @returns AsyncGenerator that yields keys from the store.\n   */\n  async *yieldKeys(prefix?: string): AsyncGenerator<string> {\n    const allFiles: string[] = await fs.readdir(this.rootPath);\n    const allKeys = allFiles\n      .filter((file) => file.endsWith(\".txt\"))\n      .map((file) => file.replace(/\\.txt$/, \"\"));\n    for (const key of allKeys) {\n      if (prefix === undefined || key.startsWith(prefix)) {\n        yield key;\n      }\n    }\n  }\n\n  /**\n   * Static method for initializing the class.\n   * Preforms a check to see if the directory exists, and if not, creates it.\n   * @param path Path to the directory.\n   * @returns Promise that resolves to an instance of the class.\n   */\n  static async fromPath(rootPath: string): Promise<LocalFileStore> {\n    try {\n      // Verifies the directory exists at the provided path, and that it is readable and writable.\n      await fs.access(rootPath, fs.constants.R_OK | fs.constants.W_OK);\n    } catch {\n      try {\n        // Directory does not exist, create it.\n        await fs.mkdir(rootPath, { recursive: true });\n      } catch (error) {\n        throw new Error(\n          `An error occurred creating directory at: ${rootPath}.\\nError: ${JSON.stringify(\n            error\n          )}`\n        );\n      }\n    }\n\n    // Clean up orphaned temp files left by interrupted atomic writes.\n    try {\n      const entries = await fs.readdir(rootPath);\n      await Promise.all(\n        entries\n          .filter((file) => file.endsWith(\".tmp\"))\n          .map((tempFile) =>\n            fs.unlink(path.join(rootPath, tempFile)).catch(() => {})\n          )\n      );\n    } catch {\n      // Ignore cleanup errors.\n    }\n\n    return new this({ rootPath });\n  }\n\n  /**\n   * Ensures calls for the same key run sequentially by chaining promises.\n   * @param key Key to serialize operations for.\n   * @param fn Async work to execute while the lock is held.\n   * @returns Promise resolving with the callback result once the lock releases.\n   */\n  private async withKeyLock<T>(key: string, fn: () => Promise<T>): Promise<T> {\n    const previous = this.keyLocks.get(key) ?? Promise.resolve();\n    const waitForPrevious = previous.catch(() => {});\n\n    let resolveCurrent: (() => void) | undefined;\n    const current = new Promise<void>((resolve) => {\n      resolveCurrent = resolve;\n    });\n\n    const tail = waitForPrevious.then(() => current);\n    this.keyLocks.set(key, tail);\n\n    await waitForPrevious;\n    try {\n      return await fn();\n    } finally {\n      resolveCurrent?.();\n      if (this.keyLocks.get(key) === tail) {\n        this.keyLocks.delete(key);\n      }\n    }\n  }\n\n  /**\n   * Writes data to a temporary file before atomically renaming it into place.\n   * @param content Serialized value to persist.\n   * @param fullPath Destination path for the stored key.\n   */\n  private async writeFileAtomically(content: Uint8Array, fullPath: string) {\n    const directory = path.dirname(fullPath);\n    await fs.mkdir(directory, { recursive: true });\n\n    const tempPath = `${fullPath}.${Date.now()}-${Math.random()\n      .toString(16)\n      .slice(2)}.tmp`;\n\n    try {\n      await fs.writeFile(tempPath, content);\n\n      try {\n        await fs.rename(tempPath, fullPath);\n      } catch (renameError) {\n        const code = (renameError as { code?: string }).code;\n        if (renameError && (code === \"EPERM\" || code === \"EACCES\")) {\n          // Fallback for Windows where replacing an existing file can fail if the\n          // destination is locked by another process.\n          await fs.writeFile(fullPath, content);\n          await fs.unlink(tempPath).catch(() => {});\n        } else {\n          throw renameError;\n        }\n      }\n    } catch (error) {\n      await fs.unlink(tempPath).catch(() => {});\n      throw error;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,IAAa,iBAAb,cAAoCA,uBAAAA,UAA8B;CAChE,eAAe,CAAC,aAAa,SAAS;CAEtC;CAEA,2BAA+C,IAAI,IAAI;CAEvD,YAAY,QAA8B;EACxC,MAAM,MAAM;EACZ,KAAK,WAAW,OAAO;CACzB;;;;;;CAOA,MAAc,cAAc,KAA8C;EAExE,IAAI,CAAC,sBAAsB,KAAK,GAAG,GACjC,MAAM,IAAI,MACR,mGACF;EAEF,IAAI;GACF,MAAM,cAAc,MAAMC,iBAAG,SAAS,KAAK,YAAY,GAAG,CAAC;GAC3D,IAAI,CAAC,aACH;GAEF,OAAO;EAET,SAAS,GAAQ;GAEf,IAAI,UAAU,KAAK,EAAE,SAAS,UAC5B;GAEF,MAAM,IAAI,MACR,2CACE,KAAK,SACN,YAAY,KAAK,UAAU,CAAC,GAC/B;EACF;CACF;;;;;CAMA,MAAc,eAAe,SAAqB,KAAa;EAC7D,MAAM,KAAK,YAAY,KAAK,YAAY;GACtC,MAAM,WAAW,KAAK,YAAY,GAAG;GACrC,IAAI;IACF,MAAM,KAAK,oBAAoB,SAAS,QAAQ;GAClD,SAAS,OAAO;IACd,MAAM,IAAI,MACR,+BAA+B,SAAS,YAAY,KAAK,UACvD,KACF,GACF;GACF;EACF,CAAC;CACH;;;;;CAMA,YAAoB,KAAqB;EACvC,IAAI;GACF,MAAM,eAAe,GAAG,IAAI;GAG5B,IAAI,CAAC,sBAAsB,KAAK,GAAG,GACjC,MAAM,IAAI,MAAM,8BAA8B,KAAK;GAGrD,MAAM,WAAWC,UAAK,QAAQ,KAAK,QAAQ;GAC3C,MAAM,WAAWA,UAAK,QAAQ,UAAU,YAAY;GACpD,MAAM,eAAeA,UAAK,SAAS,UAAU,QAAQ;GAErD,IACE,iBAAiB,QACjB,aAAa,WAAW,KAAKA,UAAK,KAAK,KACvCA,UAAK,WAAW,YAAY,GAE5B,MAAM,IAAI,MACR,gBAAgB,IAAI,wDACJ,KAAK,SAAS,eAAe,UAC/C;GAGF,OAAO;EACT,SAAS,GAAG;GACV,MAAM,IAAI,MACR,oCAAoC,IAAI,YAAY,OAAO,CAAC,GAC9D;EACF;CACF;;;;;;CAOA,MAAM,KAAK,MAAgB;EACzB,MAAM,SAAqC,CAAC;EAC5C,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,cAAc,MAAM,KAAK,cAAc,GAAG;GAChD,OAAO,KAAK,WAAW;EACzB;EACA,OAAO;CACT;;;;;;;CAQA,MAAM,KAAK,eAAsD;EAC/D,MAAM,0BAAU,IAAI,IAAwB;EAC5C,KAAK,MAAM,CAAC,KAAK,UAAU,eACzB,QAAQ,IAAI,KAAK,KAAK;EAGxB,MAAM,QAAQ,IACZ,MAAM,KAAK,QAAQ,QAAQ,IAAI,CAAC,KAAK,WACnC,KAAK,eAAe,OAAO,GAAG,CAChC,CACF;CACF;;;;;;CAOA,MAAM,QAAQ,MAA+B;EAC3C,MAAM,QAAQ,IACZ,KAAK,KAAK,QACR,KAAK,YAAY,KAAK,YAAY;GAChC,IAAI;IACF,MAAMD,iBAAG,OAAO,KAAK,YAAY,GAAG,CAAC;GACvC,SAAS,OAAO;IAEd,IAAI,CAAC,SAAU,MAA4B,SAAS,UAClD,MAAM;GAEV;EACF,CAAC,CACH,CACF;CACF;;;;;;;CAQA,OAAO,UAAU,QAAyC;EAExD,MAAM,WAAU,MADiBA,iBAAG,QAAQ,KAAK,QAAQ,EAAA,CAEtD,QAAQ,SAAS,KAAK,SAAS,MAAM,CAAC,CAAC,CACvC,KAAK,SAAS,KAAK,QAAQ,UAAU,EAAE,CAAC;EAC3C,KAAK,MAAM,OAAO,SAChB,IAAI,WAAW,KAAA,KAAa,IAAI,WAAW,MAAM,GAC/C,MAAM;CAGZ;;;;;;;CAQA,aAAa,SAAS,UAA2C;EAC/D,IAAI;GAEF,MAAMA,iBAAG,OAAO,UAAUA,iBAAG,UAAU,OAAOA,iBAAG,UAAU,IAAI;EACjE,QAAQ;GACN,IAAI;IAEF,MAAMA,iBAAG,MAAM,UAAU,EAAE,WAAW,KAAK,CAAC;GAC9C,SAAS,OAAO;IACd,MAAM,IAAI,MACR,4CAA4C,SAAS,YAAY,KAAK,UACpE,KACF,GACF;GACF;EACF;EAGA,IAAI;GACF,MAAM,UAAU,MAAMA,iBAAG,QAAQ,QAAQ;GACzC,MAAM,QAAQ,IACZ,QACG,QAAQ,SAAS,KAAK,SAAS,MAAM,CAAC,CAAC,CACvC,KAAK,aACJA,iBAAG,OAAOC,UAAK,KAAK,UAAU,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CACzD,CACJ;EACF,QAAQ,CAER;EAEA,OAAO,IAAI,KAAK,EAAE,SAAS,CAAC;CAC9B;;;;;;;CAQA,MAAc,YAAe,KAAa,IAAkC;EAE1E,MAAM,mBADW,KAAK,SAAS,IAAI,GAAG,KAAK,QAAQ,QAAQ,EAAA,CAC1B,YAAY,CAAC,CAAC;EAE/C,IAAI;EACJ,MAAM,UAAU,IAAI,SAAe,YAAY;GAC7C,iBAAiB;EACnB,CAAC;EAED,MAAM,OAAO,gBAAgB,WAAW,OAAO;EAC/C,KAAK,SAAS,IAAI,KAAK,IAAI;EAE3B,MAAM;EACN,IAAI;GACF,OAAO,MAAM,GAAG;EAClB,UAAU;GACR,iBAAiB;GACjB,IAAI,KAAK,SAAS,IAAI,GAAG,MAAM,MAC7B,KAAK,SAAS,OAAO,GAAG;EAE5B;CACF;;;;;;CAOA,MAAc,oBAAoB,SAAqB,UAAkB;EACvE,MAAM,YAAYA,UAAK,QAAQ,QAAQ;EACvC,MAAMD,iBAAG,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;EAE7C,MAAM,WAAW,GAAG,SAAS,GAAG,KAAK,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC,CACxD,SAAS,EAAE,CAAC,CACZ,MAAM,CAAC,EAAE;EAEZ,IAAI;GACF,MAAMA,iBAAG,UAAU,UAAU,OAAO;GAEpC,IAAI;IACF,MAAMA,iBAAG,OAAO,UAAU,QAAQ;GACpC,SAAS,aAAa;IACpB,MAAM,OAAQ,YAAkC;IAChD,IAAI,gBAAgB,SAAS,WAAW,SAAS,WAAW;KAG1D,MAAMA,iBAAG,UAAU,UAAU,OAAO;KACpC,MAAMA,iBAAG,OAAO,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1C,OACE,MAAM;GAEV;EACF,SAAS,OAAO;GACd,MAAMA,iBAAG,OAAO,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;GACxC,MAAM;EACR;CACF;AACF"}