langchain/dist/storage/file_system.cjs.map

Typescript bindings for langchain

{"version":3,"file":"file_system.cjs","names":["BaseStore","fs","path"],"sources":["../../src/storage/file_system.ts"],"sourcesContent":["import * as fs from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\n/**\n * File system implementation of the BaseStore using a dictionary. Used for\n * storing key-value pairs in the file system.\n * @example\n * ```typescript\n * const store = await LocalFileStore.fromPath(\"./messages\");\n * await store.mset(\n *   Array.from({ length: 5 }).map((_, index) => [\n *     `message:id:${index}`,\n *     new TextEncoder().encode(\n *       JSON.stringify(\n *         index % 2 === 0\n *           ? new AIMessage(\"ai stuff...\")\n *           : new HumanMessage(\"human stuff...\"),\n *       ),\n *     ),\n *   ]),\n * );\n * const retrievedMessages = await store.mget([\"message:id:0\", \"message:id:1\"]);\n * console.log(retrievedMessages.map((v) => new TextDecoder().decode(v)));\n * for await (const key of store.yieldKeys(\"message:id:\")) {\n *   await store.mdelete([key]);\n * }\n * ```\n *\n * @security **Security Notice** This file store\n * can alter any text file in the provided directory and any subfolders.\n * Make sure that the path you specify when initializing the store is free\n * of other files.\n */\nexport class LocalFileStore extends BaseStore<string, Uint8Array> {\n  lc_namespace = [\"langchain\", \"storage\"];\n\n  rootPath: string;\n\n  private keyLocks: Map<string, Promise<void>> = new Map();\n\n  constructor(fields: { rootPath: string }) {\n    super(fields);\n    this.rootPath = fields.rootPath;\n  }\n\n  /**\n   * Read and parse the file at the given path.\n   * @param key The key to read the file for.\n   * @returns Promise that resolves to the parsed file content.\n   */\n  private async getParsedFile(key: string): Promise<Uint8Array | undefined> {\n    // Validate the key to prevent path traversal\n    if (!/^[a-zA-Z0-9_\\-:.]+$/.test(key)) {\n      throw new Error(\n        \"Invalid key. Only alphanumeric characters, underscores, hyphens, colons, and periods are allowed.\"\n      );\n    }\n    try {\n      const fileContent = await fs.readFile(this.getFullPath(key));\n      if (!fileContent) {\n        return undefined;\n      }\n      return fileContent;\n      // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    } catch (e: any) {\n      // File does not exist yet.\n      if (\"code\" in e && e.code === \"ENOENT\") {\n        return undefined;\n      }\n      throw new Error(\n        `Error reading and parsing file at path: ${\n          this.rootPath\n        }.\\nError: ${JSON.stringify(e)}`\n      );\n    }\n  }\n\n  /**\n   * Writes the given key-value pairs to the file at the given path.\n   * @param fileContent An object with the key-value pairs to be written to the file.\n   */\n  private async setFileContent(content: Uint8Array, key: string) {\n    await this.withKeyLock(key, async () => {\n      const fullPath = this.getFullPath(key);\n      try {\n        await this.writeFileAtomically(content, fullPath);\n      } catch (error) {\n        throw new Error(\n          `Error writing file at path: ${fullPath}.\\nError: ${JSON.stringify(\n            error\n          )}`\n        );\n      }\n    });\n  }\n\n  /**\n   * Returns the full path of the file where the value of the given key is stored.\n   * @param key the key to get the full path for\n   */\n  private getFullPath(key: string): string {\n    try {\n      const keyAsTxtFile = `${key}.txt`;\n\n      // Validate the key to prevent path traversal\n      if (!/^[a-zA-Z0-9_.\\-/]+$/.test(key)) {\n        throw new Error(`Invalid characters in key: ${key}`);\n      }\n\n      const fullPath = path.resolve(this.rootPath, keyAsTxtFile);\n      const commonPath = path.resolve(this.rootPath);\n\n      if (!fullPath.startsWith(commonPath)) {\n        throw new Error(\n          `Invalid key: ${key}. Key should be relative to the root path. ` +\n            `Root path: ${this.rootPath}, Full path: ${fullPath}`\n        );\n      }\n\n      return fullPath;\n    } catch (e) {\n      throw new Error(\n        `Error getting full path for key: ${key}.\\nError: ${String(e)}`\n      );\n    }\n  }\n\n  /**\n   * Retrieves the values associated with the given keys from the store.\n   * @param keys Keys to retrieve values for.\n   * @returns Array of values associated with the given keys.\n   */\n  async mget(keys: string[]) {\n    const values: (Uint8Array | undefined)[] = [];\n    for (const key of keys) {\n      const fileContent = await this.getParsedFile(key);\n      values.push(fileContent);\n    }\n    return values;\n  }\n\n  /**\n   * Sets the values for the given keys in the store.\n   * The last value for duplicate keys will be used.\n   * @param keyValuePairs Array of key-value pairs to set in the store.\n   * @returns Promise that resolves when all key-value pairs have been set.\n   */\n  async mset(keyValuePairs: [string, Uint8Array][]): Promise<void> {\n    const deduped = new Map<string, Uint8Array>();\n    for (const [key, value] of keyValuePairs) {\n      deduped.set(key, value);\n    }\n\n    await Promise.all(\n      Array.from(deduped.entries(), ([key, value]) =>\n        this.setFileContent(value, key)\n      )\n    );\n  }\n\n  /**\n   * Deletes the given keys and their associated values from the store.\n   * @param keys Keys to delete from the store.\n   * @returns Promise that resolves when all keys have been deleted.\n   */\n  async mdelete(keys: string[]): Promise<void> {\n    await Promise.all(\n      keys.map((key) =>\n        this.withKeyLock(key, async () => {\n          try {\n            await fs.unlink(this.getFullPath(key));\n          } catch (error) {\n            // Ignore missing files so deletes remain idempotent.\n            if (!error || (error as { code?: string }).code !== \"ENOENT\") {\n              throw error;\n            }\n          }\n        })\n      )\n    );\n  }\n\n  /**\n   * Asynchronous generator that yields keys from the store. If a prefix is\n   * provided, it only yields keys that start with the prefix.\n   * @param prefix Optional prefix to filter keys.\n   * @returns AsyncGenerator that yields keys from the store.\n   */\n  async *yieldKeys(prefix?: string): AsyncGenerator<string> {\n    const allFiles: string[] = await fs.readdir(this.rootPath);\n    const allKeys = allFiles\n      .filter((file) => file.endsWith(\".txt\"))\n      .map((file) => file.replace(/\\.txt$/, \"\"));\n    for (const key of allKeys) {\n      if (prefix === undefined || key.startsWith(prefix)) {\n        yield key;\n      }\n    }\n  }\n\n  /**\n   * Static method for initializing the class.\n   * Preforms a check to see if the directory exists, and if not, creates it.\n   * @param path Path to the directory.\n   * @returns Promise that resolves to an instance of the class.\n   */\n  static async fromPath(rootPath: string): Promise<LocalFileStore> {\n    try {\n      // Verifies the directory exists at the provided path, and that it is readable and writable.\n      await fs.access(rootPath, fs.constants.R_OK | fs.constants.W_OK);\n    } catch {\n      try {\n        // Directory does not exist, create it.\n        await fs.mkdir(rootPath, { recursive: true });\n      } catch (error) {\n        throw new Error(\n          `An error occurred creating directory at: ${rootPath}.\\nError: ${JSON.stringify(\n            error\n          )}`\n        );\n      }\n    }\n\n    // Clean up orphaned temp files left by interrupted atomic writes.\n    try {\n      const entries = await fs.readdir(rootPath);\n      await Promise.all(\n        entries\n          .filter((file) => file.endsWith(\".tmp\"))\n          .map((tempFile) =>\n            fs.unlink(path.join(rootPath, tempFile)).catch(() => {})\n          )\n      );\n    } catch {\n      // Ignore cleanup errors.\n    }\n\n    return new this({ rootPath });\n  }\n\n  /**\n   * Ensures calls for the same key run sequentially by chaining promises.\n   * @param key Key to serialize operations for.\n   * @param fn Async work to execute while the lock is held.\n   * @returns Promise resolving with the callback result once the lock releases.\n   */\n  private async withKeyLock<T>(key: string, fn: () => Promise<T>): Promise<T> {\n    const previous = this.keyLocks.get(key) ?? Promise.resolve();\n    const waitForPrevious = previous.catch(() => {});\n\n    let resolveCurrent: (() => void) | undefined;\n    const current = new Promise<void>((resolve) => {\n      resolveCurrent = resolve;\n    });\n\n    const tail = waitForPrevious.then(() => current);\n    this.keyLocks.set(key, tail);\n\n    await waitForPrevious;\n    try {\n      return await fn();\n    } finally {\n      resolveCurrent?.();\n      if (this.keyLocks.get(key) === tail) {\n        this.keyLocks.delete(key);\n      }\n    }\n  }\n\n  /**\n   * Writes data to a temporary file before atomically renaming it into place.\n   * @param content Serialized value to persist.\n   * @param fullPath Destination path for the stored key.\n   */\n  private async writeFileAtomically(content: Uint8Array, fullPath: string) {\n    const directory = path.dirname(fullPath);\n    await fs.mkdir(directory, { recursive: true });\n\n    const tempPath = `${fullPath}.${Date.now()}-${Math.random()\n      .toString(16)\n      .slice(2)}.tmp`;\n\n    try {\n      await fs.writeFile(tempPath, content);\n\n      try {\n        await fs.rename(tempPath, fullPath);\n      } catch (renameError) {\n        const code = (renameError as { code?: string }).code;\n        if (renameError && (code === \"EPERM\" || code === \"EACCES\")) {\n          // Fallback for Windows where replacing an existing file can fail if the\n          // destination is locked by another process.\n          await fs.writeFile(fullPath, content);\n          await fs.unlink(tempPath).catch(() => {});\n        } else {\n          throw renameError;\n        }\n      }\n    } catch (error) {\n      await fs.unlink(tempPath).catch(() => {});\n      throw error;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,IAAa,iBAAb,cAAoCA,uBAAAA,UAA8B;CAChE,eAAe,CAAC,aAAa,UAAU;CAEvC;CAEA,2BAA+C,IAAI,KAAK;CAExD,YAAY,QAA8B;AACxC,QAAM,OAAO;AACb,OAAK,WAAW,OAAO;;;;;;;CAQzB,MAAc,cAAc,KAA8C;AAExE,MAAI,CAAC,sBAAsB,KAAK,IAAI,CAClC,OAAM,IAAI,MACR,oGACD;AAEH,MAAI;GACF,MAAM,cAAc,MAAMC,iBAAG,SAAS,KAAK,YAAY,IAAI,CAAC;AAC5D,OAAI,CAAC,YACH;AAEF,UAAO;WAEA,GAAQ;AAEf,OAAI,UAAU,KAAK,EAAE,SAAS,SAC5B;AAEF,SAAM,IAAI,MACR,2CACE,KAAK,SACN,YAAY,KAAK,UAAU,EAAE,GAC/B;;;;;;;CAQL,MAAc,eAAe,SAAqB,KAAa;AAC7D,QAAM,KAAK,YAAY,KAAK,YAAY;GACtC,MAAM,WAAW,KAAK,YAAY,IAAI;AACtC,OAAI;AACF,UAAM,KAAK,oBAAoB,SAAS,SAAS;YAC1C,OAAO;AACd,UAAM,IAAI,MACR,+BAA+B,SAAS,YAAY,KAAK,UACvD,MACD,GACF;;IAEH;;;;;;CAOJ,YAAoB,KAAqB;AACvC,MAAI;GACF,MAAM,eAAe,GAAG,IAAI;AAG5B,OAAI,CAAC,sBAAsB,KAAK,IAAI,CAClC,OAAM,IAAI,MAAM,8BAA8B,MAAM;GAGtD,MAAM,WAAWC,UAAK,QAAQ,KAAK,UAAU,aAAa;GAC1D,MAAM,aAAaA,UAAK,QAAQ,KAAK,SAAS;AAE9C,OAAI,CAAC,SAAS,WAAW,WAAW,CAClC,OAAM,IAAI,MACR,gBAAgB,IAAI,wDACJ,KAAK,SAAS,eAAe,WAC9C;AAGH,UAAO;WACA,GAAG;AACV,SAAM,IAAI,MACR,oCAAoC,IAAI,YAAY,OAAO,EAAE,GAC9D;;;;;;;;CASL,MAAM,KAAK,MAAgB;EACzB,MAAM,SAAqC,EAAE;AAC7C,OAAK,MAAM,OAAO,MAAM;GACtB,MAAM,cAAc,MAAM,KAAK,cAAc,IAAI;AACjD,UAAO,KAAK,YAAY;;AAE1B,SAAO;;;;;;;;CAST,MAAM,KAAK,eAAsD;EAC/D,MAAM,0BAAU,IAAI,KAAyB;AAC7C,OAAK,MAAM,CAAC,KAAK,UAAU,cACzB,SAAQ,IAAI,KAAK,MAAM;AAGzB,QAAM,QAAQ,IACZ,MAAM,KAAK,QAAQ,SAAS,GAAG,CAAC,KAAK,WACnC,KAAK,eAAe,OAAO,IAAI,CAChC,CACF;;;;;;;CAQH,MAAM,QAAQ,MAA+B;AAC3C,QAAM,QAAQ,IACZ,KAAK,KAAK,QACR,KAAK,YAAY,KAAK,YAAY;AAChC,OAAI;AACF,UAAMD,iBAAG,OAAO,KAAK,YAAY,IAAI,CAAC;YAC/B,OAAO;AAEd,QAAI,CAAC,SAAU,MAA4B,SAAS,SAClD,OAAM;;IAGV,CACH,CACF;;;;;;;;CASH,OAAO,UAAU,QAAyC;EAExD,MAAM,WAAU,MADiBA,iBAAG,QAAQ,KAAK,SAAS,EAEvD,QAAQ,SAAS,KAAK,SAAS,OAAO,CAAC,CACvC,KAAK,SAAS,KAAK,QAAQ,UAAU,GAAG,CAAC;AAC5C,OAAK,MAAM,OAAO,QAChB,KAAI,WAAW,KAAA,KAAa,IAAI,WAAW,OAAO,CAChD,OAAM;;;;;;;;CAWZ,aAAa,SAAS,UAA2C;AAC/D,MAAI;AAEF,SAAMA,iBAAG,OAAO,UAAUA,iBAAG,UAAU,OAAOA,iBAAG,UAAU,KAAK;UAC1D;AACN,OAAI;AAEF,UAAMA,iBAAG,MAAM,UAAU,EAAE,WAAW,MAAM,CAAC;YACtC,OAAO;AACd,UAAM,IAAI,MACR,4CAA4C,SAAS,YAAY,KAAK,UACpE,MACD,GACF;;;AAKL,MAAI;GACF,MAAM,UAAU,MAAMA,iBAAG,QAAQ,SAAS;AAC1C,SAAM,QAAQ,IACZ,QACG,QAAQ,SAAS,KAAK,SAAS,OAAO,CAAC,CACvC,KAAK,aACJA,iBAAG,OAAOC,UAAK,KAAK,UAAU,SAAS,CAAC,CAAC,YAAY,GAAG,CACzD,CACJ;UACK;AAIR,SAAO,IAAI,KAAK,EAAE,UAAU,CAAC;;;;;;;;CAS/B,MAAc,YAAe,KAAa,IAAkC;EAE1E,MAAM,mBADW,KAAK,SAAS,IAAI,IAAI,IAAI,QAAQ,SAAS,EAC3B,YAAY,GAAG;EAEhD,IAAI;EACJ,MAAM,UAAU,IAAI,SAAe,YAAY;AAC7C,oBAAiB;IACjB;EAEF,MAAM,OAAO,gBAAgB,WAAW,QAAQ;AAChD,OAAK,SAAS,IAAI,KAAK,KAAK;AAE5B,QAAM;AACN,MAAI;AACF,UAAO,MAAM,IAAI;YACT;AACR,qBAAkB;AAClB,OAAI,KAAK,SAAS,IAAI,IAAI,KAAK,KAC7B,MAAK,SAAS,OAAO,IAAI;;;;;;;;CAU/B,MAAc,oBAAoB,SAAqB,UAAkB;EACvE,MAAM,YAAYA,UAAK,QAAQ,SAAS;AACxC,QAAMD,iBAAG,MAAM,WAAW,EAAE,WAAW,MAAM,CAAC;EAE9C,MAAM,WAAW,GAAG,SAAS,GAAG,KAAK,KAAK,CAAC,GAAG,KAAK,QAAQ,CACxD,SAAS,GAAG,CACZ,MAAM,EAAE,CAAC;AAEZ,MAAI;AACF,SAAMA,iBAAG,UAAU,UAAU,QAAQ;AAErC,OAAI;AACF,UAAMA,iBAAG,OAAO,UAAU,SAAS;YAC5B,aAAa;IACpB,MAAM,OAAQ,YAAkC;AAChD,QAAI,gBAAgB,SAAS,WAAW,SAAS,WAAW;AAG1D,WAAMA,iBAAG,UAAU,UAAU,QAAQ;AACrC,WAAMA,iBAAG,OAAO,SAAS,CAAC,YAAY,GAAG;UAEzC,OAAM;;WAGH,OAAO;AACd,SAAMA,iBAAG,OAAO,SAAS,CAAC,YAAY,GAAG;AACzC,SAAM"}