@langchain/core/dist/utils/tiktoken.cjs.map

Core LangChain.js abstractions and schemas

{"version":3,"file":"tiktoken.cjs","names":["AsyncCaller","Tiktoken"],"sources":["../../src/utils/tiktoken.ts"],"sourcesContent":["import {\n  Tiktoken,\n  TiktokenEncoding,\n  TiktokenModel,\n  getEncodingNameForModel,\n} from \"js-tiktoken/lite\";\nimport { AsyncCaller } from \"./async_caller.js\";\n\nconst cache: Record<string, Promise<Tiktoken>> = {};\n\nconst caller = /* #__PURE__ */ new AsyncCaller({});\n\nexport async function getEncoding(encoding: TiktokenEncoding) {\n  if (!(encoding in cache)) {\n    cache[encoding] = caller\n      .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)\n      .then((res) => res.json())\n      .then((data) => new Tiktoken(data))\n      .catch((e) => {\n        delete cache[encoding];\n        throw e;\n      });\n  }\n\n  return await cache[encoding];\n}\n\nexport async function encodingForModel(model: TiktokenModel) {\n  return getEncoding(getEncodingNameForModel(model));\n}\n"],"mappings":";;;;;;;;;AAQA,MAAM,QAA2C,CAAC;AAElD,MAAM,yBAAyB,IAAIA,2BAAAA,YAAY,CAAC,CAAC;AAEjD,eAAsB,YAAY,UAA4B;CAC5D,IAAI,EAAE,YAAY,QAChB,MAAM,YAAY,OACf,MAAM,iCAAiC,SAAS,MAAM,CAAC,CACvD,MAAM,QAAQ,IAAI,KAAK,CAAC,CAAC,CACzB,MAAM,SAAS,IAAIC,iBAAAA,SAAS,IAAI,CAAC,CAAC,CAClC,OAAO,MAAM;EACZ,OAAO,MAAM;EACb,MAAM;CACR,CAAC;CAGL,OAAO,MAAM,MAAM;AACrB;AAEA,eAAsB,iBAAiB,OAAsB;CAC3D,OAAO,aAAA,GAAA,iBAAA,wBAAA,CAAoC,KAAK,CAAC;AACnD"}