@langchain/core/dist/utils/tiktoken.cjs.map

Core LangChain.js abstractions and schemas

{"version":3,"file":"tiktoken.cjs","names":["cache: Record<string, Promise<Tiktoken>>","AsyncCaller","encoding: TiktokenEncoding","Tiktoken","model: TiktokenModel"],"sources":["../../src/utils/tiktoken.ts"],"sourcesContent":["import {\n  Tiktoken,\n  TiktokenEncoding,\n  TiktokenModel,\n  getEncodingNameForModel,\n} from \"js-tiktoken/lite\";\nimport { AsyncCaller } from \"./async_caller.js\";\n\nconst cache: Record<string, Promise<Tiktoken>> = {};\n\nconst caller = /* #__PURE__ */ new AsyncCaller({});\n\nexport async function getEncoding(encoding: TiktokenEncoding) {\n  if (!(encoding in cache)) {\n    cache[encoding] = caller\n      .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)\n      .then((res) => res.json())\n      .then((data) => new Tiktoken(data))\n      .catch((e) => {\n        delete cache[encoding];\n        throw e;\n      });\n  }\n\n  return await cache[encoding];\n}\n\nexport async function encodingForModel(model: TiktokenModel) {\n  return getEncoding(getEncodingNameForModel(model));\n}\n"],"mappings":";;;;;;;;;;AAQA,MAAMA,QAA2C,CAAE;AAEnD,MAAM,yBAAyB,IAAIC,uCAAY,CAAE;AAEjD,eAAsB,YAAYC,UAA4B;AAC5D,KAAI,EAAE,YAAY,QAChB,MAAM,YAAY,OACf,MAAM,CAAC,8BAA8B,EAAE,SAAS,KAAK,CAAC,CAAC,CACvD,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,CACzB,KAAK,CAAC,SAAS,IAAIC,0BAAS,MAAM,CAClC,MAAM,CAAC,MAAM;EACZ,OAAO,MAAM;AACb,QAAM;CACP,EAAC;AAGN,QAAO,MAAM,MAAM;AACpB;AAED,eAAsB,iBAAiBC,OAAsB;AAC3D,QAAO,0DAAoC,MAAM,CAAC;AACnD"}