@langchain/core/dist/utils/tiktoken.cjs.map

Core LangChain.js abstractions and schemas

{"version":3,"file":"tiktoken.cjs","names":["AsyncCaller","Tiktoken"],"sources":["../../src/utils/tiktoken.ts"],"sourcesContent":["import {\n  Tiktoken,\n  TiktokenEncoding,\n  TiktokenModel,\n  getEncodingNameForModel,\n} from \"js-tiktoken/lite\";\nimport { AsyncCaller } from \"./async_caller.js\";\n\nconst cache: Record<string, Promise<Tiktoken>> = {};\n\nconst caller = /* #__PURE__ */ new AsyncCaller({});\n\nexport async function getEncoding(encoding: TiktokenEncoding) {\n  if (!(encoding in cache)) {\n    cache[encoding] = caller\n      .fetch(`https://tiktoken.pages.dev/js/${encoding}.json`)\n      .then((res) => res.json())\n      .then((data) => new Tiktoken(data))\n      .catch((e) => {\n        delete cache[encoding];\n        throw e;\n      });\n  }\n\n  return await cache[encoding];\n}\n\nexport async function encodingForModel(model: TiktokenModel) {\n  return getEncoding(getEncodingNameForModel(model));\n}\n"],"mappings":";;;;;;;;;;AAQA,MAAM,QAA2C,EAAE;AAEnD,MAAM,yBAAyB,IAAIA,uCAAY,EAAE,CAAC;AAElD,eAAsB,YAAY,UAA4B;AAC5D,KAAI,EAAE,YAAY,OAChB,OAAM,YAAY,OACf,MAAM,iCAAiC,SAAS,OAAO,CACvD,MAAM,QAAQ,IAAI,MAAM,CAAC,CACzB,MAAM,SAAS,IAAIC,0BAAS,KAAK,CAAC,CAClC,OAAO,MAAM;AACZ,SAAO,MAAM;AACb,QAAM;GACN;AAGN,QAAO,MAAM,MAAM;;AAGrB,eAAsB,iBAAiB,OAAsB;AAC3D,QAAO,0DAAoC,MAAM,CAAC"}