react-native-executorch/lib/module/modules/natural_language_processing/TokenizerModule.js

An easy way to run AI models in React Native with ExecuTorch

"use strict";

import { ResourceFetcher } from '../../utils/ResourceFetcher';
export class TokenizerModule {
  async load(tokenizer, onDownloadProgressCallback = () => {}) {
    const paths = await ResourceFetcher.fetch(onDownloadProgressCallback, tokenizer.tokenizerSource);
    const path = paths?.[0];
    if (!path) {
      throw new Error('Download interrupted.');
    }
    this.nativeModule = global.loadTokenizerModule(path);
  }
  async encode(s) {
    return await this.nativeModule.encode(s);
  }
  async decode(tokens, skipSpecialTokens = true) {
    if (tokens.length === 0) {
      return '';
    }
    return await this.nativeModule.decode(tokens, skipSpecialTokens);
  }
  async getVocabSize() {
    return await this.nativeModule.getVocabSize();
  }
  async idToToken(tokenId) {
    return this.nativeModule.idToToken(tokenId);
  }
  async tokenToId(token) {
    return await this.nativeModule.tokenToId(token);
  }
}
//# sourceMappingURL=TokenizerModule.js.map