react-native-executorch/lib/module/modules/natural_language_processing/TokenizerModule.js

An easy way to run AI models in react native with ExecuTorch

"use strict";

import { TokenizerNativeModule } from '../../native/RnExecutorchModules';
import { BaseModule } from '../BaseModule';
export class TokenizerModule extends BaseModule {
  static nativeModule = TokenizerNativeModule;
  static async load(tokenizerSource) {
    await super.load(tokenizerSource);
  }
  static async decode(input, skipSpecialTokens = false) {
    return await this.nativeModule.decode(input, skipSpecialTokens);
  }
  static async encode(input) {
    return await this.nativeModule.encode(input);
  }
  static async getVocabSize() {
    return await this.nativeModule.getVocabSize();
  }
  static async idToToken(tokenId) {
    return await this.nativeModule.idToToken(tokenId);
  }
  static async tokenToId(token) {
    return await this.nativeModule.tokenToId(token);
  }
}
//# sourceMappingURL=TokenizerModule.js.map