@readium/shared/types/src/util/tokenizer/Tokenizer.d.ts

Shared models to be used across other Readium projects and implementations in Typescript

/**
 * A tokenizer splits a piece of data [D] into a list of [T] tokens.
 */
export interface Tokenizer<D, T> {
    tokenize(data: D): T[];
}