@nanocollective/nanocoder
Version:
A local-first CLI coding agent that brings the power of agentic coding tools like Claude Code and Gemini CLI to local models or controlled APIs like OpenRouter
31 lines • 798 B
TypeScript
/**
* Tokenizer interface and types
*/
import type { Message } from '../types/core.js';
/**
* Tokenizer interface for encoding text and counting tokens
*/
export interface Tokenizer {
/**
* Encode text and return token count
*/
encode(text: string): number;
/**
* Count tokens in a message (content + role)
*/
countTokens(message: Message): number;
/**
* Get the tokenizer name/type
*/
getName(): string;
/**
* Optional cleanup method for releasing resources
* Should be called when the tokenizer is no longer needed
*/
free?(): void;
}
/**
* Provider types for tokenizer selection
*/
export type TokenizerProvider = 'openai' | 'anthropic' | 'llama' | 'fallback' | 'auto';
//# sourceMappingURL=tokenization.d.ts.map