@ai2070/l0
Version:
L0: The Missing Reliability Substrate for AI
35 lines • 1.84 kB
TypeScript
export declare function isMeaningfulToken(token: string): boolean;
export declare function hasMeaningfulContent(content: string): boolean;
export declare function countMeaningfulTokens(content: string): number;
export declare function extractMeaningfulTokens(content: string): string[];
export declare function isPunctuationOnly(token: string): boolean;
export declare function isAlphanumeric(token: string): boolean;
export declare function normalizeToken(token: string): string;
export declare function tokensEqual(token1: string, token2: string): boolean;
export declare function detectRepeatedTokens(content: string, threshold?: number): string[];
export declare function calculateTokenDensity(content: string): number;
export declare function estimateTokenCount(content: string): number;
export declare function startsWithMeaningfulToken(content: string): boolean;
export declare function getFirstMeaningfulToken(content: string): string | null;
export declare function getLastMeaningfulToken(content: string): string | null;
export declare function endsAbruptly(content: string): boolean;
export declare function chunkByTokens(content: string, chunkSize: number): string[];
export interface OverlapDetectionResult {
overlapLength: number;
overlapText: string;
deduplicatedContinuation: string;
hasOverlap: boolean;
}
export declare function detectOverlap(checkpoint: string, continuation: string, options?: {
minOverlap?: number;
maxOverlap?: number;
caseSensitive?: boolean;
normalizeWhitespace?: boolean;
}): OverlapDetectionResult;
export declare function deduplicateContinuation(checkpoint: string, continuation: string, options?: {
minOverlap?: number;
maxOverlap?: number;
caseSensitive?: boolean;
normalizeWhitespace?: boolean;
}): string;
//# sourceMappingURL=tokens.d.ts.map