pipe-protocol
Version:
A protocol for large scale Interplanetary Intertool Agent Context
109 lines • 4.6 kB
JavaScript
;
/**
* @file Token Counting Test Suite
* @version 1.0.0
* @status IN_DEVELOPMENT
* @lastModified 2024-02-04
*
* Tests for token counting functionality
*
* Test Coverage:
* - Basic token counting
* - Complex data structure handling
* - Token limit enforcement
* - Edge cases
*/
Object.defineProperty(exports, "__esModule", { value: true });
const vitest_1 = require("vitest");
const tokenCounting_1 = require("../tokenCounting");
(0, vitest_1.describe)('Token Counting', () => {
(0, vitest_1.describe)('Basic Token Counting', () => {
(0, vitest_1.it)('should count tokens in strings', () => {
const data = 'Hello world';
const count = (0, tokenCounting_1.countTokens)(data);
(0, vitest_1.expect)(count).toBeGreaterThan(0);
});
(0, vitest_1.it)('should count tokens in numbers', () => {
const data = 12345;
const count = (0, tokenCounting_1.countTokens)(data);
(0, vitest_1.expect)(count).toBeGreaterThan(0);
});
(0, vitest_1.it)('should handle null and undefined', () => {
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)(null)).toBe(0);
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)(undefined)).toBe(0);
});
});
(0, vitest_1.describe)('Complex Data Structure Handling', () => {
(0, vitest_1.it)('should count tokens in objects', () => {
const data = {
name: 'test',
value: 123,
nested: {
field: 'nested value'
}
};
const count = (0, tokenCounting_1.countTokens)(data);
(0, vitest_1.expect)(count).toBeGreaterThan(0);
});
(0, vitest_1.it)('should count tokens in arrays', () => {
const data = ['first', 'second', { name: 'third' }];
const count = (0, tokenCounting_1.countTokens)(data);
(0, vitest_1.expect)(count).toBeGreaterThan(0);
});
});
(0, vitest_1.describe)('Token Limit Enforcement', () => {
(0, vitest_1.it)('should enforce token limits on strings', () => {
const data = 'This is a long string that might exceed the token limit';
const result = (0, tokenCounting_1.enforceTokenLimit)(data, 5);
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)(result)).toBeLessThanOrEqual(5);
});
(0, vitest_1.it)('should enforce token limits on objects', () => {
const data = {
field1: 'value1',
field2: 'value2',
field3: {
nested: 'nested value'
}
};
const result = (0, tokenCounting_1.enforceTokenLimit)(data, 10);
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)(result)).toBeLessThanOrEqual(10);
});
(0, vitest_1.it)('should preserve structure when possible', () => {
const data = {
important: 'keep this',
lessImportant: 'might trim this'
};
const result = (0, tokenCounting_1.enforceTokenLimit)(data, 5);
(0, vitest_1.expect)(result).toHaveProperty('important');
});
(0, vitest_1.it)('should return null when limit is too small', () => {
const data = 'Cannot be smaller';
const result = (0, tokenCounting_1.enforceTokenLimit)(data, 1);
(0, vitest_1.expect)(result).toBeNull();
});
});
(0, vitest_1.describe)('Edge Cases', () => {
(0, vitest_1.it)('should handle empty objects and arrays', () => {
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)({})).toBe(0);
(0, vitest_1.expect)((0, tokenCounting_1.countTokens)([])).toBe(0);
});
(0, vitest_1.it)('should handle circular references', () => {
const circular = { name: 'test' };
circular.self = circular;
(0, vitest_1.expect)(() => (0, tokenCounting_1.countTokens)(circular)).not.toThrow();
});
(0, vitest_1.it)('should handle mixed content types', () => {
const data = {
string: 'text',
number: 123,
bool: true,
array: [1, 'two', false],
nested: {
more: 'content'
}
};
(0, vitest_1.expect)(() => (0, tokenCounting_1.countTokens)(data)).not.toThrow();
});
});
});
//# sourceMappingURL=tokenCounting.test.js.map