rxcc
Version:
A tool to pack repository contents to single file for AI consumption
35 lines • 1.68 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { logger } from '../../../shared/logger.js';
import { TokenCounter } from '../../tokenCount/tokenCount.js';
// Worker-level singleton for TokenCounter
let tokenCounter = null;
const getTokenCounter = (encoding) => {
if (!tokenCounter) {
tokenCounter = new TokenCounter(encoding);
}
return tokenCounter;
};
export default (_a) => __awaiter(void 0, [_a], void 0, function* ({ content, encoding, path }) {
const processStartAt = process.hrtime.bigint();
const counter = getTokenCounter(encoding);
const tokenCount = counter.countTokens(content, path);
const processEndAt = process.hrtime.bigint();
logger.trace(`Counted output tokens. Count: ${tokenCount}. Took: ${(Number(processEndAt - processStartAt) / 1e6).toFixed(2)}ms`);
return tokenCount;
});
// Cleanup when worker is terminated
process.on('exit', () => {
if (tokenCounter) {
tokenCounter.free();
tokenCounter = null;
}
});
//# sourceMappingURL=outputMetricsWorker.js.map