gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
16 lines (15 loc) • 710 B
JavaScript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { readFileSync, writeFileSync } from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'url';
import { parseTikTokenFile } from '../node/parsers.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const tikTokenFilePath = path.join(__dirname, 'cl100k_base.tiktoken');
const tikTokenFileContents = readFileSync(tikTokenFilePath, 'utf8');
const entries = parseTikTokenFile(tikTokenFileContents);
writeFileSync(path.join(__dirname, 'cl100k_base.json'), JSON.stringify(entries, null, 2));