n8n-nodes-gpt-tokenizer
Version:
n8n node for working with BPE Tokens with OpenAI's GPT models in mind.
238 lines • 11.7 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.GptTokenizer = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const gpt_tokenizer_1 = require("gpt-tokenizer");
class GptTokenizer {
constructor() {
this.description = {
displayName: 'GPT-Tokenizer',
name: 'gptTokenizer',
group: ['transform'],
version: 1,
icon: 'file:TokenIcon.svg',
description: 'Encode / decodes BPE Tokens or check Token Limits before working with the OpenAI GPT models.',
defaults: {
name: 'GPT-Tokenizer',
},
inputs: ['main'],
outputs: ['main'],
properties: [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Encode',
value: 'encode',
description: 'Encode a string into BPE tokens. Returns an array of tokens.',
action: 'Encode a string into BPE tokens',
},
{
name: 'Decode',
value: 'decode',
description: 'Decode an array of BPE tokens. Returns the decoded string.',
action: 'Decode a string into BPE tokens',
},
{
name: 'Count Tokens',
value: 'countTokens',
description: 'Determines the amount of tokens the string produces. Returns the number of tokens.',
action: 'Count tokens of a string',
},
{
name: 'Check Token Limit',
value: 'isWithinTokenLimit',
description: 'Check if the string is within the provided token limit. Returns true or false.',
action: 'Check if string is within token limit',
},
{
name: 'Slice to Max Token Limit',
value: 'sliceMatchingTokenLimit',
description: 'Slice the string into blocks with a max token limit. Returns an array of strings.',
action: 'Slice string into sections matching a max token limit',
},
],
default: 'encode',
},
{
displayName: 'Input String',
name: 'inputString',
type: 'string',
default: '',
placeholder: '',
required: true,
description: 'String to process',
displayOptions: {
show: {
operation: ['encode', 'countTokens', 'isWithinTokenLimit', 'sliceMatchingTokenLimit'],
},
},
},
{
displayName: 'Max Tokens',
name: 'maxTokens',
type: 'number',
default: 2048,
placeholder: '2048',
required: true,
description: 'The max number of tokens to allow',
displayOptions: {
show: {
operation: ['isWithinTokenLimit', 'sliceMatchingTokenLimit'],
},
},
},
{
displayName: 'Error When Exceeding Token Limit',
name: 'errorTokenLimit',
type: 'boolean',
default: false,
placeholder: '',
description: 'Whether to throw an error when the string exceeds the token limit',
displayOptions: {
show: {
operation: ['isWithinTokenLimit'],
},
}
},
{
displayName: 'Input Tokens',
name: 'inputTokens',
type: 'string',
default: '',
required: true,
placeholder: '[5661,318,1337]',
description: 'Array of BPE tokens to decode',
displayOptions: {
show: {
operation: ['decode'],
},
},
},
{
displayName: 'Destination Key',
name: 'destinationKey',
type: 'string',
default: '',
placeholder: '',
description: 'The key to write the results to. Leave empty to use default destination keys.',
}
],
};
}
async execute() {
const items = this.getInputData();
let item;
let inputString;
let inputTokens;
let maxTokens;
let destinationKey;
let shouldThrowErrorOnTokenLimit;
const operation = this.getNodeParameter('operation', 0);
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
try {
inputString = this.getNodeParameter('inputString', itemIndex, '');
inputTokens = this.getNodeParameter('inputTokens', itemIndex, null);
maxTokens = this.getNodeParameter('maxTokens', itemIndex, 2048);
destinationKey = this.getNodeParameter('destinationKey', itemIndex, '');
shouldThrowErrorOnTokenLimit = this.getNodeParameter('errorTokenLimit', itemIndex, false);
item = items[itemIndex];
if (operation === 'encode') {
if (typeof inputString !== 'string')
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String is not a string', { itemIndex: itemIndex });
if (!inputString)
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String field is empty', { itemIndex: itemIndex });
if (!destinationKey)
destinationKey = 'tokens';
item.json[destinationKey] = (0, gpt_tokenizer_1.encode)(inputString);
}
else if (operation == 'decode') {
if (!destinationKey)
destinationKey = 'data';
if (!Array.isArray(inputTokens)) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input Tokens is not an array', { itemIndex: itemIndex });
}
else if (inputTokens.length == 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input Tokens field is empty', { itemIndex: itemIndex });
}
else {
item.json[destinationKey] = (0, gpt_tokenizer_1.decode)(inputTokens);
}
}
else if (operation == 'countTokens') {
if (typeof inputString !== 'string')
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String is not a string', { itemIndex: itemIndex });
if (!inputString)
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String field is empty', { itemIndex: itemIndex });
if (!destinationKey)
destinationKey = 'tokenCount';
item.json[destinationKey] = (0, gpt_tokenizer_1.encode)(inputString).length;
}
else if (operation == 'isWithinTokenLimit') {
if (typeof inputString !== 'string')
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String is not a string', { itemIndex: itemIndex });
if (!inputString)
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String field is empty', { itemIndex: itemIndex });
if (!destinationKey)
destinationKey = 'isWithinTokenLimit';
if (maxTokens <= 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Provide Max Tokens. (bigger then 0)', { itemIndex: itemIndex });
}
else if ((0, gpt_tokenizer_1.isWithinTokenLimit)(inputString, maxTokens)) {
item.json[destinationKey] = true;
}
else {
item.json[destinationKey] = false;
if (shouldThrowErrorOnTokenLimit) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'String exceeds token limit', { itemIndex: itemIndex });
}
}
}
else if (operation == 'sliceMatchingTokenLimit') {
if (!destinationKey)
destinationKey = 'slices';
if (typeof inputString !== 'string')
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String is not a string', { itemIndex: itemIndex });
if (!inputString)
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Input String field is empty', { itemIndex: itemIndex });
if (maxTokens <= 0) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Provide Max Tokens. (bigger then 0)', { itemIndex: itemIndex });
}
else {
if (!(0, gpt_tokenizer_1.isWithinTokenLimit)(inputString, maxTokens)) {
let temporaryTokens = (0, gpt_tokenizer_1.encode)(inputString);
let blocks = [];
for (let i = 0; i < temporaryTokens.length; i += maxTokens) {
blocks.push((0, gpt_tokenizer_1.decode)(temporaryTokens.slice(i, i + maxTokens)));
}
item.json[destinationKey] = blocks;
}
else {
item.json[destinationKey] = [inputString];
}
}
}
}
catch (error) {
if (this.continueOnFail()) {
items.push({ json: this.getInputData(itemIndex)[0].json, error, pairedItem: itemIndex });
}
else {
if (error.context) {
error.context.itemIndex = itemIndex;
throw error;
}
throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
itemIndex,
});
}
}
}
return this.prepareOutputData(items);
}
}
exports.GptTokenizer = GptTokenizer;
//# sourceMappingURL=GptTokenizer.node.js.map