@n8n/n8n-nodes-langchain
Version:
73 lines • 2.85 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TextSplitterTokenSplitter = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const ai_utilities_1 = require("@n8n/ai-utilities");
const TokenTextSplitter_1 = require("./TokenTextSplitter");
class TextSplitterTokenSplitter {
constructor() {
this.description = {
displayName: 'Token Splitter',
name: 'textSplitterTokenSplitter',
icon: 'node:token-splitter',
iconColor: 'black',
group: ['transform'],
version: 1,
description: 'Split text into chunks by tokens',
defaults: {
name: 'Token Splitter',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Text Splitters'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/',
},
],
},
},
inputs: [],
outputs: [n8n_workflow_1.NodeConnectionTypes.AiTextSplitter],
outputNames: ['Text Splitter'],
properties: [
(0, ai_utilities_1.getConnectionHintNoticeField)([n8n_workflow_1.NodeConnectionTypes.AiDocument]),
{
displayName: 'Chunk Size',
name: 'chunkSize',
type: 'number',
default: 1000,
description: 'Maximum number of tokens per chunk',
},
{
displayName: 'Chunk Overlap',
name: 'chunkOverlap',
type: 'number',
default: 0,
description: 'Number of tokens shared between consecutive chunks to preserve context',
},
],
};
}
async supplyData(itemIndex) {
this.logger.debug('Supply Data for Text Splitter');
const chunkSize = this.getNodeParameter('chunkSize', itemIndex);
const chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex);
const splitter = new TokenTextSplitter_1.TokenTextSplitter({
chunkSize,
chunkOverlap,
allowedSpecial: 'all',
disallowedSpecial: 'all',
encodingName: 'cl100k_base',
keepSeparator: false,
});
return {
response: (0, ai_utilities_1.logWrapper)(splitter, this),
};
}
}
exports.TextSplitterTokenSplitter = TextSplitterTokenSplitter;
//# sourceMappingURL=TextSplitterTokenSplitter.node.js.map