n8n-nodes-semantic-splitter-with-context
Version:
Semantic Splitter with Context for n8n with LangChain integration
552 lines (551 loc) • 25 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SemanticSplitterWithContext = void 0;
const documents_1 = require("@langchain/core/documents");
const textsplitters_1 = require("@langchain/textsplitters");
const logWrapper_1 = require("../../utils/logWrapper");
class SemanticDoublePassMergingSplitterWithContext extends textsplitters_1.TextSplitter {
constructor(embeddings, chatModel, options = {}) {
var _a, _b, _c, _d, _e, _f;
super();
this.embeddings = embeddings;
this.chatModel = chatModel;
this.bufferSize = (_a = options.bufferSize) !== null && _a !== void 0 ? _a : 1;
this.breakpointThresholdType = (_b = options.breakpointThresholdType) !== null && _b !== void 0 ? _b : 'percentile';
this.breakpointThresholdAmount = options.breakpointThresholdAmount;
this.numberOfChunks = options.numberOfChunks;
this.sentenceSplitRegex = new RegExp((_c = options.sentenceSplitRegex) !== null && _c !== void 0 ? _c : '(?<=[.?!])\\s+');
this.minChunkSize = options.minChunkSize;
this.maxChunkSize = options.maxChunkSize;
this.secondPassThreshold = (_d = options.secondPassThreshold) !== null && _d !== void 0 ? _d : 0.8;
this.contextPrompt = (_e = options.contextPrompt) !== null && _e !== void 0 ? _e : `Generate a brief contextual summary for this text chunk to enhance search retrieval, two to three short sentences max. The chunk contains merged content from different document sections, so focus on the main topics and concepts rather than the sequential flow. Answer only with the succinct context and nothing else.`;
this.includeLabels = (_f = options.includeLabels) !== null && _f !== void 0 ? _f : false;
}
async splitText(text) {
if (!text || text.trim().length === 0)
return [];
const sentences = this._splitTextIntoSentences(text);
if (sentences.length === 0)
return [];
if (sentences.length === 1) {
const singleSentence = sentences[0].trim();
if (!singleSentence)
return [];
if (this.maxChunkSize && singleSentence.length > this.maxChunkSize) {
const words = singleSentence.split(/\s+/);
const chunks = [];
let currentChunk = '';
for (const word of words) {
if (currentChunk.length + word.length + 1 <= this.maxChunkSize) {
currentChunk = currentChunk ? currentChunk + ' ' + word : word;
}
else {
if (currentChunk)
chunks.push(currentChunk);
currentChunk = word;
}
}
if (currentChunk)
chunks.push(currentChunk);
return chunks;
}
return [singleSentence];
}
const combinedSentences = await this._combineSentences(sentences);
const embeddings = await this._embedSentences(combinedSentences);
const distances = this._calculateDistances(embeddings);
if (distances.length === 0) {
return sentences.map(s => s.trim()).filter(s => s.length > 0);
}
const breakpoints = this._calculateBreakpoints(distances);
let chunks = this._createChunks(sentences, breakpoints);
chunks = await this._secondPassMerge(chunks);
chunks = this._applySizeConstraints(chunks);
return chunks;
}
async splitDocuments(documents) {
const splitDocuments = [];
for (const document of documents) {
const chunks = await this.splitText(document.pageContent);
for (const chunk of chunks) {
const contextualContent = await this._generateContextualContent(document.pageContent, chunk);
splitDocuments.push(new documents_1.Document({
pageContent: contextualContent,
metadata: { ...document.metadata },
}));
}
}
return splitDocuments;
}
async _generateContextualContent(wholeDocument, chunk) {
try {
const fullPrompt = `<document>
${wholeDocument}
</document>
Here is the chunk we want to situate within the whole document
<chunk>
${chunk}
</chunk>
${this.contextPrompt}`;
const response = await this.chatModel.invoke(fullPrompt);
const context = typeof response === 'string' ? response : response.content;
return this._formatContextualOutput(context, chunk);
}
catch (error) {
console.error('Error generating contextual content:', error);
return chunk;
}
}
_formatContextualOutput(context, chunk) {
if (this.includeLabels) {
return `Context: ${context}\n\nChunk: ${chunk}`;
}
else {
return `${context}\n\n${chunk}`;
}
}
_splitTextIntoSentences(text) {
const sentences = text.split(this.sentenceSplitRegex).filter((s) => s.trim().length > 0);
return sentences;
}
async _combineSentences(sentences) {
const combined = [];
const bufferSize = Math.min(this.bufferSize, sentences.length);
for (let i = 0; i < sentences.length; i++) {
const start = Math.max(0, i - bufferSize);
const end = Math.min(sentences.length, i + bufferSize + 1);
const combinedText = sentences.slice(start, end).join(' ');
combined.push(combinedText);
}
return combined;
}
async _embedSentences(sentences) {
const embeddings = await this.embeddings.embedDocuments(sentences);
return embeddings;
}
_calculateDistances(embeddings) {
const distances = [];
for (let i = 0; i < embeddings.length - 1; i++) {
const distance = this._cosineDistance(embeddings[i], embeddings[i + 1]);
distances.push(distance);
}
return distances;
}
_cosineDistance(vec1, vec2) {
const dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);
const magnitude1 = Math.sqrt(vec1.reduce((sum, val) => sum + val * val, 0));
const magnitude2 = Math.sqrt(vec2.reduce((sum, val) => sum + val * val, 0));
if (magnitude1 === 0 || magnitude2 === 0) {
return 1;
}
const similarity = dotProduct / (magnitude1 * magnitude2);
const clampedSimilarity = Math.max(-1, Math.min(1, similarity));
return 1 - clampedSimilarity;
}
_calculateBreakpoints(distances) {
if (distances.length === 0)
return [];
let threshold;
if (this.numberOfChunks) {
const sortedDistances = [...distances].sort((a, b) => b - a);
const index = Math.min(this.numberOfChunks - 1, sortedDistances.length - 1);
threshold = sortedDistances[index];
}
else if (this.breakpointThresholdAmount !== undefined) {
threshold = this.breakpointThresholdAmount;
}
else {
switch (this.breakpointThresholdType) {
case 'percentile': {
const percentile = 0.95;
const sortedDist = [...distances].sort((a, b) => a - b);
const index = Math.floor(sortedDist.length * percentile);
const safeIndex = Math.min(index, sortedDist.length - 1);
threshold = sortedDist[safeIndex];
break;
}
case 'standard_deviation': {
const mean = distances.reduce((a, b) => a + b, 0) / distances.length;
const variance = distances.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / distances.length;
const stdDev = Math.sqrt(variance);
threshold = mean + stdDev;
break;
}
case 'interquartile': {
const sorted = [...distances].sort((a, b) => a - b);
const q1Index = Math.floor(sorted.length * 0.25);
const q3Index = Math.floor(sorted.length * 0.75);
const safeQ1Index = Math.min(q1Index, sorted.length - 1);
const safeQ3Index = Math.min(q3Index, sorted.length - 1);
const q1 = sorted[safeQ1Index];
const q3 = sorted[safeQ3Index];
const iqr = q3 - q1;
threshold = q3 + 1.5 * iqr;
break;
}
case 'gradient': {
const gradients = [];
for (let i = 1; i < distances.length; i++) {
gradients.push(Math.abs(distances[i] - distances[i - 1]));
}
if (gradients.length > 0) {
const maxGradientIndex = gradients.indexOf(Math.max(...gradients));
const thresholdIndex = Math.min(maxGradientIndex + 1, distances.length - 1);
threshold = distances[thresholdIndex];
}
else {
threshold = 0.5;
}
break;
}
default:
threshold = 0.5;
}
}
const breakpoints = [];
for (let i = 0; i < distances.length; i++) {
if (distances[i] > threshold) {
breakpoints.push(i + 1);
}
}
return breakpoints;
}
_createChunks(sentences, breakpoints) {
const chunks = [];
let start = 0;
for (const breakpoint of breakpoints) {
const chunk = sentences.slice(start, breakpoint).join(' ');
if (chunk.trim()) {
chunks.push(chunk.trim());
}
start = breakpoint;
}
if (start < sentences.length) {
const chunk = sentences.slice(start).join(' ');
if (chunk.trim()) {
chunks.push(chunk.trim());
}
}
return chunks;
}
async _secondPassMerge(chunks) {
if (chunks.length <= 1)
return chunks;
const chunkEmbeddings = await this.embeddings.embedDocuments(chunks);
const mergedChunks = [];
let currentChunk = chunks[0];
let currentEmbedding = chunkEmbeddings[0];
let needsEmbeddingUpdate = false;
for (let i = 1; i < chunks.length; i++) {
const similarity = 1 - this._cosineDistance(currentEmbedding, chunkEmbeddings[i]);
if (similarity >= this.secondPassThreshold) {
currentChunk = currentChunk + ' ' + chunks[i];
needsEmbeddingUpdate = true;
}
else {
if (needsEmbeddingUpdate) {
const [newEmbedding] = await this.embeddings.embedDocuments([currentChunk]);
currentEmbedding = newEmbedding;
needsEmbeddingUpdate = false;
}
mergedChunks.push(currentChunk);
currentChunk = chunks[i];
currentEmbedding = chunkEmbeddings[i];
}
}
if (needsEmbeddingUpdate) {
const [newEmbedding] = await this.embeddings.embedDocuments([currentChunk]);
currentEmbedding = newEmbedding;
}
mergedChunks.push(currentChunk);
return mergedChunks;
}
_applySizeConstraints(chunks) {
if (!this.minChunkSize && !this.maxChunkSize)
return chunks;
const constrainedChunks = [];
let currentChunk = '';
for (const chunk of chunks) {
const chunkLength = chunk.length;
if (this.maxChunkSize && chunkLength > this.maxChunkSize) {
const sentences = this._splitTextIntoSentences(chunk);
let tempChunk = '';
for (const sentence of sentences) {
if (tempChunk.length + sentence.length + 1 <= this.maxChunkSize) {
tempChunk = tempChunk ? tempChunk + ' ' + sentence : sentence;
}
else {
if (tempChunk && (!this.minChunkSize || tempChunk.length >= this.minChunkSize)) {
constrainedChunks.push(tempChunk);
tempChunk = sentence;
}
else {
if (currentChunk) {
currentChunk = currentChunk + ' ' + tempChunk + ' ' + sentence;
}
else {
tempChunk = tempChunk ? tempChunk + ' ' + sentence : sentence;
}
}
}
}
if (tempChunk) {
if (!this.minChunkSize || tempChunk.length >= this.minChunkSize) {
constrainedChunks.push(tempChunk);
}
else {
if (currentChunk) {
currentChunk = currentChunk + ' ' + tempChunk;
}
else {
currentChunk = tempChunk;
}
}
}
}
else if (this.minChunkSize && chunkLength < this.minChunkSize) {
if (currentChunk) {
currentChunk = currentChunk + ' ' + chunk;
}
else {
currentChunk = chunk;
}
if (currentChunk.length >= this.minChunkSize) {
constrainedChunks.push(currentChunk);
currentChunk = '';
}
}
else {
if (currentChunk) {
if (!this.minChunkSize || currentChunk.length >= this.minChunkSize) {
constrainedChunks.push(currentChunk);
currentChunk = '';
constrainedChunks.push(chunk);
}
else {
currentChunk = currentChunk + ' ' + chunk;
if (currentChunk.length >= this.minChunkSize) {
constrainedChunks.push(currentChunk);
currentChunk = '';
}
}
}
else {
constrainedChunks.push(chunk);
}
}
}
if (currentChunk) {
if (!this.minChunkSize || currentChunk.length >= this.minChunkSize) {
constrainedChunks.push(currentChunk);
}
else {
if (constrainedChunks.length > 0) {
const lastChunk = constrainedChunks.pop();
const mergedChunk = lastChunk + ' ' + currentChunk;
constrainedChunks.push(mergedChunk);
}
else {
constrainedChunks.push(currentChunk);
}
}
}
return constrainedChunks;
}
}
class SemanticSplitterWithContext {
constructor() {
this.description = {
displayName: 'Semantic Splitter with Context',
name: 'contextualSemanticTextSplitterWithContext',
icon: 'fa:cut',
group: ['transform'],
version: 1,
description: 'Split text using semantic similarity with contextual enhancement for improved retrieval',
defaults: {
name: 'Semantic Splitter with Context',
},
codex: {
categories: ['AI'],
subcategories: {
AI: ['Text Splitters'],
},
resources: {
primaryDocumentation: [
{
url: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittercontextualsemantic/',
},
],
},
},
inputs: [
{
displayName: 'Chat Model',
maxConnections: 1,
type: "ai_languageModel",
required: true,
},
{
displayName: 'Embeddings',
maxConnections: 1,
type: "ai_embedding",
required: true,
},
],
outputs: [
{
displayName: 'Text Splitter',
maxConnections: 1,
type: "ai_textSplitter",
},
],
properties: [
{
displayName: 'Context Prompt',
name: 'contextPrompt',
type: 'string',
typeOptions: {
rows: 4,
},
default: `Please generate a short succinct context summary to situate this text chunk within the overall document to enhance search retrieval, two or three sentances max. The chunk contains merged content from different document sections, so focus on the main topics and concepts rather than sequential flow. Answer only with the succinct context and nothing else.`,
description: 'Instructions for the AI model on how to generate contextual descriptions. The document and chunk will be automatically provided in the prompt structure.',
},
{
displayName: 'Include Labels in Output',
name: 'includeLabels',
type: 'boolean',
default: false,
description: 'Whether to include "Context:" and "Chunk:" labels in the output. When disabled, only the context and chunk content are included without labels.',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
default: {},
placeholder: 'Add Option',
options: [
{
displayName: 'Buffer Size',
name: 'bufferSize',
type: 'number',
default: 1,
description: 'Number of sentences to combine for context when creating embeddings',
},
{
displayName: 'Breakpoint Threshold Type',
name: 'breakpointThresholdType',
type: 'options',
default: 'percentile',
options: [
{
name: 'Percentile',
value: 'percentile',
description: 'Use percentile of distances as threshold',
},
{
name: 'Standard Deviation',
value: 'standard_deviation',
description: 'Use mean + standard deviation as threshold',
},
{
name: 'Interquartile',
value: 'interquartile',
description: 'Use interquartile range method',
},
{
name: 'Gradient',
value: 'gradient',
description: 'Use maximum gradient change as threshold',
},
],
},
{
displayName: 'Breakpoint Threshold Amount',
name: 'breakpointThresholdAmount',
type: 'number',
default: 0.5,
typeOptions: {
minValue: 0,
maxValue: 1,
numberStepSize: 0.01,
},
description: 'Manual threshold for determining chunk boundaries (0-1). If set, overrides threshold type.',
displayOptions: {
show: {
'/breakpointThresholdType': ['percentile', 'standard_deviation', 'interquartile', 'gradient'],
},
},
},
{
displayName: 'Number of Chunks',
name: 'numberOfChunks',
type: 'number',
default: 0,
description: 'Target number of chunks to create. If set, overrides threshold settings. Set to 0 to use threshold.',
},
{
displayName: 'Second Pass Threshold',
name: 'secondPassThreshold',
type: 'number',
default: 0.8,
typeOptions: {
minValue: 0,
maxValue: 1,
numberStepSize: 0.01,
},
description: 'Similarity threshold for merging chunks in the second pass (0-1). Higher values require more similarity to merge.',
},
{
displayName: 'Min Chunk Size',
name: 'minChunkSize',
type: 'number',
default: 100,
description: 'Minimum number of characters per chunk',
},
{
displayName: 'Max Chunk Size',
name: 'maxChunkSize',
type: 'number',
default: 2000,
description: 'Maximum number of characters per chunk',
},
{
displayName: 'Sentence Split Regex',
name: 'sentenceSplitRegex',
type: 'string',
default: '(?<=[.?!])\\s+',
description: 'Regular expression to split text into sentences',
},
],
},
],
};
}
async supplyData(itemIndex) {
console.log('ContextualSemanticSplitter: supplyData called!');
const chatModel = (await this.getInputConnectionData("ai_languageModel", itemIndex));
const embeddings = (await this.getInputConnectionData("ai_embedding", itemIndex));
const contextPrompt = this.getNodeParameter('contextPrompt', itemIndex, '');
const includeLabels = this.getNodeParameter('includeLabels', itemIndex, false);
const options = this.getNodeParameter('options', itemIndex, {});
const splitter = new SemanticDoublePassMergingSplitterWithContext(embeddings, chatModel, {
bufferSize: options.bufferSize,
breakpointThresholdType: options.breakpointThresholdType,
breakpointThresholdAmount: options.breakpointThresholdAmount,
numberOfChunks: options.numberOfChunks,
secondPassThreshold: options.secondPassThreshold,
minChunkSize: options.minChunkSize,
maxChunkSize: options.maxChunkSize,
sentenceSplitRegex: options.sentenceSplitRegex,
contextPrompt,
includeLabels,
});
console.log('ContextualSemanticSplitter: About to wrap splitter with logWrapper');
const wrappedSplitter = (0, logWrapper_1.logWrapper)(splitter, this);
console.log('ContextualSemanticSplitter: Wrapped splitter created');
return {
response: wrappedSplitter,
};
}
}
exports.SemanticSplitterWithContext = SemanticSplitterWithContext;