chonkie
Version:
🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.
131 lines • 6.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.SentenceChunk = exports.Sentence = void 0;
const base_1 = require("./base");
/**
* Class to represent a sentence.
*
* Represents a single sentence within a text, including its text, position, and token count.
*
* @class
* @param {SentenceData} data - The data required to construct a Sentence instance.
* @property {string} text - The text of the sentence.
* @property {number} startIndex - The starting index of the sentence in the original text.
* @property {number} endIndex - The ending index of the sentence in the original text.
* @property {number} tokenCount - The number of tokens in the sentence.
* @property {number[]} [embedding] - The embedding vector for the sentence (array of numbers, or null if not present).
*
* @method toString Returns a string representation of the Sentence.
* @returns {string}
*
* @method toDict Returns the Sentence as a dictionary-like object.
* @returns {SentenceData}
*
* @method static fromDict Creates a Sentence object from a dictionary-like object.
* @param {SentenceData} data - The data to create the Sentence from.
* @returns {Sentence}
*/
class Sentence {
constructor(data) {
this.text = data.text;
this.startIndex = data.startIndex;
this.endIndex = data.endIndex;
this.tokenCount = data.tokenCount;
}
/** Return a string representation of the Sentence */
toString() {
return `Sentence(text=${this.text}, startIndex=${this.startIndex}, endIndex=${this.endIndex}, tokenCount=${this.tokenCount})`;
}
/** Return the Sentence as a dictionary-like object */
toDict() {
return {
text: this.text,
startIndex: this.startIndex,
endIndex: this.endIndex,
tokenCount: this.tokenCount,
};
}
/** Create a Sentence object from a dictionary-like object */
static fromDict(data) {
return new Sentence(data);
}
}
exports.Sentence = Sentence;
/**
* Represents a chunk of one or more sentences within a text.
*
* A SentenceChunk groups together multiple {@link Sentence} objects, providing their combined text, position, and token count within the original text.
*
* @class
* @extends Chunk
*
* @param {Object} data - Data to construct a SentenceChunk instance.
* @param {string} data.text - Combined text of all sentences in the chunk.
* @param {number} data.startIndex - Zero-based index where the chunk starts in the original text.
* @param {number} data.endIndex - Zero-based index where the chunk ends in the original text (inclusive).
* @param {number} data.tokenCount - Total number of tokens in the chunk.
* @param {Sentence[]} data.sentences - Array of {@link Sentence} objects in the chunk.
*
* @property {string} text - Combined text of all sentences in the chunk.
* @property {number} startIndex - Starting index of the chunk in the original text.
* @property {number} endIndex - Ending index of the chunk in the original text.
* @property {number} tokenCount - Total number of tokens in the chunk.
* @property {Sentence[]} sentences - List of {@link Sentence} objects in the chunk.
*
* @method toString Returns a detailed string representation of the SentenceChunk, including its text, start and end indices, token count, and a list of all contained sentences with their metadata.
* @method toDict Returns the SentenceChunk as a plain object (see {@link SentenceChunkData}).
* @method static fromDict Creates a SentenceChunk from a {@link SentenceChunkData} object.
*/
class SentenceChunk extends base_1.Chunk {
constructor(data) {
var _a;
super(data);
this.sentences = data.sentences;
this.embedding = (_a = data.embedding) !== null && _a !== void 0 ? _a : undefined;
}
/**
* Returns a detailed string representation of the SentenceChunk, including its text, start and end indices, token count, and a list of all contained sentences with their metadata.
*
* This method overrides the base {@link Chunk} toString method to provide a more informative output, which is especially useful for debugging and logging. Each sentence in the chunk is represented using its own toString method, and all sentences are included in the output.
*
* @returns {string} A string describing the SentenceChunk and all its sentences, e.g.,
* SentenceChunk(text=..., startIndex=..., endIndex=..., tokenCount=..., sentences=[Sentence(...), ...])
*/
toString() {
const sentencesStr = this.sentences.map(s => s.toString()).join(', ');
return `SentenceChunk(text=${this.text}, startIndex=${this.startIndex}, endIndex=${this.endIndex}, tokenCount=${this.tokenCount}, sentences=[${sentencesStr}])`;
}
/**
* Returns the SentenceChunk as a dictionary-like object.
*
* This method extends the base {@link Chunk} toDict method to include the sentences in the chunk.
*
* @returns {SentenceChunkData} A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
/** Return the SentenceChunk as a dictionary-like object */
toDict() {
const baseDict = super.toDict();
return Object.assign(Object.assign({}, baseDict), { sentences: this.sentences.map(sentence => sentence.toDict()) });
}
/**
* Creates a SentenceChunk object from a dictionary-like object.
*
* This method extends the base {@link Chunk} fromDict method to include the sentences in the chunk.
*
* @param {SentenceChunkData} data - A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
* @returns {SentenceChunk} A new SentenceChunk object created from the provided dictionary-like object.
*/
static fromDict(data) {
var _a;
const sentences = data.sentences.map(sentence => Sentence.fromDict(sentence));
return new SentenceChunk({
text: data.text,
startIndex: data.startIndex,
endIndex: data.endIndex,
tokenCount: data.tokenCount,
sentences,
embedding: (_a = data.embedding) !== null && _a !== void 0 ? _a : undefined,
});
}
}
exports.SentenceChunk = SentenceChunk;
//# sourceMappingURL=sentence.js.map