chonkie
Version:
🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.
150 lines • 6.56 kB
JavaScript
;
/** Base Chunking Class. **/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BaseChunker = void 0;
/**
* Base class for all chunking classes.
*
* This abstract class provides a common interface and shared logic for all chunking implementations.
* It supports chunking a single text or a batch of texts, with optional concurrency and progress reporting.
*
* Subclasses must implement the `chunk` method to define how a single text is chunked.
*
* @template T - The type of chunk produced (usually `Chunk[]` or `string[]`).
*
* @property {Tokenizer} tokenizer - The tokenizer instance used for chunking operations.
* @property {boolean} _useConcurrency - Whether to use concurrent processing for batch chunking (default: true).
*
* @example
* class MyChunker extends BaseChunker {
* async chunk(text: string): Promise<Chunk[]> {
* // ... implementation ...
* }
* }
*
* const chunker = new MyChunker(tokenizer);
* const chunks = await chunker.call("Some text");
* const batchChunks = await chunker.call(["Text 1", "Text 2"], true);
*/
class BaseChunker {
constructor(tokenizer) {
this._useConcurrency = true; // Determines if batch processing uses Promise.all
this.tokenizer = tokenizer;
}
/**
* Returns a string representation of the chunker instance.
*
* @returns {string} The class name and constructor signature.
*/
toString() {
return `${this.constructor.name}()`;
}
call(textOrTexts_1) {
return __awaiter(this, arguments, void 0, function* (textOrTexts, showProgress = false) {
if (typeof textOrTexts === 'string') {
return this.chunk(textOrTexts);
}
else if (Array.isArray(textOrTexts)) {
return this.chunkBatch(textOrTexts, showProgress);
}
else {
// This case should ideally not be reached due to TypeScript's type checking
// if the public overloads are used correctly.
throw new Error("Input must be a string or an array of strings.");
}
});
}
/**
* Process a batch of texts sequentially (one after another).
*
* @protected
* @param {string[]} texts - The texts to chunk.
* @param {boolean} [showProgress=false] - Whether to display progress in the console.
* @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
*/
_sequential_batch_processing(texts_1) {
return __awaiter(this, arguments, void 0, function* (texts, showProgress = false) {
const results = [];
const total = texts.length;
for (let i = 0; i < total; i++) {
if (showProgress && total > 1) {
const progress = Math.round(((i + 1) / total) * 100);
process.stdout.write(`Sequential processing: Document ${i + 1}/${total} (${progress}%)\r`);
}
results.push(yield this.chunk(texts[i]));
}
if (showProgress && total > 1) {
process.stdout.write("\n"); // Newline after progress
}
return results;
});
}
/**
* Process a batch of texts concurrently using Promise.all.
*
* @protected
* @param {string[]} texts - The texts to chunk.
* @param {boolean} [showProgress=false] - Whether to display progress in the console.
* @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
*/
_concurrent_batch_processing(texts_1) {
return __awaiter(this, arguments, void 0, function* (texts, showProgress = false) {
const total = texts.length;
let completedCount = 0;
const updateProgress = () => {
if (showProgress && total > 1) {
completedCount++;
const progress = Math.round((completedCount / total) * 100);
process.stdout.write(`Concurrent processing: Document ${completedCount}/${total} (${progress}%)\r`);
}
};
const chunkPromises = texts.map(text => this.chunk(text).then(result => {
updateProgress();
return result;
}));
const results = yield Promise.all(chunkPromises);
if (showProgress && total > 1 && completedCount > 0) { // ensure newline only if progress was shown
process.stdout.write("\n"); // Newline after progress
}
return results;
});
}
/**
* Chunk a batch of texts, using either concurrent or sequential processing.
*
* If only one text is provided, processes it directly without batch overhead.
*
* @param {string[]} texts - The texts to chunk.
* @param {boolean} [showProgress=true] - Whether to display progress in the console.
* @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
*/
chunkBatch(texts_1) {
return __awaiter(this, arguments, void 0, function* (texts, showProgress = true) {
if (texts.length === 0) {
return [];
}
// If only one text, process it directly without batch overhead, progress not shown for single item.
if (texts.length === 1) {
return [yield this.chunk(texts[0])];
}
// For multiple texts, use selected batch processing strategy
if (this._useConcurrency) {
return this._concurrent_batch_processing(texts, showProgress);
}
else {
return this._sequential_batch_processing(texts, showProgress);
}
});
}
}
exports.BaseChunker = BaseChunker;
//# sourceMappingURL=base.js.map