chonkie
Version:
🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.
64 lines • 3.06 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.OverlapRefinery = void 0;
const base_1 = require("./base");
class OverlapRefinery extends base_1.CloudClient {
constructor(apiKey, config = {}) {
var _a, _b;
super({ apiKey });
this.config = {
tokenizerOrTokenCounter: config.tokenizerOrTokenCounter || "character",
contextSize: (_a = config.contextSize) !== null && _a !== void 0 ? _a : 0.25,
mode: config.mode || "token",
method: config.method || "suffix",
recipe: config.recipe || "default",
lang: config.lang || "en",
merge: (_b = config.merge) !== null && _b !== void 0 ? _b : true,
};
}
refine(chunks) {
return __awaiter(this, void 0, void 0, function* () {
// Create snake cased chunks for the request
const snakeCasedChunks = chunks.map(chunk => {
return {
text: chunk.text,
start_index: chunk.startIndex,
end_index: chunk.endIndex,
token_count: chunk.tokenCount,
};
});
const response = yield this.request("/v1/refine/overlap", {
body: {
chunks: snakeCasedChunks,
tokenizer_or_token_counter: this.config.tokenizerOrTokenCounter,
context_size: this.config.contextSize,
mode: this.config.mode,
method: this.config.method,
recipe: this.config.recipe,
lang: this.config.lang,
merge: this.config.merge,
},
headers: {
"Content-Type": "application/json",
},
});
// Merge the response chunks with the original chunks
const mergedChunks = response.map((chunk, index) => {
const originalChunk = chunks[index];
return Object.assign(Object.assign({}, originalChunk), { text: chunk.text, startIndex: chunk.start_index, endIndex: chunk.end_index, tokenCount: chunk.token_count });
});
return mergedChunks;
});
}
}
exports.OverlapRefinery = OverlapRefinery;
//# sourceMappingURL=overlap_refinery.js.map