chonkie
Version:
🦛 CHONK your texts in TS with Chonkie!✨The no-nonsense lightweight and efficient chunking library.
155 lines • 6.37 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Hubbie = void 0;
const hub_1 = require("@huggingface/hub");
const fs = __importStar(require("fs"));
const jsonschema = __importStar(require("jsonschema"));
/**
* Hubbie is a Huggingface hub manager for Chonkie.
*/
class Hubbie {
constructor() {
// Define the path to the recipes
this.getRecipeConfig = {
repo: "chonkie-ai/recipes",
subfolder: "recipes",
repoType: "dataset",
};
// Fetch the current recipe schema from the hub
this.recipeSchema = this.getRecipeSchema();
}
/**
* Get the current recipe schema from the hub.
*/
getRecipeSchema() {
return __awaiter(this, void 0, void 0, function* () {
const schemaBlob = yield (0, hub_1.downloadFile)({
repo: {
name: "chonkie-ai/recipes",
type: "dataset",
},
path: `${Hubbie.SCHEMA_VERSION}.schema.json`,
});
if (!schemaBlob) {
throw new Error("Failed to download schema file");
}
const schemaContent = yield schemaBlob.text();
return JSON.parse(schemaContent);
});
}
/**
* Validate a recipe against the current schema.
*/
validateRecipe(recipe) {
try {
jsonschema.validate(recipe, this.recipeSchema);
return true;
}
catch (error) {
throw new Error(`Recipe is invalid. Please check the recipe and try again. Error: ${error}`);
}
}
/**
* Get a recipe from the hub.
*
* @param name - The name of the recipe to get
* @param language - The language of the recipe to get
* @param filePath - Optionally, provide the path to the recipe
* @returns The recipe
* @throws Error if the recipe is not found or invalid
*/
getRecipe() {
return __awaiter(this, arguments, void 0, function* (name = 'default', language = 'en', filePath) {
// Check if either (name & language) or path is provided
if ((!name || !language) && !filePath) {
throw new Error("Either (name & language) or path must be provided.");
}
let recipeContent;
// If path is not provided, download the recipe from the hub
if (!filePath && name && language) {
try {
const recipeBlob = yield (0, hub_1.downloadFile)({
repo: {
name: this.getRecipeConfig.repo,
type: this.getRecipeConfig.repoType,
},
path: `${this.getRecipeConfig.subfolder}/${name}_${language}.json`,
});
if (!recipeBlob) {
throw new Error(`Could not download recipe '${name}_${language}'`);
}
recipeContent = yield recipeBlob.text();
}
catch (error) {
throw new Error(`Could not download recipe '${name}_${language}'. Ensure name and language are correct or provide a valid path. Error: ${error}`);
}
}
else {
// Read from local file
try {
recipeContent = fs.readFileSync(filePath, 'utf-8');
}
catch (error) {
throw new Error(`Failed to read the file ${filePath} —— please check if the file exists and if the path is correct. Error: ${error}`);
}
}
// Parse and validate the recipe
try {
const recipe = JSON.parse(recipeContent);
// Validate the recipe
if (!this.validateRecipe(recipe)) {
throw new Error("Recipe is invalid. Please check the recipe and try again.");
}
return recipe;
}
catch (error) {
throw new Error(`Failed to parse recipe JSON. Error: ${error}`);
}
});
}
}
exports.Hubbie = Hubbie;
Hubbie.SCHEMA_VERSION = "v1";
//# sourceMappingURL=hub.js.map