embeddings-splitter
Version:
A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings API
44 lines (43 loc) • 1.91 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getAllFilesFromGithubRepo = void 0;
const cross_fetch_1 = __importDefault(require("cross-fetch"));
// get all files from agithub repo
const getAllFilesFromGithubRepo = (url, githubToken) => __awaiter(void 0, void 0, void 0, function* () {
if (!url) {
throw new Error('No url provided');
}
if (!githubToken) {
throw new Error('No github token provided');
}
const response = yield (0, cross_fetch_1.default)(url, {
headers: {
Authorization: `token ${githubToken}`,
},
});
const data = yield response.json();
const dataList = [];
for (const item of data) {
if (item.type === 'file') {
dataList.push(item);
}
else if (item.type === 'dir') {
const subdirFiles = yield (0, exports.getAllFilesFromGithubRepo)(item._links.self, githubToken);
dataList.push(...subdirFiles);
}
}
return dataList;
});
exports.getAllFilesFromGithubRepo = getAllFilesFromGithubRepo;