embeddings-splitter/lib/github.js

A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings API

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const cross_fetch_1 = __importDefault(require("cross-fetch"));
const githubToken = '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';
// get all files from agithub repo
const getAllFilesFromGithubRepo = (path) => __awaiter(void 0, void 0, void 0, function* () {
    const response = yield (0, cross_fetch_1.default)(path, {
        headers: {
            Authorization: `token ${githubToken}`,
        },
    });
    const data = yield response.json();
    const dataList = [];
    data.forEach((data) => __awaiter(void 0, void 0, void 0, function* () {
        // if dir then get all files from that dir
        if (data.type === 'dir') {
            return yield getAllFilesFromGithubRepo(data._links.self); // recursive call
        }
        console.log(data);
        dataList.push(data);
    }));
    return Promise.all(dataList);
});
const main = (repo) => __awaiter(void 0, void 0, void 0, function* () {
    // get all files from github repo
    const files = yield getAllFilesFromGithubRepo(`https://api.github.com/repos/${repo}/contents/`);
    // keep only python files
    console.log(files);
});
main('different-ai/embedbase');
// split the files in token chunks of 1000
//