repomix/lib/core/git/gitHubArchive.js

A tool to pack repository contents to single file for AI consumption

import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import * as zlib from 'node:zlib';
import { extract as tarExtract } from 'tar';
import { RepomixError } from '../../shared/errorHandle.js';
import { logger } from '../../shared/logger.js';
import { createArchiveEntryFilter } from './archiveEntryFilter.js';
import { buildGitHubArchiveUrl, buildGitHubMasterArchiveUrl, buildGitHubTagArchiveUrl, checkGitHubResponse, } from './gitHubArchiveApi.js';
const defaultDeps = {
    fetch: globalThis.fetch,
    pipeline,
    Transform,
    tarExtract,
    createGunzip: zlib.createGunzip,
    createArchiveEntryFilter,
};
export const downloadGitHubArchive = async (repoInfo, targetDirectory, options = {}, onProgress, deps = defaultDeps) => {
    const { timeout = 30000, retries = 3 } = options;
    let lastError = null;
    const archiveUrls = [
        buildGitHubArchiveUrl(repoInfo),
        buildGitHubMasterArchiveUrl(repoInfo),
        buildGitHubTagArchiveUrl(repoInfo),
    ].filter(Boolean);
    for (const archiveUrl of archiveUrls) {
        for (let attempt = 1; attempt <= retries; attempt++) {
            try {
                logger.trace(`Downloading GitHub archive from: ${archiveUrl} (attempt ${attempt}/${retries})`);
                await downloadAndExtractArchive(archiveUrl, targetDirectory, timeout, onProgress, deps);
                logger.trace('Successfully downloaded and extracted GitHub archive');
                return;
            }
            catch (error) {
                lastError = error;
                logger.trace(`Archive download attempt ${attempt} failed:`, lastError.message);
                const isNotFoundError = lastError instanceof RepomixError &&
                    (lastError.message.includes('not found') || lastError.message.includes('404'));
                if (isNotFoundError && archiveUrls.length > 1) {
                    break;
                }
                if (attempt < retries) {
                    const delay = Math.min(1000 * 2 ** (attempt - 1), 5000);
                    logger.trace(`Retrying in ${delay}ms...`);
                    await new Promise((resolve) => setTimeout(resolve, delay));
                }
            }
        }
    }
    throw new RepomixError(`Failed to download GitHub archive after ${retries} attempts. ${lastError?.message || 'Unknown error'}`);
};
const downloadAndExtractArchive = async (archiveUrl, targetDirectory, timeout, onProgress, deps = defaultDeps) => {
    const controller = new AbortController();
    const timeoutId = setTimeout(controller.abort.bind(controller), timeout);
    try {
        const response = await deps.fetch(archiveUrl, {
            signal: controller.signal,
        });
        checkGitHubResponse(response);
        if (!response.body) {
            throw new RepomixError('No response body received');
        }
        const totalSize = response.headers.get('content-length');
        const total = totalSize ? Number.parseInt(totalSize, 10) : null;
        let downloaded = 0;
        let lastProgressUpdate = 0;
        const nodeStream = Readable.fromWeb(response.body);
        const progressStream = new deps.Transform({
            transform(chunk, _encoding, callback) {
                downloaded += chunk.length;
                const now = Date.now();
                if (onProgress && now - lastProgressUpdate > 100) {
                    lastProgressUpdate = now;
                    onProgress({
                        downloaded,
                        total,
                        percentage: total ? Math.round((downloaded / total) * 100) : null,
                    });
                }
                callback(null, chunk);
            },
            flush(callback) {
                if (onProgress) {
                    onProgress({
                        downloaded,
                        total,
                        percentage: total ? 100 : null,
                    });
                }
                callback();
            },
        });
        const entryFilter = deps.createArchiveEntryFilter();
        const extractStream = deps.tarExtract({
            cwd: targetDirectory,
            strip: 1,
            filter: (entryPath) => entryFilter(entryPath),
        });
        const gunzipStream = deps.createGunzip();
        try {
            await deps.pipeline(nodeStream, progressStream, gunzipStream, extractStream);
        }
        finally {
            nodeStream.destroy();
            progressStream.destroy();
            gunzipStream.destroy();
        }
    }
    finally {
        clearTimeout(timeoutId);
    }
};
export const isArchiveDownloadSupported = (_repoInfo) => {
    return true;
};