@controlplane/cli/dist/src/image/remote/folder/sync.js

Control Plane Corporation CLI

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RemoteFolderSync = void 0;
const fs = require("fs");
const https = require("https");
const path = require("path");
const axios_1 = require("axios");
const format_1 = require("../../../util/format");
const time_1 = require("../../../util/time");
const config_1 = require("../config");
const errors_1 = require("../errors");
const archive_1 = require("./archive");
// ANCHOR - Constants
const UPLOAD_AGENT = new https.Agent({ keepAlive: true, maxSockets: config_1.UPLOAD_CONCURRENCY });
// ANCHOR - RemoteFolderSync
/**
 * Makes the service's staged context match the local folder: PUTs the plan's
 * changed files with bounded concurrency, or packs and uploads one seed archive.
 * Owns the archive's lifecycle; the orchestrator never sees tar files.
 */
class RemoteFolderSync {
    constructor(progress, deps = {}) {
        var _a, _b;
        this.progress = progress;
        this.put = (_a = deps.put) !== null && _a !== void 0 ? _a : defaultPut;
        this.sleep = (_b = deps.sleep) !== null && _b !== void 0 ? _b : time_1.sleep;
        this.tmpFile = deps.tmpFile;
    }
    // Public Methods //
    /**
     * Uploads whatever the plan says the service is missing: the changed files for a
     * delta plan, or the whole folder as one seed archive.
     *
     * @param {FolderIndex} index - The scanned folder.
     * @param {ContextPlan} plan - The service's upload plan.
     * @returns {Promise<void>} Resolves once the staged context matches the folder.
     */
    async push(index, plan) {
        if (plan.mode === 'seed') {
            await this.pushSeed(index, plan);
            return;
        }
        await this.pushDelta(index, plan);
    }
    // Private Methods //
    /**
     * PUTs each changed file to its presigned URL with bounded concurrency, streaming
     * byte progress as the files transfer. A 403 means the upload window expired, so
     * the queue short-circuits instead of burning a doomed request per remaining file.
     *
     * @param {FolderIndex} index - The scanned folder.
     * @param {DeltaPlan} plan - The delta plan with presigned URLs.
     * @returns {Promise<void>}
     */
    async pushDelta(index, plan) {
        this.assertPlanMatchesIndex(index, plan);
        if (plan.uploads.length === 0) {
            return;
        }
        for (const upload of plan.uploads) {
            assertHttpsUploadUrl(upload.url);
        }
        const state = {
            expired: false,
            completedFiles: 0,
            completedBytes: 0,
            next: 0,
            totalBytes: plan.uploads.reduce((sum, upload) => { var _a, _b; return sum + ((_b = (_a = index.manifest[upload.path]) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : 0); }, 0),
            inflightBytes: new Map(),
            uploads: plan.uploads,
            failures: [],
        };
        const workers = Math.min(config_1.UPLOAD_CONCURRENCY, plan.uploads.length);
        await Promise.all(Array.from({ length: workers }, () => this.runDeltaWorker(index, state)));
        if (state.failures.length === 0) {
            return;
        }
        // An expired window stops the queue, so the files never claimed are part of the failure.
        const abandoned = state.uploads.length - state.completedFiles;
        const shown = state.failures
            .slice(0, 5)
            .map((failure) => `${failure.path} (${failure.reason})`)
            .join(', ');
        throw new errors_1.RemoteBuildError('request-failed', `failed to upload ${state.failures.length} file(s): ${shown}${abandoned > 0 ? `; ${abandoned} more were not attempted` : ''}`, state.expired ? 'The upload window expired. Re-run the build.' : 'Please try again.');
    }
    /**
     * One upload worker: claims the next file in the queue, streams it to its presigned
     * URL, and advances the shared state until the queue drains or the window expires.
     *
     * @param {FolderIndex} index - The scanned folder.
     * @param {DeltaUploadState} state - The upload's shared state.
     * @returns {Promise<void>}
     */
    async runDeltaWorker(index, state) {
        var _a, _b;
        while (!state.expired) {
            const position = state.next;
            state.next += 1;
            if (position >= state.uploads.length) {
                return;
            }
            const upload = state.uploads[position];
            const size = (_b = (_a = index.manifest[upload.path]) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : 0;
            try {
                await this.pushWithRetry(upload.url, path.join(index.root, ...upload.path.split('/')), 'application/octet-stream', (sent) => {
                    // A retried attempt restarts its count, so the entry is replaced, never summed.
                    state.inflightBytes.set(position, Math.min(sent, size));
                    this.emitDeltaProgress(state);
                }, () => state.expired);
                state.completedBytes += size;
            }
            catch (e) {
                state.expired = state.expired || (0, errors_1.httpStatus)(e) === 403;
                state.failures.push({ path: upload.path, reason: (0, errors_1.describeRequestError)(e) });
            }
            state.inflightBytes.delete(position);
            state.completedFiles += 1;
            this.emitDeltaProgress(state);
        }
    }
    /**
     * Emits one live snapshot: the bytes of finished files plus whatever the in-flight
     * streams have sent so far.
     *
     * @param {DeltaUploadState} state - The upload's shared state.
     * @returns {void}
     */
    emitDeltaProgress(state) {
        let inflight = 0;
        for (const sent of state.inflightBytes.values()) {
            inflight += sent;
        }
        this.progress.onUploadProgress({
            isArchive: false,
            completedFiles: state.completedFiles,
            totalFiles: state.uploads.length,
            sentBytes: state.completedBytes + inflight,
            totalBytes: state.totalBytes,
        });
    }
    /**
     * Packs the folder into a seed archive, uploads it, and always removes the
     * archive afterwards.
     *
     * @param {FolderIndex} index - The scanned folder.
     * @param {SeedPlan} plan - The seed plan with the archive's presigned URL.
     * @returns {Promise<void>}
     */
    async pushSeed(index, plan) {
        assertHttpsUploadUrl(plan.seedUrl);
        const archive = await archive_1.RemoteFolderArchive.create(index, this.tmpFile);
        try {
            await this.pushWithRetry(plan.seedUrl, archive.filePath, 'application/gzip', (sent) => {
                this.progress.onUploadProgress({
                    isArchive: true,
                    completedFiles: 0,
                    totalFiles: 1,
                    sentBytes: Math.min(sent, archive.bytes),
                    totalBytes: archive.bytes,
                });
            });
            this.progress.onUploadProgress({
                isArchive: true,
                completedFiles: 1,
                totalFiles: 1,
                sentBytes: archive.bytes,
                totalBytes: archive.bytes,
            });
        }
        catch (e) {
            throw new errors_1.RemoteBuildError('request-failed', `failed to upload the folder archive (${(0, errors_1.describeRequestError)(e)})`, seedFailureHint(e, archive.bytes));
        }
        finally {
            archive.dispose();
        }
    }
    /**
     * Streams a file to a presigned PUT URL, re-opening the stream per attempt and
     * backing off between transient failures. The stream is destroyed on failure so a
     * failed attempt never leaks its file descriptor. A 403 (expired presigned URL) or
     * an already-expired window stops the retries — neither can succeed.
     *
     * @param {string} url - The presigned PUT URL.
     * @param {string} filePath - The absolute file path.
     * @param {string} contentType - The Content-Type header value.
     * @param {(sent: number) => void} onBytes - (Optional) Receives bytes sent so far.
     * @param {() => boolean} isExpired - (Optional) Reports whether the upload window has expired.
     * @returns {Promise<void>}
     */
    async pushWithRetry(url, filePath, contentType, onBytes, isExpired) {
        const size = fs.statSync(filePath).size;
        for (let attempt = 1;; attempt++) {
            const body = fs.createReadStream(filePath);
            try {
                await this.put(url, body, { 'Content-Type': contentType, 'Content-Length': String(size) }, onBytes);
                return;
            }
            catch (e) {
                body.destroy();
                if ((0, errors_1.httpStatus)(e) === 403 || attempt >= config_1.UPLOAD_ATTEMPTS || (isExpired === null || isExpired === void 0 ? void 0 : isExpired())) {
                    throw e;
                }
                await this.sleep(attempt * 1000);
            }
        }
    }
    /**
     * Rejects a plan that references paths outside the scanned folder — the staged
     * state and the local index have diverged.
     *
     * @param {FolderIndex} index - The scanned folder.
     * @param {DeltaPlan} plan - The delta plan to validate.
     * @returns {void}
     */
    assertPlanMatchesIndex(index, plan) {
        // Own-property check: inherited keys like __proto__ must not pass as scanned paths.
        const unknown = plan.uploads.filter((upload) => !Object.prototype.hasOwnProperty.call(index.manifest, upload.path));
        if (unknown.length > 0) {
            const shown = unknown
                .slice(0, 3)
                .map((upload) => upload.path)
                .join(', ');
            throw new errors_1.RemoteBuildError('request-failed', `the upload plan referenced ${unknown.length} file(s) outside the scanned folder (${shown})`, 'Re-run the build.');
        }
    }
}
exports.RemoteFolderSync = RemoteFolderSync;
// SECTION - Functions
// Validators
/**
 * Rejects an upload URL that is not https. The service chooses the scheme, so an
 * http URL would stream the whole build context over the network in cleartext.
 *
 * @param {string} url - The presigned upload URL as the service sent it.
 * @returns {void}
 */
function assertHttpsUploadUrl(url) {
    let parsed;
    try {
        parsed = new URL(url);
    }
    catch (_a) {
        throw new errors_1.RemoteBuildError('request-failed', 'the upload plan contained an invalid upload URL', 'Re-run the build.');
    }
    if (parsed.protocol !== 'https:') {
        throw new errors_1.RemoteBuildError('request-failed', `the upload plan asked for an insecure ${parsed.protocol}// upload`, 'Re-run the build; the build context is only uploaded over https.');
    }
}
// Failure Hints
/**
 * Maps a seed upload failure to its next step: an expired window and a transfer that
 * outran the timeout each need a different answer than a plain retry.
 *
 * @param {unknown} e - The thrown upload error.
 * @param {number} bytes - The archive size, named in the timeout hint.
 * @returns {string} The hint.
 */
function seedFailureHint(e, bytes) {
    if ((0, errors_1.httpStatus)(e) === 403) {
        return 'The upload window expired. Re-run the build.';
    }
    if (isTimeout(e)) {
        return `The ${(0, format_1.humanSize)(bytes)} archive did not finish uploading in time. Exclude large paths in .dockerignore, then re-run the build.`;
    }
    return 'Please try again.';
}
/**
 * Reports whether a request error is a client-side timeout rather than a service reply.
 *
 * @param {unknown} e - The thrown request error.
 * @returns {boolean} True when the request timed out.
 */
function isTimeout(e) {
    var _a;
    const error = (_a = e) !== null && _a !== void 0 ? _a : {};
    return error.code === 'ECONNABORTED' || error.code === 'ETIMEDOUT';
}
// Uploads
/**
 * Streams a file to a presigned PUT URL on the shared keep-alive pool. Opts out of
 * the global retry policy: a replay would send an already-drained stream, so retries
 * re-open the stream in pushWithRetry instead.
 *
 * @param {string} url - The presigned PUT URL.
 * @param {fs.ReadStream} body - The file stream.
 * @param {Record<string, string>} headers - The request headers.
 * @param {(sent: number) => void} onBytes - (Optional) Receives bytes sent so far.
 * @returns {Promise<void>}
 */
async function defaultPut(url, body, headers, onBytes) {
    await axios_1.default.put(url, body, {
        headers,
        httpsAgent: UPLOAD_AGENT,
        maxBodyLength: Infinity,
        maxContentLength: Infinity,
        maxRedirects: 0, // presigned PUTs never redirect
        timeout: config_1.UPLOAD_TIMEOUT_MS,
        'axios-retry': { retries: 0 },
        onUploadProgress: (event) => {
            var _a;
            if (onBytes) {
                onBytes((_a = event.loaded) !== null && _a !== void 0 ? _a : 0);
            }
        },
    });
}
// !SECTION
//# sourceMappingURL=sync.js.map