@pnpm/tarball-fetcher
Version:
Fetcher for packages hosted as tarballs
158 lines • 7.12 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createDownloader = createDownloader;
const assert_1 = __importDefault(require("assert"));
const util_1 = __importDefault(require("util"));
const core_loggers_1 = require("@pnpm/core-loggers");
const error_1 = require("@pnpm/error");
const logger_1 = require("@pnpm/logger");
const worker_1 = require("@pnpm/worker");
const retry = __importStar(require("@zkochan/retry"));
const lodash_throttle_1 = __importDefault(require("lodash.throttle"));
const index_js_1 = require("./errorTypes/index.js");
const BIG_TARBALL_SIZE = 1024 * 1024 * 5; // 5 MB
function createDownloader(fetchFromRegistry, gotOpts) {
const retryOpts = {
factor: 10,
maxTimeout: 6e4, // 1 minute
minTimeout: 1e4, // 10 seconds
retries: 2,
...gotOpts.retry,
};
const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50; // 50 KiB/s
return async function download(url, opts) {
const authHeaderValue = opts.getAuthHeaderByURI(url);
const op = retry.operation(retryOpts);
return new Promise((resolve, reject) => {
op.attempt(async (attempt) => {
try {
resolve(await fetch(attempt));
}
catch (error) { // eslint-disable-line
if (error.response?.status === 401 ||
error.response?.status === 403 ||
error.response?.status === 404 ||
error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE') {
reject(error);
return;
}
const timeout = op.retry(error);
if (timeout === false) {
reject(op.mainError());
return;
}
core_loggers_1.requestRetryLogger.debug({
attempt,
error,
maxRetries: retryOpts.retries,
method: 'GET',
timeout,
url,
});
}
});
});
async function fetch(currentAttempt) {
let data;
try {
const res = await fetchFromRegistry(url, {
authHeaderValue,
// The fetch library can retry requests on bad HTTP responses.
// However, it is not enough to retry on bad HTTP responses only.
// Requests should also be retried when the tarball's integrity check fails.
// Hence, we tell fetch to not retry,
// and we perform the retries from this function instead.
retry: { retries: 0 },
timeout: gotOpts.timeout,
});
if (res.status !== 200) {
throw new error_1.FetchError({ url, authHeaderValue }, res);
}
const contentLength = res.headers.has('content-length') && res.headers.get('content-length');
const size = typeof contentLength === 'string'
? parseInt(contentLength, 10)
: null;
if (opts.onStart != null) {
opts.onStart(size, currentAttempt);
}
// In order to reduce the amount of logs, we only report the download progress of big tarballs
const onProgress = (size != null && size >= BIG_TARBALL_SIZE && opts.onProgress)
? (0, lodash_throttle_1.default)(opts.onProgress, 500)
: undefined;
const startTime = Date.now();
let downloaded = 0;
const chunks = [];
// This will handle the 'data', 'error', and 'end' events.
for await (const chunk of res.body) {
chunks.push(chunk);
downloaded += chunk.length;
onProgress?.(downloaded);
}
if (size !== null && size !== downloaded) {
throw new index_js_1.BadTarballError({
expectedSize: size,
receivedSize: downloaded,
tarballUrl: url,
});
}
const elapsedSec = (Date.now() - startTime) / 1000;
const avgKiBps = Math.floor((downloaded / elapsedSec) / 1024);
if (downloaded > 0 && elapsedSec > 1 && avgKiBps < fetchMinSpeedKiBps) {
const sizeKb = Math.floor(downloaded / 1024);
(0, logger_1.globalWarn)(`Tarball download average speed ${avgKiBps} KiB/s (size ${sizeKb} KiB) is below ${fetchMinSpeedKiBps} KiB/s: ${url} (GET)`);
}
data = Buffer.from(new SharedArrayBuffer(downloaded));
let offset = 0;
for (const chunk of chunks) {
chunk.copy(data, offset);
offset += chunk.length;
}
}
catch (err) {
(0, assert_1.default)(util_1.default.types.isNativeError(err));
Object.assign(err, {
attempts: currentAttempt,
resource: url,
});
throw err;
}
return (0, worker_1.addFilesFromTarball)({
buffer: data,
storeDir: opts.cafs.storeDir,
readManifest: opts.readManifest,
integrity: opts.integrity,
filesIndexFile: opts.filesIndexFile,
url,
pkg: opts.pkg,
appendManifest: opts.appendManifest,
});
}
};
}
//# sourceMappingURL=remoteTarballFetcher.js.map