verdaccio
Version:
A lightweight private npm proxy registry
484 lines (482 loc) • 16.1 kB
JavaScript
import { API_ERROR, CHARACTER_ENCODING, ERROR_CODE, HEADERS, HEADER_TYPE, HTTP_STATUS, TOKEN_BASIC, TOKEN_BEARER } from "./constants.mjs";
import { logger } from "./logger/index.mjs";
import { ErrorCode, isObject, isObjectOrArray, parseInterval } from "./utils.mjs";
import _ from "lodash";
import { buildToken } from "@verdaccio/utils";
import createDebug from "debug";
import Stream from "stream";
import { ReadTarball } from "@verdaccio/streams";
import request from "@cypress/request";
import JSONStream from "JSONStream";
import zlib from "zlib";
//#region src/lib/up-storage.ts
var debug = createDebug("verdaccio:proxy");
var encode = function(thing) {
return encodeURIComponent(thing).replace(/^%40/, "@");
};
var contentTypeAccept = `${HEADERS.JSON};`;
/**
* Just a helper (`config[key] || default` doesn't work because of zeroes)
*/
var setConfig = (config, key, def) => {
return _.isNil(config[key]) === false ? config[key] : def;
};
/**
* Implements Storage interface
* (same for storage.js, local-storage.js, up-storage.js)
*/
var ProxyStorage = class {
config;
failed_requests;
userAgent;
ca;
logger;
server_id;
url;
maxage;
timeout;
max_fails;
fail_timeout;
agent_options;
upname;
proxy;
last_request_time;
strict_ssl;
/**
* Constructor
* @param {*} config
* @param {*} mainConfig
*/
constructor(config, mainConfig) {
this.config = config;
this.failed_requests = 0;
this.userAgent = mainConfig.user_agent ?? "hidden";
this.ca = config.ca;
this.logger = logger;
this.server_id = mainConfig.server_id;
try {
this.url = new URL(this.config.url);
} catch (err) {
const redactedUrl = String(this.config.url).replace(/\/\/[^/@]+@/, "//***@");
throw new Error(`invalid uplink url: ${redactedUrl}`, { cause: err });
}
this._setupProxy(this.url.hostname, config, mainConfig, this.url.protocol === "https:");
this.config.url = this.config.url.replace(/\/$/, "");
if (this.config.timeout && Number(this.config.timeout) >= 1e3) this.logger.warn([
"Too big timeout value: " + this.config.timeout,
"We changed time format to nginx-like one",
"(see http://nginx.org/en/docs/syntax.html)",
"so please update your config accordingly"
].join("\n"));
this.maxage = parseInterval(setConfig(this.config, "maxage", "2m"));
this.timeout = parseInterval(setConfig(this.config, "timeout", "30s"));
this.max_fails = Number(setConfig(this.config, "max_fails", 2));
this.fail_timeout = parseInterval(setConfig(this.config, "fail_timeout", "5m"));
this.strict_ssl = Boolean(setConfig(this.config, "strict_ssl", true));
this.agent_options = setConfig(this.config, "agent_options", {
keepAlive: true,
maxSockets: 40,
maxFreeSockets: 10
});
}
/**
* Fetch an asset.
* @param {*} options
* @param {*} cb
* @return {Request}
*/
request(options, cb) {
let json;
if (this._statusCheck() === false) {
const streamRead = new Stream.Readable();
process.nextTick(function() {
if (cb) cb(ErrorCode.getInternalError(API_ERROR.UPLINK_OFFLINE));
streamRead.emit("error", ErrorCode.getInternalError(API_ERROR.UPLINK_OFFLINE));
});
streamRead._read = function() {};
streamRead.on("error", function() {});
return streamRead;
}
const self = this;
const headers = this._setHeaders(options);
this._addProxyHeaders(options.req, headers);
this._overrideWithUpLinkConfigHeaders(headers);
const method = options.method || "GET";
const uri = options.uri_full || this.config.url + options.uri;
self.logger.info({
method,
uri
}, "making request: '@{method} @{uri}'");
if (isObject(options.json)) {
json = JSON.stringify(options.json);
headers["Content-Type"] = headers["Content-Type"] || HEADERS.JSON;
}
const requestCallback = cb ? function(err, res, body) {
let error;
const responseLength = err ? 0 : body.length;
processBody();
logActivity();
cb(err, res, body);
/**
* Perform a decode.
*/
function processBody() {
if (err) {
error = err.message;
return;
}
if (options.json && res.statusCode < 300) try {
body = JSON.parse(body.toString(CHARACTER_ENCODING.UTF8));
} catch (_err) {
body = {};
err = _err;
error = err.message;
}
if (!err && isObject(body)) {
if (_.isString(body.error)) error = body.error;
}
}
/**
* Perform a log.
*/
function logActivity() {
let message = "@{!status}, req: '@{request.method} @{request.url}'";
message += error ? ", error: @{!error}" : ", bytes: @{bytes.in}/@{bytes.out}";
self.logger.http({
err: err || void 0,
request: {
method,
url: uri
},
status: res != null ? res.statusCode : "ERR",
error,
bytes: {
in: json ? json.length : 0,
out: responseLength || 0
}
}, message);
}
} : void 0;
let requestOptions = {
url: uri,
method,
headers,
body: json,
proxy: this.proxy,
encoding: null,
gzip: true,
timeout: this.timeout,
strictSSL: this.strict_ssl,
agentOptions: this.agent_options
};
if (this.ca) requestOptions = Object.assign({}, requestOptions, { ca: this.ca });
const req = request(requestOptions, requestCallback);
let statusCalled = false;
req.on("response", function(res) {
if (!req._verdaccio_aborted && !statusCalled) {
statusCalled = true;
self._statusCheck(true);
}
if (_.isNil(requestCallback) === false) (function do_log() {
self.logger.http({
request: {
method,
url: uri
},
status: _.isNull(res) === false ? res.statusCode : "ERR"
}, "@{!status}, req: '@{request.method} @{request.url}' (streaming)");
})();
});
req.on("error", function(err) {
debug("_verdaccio_aborted: %o", err);
if (!req._verdaccio_aborted && !statusCalled) {
statusCalled = true;
self._statusCheck(false);
}
});
return req;
}
/**
* Set default headers.
* @param {Object} options
* @return {Object}
* @private
*/
_setHeaders(options) {
const headers = options.headers || {};
const accept = HEADERS.ACCEPT;
const acceptEncoding = HEADERS.ACCEPT_ENCODING;
const userAgent = HEADERS.USER_AGENT;
headers[accept] = headers[accept] || contentTypeAccept;
headers[acceptEncoding] = headers[acceptEncoding] || "gzip";
headers[userAgent] = this.userAgent ? `npm (${this.userAgent})` : options.req?.get("user-agent");
return this._setAuth(headers);
}
/**
* Validate configuration auth and assign Header authorization
* @param {Object} headers
* @return {Object}
* @private
*/
_setAuth(headers) {
const { auth } = this.config;
if (_.isNil(auth) || headers[HEADERS.AUTHORIZATION]) return headers;
if (_.isObject(auth) === false && _.isObject(auth.token) === false) this._throwErrorAuth("Auth invalid");
let token;
const tokenConf = auth;
if (_.isNil(tokenConf.token) === false && _.isString(tokenConf.token)) token = tokenConf.token;
else if (_.isNil(tokenConf.token_env) === false) if (_.isString(tokenConf.token_env)) token = process.env[tokenConf.token_env];
else if (_.isBoolean(tokenConf.token_env) && tokenConf.token_env) token = process.env.NPM_TOKEN;
else {
this.logger.error(ERROR_CODE.token_required);
this._throwErrorAuth(ERROR_CODE.token_required);
}
else token = process.env.NPM_TOKEN;
if (_.isNil(token)) this._throwErrorAuth(ERROR_CODE.token_required);
const type = tokenConf.type || TOKEN_BASIC;
this._setHeaderAuthorization(headers, type, token);
return headers;
}
/**
* @param {string} message
* @throws {Error}
* @private
*/
_throwErrorAuth(message) {
this.logger.error(message);
throw new Error(message);
}
/**
* Assign Header authorization with type authentication
* @param {Object} headers
* @param {string} type
* @param {string} token
* @private
*/
_setHeaderAuthorization(headers, type, token) {
const _type = type.toLowerCase();
if ([TOKEN_BEARER.toLowerCase(), TOKEN_BASIC.toLowerCase()].includes(_type) === false) this._throwErrorAuth(`Auth type '${_type}' not allowed`);
type = _.upperFirst(type);
headers[HEADERS.AUTHORIZATION] = buildToken(type, token);
}
/**
* It will add or override specified headers from config file.
*
* Eg:
*
* uplinks:
npmjs:
url: https://registry.npmjs.org/
headers:
Accept: "application/vnd.npm.install-v2+json; q=1.0"
verdaccio-staging:
url: https://mycompany.com/npm
headers:
Accept: "application/json"
authorization: "Basic YourBase64EncodedCredentials=="
* @param {Object} headers
* @private
*/
_overrideWithUpLinkConfigHeaders(headers) {
if (!this.config.headers) return headers;
for (const key in this.config.headers) headers[key] = this.config.headers[key];
}
/**
* Determine whether can fetch from the provided URL
* @param {*} url
* @return {Boolean}
*/
isUplinkValid(url) {
const urlParsed = URL.parse(url);
if (urlParsed === null) return false;
const isHTTPS = (urlDomainParsed) => urlDomainParsed.protocol === "https:" && (urlDomainParsed.port === "" || urlDomainParsed.port === "443");
const getHost = (urlDomainParsed) => isHTTPS(urlDomainParsed) ? urlDomainParsed.hostname : urlDomainParsed.host;
const getPath = (urlDomainParsed) => urlDomainParsed.pathname + urlDomainParsed.search;
const isMatchProtocol = urlParsed.protocol === this.url.protocol;
const isMatchHost = getHost(urlParsed) === getHost(this.url);
const isMatchPath = getPath(urlParsed).indexOf(getPath(this.url)) === 0;
return isMatchProtocol && isMatchHost && isMatchPath;
}
/**
* Get a remote package metadata
* @param {*} name package name
* @param {*} options request options, eg: eTag.
* @param {*} callback
*/
getRemoteMetadata(name, options, callback) {
const headers = {};
if (_.isNil(options.etag) === false) {
headers["If-None-Match"] = options.etag;
headers[HEADERS.ACCEPT] = contentTypeAccept;
}
this.request({
uri: `/${encode(name)}`,
json: true,
headers,
req: options.req
}, (err, res, body) => {
if (err) return callback(err);
if (res.statusCode === HTTP_STATUS.NOT_FOUND) return callback(ErrorCode.getNotFound(API_ERROR.NOT_PACKAGE_UPLINK));
if (!(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES)) {
const error = ErrorCode.getInternalError(`${API_ERROR.BAD_STATUS_CODE}: ${res.statusCode}`);
error.remoteStatus = res.statusCode;
return callback(error);
}
callback(null, body, res.headers.etag);
});
}
/**
* Fetch a tarball from the uplink.
* @param {String} url
* @return {Stream}
*/
fetchTarball(url) {
const stream = new ReadTarball({});
let current_length = 0;
let expected_length;
stream.abort = () => {};
const readStream = this.request({
uri_full: url,
encoding: null,
headers: { Accept: contentTypeAccept }
});
readStream.on("response", function(res) {
if (res.statusCode === HTTP_STATUS.NOT_FOUND) return stream.emit("error", ErrorCode.getNotFound(API_ERROR.NOT_FILE_UPLINK));
if (res.statusCode === HTTP_STATUS.FORBIDDEN) {
const chunks = [];
readStream.on("data", (chunk) => chunks.push(chunk));
readStream.once("end", () => {
let body;
const raw = Buffer.concat(chunks);
if (raw.length > 0) try {
body = JSON.parse(raw.toString("utf8"));
} catch {}
const message = body?.detail || body?.title || body?.message || "tarball not available: forbidden by uplink";
stream.emit("error", ErrorCode.getForbidden(message));
});
return;
}
if (!(res.statusCode >= HTTP_STATUS.OK && res.statusCode < HTTP_STATUS.MULTIPLE_CHOICES)) return stream.emit("error", ErrorCode.getInternalError(`bad uplink status code: ${res.statusCode}`));
if (res.headers[HEADER_TYPE.CONTENT_LENGTH]) {
expected_length = res.headers[HEADER_TYPE.CONTENT_LENGTH];
stream.emit(HEADER_TYPE.CONTENT_LENGTH, res.headers[HEADER_TYPE.CONTENT_LENGTH]);
}
readStream.pipe(stream);
});
readStream.on("error", function(err) {
stream.emit("error", err);
});
readStream.on("data", function(data) {
current_length += data.length;
});
readStream.on("end", function(data) {
if (data) current_length += data.length;
if (expected_length && current_length != expected_length) stream.emit("error", ErrorCode.getInternalError(API_ERROR.CONTENT_MISMATCH));
});
return stream;
}
/**
* Perform a stream search.
* @param {*} options request options
* @return {Stream}
*/
search(options) {
const transformStream = new Stream.PassThrough({ objectMode: true });
const requestStream = this.request({
uri: options.url || options.req.url,
req: options.req,
headers: { referer: options.req.get("referer") }
});
const parsePackage = (pkg) => {
if (isObjectOrArray(pkg)) transformStream.emit("data", pkg);
};
requestStream.on("response", (res) => {
if (!String(res.statusCode).match(/^2\d\d$/)) return transformStream.emit("error", ErrorCode.getInternalError(`bad status code ${res.statusCode} from uplink`));
let jsonStream;
if (res.headers[HEADER_TYPE.CONTENT_ENCODING] === HEADERS.GZIP) jsonStream = res.pipe(zlib.createUnzip());
else jsonStream = res;
jsonStream.pipe(JSONStream.parse("*")).on("data", parsePackage);
jsonStream.on("end", () => {
transformStream.emit("end");
});
});
requestStream.on("error", (err) => {
transformStream.emit("error", err);
});
transformStream.abort = () => {
requestStream.abort();
transformStream.emit("end");
};
return transformStream;
}
/**
* Add proxy headers.
* FIXME: object mutations, it should return an new object
* @param {*} req the http request
* @param {*} headers the request headers
*/
_addProxyHeaders(req, headers) {
if (req) {
if (!this.proxy) headers["x-forwarded-for"] = (req.get("x-forwarded-for") ? req.get("x-forwarded-for") + ", " : "") + req.connection.remoteAddress;
}
headers["via"] = req && req.get("via") ? req.get("via") + ", " : "";
headers["via"] += "1.1 " + this.server_id + " (Verdaccio)";
}
/**
* Check whether the remote host is available.
* @param {*} alive
* @return {Boolean}
*/
_statusCheck(alive) {
if (arguments.length === 0) return this._ifRequestFailure() === false;
if (alive) {
if (this.failed_requests >= this.max_fails) this.logger.warn({ host: this.url.host }, "host @{host} is back online");
this.failed_requests = 0;
} else {
this.failed_requests++;
if (this.failed_requests === this.max_fails) this.logger.warn({ host: this.url.host }, "host @{host} is now offline");
}
this.last_request_time = Date.now();
}
/**
* If the request failure.
* @return {boolean}
* @private
*/
_ifRequestFailure() {
return this.failed_requests >= this.max_fails && Math.abs(Date.now() - this.last_request_time) < this.fail_timeout;
}
/**
* Set up a proxy.
* @param {*} hostname
* @param {*} config
* @param {*} mainconfig
* @param {*} isHTTPS
*/
_setupProxy(hostname, config, mainconfig, isHTTPS) {
let noProxyList;
const proxy_key = isHTTPS ? "https_proxy" : "http_proxy";
if (proxy_key in config) this.proxy = config[proxy_key];
else if (proxy_key in mainconfig) this.proxy = mainconfig[proxy_key];
if ("no_proxy" in config) noProxyList = config.no_proxy;
else if ("no_proxy" in mainconfig) noProxyList = mainconfig.no_proxy;
if (hostname[0] !== ".") hostname = "." + hostname;
if (_.isString(noProxyList) && noProxyList.length) noProxyList = noProxyList.split(",");
if (_.isArray(noProxyList)) for (let i = 0; i < noProxyList.length; i++) {
let noProxyItem = noProxyList[i];
if (noProxyItem[0] !== ".") noProxyItem = "." + noProxyItem;
if (hostname.lastIndexOf(noProxyItem) === hostname.length - noProxyItem.length) {
if (this.proxy) {
debug("not using proxy for %o, excluded by %o rule", this.url.href, noProxyItem);
this.proxy = false;
}
break;
}
}
if (_.isString(this.proxy) === false) delete this.proxy;
else debug("using proxy %o for %o", this.url.href, this.proxy);
}
};
//#endregion
export { ProxyStorage as default };
//# sourceMappingURL=up-storage.mjs.map