verdaccio
Version:
A lightweight private npm proxy registry
665 lines (664 loc) • 25.5 kB
JavaScript
import { API_ERROR, DIST_TAGS, HTTP_STATUS } from "./constants.mjs";
import { logger } from "./logger/index.mjs";
import { ErrorCode, hasTarball, isObject, normalizeDistTags } from "./utils.mjs";
import { checkPackageLocal, checkPackageRemote, cleanUpLinksRef, convertAbbreviatedManifest, generatePackageTemplate, lookupDistFile, mergeUplinkTimeIntoLocal, publishPackage } from "./storage-utils.mjs";
import LocalStorage from "./local-storage.mjs";
import { mergeVersions } from "./metadata-utils.mjs";
import ProxyStorage from "./up-storage.mjs";
import { setupUpLinks, updateVersionsHiddenUpLink } from "./uplink-util.mjs";
import _ from "lodash";
import { PLUGIN_CATEGORY, validationUtils } from "@verdaccio/core";
import createDebug from "debug";
import { hasProxyTo } from "@verdaccio/config";
import { SearchMemoryIndexer } from "@verdaccio/search-indexer";
import { asyncLoadPlugin } from "@verdaccio/loaders";
import assert from "assert";
import async from "async";
import Stream from "stream";
import LocalDatabasePluginModule from "@verdaccio/local-storage-legacy";
import { ReadTarball } from "@verdaccio/streams";
//#region src/lib/storage.ts
var debug = createDebug("verdaccio:storage");
var Storage = class {
localStorage;
config;
logger;
uplinks;
filters;
constructor(config) {
this.config = config;
this.uplinks = setupUpLinks(config);
this.logger = logger;
this.localStorage = null;
}
/**
* Initialize the storage: load the storage plugin (or the default
* local storage) and the filter plugins. Safe to call once; subsequent
* calls only reload missing filters.
* @param {Config} config verdaccio configuration
* @param {IPluginFilters} filters preloaded filter plugins; when omitted they are loaded from the configuration
*/
async init(config, filters) {
if (this.localStorage === null) {
this.filters = filters;
const storageInstance = await this.loadStorage(config, this.logger);
this.localStorage = new LocalStorage(this.config, logger, storageInstance);
await this.localStorage.getSecret(config);
debug("initialization completed");
} else debug("storage has been already initialized");
if (!this.filters) {
this.filters = await asyncLoadPlugin(this.config.filters, {
config: this.config,
logger: this.logger
}, (plugin) => {
return typeof plugin.filter_metadata !== "undefined";
}, true, this.config?.serverSettings?.pluginPrefix, PLUGIN_CATEGORY.FILTER);
debug("filters available %o", this.filters.length);
}
}
/**
* Resolve the storage backend: a configured storage plugin when available,
* otherwise the default `@verdaccio/local-storage` on the configured path.
* @return {Promise<StoragePlugin>} the storage plugin instance
*/
async loadStorage(config, logger) {
const Storage = await this.loadStorePlugin();
if (_.isNil(Storage)) {
assert(this.config.storage, "CONFIG: storage path not defined");
debug("no custom storage found, loading default storage @verdaccio/local-storage");
const localStorage = new (LocalDatabasePluginModule.default ?? LocalDatabasePluginModule)(config, logger);
logger.info({
name: "@verdaccio/local-storage",
pluginCategory: PLUGIN_CATEGORY.STORAGE
}, "plugin @{name} successfully loaded (@{pluginCategory})");
return localStorage;
}
return Storage;
}
/**
* Load the storage plugins declared under `store` in the configuration.
* Only one storage is supported: with several plugins configured the
* first loaded one wins and a warning is logged.
* @return {Promise<StoragePluginLegacy<Config> | undefined>} the selected plugin, or undefined when none is configured
*/
async loadStorePlugin() {
const plugins = await asyncLoadPlugin(this.config.store, {
config: this.config,
logger: this.logger
}, (plugin) => {
return typeof plugin.getPackageStorage !== "undefined";
}, true, this.config?.serverSettings?.pluginPrefix, PLUGIN_CATEGORY.STORAGE);
if (plugins.length > 1) this.logger.warn("more than one storage plugins has been detected, multiple storage are not supported, one will be selected automatically");
return _.head(plugins);
}
/**
* Add a package to the system (publish flow).
* Verifies the package does not exist locally nor on any uplink before
* creating it locally; uplinks being offline rejects the publish unless
* `publish.allow_offline` is enabled.
* Used storages: local (write) && uplinks
* @param {String} name package name
* @param {Object} metadata package manifest to publish
* @param {Function} callback invoked with an error when the publish is rejected
*/
async addPackage(name, metadata, callback) {
debug("add package %o", name);
try {
await checkPackageLocal(name, this.localStorage);
await checkPackageRemote(name, this._isAllowPublishOffline(), this._syncUplinksMetadata.bind(this));
await publishPackage(name, metadata, this.localStorage);
callback();
} catch (err) {
callback(err);
}
}
/**
* Whether `publish.allow_offline` is enabled in the configuration.
* @return {Boolean} true when publishing with unreachable uplinks is allowed
*/
_isAllowPublishOffline() {
return typeof this.config.publish !== "undefined" && _.isBoolean(this.config.publish.allow_offline) && this.config.publish.allow_offline;
}
/**
* Read the npm tokens matching the filter from the token storage.
* Used storages: local (read)
*/
readTokens(filter) {
return this.localStorage.readTokens(filter);
}
/**
* Save an npm token to the token storage.
* Used storages: local (write)
*/
saveToken(token) {
return this.localStorage.saveToken(token);
}
/**
* Delete an npm token from the token storage.
* Used storages: local (write)
*/
deleteToken(user, tokenKey) {
return this.localStorage.deleteToken(user, tokenKey);
}
/**
* Add a new version of a package to the system.
* Used storages: local (write)
* @param {String} name package name
* @param {String} version version id, eg. 1.0.0
* @param {Version} metadata version metadata
* @param {String} tag dist-tag pointing to the version
* @param {Function} callback
*/
addVersion(name, version, metadata, tag, callback) {
debug("add version %s package for %s", version, name);
this.localStorage.addVersion(name, version, metadata, tag, callback);
}
/**
* Tag package versions with the provided dist-tags.
* Used storages: local (write)
* @param {String} name package name
* @param {MergeTags} tagHash dist-tag to version mapping to merge
* @param {Function} callback
*/
mergeTags(name, tagHash, callback) {
debug("merge tags for %o", name);
this.localStorage.mergeTags(name, tagHash, callback);
}
/**
* Change an existing package (eg. unpublish one version).
* Used storages: local (write)
* @param {String} name package name
* @param {Manifest} metadata updated package manifest
* @param {String} revision expected revision of the stored manifest
* @param {Function} callback
*/
changePackage(name, metadata, revision, callback) {
debug("change existing package for package %o revision %o", name, revision);
this.localStorage.changePackage(name, metadata, revision, callback);
}
/**
* Remove a package from the local storage and the search indexer.
* Used storages: local (write)
* @param {String} name package name
* @param {Function} callback
*/
removePackage(name, callback) {
debug("remove package %o", name);
this.localStorage.removePackage(name, callback);
SearchMemoryIndexer.remove(name).catch((reason) => {
debug("indexer has failed on remove item %o", reason);
logger.error("indexer has failed on remove item");
});
}
/**
* Remove a tarball from the local storage. The tarball must not be
* linked by any existing version, ie. the version should be
* unpublished first.
* Used storages: local (write)
* @param {String} name package name
* @param {String} filename tarball file name
* @param {String} revision expected revision of the stored manifest
* @param {Function} callback
*/
removeTarball(name, filename, revision, callback) {
debug("remove tarball %s for %s", filename, name);
this.localStorage.removeTarball(name, filename, revision, callback);
}
/**
* Upload a tarball (publish flow). Synchronous, returns a writable
* stream the tarball body is piped into.
* Used storages: local (write)
* @param {String} name package name
* @param {String} filename tarball file name
* @return {Stream} writable upload stream
*/
addTarball(name, filename) {
debug("add a tarball for %o", name);
return this.localStorage.addTarball(name, filename);
}
/**
* Whether a tarball exists in the local storage, without reading it
* (the stream is aborted as soon as it opens).
* Used storages: local (read)
* @param {String} name package name
* @param {String} filename tarball file name
* @return {Promise<Boolean>} true when the tarball is stored locally
*/
hasLocalTarball(name, filename) {
const self = this;
return new Promise((resolve, reject) => {
let localStream = self.localStorage.getTarball(name, filename);
let isOpen = false;
localStream.on("error", (err) => {
if (isOpen || err.status !== HTTP_STATUS.NOT_FOUND) reject(err);
if (localStream) {
localStream.abort();
localStream = null;
}
resolve(false);
});
localStream.on("open", function() {
isOpen = true;
localStream.abort();
localStream = null;
resolve(true);
});
});
}
/**
* Get a tarball. Synchronous, returns a readable stream.
* The tarball is read locally first; on a local miss the distfile record
* is resolved from the package metadata (see lookupDistFile), syncing the
* uplinks when the local metadata does not know the file, and the tarball
* is then fetched from the uplink that serves the distfile url.
* Used storages: local || uplink (just one)
* @param {String} name package name
* @param {String} filename tarball file name
* @return {Stream} readable tarball stream
*/
getTarball(name, filename) {
debug("get tarball for package %o filename %o", name, filename);
const readStream = new ReadTarball({});
readStream.abort = function() {};
const self = this;
this._isTarballAllowedByFilters(name, filename).then(async (allowed) => {
if (!allowed) {
readStream.emit("error", ErrorCode.getNotFound(API_ERROR.NO_PACKAGE));
return;
}
let localStream = self.localStorage.getTarball(name, filename);
let isOpen = false;
localStream.on("error", (err) => {
if (isOpen || err.status !== HTTP_STATUS.NOT_FOUND) return readStream.emit("error", err);
const err404 = err;
localStream.abort();
localStream = null;
const lookupFromUplinks = (info) => {
self._syncUplinksMetadata(name, info, {}, (syncErr, syncInfo) => {
if (_.isNil(syncErr) === false) return readStream.emit("error", syncErr);
if (self.filters?.length && !hasTarball(syncInfo, filename)) return readStream.emit("error", err404);
const distFile = lookupDistFile(syncInfo, filename);
if (_.isNil(distFile)) {
debug("remote tarball not found");
return readStream.emit("error", err404);
}
debug("dist file found, using it %o", distFile.url);
serveFile(distFile);
});
};
self.localStorage.getPackageMetadataAsync(name).then((info) => {
const distFile = lookupDistFile(info, filename);
if (_.isNil(distFile) === false) {
debug("dist file found, using it %o", distFile.url);
serveFile(distFile);
} else {
debug("dist file not found, proceed update upstream");
lookupFromUplinks(info);
}
}, () => {
lookupFromUplinks(null);
});
});
localStream.on("content-length", function(v) {
readStream.emit("content-length", v);
});
localStream.on("open", function() {
isOpen = true;
localStream.pipe(readStream);
});
});
return readStream;
/**
* Stream the tarball from the uplink serving the distfile, caching it
* locally (and restoring the distfile record) when the uplink has the
* cache enabled.
* @param {DistFile} file distfile record resolved for the tarball
*/
function serveFile(file) {
let uplink = null;
if (file.registry && self.uplinks[file.registry]) {
uplink = self.uplinks[file.registry];
debug("tarball %o is served by the recorded uplink %o", filename, file.registry);
} else {
const candidates = [];
for (const uplinkId in self.uplinks) if (hasProxyTo(name, uplinkId, self.config.packages)) candidates.push(self.uplinks[uplinkId]);
debug("tarball %o has %o candidate uplinks", filename, candidates.length);
for (const candidate of candidates) if (candidate.isUplinkValid(file.url)) uplink = candidate;
if (uplink !== null) debug("uplink %o url matches tarball %o", uplink.upname, file.url);
else if (candidates.length === 1) {
uplink = candidates[0];
debug("using the single configured uplink %o for tarball %o hosted elsewhere", uplink.upname, file.url);
}
}
if (uplink == null) {
debug("upstream not found, creating one for %o", name);
uplink = new ProxyStorage({
url: file.url,
cache: true,
_autogenerated: true
}, self.config);
}
let savestream = null;
if (uplink.config.cache) {
debug("cache remote tarball enabled");
let distFile = file;
if (!file.registry && uplink.upname) {
debug("recording uplink %o on the persisted distfile for %o", uplink.upname, filename);
distFile = {
...file,
registry: uplink.upname
};
}
savestream = self.localStorage.addTarball(name, filename, distFile);
} else debug("cache remote tarball disabled");
let on_open = function() {
on_open = function() {};
const rstream2 = uplink.fetchTarball(file.url);
rstream2.on("error", function(err) {
if (savestream) savestream.abort();
savestream = null;
readStream.emit("error", err);
});
rstream2.on("end", function() {
if (savestream) savestream.done();
});
rstream2.on("content-length", function(v) {
readStream.emit("content-length", v);
if (savestream) savestream.emit("content-length", v);
});
rstream2.pipe(readStream);
if (savestream) rstream2.pipe(savestream);
};
if (savestream) {
savestream.on("open", function() {
on_open();
});
savestream.on("error", function(err) {
self.logger.warn({
err,
fileName: file
}, "error saving file @{fileName}: @{err.message}\n@{err.stack}");
if (savestream) savestream.abort();
savestream = null;
on_open();
});
} else on_open();
}
}
/**
* Retrieve the package metadata: the local manifest merged with the
* manifest of every uplink with proxy access to the package.
* Used storages: local && uplink (proxy_access)
* @param {Object} options
* @property {String} options.name package name
* @property {Object} options.req Express `req` object
* @property {Boolean} options.keepUpLinkData keep the uplink info (last update, etc.) in the package metadata
* @property {Boolean} options.abbreviated serve the abbreviated manifest (npm install)
* @property {Function} options.callback invoked with the merged manifest and any uplink errors
*/
getPackage(options) {
debug("get package for %o", options.name);
this.localStorage.getPackageMetadata(options.name, (err, data) => {
if (err && (!err.status || err.status >= HTTP_STATUS.INTERNAL_ERROR)) return options.callback(err);
this._syncUplinksMetadata(options.name, data, {
req: options.req,
uplinksLook: options.uplinksLook
}, function getPackageSynUpLinksCallback(err, result, uplinkErrors) {
if (err) return options.callback(err);
normalizeDistTags(cleanUpLinksRef(options.keepUpLinkData, result));
result._attachments = {};
if (options.abbreviated === true) {
debug("get abbreviated manifest");
options.callback(null, convertAbbreviatedManifest(result), uplinkErrors);
} else {
debug("get full package manifest");
options.callback(null, result, uplinkErrors);
}
});
});
}
/**
* Retrieve remote and local packages more recent than the start key.
* All uplinks are streamed first, then the local packages; local
* packages can override registry ones just because they appear in the
* JSON last — a trade-off made to avoid memory issues.
* Used storages: local && uplink (proxy_access)
* @param {String} startkey timestamp to search from
* @param {Object} options request options; `local=1` in the query skips the uplinks
* @return {Stream} object stream of search results
*/
search(startkey, options) {
const self = this;
const searchStream = new Stream.PassThrough({ objectMode: true });
async.eachSeries(Object.keys(this.uplinks), function(up_name, cb) {
if (options.req?.query?.local !== void 0) return cb();
logger.info(`search for uplink ${up_name}`);
const uplinkStream = self.uplinks[up_name].search(options);
uplinkStream.pipe(searchStream, { end: false });
uplinkStream.on("error", function(err) {
self.logger.error({ err }, "uplink error: @{err.message}");
cb();
cb = function() {};
});
uplinkStream.on("end", function() {
cb();
cb = function() {};
});
searchStream.abort = function() {
if (uplinkStream.abort) uplinkStream.abort();
cb();
cb = function() {};
};
}, function() {
const localSearchStream = self.localStorage.search(startkey, options);
searchStream.abort = function() {
localSearchStream.abort();
};
localSearchStream.pipe(searchStream, { end: true });
localSearchStream.on("error", function(err) {
self.logger.error({ err }, "search error: @{err.message}");
searchStream.end();
});
});
return searchStream;
}
/**
* Retrieve only private local packages, as the latest version of each.
* Used storages: local (read)
* @param {Function} callback invoked with the list of latest versions
*/
getLocalDatabase(callback) {
this.localStorage.storagePlugin.get((err, locals) => {
if (err) return callback(err);
this._collectLocalPackages(locals).then((packages) => callback(null, packages), (err) => callback(err));
});
}
/**
* Read each local package name, apply filters, and collect the latest version.
*/
async _collectLocalPackages(locals) {
const packages = [];
for (const name of locals) try {
const pkgMetadata = await this.localStorage.getPackageMetadataAsync(name);
const { filteredPackage } = await this._applyFilters(pkgMetadata);
const latest = filteredPackage[DIST_TAGS]?.latest;
if (latest && filteredPackage.versions[latest]) {
const version = filteredPackage.versions[latest];
version.time = filteredPackage.time[latest];
version.users = filteredPackage.users;
packages.push(version);
} else this.logger.warn({ package: name }, "package @{package} does not have a \"latest\" tag?");
} catch (err) {
this.logger.error({
err,
package: name
}, "error reading package @{package}");
}
return packages;
}
/**
* Fetch the package metadata from every uplink with proxy access and
* synchronize it with the local data; filter plugins are applied to the
* merged result.
* @param {String} name package name
* @param {Manifest} packageInfo local manifest; MUST be provided when the package exists locally
* @param {ISyncUplinks} options uplink request options
* @param {Function} callback invoked with (err, mergedManifest, uplinkErrors)
*/
_syncUplinksMetadata(name, packageInfo, options, callback) {
let found = true;
const self = this;
const upLinks = [];
const hasToLookIntoUplinks = _.isNil(options.uplinksLook) || options.uplinksLook;
debug("sync uplinks for %o", name);
debug("is sync uplink enabled %o", hasToLookIntoUplinks);
if (!packageInfo) {
debug("local package %s not found", name);
found = false;
packageInfo = generatePackageTemplate(name);
}
for (const uplink in this.uplinks) if (hasProxyTo(name, uplink, this.config.packages) && hasToLookIntoUplinks) upLinks.push(this.uplinks[uplink]);
debug("uplinks found for %o: %o", name, upLinks.length);
async.map(upLinks, (upLink, cb) => {
const _options = Object.assign({}, options);
const upLinkMeta = packageInfo._uplinks[upLink.upname];
if (isObject(upLinkMeta)) {
const fetched = upLinkMeta.fetched;
if (fetched && Date.now() - fetched < upLink.maxage) {
debug("returning cached manifest for %o", upLink.upname);
return cb();
}
_options.etag = upLinkMeta.etag;
}
upLink.getRemoteMetadata(name, _options, (err, upLinkResponse, eTag) => {
if (err && err.remoteStatus === 304) {
debug("uplink %o responded 304 for %o, cache is up to date", upLink.upname, name);
upLinkMeta.fetched = Date.now();
}
if (err || !upLinkResponse) {
debug("error captured on uplink %o for %o: %o", upLink.upname, name, err?.message);
return cb(null, [err || ErrorCode.getInternalError("no data")]);
}
try {
upLinkResponse = validationUtils.normalizeMetadata(upLinkResponse, name);
} catch (err) {
self.logger.error({
sub: "out",
err
}, "package.json validating error @{!err.message}\n@{err.stack}");
return cb(null, [err]);
}
packageInfo._uplinks[upLink.upname] = {
etag: eTag,
fetched: Date.now()
};
packageInfo = mergeUplinkTimeIntoLocal(packageInfo, upLinkResponse);
updateVersionsHiddenUpLink(upLinkResponse.versions, upLink);
try {
mergeVersions(packageInfo, upLinkResponse);
} catch (err) {
self.logger.error({
sub: "out",
err
}, "package.json parsing error @{!err.message}\n@{err.stack}");
return cb(null, [err]);
}
debug("syncing on uplink %o", upLink.upname);
found = true;
cb();
});
}, async (err, upLinksErrors) => {
assert(!err && Array.isArray(upLinksErrors));
if (!found) {
let uplinkTimeoutError;
for (let i = 0; i < upLinksErrors.length; i++) if (upLinksErrors[i]) {
for (let j = 0; j < upLinksErrors[i].length; j++) if (upLinksErrors[i][j]) {
const code = upLinksErrors[i][j].code;
if (code === "ETIMEDOUT" || code === "ESOCKETTIMEDOUT" || code === "ECONNRESET") {
uplinkTimeoutError = true;
break;
}
}
}
if (uplinkTimeoutError) {
debug("uplinks sync failed with timeout error");
return callback(ErrorCode.getServiceUnavailable(), null, upLinksErrors);
}
debug("uplinks sync failed with no package found");
return callback(ErrorCode.getNotFound(API_ERROR.NO_PACKAGE), null, upLinksErrors);
}
if (upLinks.length === 0) {
debug("no uplinks found for %o, upstream update aborted", name);
const { filteredPackage, filterErrors } = await self._applyFilters(packageInfo);
return callback(null, filteredPackage, filterErrors);
}
try {
const packageJsonLocal = await self.localStorage.updateVersionsAsync(name, packageInfo);
const { filteredPackage, filterErrors } = await self._applyFilters(packageJsonLocal);
callback(null, filteredPackage, _.concat(upLinksErrors, filterErrors));
} catch (err) {
callback(err);
}
});
}
/**
* Apply all configured filter plugins to a package manifest sequentially.
* Each filter's output is passed as input to the next filter.
* Returns the filtered manifest and any errors that occurred.
*/
async _applyFilters(packageInfo) {
const filterErrors = [];
let filteredPackage = packageInfo;
for (const filter of this.filters ?? []) try {
filteredPackage = await filter.filter_metadata(filteredPackage);
} catch (err) {
debug("filter plugin has failed on %o: %o", packageInfo.name, err?.message);
filterErrors.push(err);
}
return {
filteredPackage,
filterErrors
};
}
/**
* Check if a tarball should be served based on filter plugins, using only
* local metadata. Returns true (defer) when the version isn't cached locally
* — the uplink-sync path in getTarball re-checks filters after sync, which
* avoids a redundant uplink round-trip and double filter application.
*/
async _isTarballAllowedByFilters(name, filename) {
if (!this.filters?.length) return true;
try {
const pkgMetadata = await this.localStorage.getPackageMetadataAsync(name);
if (!hasTarball(pkgMetadata, filename)) return true;
const { filteredPackage } = await this._applyFilters(pkgMetadata);
const allowed = hasTarball(filteredPackage, filename);
if (allowed === false) debug("tarball %o of %o is blocked by a filter plugin", filename, name);
return allowed;
} catch (err) {
if (err?.status === HTTP_STATUS.NOT_FOUND) return true;
this.logger.error({
package: name,
fileName: filename,
err
}, "error checking filters for tarball @{fileName} of package @{package}: @{err.message}");
return true;
}
}
/**
* Tag each version with the uplink it was fetched from, under a hidden
* (symbol) key. The local storage uses the tag to record the registry
* on the distfile records it creates.
* @param {Versions} versions versions fetched from the uplink
* @param {ProxyStorage} upLink uplink the versions were fetched from
*/
_updateVersionsHiddenUpLink(versions, upLink) {
for (const i in versions) if (Object.prototype.hasOwnProperty.call(versions, i)) {
const version = versions[i];
version[Symbol.for("__verdaccio_uplink")] = upLink.upname;
}
}
};
//#endregion
export { Storage as default };
//# sourceMappingURL=storage.mjs.map