UNPKG

verdaccio

Version:

A lightweight private npm proxy registry

676 lines (675 loc) 27 kB
Object.defineProperties(exports, { __esModule: { value: true }, [Symbol.toStringTag]: { value: "Module" } }); const require_runtime = require("../_virtual/_rolldown/runtime.js"); const require_lib_constants = require("./constants.js"); const require_lib_logger_index = require("./logger/index.js"); const require_lib_utils = require("./utils.js"); const require_lib_storage_utils = require("./storage-utils.js"); const require_lib_local_storage = require("./local-storage.js"); const require_lib_metadata_utils = require("./metadata-utils.js"); const require_lib_up_storage = require("./up-storage.js"); const require_lib_uplink_util = require("./uplink-util.js"); let lodash = require("lodash"); lodash = require_runtime.__toESM(lodash); let _verdaccio_core = require("@verdaccio/core"); let debug = require("debug"); debug = require_runtime.__toESM(debug); let _verdaccio_config = require("@verdaccio/config"); let _verdaccio_search_indexer = require("@verdaccio/search-indexer"); let _verdaccio_loaders = require("@verdaccio/loaders"); let assert = require("assert"); assert = require_runtime.__toESM(assert); let async = require("async"); async = require_runtime.__toESM(async); let stream = require("stream"); stream = require_runtime.__toESM(stream); let _verdaccio_local_storage_legacy = require("@verdaccio/local-storage-legacy"); _verdaccio_local_storage_legacy = require_runtime.__toESM(_verdaccio_local_storage_legacy); let _verdaccio_streams = require("@verdaccio/streams"); //#region src/lib/storage.ts var debug$1 = (0, debug.default)("verdaccio:storage"); var Storage = class { localStorage; config; logger; uplinks; filters; constructor(config) { this.config = config; this.uplinks = require_lib_uplink_util.setupUpLinks(config); this.logger = require_lib_logger_index.logger; this.localStorage = null; } /** * Initialize the storage: load the storage plugin (or the default * local storage) and the filter plugins. Safe to call once; subsequent * calls only reload missing filters. * @param {Config} config verdaccio configuration * @param {IPluginFilters} filters preloaded filter plugins; when omitted they are loaded from the configuration */ async init(config, filters) { if (this.localStorage === null) { this.filters = filters; const storageInstance = await this.loadStorage(config, this.logger); this.localStorage = new require_lib_local_storage.default(this.config, require_lib_logger_index.logger, storageInstance); await this.localStorage.getSecret(config); debug$1("initialization completed"); } else debug$1("storage has been already initialized"); if (!this.filters) { this.filters = await (0, _verdaccio_loaders.asyncLoadPlugin)(this.config.filters, { config: this.config, logger: this.logger }, (plugin) => { return typeof plugin.filter_metadata !== "undefined"; }, true, this.config?.serverSettings?.pluginPrefix, _verdaccio_core.PLUGIN_CATEGORY.FILTER); debug$1("filters available %o", this.filters.length); } } /** * Resolve the storage backend: a configured storage plugin when available, * otherwise the default `@verdaccio/local-storage` on the configured path. * @return {Promise<StoragePlugin>} the storage plugin instance */ async loadStorage(config, logger) { const Storage = await this.loadStorePlugin(); if (lodash.default.isNil(Storage)) { (0, assert.default)(this.config.storage, "CONFIG: storage path not defined"); debug$1("no custom storage found, loading default storage @verdaccio/local-storage"); const localStorage = new (_verdaccio_local_storage_legacy.default.default ?? _verdaccio_local_storage_legacy.default)(config, logger); logger.info({ name: "@verdaccio/local-storage", pluginCategory: _verdaccio_core.PLUGIN_CATEGORY.STORAGE }, "plugin @{name} successfully loaded (@{pluginCategory})"); return localStorage; } return Storage; } /** * Load the storage plugins declared under `store` in the configuration. * Only one storage is supported: with several plugins configured the * first loaded one wins and a warning is logged. * @return {Promise<StoragePluginLegacy<Config> | undefined>} the selected plugin, or undefined when none is configured */ async loadStorePlugin() { const plugins = await (0, _verdaccio_loaders.asyncLoadPlugin)(this.config.store, { config: this.config, logger: this.logger }, (plugin) => { return typeof plugin.getPackageStorage !== "undefined"; }, true, this.config?.serverSettings?.pluginPrefix, _verdaccio_core.PLUGIN_CATEGORY.STORAGE); if (plugins.length > 1) this.logger.warn("more than one storage plugins has been detected, multiple storage are not supported, one will be selected automatically"); return lodash.default.head(plugins); } /** * Add a package to the system (publish flow). * Verifies the package does not exist locally nor on any uplink before * creating it locally; uplinks being offline rejects the publish unless * `publish.allow_offline` is enabled. * Used storages: local (write) && uplinks * @param {String} name package name * @param {Object} metadata package manifest to publish * @param {Function} callback invoked with an error when the publish is rejected */ async addPackage(name, metadata, callback) { debug$1("add package %o", name); try { await require_lib_storage_utils.checkPackageLocal(name, this.localStorage); await require_lib_storage_utils.checkPackageRemote(name, this._isAllowPublishOffline(), this._syncUplinksMetadata.bind(this)); await require_lib_storage_utils.publishPackage(name, metadata, this.localStorage); callback(); } catch (err) { callback(err); } } /** * Whether `publish.allow_offline` is enabled in the configuration. * @return {Boolean} true when publishing with unreachable uplinks is allowed */ _isAllowPublishOffline() { return typeof this.config.publish !== "undefined" && lodash.default.isBoolean(this.config.publish.allow_offline) && this.config.publish.allow_offline; } /** * Read the npm tokens matching the filter from the token storage. * Used storages: local (read) */ readTokens(filter) { return this.localStorage.readTokens(filter); } /** * Save an npm token to the token storage. * Used storages: local (write) */ saveToken(token) { return this.localStorage.saveToken(token); } /** * Delete an npm token from the token storage. * Used storages: local (write) */ deleteToken(user, tokenKey) { return this.localStorage.deleteToken(user, tokenKey); } /** * Add a new version of a package to the system. * Used storages: local (write) * @param {String} name package name * @param {String} version version id, eg. 1.0.0 * @param {Version} metadata version metadata * @param {String} tag dist-tag pointing to the version * @param {Function} callback */ addVersion(name, version, metadata, tag, callback) { debug$1("add version %s package for %s", version, name); this.localStorage.addVersion(name, version, metadata, tag, callback); } /** * Tag package versions with the provided dist-tags. * Used storages: local (write) * @param {String} name package name * @param {MergeTags} tagHash dist-tag to version mapping to merge * @param {Function} callback */ mergeTags(name, tagHash, callback) { debug$1("merge tags for %o", name); this.localStorage.mergeTags(name, tagHash, callback); } /** * Change an existing package (eg. unpublish one version). * Used storages: local (write) * @param {String} name package name * @param {Manifest} metadata updated package manifest * @param {String} revision expected revision of the stored manifest * @param {Function} callback */ changePackage(name, metadata, revision, callback) { debug$1("change existing package for package %o revision %o", name, revision); this.localStorage.changePackage(name, metadata, revision, callback); } /** * Remove a package from the local storage and the search indexer. * Used storages: local (write) * @param {String} name package name * @param {Function} callback */ removePackage(name, callback) { debug$1("remove package %o", name); this.localStorage.removePackage(name, callback); _verdaccio_search_indexer.SearchMemoryIndexer.remove(name).catch((reason) => { debug$1("indexer has failed on remove item %o", reason); require_lib_logger_index.logger.error("indexer has failed on remove item"); }); } /** * Remove a tarball from the local storage. The tarball must not be * linked by any existing version, ie. the version should be * unpublished first. * Used storages: local (write) * @param {String} name package name * @param {String} filename tarball file name * @param {String} revision expected revision of the stored manifest * @param {Function} callback */ removeTarball(name, filename, revision, callback) { debug$1("remove tarball %s for %s", filename, name); this.localStorage.removeTarball(name, filename, revision, callback); } /** * Upload a tarball (publish flow). Synchronous, returns a writable * stream the tarball body is piped into. * Used storages: local (write) * @param {String} name package name * @param {String} filename tarball file name * @return {Stream} writable upload stream */ addTarball(name, filename) { debug$1("add a tarball for %o", name); return this.localStorage.addTarball(name, filename); } /** * Whether a tarball exists in the local storage, without reading it * (the stream is aborted as soon as it opens). * Used storages: local (read) * @param {String} name package name * @param {String} filename tarball file name * @return {Promise<Boolean>} true when the tarball is stored locally */ hasLocalTarball(name, filename) { const self = this; return new Promise((resolve, reject) => { let localStream = self.localStorage.getTarball(name, filename); let isOpen = false; localStream.on("error", (err) => { if (isOpen || err.status !== _verdaccio_core.HTTP_STATUS.NOT_FOUND) reject(err); if (localStream) { localStream.abort(); localStream = null; } resolve(false); }); localStream.on("open", function() { isOpen = true; localStream.abort(); localStream = null; resolve(true); }); }); } /** * Get a tarball. Synchronous, returns a readable stream. * The tarball is read locally first; on a local miss the distfile record * is resolved from the package metadata (see lookupDistFile), syncing the * uplinks when the local metadata does not know the file, and the tarball * is then fetched from the uplink that serves the distfile url. * Used storages: local || uplink (just one) * @param {String} name package name * @param {String} filename tarball file name * @return {Stream} readable tarball stream */ getTarball(name, filename) { debug$1("get tarball for package %o filename %o", name, filename); const readStream = new _verdaccio_streams.ReadTarball({}); readStream.abort = function() {}; const self = this; this._isTarballAllowedByFilters(name, filename).then(async (allowed) => { if (!allowed) { readStream.emit("error", require_lib_utils.ErrorCode.getNotFound(_verdaccio_core.API_ERROR.NO_PACKAGE)); return; } let localStream = self.localStorage.getTarball(name, filename); let isOpen = false; localStream.on("error", (err) => { if (isOpen || err.status !== _verdaccio_core.HTTP_STATUS.NOT_FOUND) return readStream.emit("error", err); const err404 = err; localStream.abort(); localStream = null; const lookupFromUplinks = (info) => { self._syncUplinksMetadata(name, info, {}, (syncErr, syncInfo) => { if (lodash.default.isNil(syncErr) === false) return readStream.emit("error", syncErr); if (self.filters?.length && !require_lib_utils.hasTarball(syncInfo, filename)) return readStream.emit("error", err404); const distFile = require_lib_storage_utils.lookupDistFile(syncInfo, filename); if (lodash.default.isNil(distFile)) { debug$1("remote tarball not found"); return readStream.emit("error", err404); } debug$1("dist file found, using it %o", distFile.url); serveFile(distFile); }); }; self.localStorage.getPackageMetadataAsync(name).then((info) => { const distFile = require_lib_storage_utils.lookupDistFile(info, filename); if (lodash.default.isNil(distFile) === false) { debug$1("dist file found, using it %o", distFile.url); serveFile(distFile); } else { debug$1("dist file not found, proceed update upstream"); lookupFromUplinks(info); } }, () => { lookupFromUplinks(null); }); }); localStream.on("content-length", function(v) { readStream.emit("content-length", v); }); localStream.on("open", function() { isOpen = true; localStream.pipe(readStream); }); }); return readStream; /** * Stream the tarball from the uplink serving the distfile, caching it * locally (and restoring the distfile record) when the uplink has the * cache enabled. * @param {DistFile} file distfile record resolved for the tarball */ function serveFile(file) { let uplink = null; if (file.registry && self.uplinks[file.registry]) { uplink = self.uplinks[file.registry]; debug$1("tarball %o is served by the recorded uplink %o", filename, file.registry); } else { const candidates = []; for (const uplinkId in self.uplinks) if ((0, _verdaccio_config.hasProxyTo)(name, uplinkId, self.config.packages)) candidates.push(self.uplinks[uplinkId]); debug$1("tarball %o has %o candidate uplinks", filename, candidates.length); for (const candidate of candidates) if (candidate.isUplinkValid(file.url)) uplink = candidate; if (uplink !== null) debug$1("uplink %o url matches tarball %o", uplink.upname, file.url); else if (candidates.length === 1) { uplink = candidates[0]; debug$1("using the single configured uplink %o for tarball %o hosted elsewhere", uplink.upname, file.url); } } if (uplink == null) { debug$1("upstream not found, creating one for %o", name); uplink = new require_lib_up_storage.default({ url: file.url, cache: true, _autogenerated: true }, self.config); } let savestream = null; if (uplink.config.cache) { debug$1("cache remote tarball enabled"); let distFile = file; if (!file.registry && uplink.upname) { debug$1("recording uplink %o on the persisted distfile for %o", uplink.upname, filename); distFile = { ...file, registry: uplink.upname }; } savestream = self.localStorage.addTarball(name, filename, distFile); } else debug$1("cache remote tarball disabled"); let on_open = function() { on_open = function() {}; const rstream2 = uplink.fetchTarball(file.url); rstream2.on("error", function(err) { if (savestream) savestream.abort(); savestream = null; readStream.emit("error", err); }); rstream2.on("end", function() { if (savestream) savestream.done(); }); rstream2.on("content-length", function(v) { readStream.emit("content-length", v); if (savestream) savestream.emit("content-length", v); }); rstream2.pipe(readStream); if (savestream) rstream2.pipe(savestream); }; if (savestream) { savestream.on("open", function() { on_open(); }); savestream.on("error", function(err) { self.logger.warn({ err, fileName: file }, "error saving file @{fileName}: @{err.message}\n@{err.stack}"); if (savestream) savestream.abort(); savestream = null; on_open(); }); } else on_open(); } } /** * Retrieve the package metadata: the local manifest merged with the * manifest of every uplink with proxy access to the package. * Used storages: local && uplink (proxy_access) * @param {Object} options * @property {String} options.name package name * @property {Object} options.req Express `req` object * @property {Boolean} options.keepUpLinkData keep the uplink info (last update, etc.) in the package metadata * @property {Boolean} options.abbreviated serve the abbreviated manifest (npm install) * @property {Function} options.callback invoked with the merged manifest and any uplink errors */ getPackage(options) { debug$1("get package for %o", options.name); this.localStorage.getPackageMetadata(options.name, (err, data) => { if (err && (!err.status || err.status >= _verdaccio_core.HTTP_STATUS.INTERNAL_ERROR)) return options.callback(err); this._syncUplinksMetadata(options.name, data, { req: options.req, uplinksLook: options.uplinksLook }, function getPackageSynUpLinksCallback(err, result, uplinkErrors) { if (err) return options.callback(err); require_lib_utils.normalizeDistTags(require_lib_storage_utils.cleanUpLinksRef(options.keepUpLinkData, result)); result._attachments = {}; if (options.abbreviated === true) { debug$1("get abbreviated manifest"); options.callback(null, require_lib_storage_utils.convertAbbreviatedManifest(result), uplinkErrors); } else { debug$1("get full package manifest"); options.callback(null, result, uplinkErrors); } }); }); } /** * Retrieve remote and local packages more recent than the start key. * All uplinks are streamed first, then the local packages; local * packages can override registry ones just because they appear in the * JSON last — a trade-off made to avoid memory issues. * Used storages: local && uplink (proxy_access) * @param {String} startkey timestamp to search from * @param {Object} options request options; `local=1` in the query skips the uplinks * @return {Stream} object stream of search results */ search(startkey, options) { const self = this; const searchStream = new stream.default.PassThrough({ objectMode: true }); async.default.eachSeries(Object.keys(this.uplinks), function(up_name, cb) { if (options.req?.query?.local !== void 0) return cb(); require_lib_logger_index.logger.info(`search for uplink ${up_name}`); const uplinkStream = self.uplinks[up_name].search(options); uplinkStream.pipe(searchStream, { end: false }); uplinkStream.on("error", function(err) { self.logger.error({ err }, "uplink error: @{err.message}"); cb(); cb = function() {}; }); uplinkStream.on("end", function() { cb(); cb = function() {}; }); searchStream.abort = function() { if (uplinkStream.abort) uplinkStream.abort(); cb(); cb = function() {}; }; }, function() { const localSearchStream = self.localStorage.search(startkey, options); searchStream.abort = function() { localSearchStream.abort(); }; localSearchStream.pipe(searchStream, { end: true }); localSearchStream.on("error", function(err) { self.logger.error({ err }, "search error: @{err.message}"); searchStream.end(); }); }); return searchStream; } /** * Retrieve only private local packages, as the latest version of each. * Used storages: local (read) * @param {Function} callback invoked with the list of latest versions */ getLocalDatabase(callback) { this.localStorage.storagePlugin.get((err, locals) => { if (err) return callback(err); this._collectLocalPackages(locals).then((packages) => callback(null, packages), (err) => callback(err)); }); } /** * Read each local package name, apply filters, and collect the latest version. */ async _collectLocalPackages(locals) { const packages = []; for (const name of locals) try { const pkgMetadata = await this.localStorage.getPackageMetadataAsync(name); const { filteredPackage } = await this._applyFilters(pkgMetadata); const latest = filteredPackage[require_lib_constants.DIST_TAGS]?.latest; if (latest && filteredPackage.versions[latest]) { const version = filteredPackage.versions[latest]; version.time = filteredPackage.time[latest]; version.users = filteredPackage.users; packages.push(version); } else this.logger.warn({ package: name }, "package @{package} does not have a \"latest\" tag?"); } catch (err) { this.logger.error({ err, package: name }, "error reading package @{package}"); } return packages; } /** * Fetch the package metadata from every uplink with proxy access and * synchronize it with the local data; filter plugins are applied to the * merged result. * @param {String} name package name * @param {Manifest} packageInfo local manifest; MUST be provided when the package exists locally * @param {ISyncUplinks} options uplink request options * @param {Function} callback invoked with (err, mergedManifest, uplinkErrors) */ _syncUplinksMetadata(name, packageInfo, options, callback) { let found = true; const self = this; const upLinks = []; const hasToLookIntoUplinks = lodash.default.isNil(options.uplinksLook) || options.uplinksLook; debug$1("sync uplinks for %o", name); debug$1("is sync uplink enabled %o", hasToLookIntoUplinks); if (!packageInfo) { debug$1("local package %s not found", name); found = false; packageInfo = require_lib_storage_utils.generatePackageTemplate(name); } for (const uplink in this.uplinks) if ((0, _verdaccio_config.hasProxyTo)(name, uplink, this.config.packages) && hasToLookIntoUplinks) upLinks.push(this.uplinks[uplink]); debug$1("uplinks found for %o: %o", name, upLinks.length); async.default.map(upLinks, (upLink, cb) => { const _options = Object.assign({}, options); const upLinkMeta = packageInfo._uplinks[upLink.upname]; if (require_lib_utils.isObject(upLinkMeta)) { const fetched = upLinkMeta.fetched; if (fetched && Date.now() - fetched < upLink.maxage) { debug$1("returning cached manifest for %o", upLink.upname); return cb(); } _options.etag = upLinkMeta.etag; } upLink.getRemoteMetadata(name, _options, (err, upLinkResponse, eTag) => { if (err && err.remoteStatus === 304) { debug$1("uplink %o responded 304 for %o, cache is up to date", upLink.upname, name); upLinkMeta.fetched = Date.now(); } if (err || !upLinkResponse) { debug$1("error captured on uplink %o for %o: %o", upLink.upname, name, err?.message); return cb(null, [err || require_lib_utils.ErrorCode.getInternalError("no data")]); } try { upLinkResponse = _verdaccio_core.validationUtils.normalizeMetadata(upLinkResponse, name); } catch (err) { self.logger.error({ sub: "out", err }, "package.json validating error @{!err.message}\n@{err.stack}"); return cb(null, [err]); } packageInfo._uplinks[upLink.upname] = { etag: eTag, fetched: Date.now() }; packageInfo = require_lib_storage_utils.mergeUplinkTimeIntoLocal(packageInfo, upLinkResponse); require_lib_uplink_util.updateVersionsHiddenUpLink(upLinkResponse.versions, upLink); try { require_lib_metadata_utils.mergeVersions(packageInfo, upLinkResponse); } catch (err) { self.logger.error({ sub: "out", err }, "package.json parsing error @{!err.message}\n@{err.stack}"); return cb(null, [err]); } debug$1("syncing on uplink %o", upLink.upname); found = true; cb(); }); }, async (err, upLinksErrors) => { (0, assert.default)(!err && Array.isArray(upLinksErrors)); if (!found) { let uplinkTimeoutError; for (let i = 0; i < upLinksErrors.length; i++) if (upLinksErrors[i]) { for (let j = 0; j < upLinksErrors[i].length; j++) if (upLinksErrors[i][j]) { const code = upLinksErrors[i][j].code; if (code === "ETIMEDOUT" || code === "ESOCKETTIMEDOUT" || code === "ECONNRESET") { uplinkTimeoutError = true; break; } } } if (uplinkTimeoutError) { debug$1("uplinks sync failed with timeout error"); return callback(require_lib_utils.ErrorCode.getServiceUnavailable(), null, upLinksErrors); } debug$1("uplinks sync failed with no package found"); return callback(require_lib_utils.ErrorCode.getNotFound(_verdaccio_core.API_ERROR.NO_PACKAGE), null, upLinksErrors); } if (upLinks.length === 0) { debug$1("no uplinks found for %o, upstream update aborted", name); const { filteredPackage, filterErrors } = await self._applyFilters(packageInfo); return callback(null, filteredPackage, filterErrors); } try { const packageJsonLocal = await self.localStorage.updateVersionsAsync(name, packageInfo); const { filteredPackage, filterErrors } = await self._applyFilters(packageJsonLocal); callback(null, filteredPackage, lodash.default.concat(upLinksErrors, filterErrors)); } catch (err) { callback(err); } }); } /** * Apply all configured filter plugins to a package manifest sequentially. * Each filter's output is passed as input to the next filter. * Returns the filtered manifest and any errors that occurred. */ async _applyFilters(packageInfo) { const filterErrors = []; let filteredPackage = packageInfo; for (const filter of this.filters ?? []) try { filteredPackage = await filter.filter_metadata(filteredPackage); } catch (err) { debug$1("filter plugin has failed on %o: %o", packageInfo.name, err?.message); filterErrors.push(err); } return { filteredPackage, filterErrors }; } /** * Check if a tarball should be served based on filter plugins, using only * local metadata. Returns true (defer) when the version isn't cached locally * — the uplink-sync path in getTarball re-checks filters after sync, which * avoids a redundant uplink round-trip and double filter application. */ async _isTarballAllowedByFilters(name, filename) { if (!this.filters?.length) return true; try { const pkgMetadata = await this.localStorage.getPackageMetadataAsync(name); if (!require_lib_utils.hasTarball(pkgMetadata, filename)) return true; const { filteredPackage } = await this._applyFilters(pkgMetadata); const allowed = require_lib_utils.hasTarball(filteredPackage, filename); if (allowed === false) debug$1("tarball %o of %o is blocked by a filter plugin", filename, name); return allowed; } catch (err) { if (err?.status === _verdaccio_core.HTTP_STATUS.NOT_FOUND) return true; this.logger.error({ package: name, fileName: filename, err }, "error checking filters for tarball @{fileName} of package @{package}: @{err.message}"); return true; } } /** * Tag each version with the uplink it was fetched from, under a hidden * (symbol) key. The local storage uses the tag to record the registry * on the distfile records it creates. * @param {Versions} versions versions fetched from the uplink * @param {ProxyStorage} upLink uplink the versions were fetched from */ _updateVersionsHiddenUpLink(versions, upLink) { for (const i in versions) if (Object.prototype.hasOwnProperty.call(versions, i)) { const version = versions[i]; version[Symbol.for("__verdaccio_uplink")] = upLink.upname; } } }; //#endregion exports.default = Storage; //# sourceMappingURL=storage.js.map