cnpmcore
Version:
Private NPM Registry for Enterprise
979 lines • 128 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var PackageManagerService_1;
import assert from 'node:assert/strict';
import { readFile, stat } from 'node:fs/promises';
import { JSONBuilder } from '@cnpmjs/packument';
import { AccessLevel, Inject, SingletonProto } from 'egg';
import { BadRequestError, ForbiddenError, NotFoundError } from 'egg/errors';
import npa from 'npm-package-arg';
import pMap from 'p-map';
import semver from 'semver';
import { AbstractService } from "../../common/AbstractService.js";
import { calculateIntegrity, detectInstallScript, formatTarball, getFullname, getScopeAndName, hasShrinkWrapInTgz, } from "../../common/PackageUtil.js";
import { isDuplicateKeyError } from "../../repository/util/ErrorUtil.js";
import { Package } from "../entity/Package.js";
import { PackageTag } from "../entity/PackageTag.js";
import { PackageVersion } from "../entity/PackageVersion.js";
import { PackageVersionBlock } from "../entity/PackageVersionBlock.js";
import { PACKAGE_ADDED, PACKAGE_BLOCKED, PACKAGE_MAINTAINER_CHANGED, PACKAGE_MAINTAINER_REMOVED, PACKAGE_META_CHANGED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED, PACKAGE_TAG_REMOVED, PACKAGE_UNBLOCKED, PACKAGE_UNPUBLISHED, PACKAGE_VERSION_ADDED, PACKAGE_VERSION_REMOVED, } from "../event/index.js";
const TOTAL = '@@TOTAL@@';
const SCOPE_TOTAL_PREFIX = '@@SCOPE@@:';
const DESCRIPTION_LIMIT = 1024 * 10;
let PackageManagerService = PackageManagerService_1 = class PackageManagerService extends AbstractService {
// Reset in-memory counters, used by test teardown to prevent cross-test pollution
static resetDownloadCounters() {
PackageManagerService_1.downloadCounters = {};
}
// support user publish private package and sync worker publish public package
async publish(cmd, publisher) {
if (this.config.cnpmcore.strictValidatePackageDeps) {
await this._checkPackageDepsVersion(cmd.packageJson);
}
let pkg = await this.packageRepository.findPackage(cmd.scope, cmd.name);
let isNewPackage = !pkg;
if (pkg) {
// update description
// will read database twice to update description by model to entity and entity to model
if (pkg.description !== cmd.description) {
pkg.description = cmd.description || '';
}
/* c8 ignore next 3 */
// package can be migrated into another registry
if (cmd.registryId) {
pkg.registryId = cmd.registryId;
}
}
else {
pkg = Package.create({
scope: cmd.scope,
name: cmd.name,
isPrivate: cmd.isPrivate,
description: cmd.description || '',
registryId: cmd.registryId,
});
}
// 防止 description 长度超过 db 限制
if (pkg.description?.length > DESCRIPTION_LIMIT) {
pkg.description = pkg.description.slice(0, DESCRIPTION_LIMIT);
}
await this.packageRepository.savePackage(pkg);
// create maintainer
await this.packageRepository.savePackageMaintainer(pkg.packageId, publisher.userId);
let pkgVersion = await this.packageRepository.findPackageVersion(pkg.packageId, cmd.version);
if (pkgVersion) {
throw new ForbiddenError(`Can't modify pre-existing version: ${pkg.fullname}@${pkgVersion.version}`);
}
// make sure cmd.packageJson.readme is deleted
if ('readme' in cmd.packageJson) {
delete cmd.packageJson.readme;
}
const publishTime = cmd.publishTime || new Date();
// add _cnpmcore_publish_time field to cmd.packageJson
if (!cmd.packageJson._cnpmcore_publish_time) {
cmd.packageJson._cnpmcore_publish_time = publishTime;
}
if (!cmd.packageJson.publish_time) {
cmd.packageJson.publish_time = publishTime.getTime();
}
if (cmd.packageJson._hasShrinkwrap === undefined) {
const contentOrFile = cmd.dist.content || cmd.dist.localFile;
if (contentOrFile) {
cmd.packageJson._hasShrinkwrap = await hasShrinkWrapInTgz(contentOrFile);
}
}
// override _npmUser field if is private package or _npmUser field not exists
// otherwise avoid overriding _npmUser field from sync package, e.g.: oidc publish package
// "_npmUser": {
// "name": "GitHub Actions",
// "email": "npm-oidc-no-reply@github.com",
// "trustedPublisher": {
// "id": "github",
// "oidcConfigId": "oidc:d0f693c3-ada3-4197-b34c-b7aaeb524f11"
// }
// }
if (cmd.isPrivate || !cmd.packageJson._npmUser) {
// set _npmUser field to cmd.packageJson
cmd.packageJson._npmUser = {
// clean user scope prefix
name: publisher.displayName,
email: publisher.email,
};
}
// add _registry_name field to cmd.packageJson
const registry = await this.getSourceRegistry(pkg);
if (registry) {
cmd.packageJson._source_registry_name = registry.name;
}
// https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
const hasInstallScript = detectInstallScript(cmd.packageJson) ? true : undefined;
let tarDistIntegrity;
let tarDistSize = 0;
if (cmd.dist.content) {
const tarDistBytes = cmd.dist.content;
tarDistIntegrity = await calculateIntegrity(tarDistBytes);
tarDistSize = tarDistBytes.length;
}
else {
// should has localFile
const localFile = cmd.dist.localFile;
const fileStat = await stat(localFile);
tarDistIntegrity = await calculateIntegrity(localFile);
tarDistSize = fileStat.size;
}
const tarDist = pkg.createTar(cmd.version, {
size: tarDistSize,
shasum: tarDistIntegrity.shasum,
integrity: tarDistIntegrity.integrity,
});
if (cmd.dist.content) {
await this.distRepository.saveDist(tarDist, cmd.dist.content);
}
else if (cmd.dist.localFile) {
await this.distRepository.saveDist(tarDist, cmd.dist.localFile);
}
cmd.packageJson.dist = {
...cmd.packageJson.dist,
tarball: formatTarball(this.config.cnpmcore.registry, pkg.scope, pkg.name, cmd.version),
size: tarDistSize,
shasum: tarDistIntegrity.shasum,
integrity: tarDistIntegrity.integrity,
};
// https://github.com/npm/registry/blob/main/docs/responses/package-metadata.md#abbreviated-version-object
// Abbreviated version object
const abbreviated = JSON.stringify({
name: cmd.packageJson.name,
version: cmd.packageJson.version,
deprecated: cmd.packageJson.deprecated,
dependencies: cmd.packageJson.dependencies,
acceptDependencies: cmd.packageJson.acceptDependencies,
optionalDependencies: cmd.packageJson.optionalDependencies,
devDependencies: cmd.packageJson.devDependencies,
bundleDependencies: cmd.packageJson.bundleDependencies,
peerDependencies: cmd.packageJson.peerDependencies,
peerDependenciesMeta: cmd.packageJson.peerDependenciesMeta,
bin: cmd.packageJson.bin,
directories: cmd.packageJson.directories,
os: cmd.packageJson.os,
cpu: cmd.packageJson.cpu,
libc: cmd.packageJson.libc,
workspaces: cmd.packageJson.workspaces,
dist: cmd.packageJson.dist,
engines: cmd.packageJson.engines,
_hasShrinkwrap: cmd.packageJson._hasShrinkwrap,
hasInstallScript,
funding: cmd.packageJson.funding,
// https://github.com/cnpm/npminstall/blob/13efc7eec21a61e509226e3772bfb75cd5605612/lib/install_package.js#L176
// npminstall require publish time to show the recently update versions
publish_time: cmd.packageJson.publish_time,
_source_registry_name: cmd.packageJson._source_registry_name,
_npmUser: cmd.packageJson._npmUser,
});
const abbreviatedDistBytes = Buffer.from(abbreviated);
const abbreviatedDistIntegrity = await calculateIntegrity(abbreviatedDistBytes);
const readmeDistBytes = Buffer.from(cmd.readme);
const readmeDistIntegrity = await calculateIntegrity(readmeDistBytes);
const manifestDistBytes = Buffer.from(JSON.stringify(cmd.packageJson));
const manifestDistIntegrity = await calculateIntegrity(manifestDistBytes);
pkgVersion = PackageVersion.create({
packageId: pkg.packageId,
version: cmd.version,
publishTime,
manifestDist: pkg.createManifest(cmd.version, {
size: manifestDistBytes.length,
shasum: manifestDistIntegrity.shasum,
integrity: manifestDistIntegrity.integrity,
}),
readmeDist: pkg.createReadme(cmd.version, {
size: readmeDistBytes.length,
shasum: readmeDistIntegrity.shasum,
integrity: readmeDistIntegrity.integrity,
}),
abbreviatedDist: pkg.createAbbreviated(cmd.version, {
size: abbreviatedDistBytes.length,
shasum: abbreviatedDistIntegrity.shasum,
integrity: abbreviatedDistIntegrity.integrity,
}),
tarDist,
});
await Promise.all([
this.distRepository.saveDist(pkgVersion.abbreviatedDist, abbreviatedDistBytes),
this.distRepository.saveDist(pkgVersion.manifestDist, manifestDistBytes),
this.distRepository.saveDist(pkgVersion.readmeDist, readmeDistBytes),
]);
try {
await this.packageRepository.createPackageVersion(pkgVersion);
}
catch (e) {
if (isDuplicateKeyError(e)) {
throw new ForbiddenError(`Can't modify pre-existing version: ${pkg.fullname}@${cmd.version}`);
}
throw e;
}
if (cmd.skipRefreshPackageManifests !== true) {
await this.refreshPackageChangeVersionsToDists(pkg, [pkgVersion.version]);
}
if (cmd.tags) {
for (const tag of cmd.tags) {
await this.savePackageTag(pkg, tag, cmd.version, true);
this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, tag);
}
}
else {
this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, undefined);
}
if (isNewPackage) {
this.eventBus.emit(PACKAGE_ADDED, pkg.fullname);
}
return pkgVersion;
}
async blockPackageByFullname(name, reason) {
const [scope, pkgName] = getScopeAndName(name);
const pkg = await this.packageRepository.findPackage(scope, pkgName);
if (!pkg) {
throw new NotFoundError(`Package name(${name}) not found`);
}
return await this.blockPackage(pkg, reason);
}
async blockPackage(pkg, reason) {
let block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
if (block) {
block.reason = reason;
}
else {
block = PackageVersionBlock.create({
packageId: pkg.packageId,
version: '*',
reason,
});
}
await this.packageVersionBlockRepository.savePackageVersionBlock(block);
if (pkg.manifestsDist && pkg.abbreviatedsDist) {
if (this.config.cnpmcore.experimental.enableJSONBuilder) {
const fullManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.manifestsDist);
if (fullManifestsBuilder) {
fullManifestsBuilder.setIn(['block'], reason);
}
const abbreviatedManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.abbreviatedsDist);
if (abbreviatedManifestsBuilder) {
abbreviatedManifestsBuilder.setIn(['block'], reason);
}
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
}
else {
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
if (fullManifests) {
fullManifests.block = reason;
}
const abbreviatedManifests = await this.distRepository.readDistBytesToJSON(pkg.abbreviatedsDist);
if (abbreviatedManifests) {
abbreviatedManifests.block = reason;
}
await this._updatePackageManifestsToDists(pkg, fullManifests || null, abbreviatedManifests || null);
}
this.eventBus.emit(PACKAGE_BLOCKED, pkg.fullname);
this.logger.info('[packageManagerService.blockPackage:success] packageId: %s, reason: %j', pkg.packageId, reason);
}
return block;
}
async unblockPackageByFullname(name) {
const [scope, pkgName] = getScopeAndName(name);
const pkg = await this.packageRepository.findPackage(scope, pkgName);
if (!pkg) {
throw new NotFoundError(`Package name(${name}) not found`);
}
return await this.unblockPackage(pkg);
}
async unblockPackage(pkg) {
const block = await this.packageVersionBlockRepository.findPackageVersionBlock(pkg.packageId, '*');
if (block) {
await this.packageVersionBlockRepository.removePackageVersionBlock(block.packageVersionBlockId);
}
if (pkg.manifestsDist && pkg.abbreviatedsDist) {
if (this.config.cnpmcore.experimental.enableJSONBuilder) {
const fullManifestsBuffer = await this.distRepository.readDistBytesToBuffer(pkg.manifestsDist);
let fullManifestsBuilder;
if (fullManifestsBuffer) {
fullManifestsBuilder = new JSONBuilder(fullManifestsBuffer);
fullManifestsBuilder.deleteIn(['block']);
}
const abbreviatedManifestsBuffer = await this.distRepository.readDistBytesToBuffer(pkg.abbreviatedsDist);
let abbreviatedManifestsBuilder;
if (abbreviatedManifestsBuffer) {
abbreviatedManifestsBuilder = new JSONBuilder(abbreviatedManifestsBuffer);
abbreviatedManifestsBuilder.deleteIn(['block']);
}
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
}
else {
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
if (fullManifests) {
fullManifests.block = undefined;
}
const abbreviatedManifests = await this.distRepository.readDistBytesToJSON(pkg.abbreviatedsDist);
if (abbreviatedManifests) {
abbreviatedManifests.block = undefined;
}
await this._updatePackageManifestsToDists(pkg, fullManifests || null, abbreviatedManifests || null);
}
this.eventBus.emit(PACKAGE_UNBLOCKED, pkg.fullname);
this.logger.info('[packageManagerService.unblockPackage:success] packageId: %s', pkg.packageId);
}
}
async replacePackageMaintainersAndDist(pkg, maintainers) {
await this.packageRepository.replacePackageMaintainers(pkg.packageId, maintainers.map((m) => m.userId));
await this.refreshPackageMaintainersToDists(pkg);
this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
}
async savePackageMaintainers(pkg, maintainers) {
let hasNewRecord = false;
for (const maintainer of maintainers) {
const newRecord = await this.packageRepository.savePackageMaintainer(pkg.packageId, maintainer.userId);
if (newRecord) {
hasNewRecord = true;
}
}
if (hasNewRecord) {
this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
}
return hasNewRecord;
}
async removePackageMaintainer(pkg, maintainer) {
await this.packageRepository.removePackageMaintainer(pkg.packageId, maintainer.userId);
this.eventBus.emit(PACKAGE_MAINTAINER_REMOVED, pkg.fullname, maintainer.name);
}
async refreshPackageMaintainersToDists(pkg) {
await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
}
async refreshPackageDistTagsToDists(pkg) {
await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'dist-tags');
}
async listPackageFullManifests(scope, name, isSync = false) {
return await this._listPackageFullOrAbbreviatedManifests(scope, name, true, isSync);
}
async listPackageFullManifestsBuffer(scope, name) {
return await this._listPackageFullOrAbbreviatedManifestsBuffer(scope, name, true);
}
async listPackageAbbreviatedManifests(scope, name, isSync = false) {
return await this._listPackageFullOrAbbreviatedManifests(scope, name, false, isSync);
}
async listPackageAbbreviatedManifestsBuffer(scope, name) {
return await this._listPackageFullOrAbbreviatedManifestsBuffer(scope, name, false);
}
async showPackageVersionByVersionOrTag(scope, name, spec) {
const pkg = await this.packageRepository.findPackage(scope, name);
if (!pkg)
return {};
const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
if (block) {
return { blockReason: block.reason, pkg };
}
const fullname = getFullname(scope, name);
const result = npa(`${fullname}@${spec}`);
const version = await this.packageVersionService.getVersion(result);
if (!version) {
return {};
}
const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
return { packageVersion, pkg };
}
async showPackageVersionManifest(scope, name, spec, isSync = false, isFullManifests = false) {
const pkg = await this.packageRepository.findPackage(scope, name);
if (!pkg)
return {};
const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
if (block) {
return { blockReason: block.reason, pkg };
}
const fullname = getFullname(scope, name);
const result = npa(`${fullname}@${spec}`);
const manifest = await this.packageVersionService.readManifest(pkg.packageId, result, isFullManifests, !isSync);
return { manifest, blockReason: null, pkg };
}
async downloadPackageVersionTar(packageVersion) {
return await this.distRepository.downloadDist(packageVersion.tarDist);
}
plusPackageVersionCounter(fullname, version) {
// set counter + 1, schedule will store them into database
const counters = PackageManagerService_1.downloadCounters;
if (!counters[fullname])
counters[fullname] = {};
counters[fullname][version] = (counters[fullname][version] || 0) + 1;
// Total
const ALL = '*';
if (!counters[TOTAL])
counters[TOTAL] = {};
counters[TOTAL][ALL] = (counters[TOTAL][ALL] || 0) + 1;
// scope total
const scope = getScopeAndName(fullname)[0];
if (scope) {
const scopeKey = `${SCOPE_TOTAL_PREFIX}${scope}`;
if (!counters[scopeKey])
counters[scopeKey] = {};
counters[scopeKey][ALL] = (counters[scopeKey][ALL] || 0) + 1;
}
}
// will be call by schedule/SavePackageVersionDownloadCounter.ts
async savePackageVersionCounters() {
// { [fullname]: { [version]: number } }
const counters = PackageManagerService_1.downloadCounters;
const fullnames = Object.keys(counters);
if (fullnames.length === 0)
return;
PackageManagerService_1.downloadCounters = {};
this.logger.info('[packageManagerService.savePackageVersionCounters:saving] %d fullnames', fullnames.length);
let total = 0;
for (const fullname in counters) {
const versions = counters[fullname];
let packageId = null;
if (fullname === TOTAL) {
packageId = 'total';
}
else if (fullname.startsWith(SCOPE_TOTAL_PREFIX)) {
packageId = fullname.replace(SCOPE_TOTAL_PREFIX, '');
}
else {
// find packageId from fullname
const [scope, name] = getScopeAndName(fullname);
packageId = await this.packageRepository.findPackageId(scope, name);
}
if (!packageId)
continue;
for (const version in versions) {
const counter = versions[version];
await this.packageVersionDownloadRepository.plus(packageId, version, counter);
total += counter;
}
}
this.logger.info('[packageManagerService.savePackageVersionCounters:saved] %d total', total);
}
async saveDeprecatedVersions(pkg, deprecatedList) {
const updateVersions = [];
for (const { version, deprecated } of deprecatedList) {
const pkgVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
if (!pkgVersion)
continue;
const message = deprecated === '' ? undefined : deprecated;
await this._mergeManifestDist(pkgVersion.manifestDist, {
deprecated: message,
});
await this._mergeManifestDist(pkgVersion.abbreviatedDist, {
deprecated: message,
});
await this.packageRepository.savePackageVersion(pkgVersion);
updateVersions.push(version);
}
await this.refreshPackageChangeVersionsToDists(pkg, updateVersions);
this.eventBus.emit(PACKAGE_META_CHANGED, pkg.fullname, {
deprecateds: deprecatedList,
});
}
async savePackageVersionManifest(pkgVersion, mergeManifest, mergeAbbreviated) {
await this._mergeManifestDist(pkgVersion.manifestDist, mergeManifest);
await this._mergeManifestDist(pkgVersion.abbreviatedDist, mergeAbbreviated);
}
async savePackageVersionManifestWithBuilder(pkgVersion, manifestBuilder, abbreviatedBuilder) {
const manifestBytes = manifestBuilder.build();
const manifestIntegrity = await calculateIntegrity(manifestBytes);
pkgVersion.manifestDist.size = manifestBytes.length;
pkgVersion.manifestDist.shasum = manifestIntegrity.shasum;
pkgVersion.manifestDist.integrity = manifestIntegrity.integrity;
await this.distRepository.saveDist(pkgVersion.manifestDist, manifestBytes);
if (abbreviatedBuilder) {
const abbreviatedBytes = abbreviatedBuilder.build();
const abbreviatedIntegrity = await calculateIntegrity(abbreviatedBytes);
pkgVersion.abbreviatedDist.size = abbreviatedBytes.length;
pkgVersion.abbreviatedDist.shasum = abbreviatedIntegrity.shasum;
pkgVersion.abbreviatedDist.integrity = abbreviatedIntegrity.integrity;
await this.distRepository.saveDist(pkgVersion.abbreviatedDist, abbreviatedBytes);
}
await this.packageRepository.savePackageVersion(pkgVersion);
}
/**
* save package version readme
*/
async savePackageVersionReadme(pkgVersion, readmeFile) {
await this.distRepository.saveDist(pkgVersion.readmeDist, readmeFile);
this.logger.info('[PackageManagerService.savePackageVersionReadme] save packageVersionId:%s readme:%s to dist:%s', pkgVersion.packageVersionId, readmeFile, pkgVersion.readmeDist.distId);
}
async savePackageReadme(pkg, readmeFile) {
if (!pkg.manifestsDist)
return;
const readme = await readFile(readmeFile, 'utf8');
if (this.config.cnpmcore.experimental.enableJSONBuilder) {
const fullManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.manifestsDist);
if (!fullManifestsBuilder)
return;
fullManifestsBuilder.setIn(['readme'], readme);
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, undefined);
}
else {
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
if (!fullManifests)
return;
fullManifests.readme = readme;
await this._updatePackageManifestsToDists(pkg, fullManifests, null);
}
this.logger.info('[PackageManagerService.savePackageReadme] save packageId:%s readme, size: %s', pkg.packageId, readme.length);
}
async _removePackageVersionAndDist(pkgVersion) {
// remove nfs dists
await Promise.all([
this.distRepository.destroyDist(pkgVersion.abbreviatedDist),
this.distRepository.destroyDist(pkgVersion.manifestDist),
this.distRepository.destroyDist(pkgVersion.readmeDist),
this.distRepository.destroyDist(pkgVersion.tarDist),
]);
// remove from repository
await this.packageRepository.removePackageVersion(pkgVersion);
this.logger.info('[packageManagerService.removePackageVersionAndDist:success] packageVersionId: %s', pkgVersion.packageVersionId);
}
async unpublishPackage(pkg) {
const pkgVersions = await this.packageRepository.listPackageVersions(pkg.packageId);
// already unpublished
if (pkgVersions.length === 0) {
this.logger.info(`[packageManagerService.unpublishPackage:skip] ${pkg.packageId} already unpublished`);
return;
}
for (const pkgVersion of pkgVersions) {
await this._removePackageVersionAndDist(pkgVersion);
}
// set unpublished dist to package's manifestDist and abbreviatedDist
const unpublishedInfo = {
_id: pkg.fullname,
name: pkg.fullname,
time: {
created: pkg.createdAt,
modified: pkg.updatedAt,
unpublished: new Date(),
},
// keep this property exists for forward compatibility
// https://github.com/cnpm/cnpmjs.org/blob/ad622d55e384743b48e79bb6aec574a7f354ee9f/controllers/sync_module_worker.js#L828
'dist-tags': {},
};
await this._mergeManifestDist(
// oxlint-disable-next-line typescript-eslint/no-non-null-assertion
pkg.manifestsDist, undefined, unpublishedInfo);
await this._mergeManifestDist(
// oxlint-disable-next-line typescript-eslint/no-non-null-assertion
pkg.abbreviatedsDist, undefined, unpublishedInfo);
this.eventBus.emit(PACKAGE_UNPUBLISHED, pkg.fullname);
this.logger.info('[packageManagerService.unpublishPackage:success] packageId: %s, fullname: %s', pkg.packageId, pkg.fullname);
}
async removePackageVersion(pkg, pkgVersion, skipRefreshPackageManifests = false) {
const currentVersions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
// only one version, unpublish the package
if (currentVersions.length === 1 && currentVersions[0] === pkgVersion.version) {
await this.unpublishPackage(pkg);
return;
}
// remove version & update tags
await this._removePackageVersionAndDist(pkgVersion);
const versions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
if (versions.length > 0) {
let updateTag;
// make sure latest tag exists
const latestTag = await this.packageRepository.findPackageTag(pkg.packageId, 'latest');
if (latestTag?.version === pkgVersion.version) {
// change latest version
// https://github.com/npm/libnpmpublish/blob/main/unpublish.js#L62
const latestVersion = versions.sort(semver.compareLoose).pop();
if (latestVersion) {
updateTag = latestTag.tag;
await this.savePackageTag(pkg, latestTag.tag, latestVersion, true);
}
}
if (skipRefreshPackageManifests !== true) {
await this.refreshPackageChangeVersionsToDists(pkg, undefined, [pkgVersion.version]);
this.eventBus.emit(PACKAGE_VERSION_REMOVED, pkg.fullname, pkgVersion.version, updateTag);
}
return;
}
}
async savePackageTag(pkg, tag, version, skipEvent = false) {
let tagEntity = await this.packageRepository.findPackageTag(pkg.packageId, tag);
if (!tagEntity) {
tagEntity = PackageTag.create({
packageId: pkg.packageId,
tag,
version,
});
await this.packageRepository.savePackageTag(tagEntity);
await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'dist-tags');
if (!skipEvent) {
this.eventBus.emit(PACKAGE_TAG_ADDED, pkg.fullname, tagEntity.tag);
}
return true;
}
if (tagEntity.version === version) {
// nothing change
return false;
}
tagEntity.version = version;
await this.packageRepository.savePackageTag(tagEntity);
await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'dist-tags');
if (!skipEvent) {
this.eventBus.emit(PACKAGE_TAG_CHANGED, pkg.fullname, tagEntity.tag);
}
return true;
}
async removePackageTag(pkg, tag) {
const tagEntity = await this.packageRepository.findPackageTag(pkg.packageId, tag);
if (!tagEntity)
return false;
await this.packageRepository.removePackageTag(tagEntity);
await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'dist-tags');
this.eventBus.emit(PACKAGE_TAG_REMOVED, pkg.fullname, tagEntity.tag);
return true;
}
async refreshPackageChangeVersionsToDists(pkg, updateVersions, removeVersions) {
if (this.config.cnpmcore.experimental.enableJSONBuilder) {
return await this._refreshPackageChangeVersionsToDistsWithBuilder(pkg, updateVersions, removeVersions);
}
if (!pkg.manifestsDist?.distId || !pkg.abbreviatedsDist?.distId) {
// no dists, refresh all again, the first time sync package will not have dists
return await this._refreshPackageManifestsToDists(pkg);
}
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
const abbreviatedManifests = await this.distRepository.readDistBytesToJSON(pkg.abbreviatedsDist);
if (!fullManifests?.versions || !abbreviatedManifests?.versions) {
// is unpublished, refresh all again
return await this._refreshPackageManifestsToDists(pkg);
}
if (updateVersions) {
for (const version of updateVersions) {
const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
if (packageVersion) {
const manifest = await this.distRepository.readDistBytesToJSON(packageVersion.manifestDist);
if (!manifest)
continue;
if ('readme' in manifest) {
delete manifest.readme;
}
fullManifests.versions[packageVersion.version] = manifest;
fullManifests.time[packageVersion.version] = packageVersion.publishTime;
const abbreviatedManifest = await this.distRepository.readDistBytesToJSON(packageVersion.abbreviatedDist);
if (abbreviatedManifest) {
abbreviatedManifests.versions[packageVersion.version] = abbreviatedManifest;
// abbreviatedManifests.time is guaranteed to exist since it's initialized in _listPackageAbbreviatedManifests
if (abbreviatedManifests.time) {
abbreviatedManifests.time[packageVersion.version] = packageVersion.publishTime;
}
}
}
}
}
if (removeVersions) {
for (const version of removeVersions) {
delete fullManifests.versions[version];
delete fullManifests.time[version];
delete abbreviatedManifests.versions[version];
delete abbreviatedManifests.time?.[version];
}
}
// update dist-tags
await this._setPackageDistTagsAndLatestInfos(pkg, fullManifests, abbreviatedManifests);
// store to nfs dist
await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
}
async _refreshPackageChangeVersionsToDistsWithBuilder(pkg, updateVersions, removeVersions) {
if (!pkg.manifestsDist?.distId || !pkg.abbreviatedsDist?.distId) {
// no dists, refresh all again, the first time sync package will not have dists
return await this._refreshPackageManifestsToDists(pkg);
}
const fullManifestsBuffer = await this.distRepository.readDistBytesToBuffer(pkg.manifestsDist);
const abbreviatedManifestsBuffer = await this.distRepository.readDistBytesToBuffer(pkg.abbreviatedsDist);
if (!fullManifestsBuffer || !abbreviatedManifestsBuffer) {
// is unpublished, refresh all again
return await this._refreshPackageManifestsToDists(pkg);
}
const fullManifestsBuilder = new JSONBuilder(fullManifestsBuffer);
const abbreviatedManifestsBuilder = new JSONBuilder(abbreviatedManifestsBuffer);
if (!fullManifestsBuilder.hasIn(['versions']) || !abbreviatedManifestsBuilder.hasIn(['versions'])) {
// is unpublished, refresh all again
return await this._refreshPackageManifestsToDists(pkg);
}
if (updateVersions) {
for (const version of updateVersions) {
const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
if (packageVersion) {
const manifest = await this.distRepository.readDistBytesToJSON(packageVersion.manifestDist);
if (!manifest)
continue;
if ('readme' in manifest) {
delete manifest.readme;
}
fullManifestsBuilder.setIn(['versions', packageVersion.version], manifest);
fullManifestsBuilder.setIn(['time', packageVersion.version], packageVersion.publishTime);
const abbreviatedManifest = await this.distRepository.readDistBytesToJSON(packageVersion.abbreviatedDist);
if (abbreviatedManifest) {
abbreviatedManifestsBuilder.setIn(['versions', packageVersion.version], abbreviatedManifest);
// abbreviatedManifests.time is guaranteed to exist since it's initialized in _listPackageAbbreviatedManifests
abbreviatedManifestsBuilder.setIn(['time', packageVersion.version], packageVersion.publishTime);
}
}
}
}
if (removeVersions) {
for (const version of removeVersions) {
fullManifestsBuilder.deleteIn(['versions', version]);
fullManifestsBuilder.deleteIn(['time', version]);
abbreviatedManifestsBuilder.deleteIn(['versions', version]);
abbreviatedManifestsBuilder.deleteIn(['time', version]);
}
}
// update dist-tags
await this._setPackageDistTagsAndLatestInfosWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
// store to nfs dist
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
}
async getSourceRegistry(pkg) {
let registry;
if (pkg.registryId) {
registry = await this.registryManagerService.findByRegistryId(pkg.registryId);
}
else {
registry = await this.registryManagerService.ensureDefaultRegistry();
}
return registry;
}
async distTags(pkg) {
const entities = await this.packageRepository.listPackageTags(pkg.packageId);
const distTags = {};
for (const entity of entities) {
distTags[entity.tag] = entity.version;
}
return distTags;
}
// refresh package full manifests and abbreviated manifests to NFS
async _refreshPackageManifestsToDists(pkg) {
const [fullManifests, abbreviatedManifests] = await Promise.all([
this._listPackageFullManifests(pkg),
this._listPackageAbbreviatedManifests(pkg),
]);
await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
}
// only refresh root attributes only, e.g.: dist-tags, maintainers
async _refreshPackageManifestRootAttributeOnlyToDists(pkg, refreshAttr) {
if (this.config.cnpmcore.experimental.enableJSONBuilder) {
return await this._refreshPackageManifestRootAttributeOnlyToDistsWithBuilder(pkg, refreshAttr);
}
if (refreshAttr === 'maintainers') {
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
if (fullManifests) {
const maintainers = await this.maintainers(pkg);
fullManifests.maintainers = maintainers;
await this._updatePackageManifestsToDists(pkg, fullManifests, null);
}
}
else if (refreshAttr === 'dist-tags') {
const fullManifests = await this.distRepository.readDistBytesToJSON(pkg.manifestsDist);
if (fullManifests) {
const abbreviatedManifests = await this.distRepository.readDistBytesToJSON(pkg.abbreviatedsDist);
if (abbreviatedManifests) {
await this._setPackageDistTagsAndLatestInfos(pkg, fullManifests, abbreviatedManifests);
}
await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
}
}
}
async _refreshPackageManifestRootAttributeOnlyToDistsWithBuilder(pkg, refreshAttr) {
if (refreshAttr === 'maintainers') {
const fullManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.manifestsDist);
if (fullManifestsBuilder) {
const maintainers = await this.maintainers(pkg);
fullManifestsBuilder.setIn(['maintainers'], maintainers);
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, undefined);
}
}
else if (refreshAttr === 'dist-tags') {
const fullManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.manifestsDist);
if (fullManifestsBuilder) {
const abbreviatedManifestsBuilder = await this.distRepository.readDistBytesToJSONBuilder(pkg.abbreviatedsDist);
if (abbreviatedManifestsBuilder) {
await this._setPackageDistTagsAndLatestInfosWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
}
await this._updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder);
}
}
}
_mergeLatestManifestFields(fullManifests, latestManifest) {
if (!latestManifest)
return;
// https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
const fieldsFromLatestManifest = [
'author',
'bugs',
'contributors',
'description',
'homepage',
'keywords',
'license',
'readmeFilename',
'repository',
];
// the latest version metas
for (const field of fieldsFromLatestManifest) {
if (latestManifest[field]) {
Reflect.set(fullManifests, field, latestManifest[field]);
}
}
}
_mergeLatestManifestFieldsWithBuilder(fullManifestsBuilder, latestManifest) {
if (!latestManifest)
return;
// https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
const fieldsFromLatestManifest = [
'author',
'bugs',
'contributors',
'description',
'homepage',
'keywords',
'license',
'readmeFilename',
'repository',
];
// the latest version metas
for (const field of fieldsFromLatestManifest) {
if (latestManifest[field]) {
fullManifestsBuilder.setIn([field], latestManifest[field]);
}
}
}
async _setPackageDistTagsAndLatestInfos(pkg, fullManifests, abbreviatedManifests) {
const distTags = await this.distTags(pkg);
if (distTags.latest) {
const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, distTags.latest);
if (packageVersion) {
fullManifests.readme = await this.distRepository.readDistBytesToString(packageVersion.readmeDist);
const latestManifest = await this.distRepository.readDistBytesToJSON(packageVersion.manifestDist);
this._mergeLatestManifestFields(fullManifests, latestManifest);
}
}
fullManifests['dist-tags'] = distTags;
abbreviatedManifests['dist-tags'] = distTags;
}
async _setPackageDistTagsAndLatestInfosWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder) {
const distTags = await this.distTags(pkg);
if (distTags.latest) {
const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, distTags.latest);
if (packageVersion) {
const readme = await this.distRepository.readDistBytesToString(packageVersion.readmeDist);
fullManifestsBuilder.setIn(['readme'], readme);
const latestManifest = await this.distRepository.readDistBytesToJSON(packageVersion.manifestDist);
this._mergeLatestManifestFieldsWithBuilder(fullManifestsBuilder, latestManifest);
}
}
fullManifestsBuilder.setIn(['dist-tags'], distTags);
abbreviatedManifestsBuilder.setIn(['dist-tags'], distTags);
}
async _mergeManifestDist(manifestDist, mergeData, replaceData) {
let manifest = await this.distRepository.readDistBytesToJSON(manifestDist);
if (mergeData && manifest) {
Object.assign(manifest, mergeData);
}
if (replaceData) {
manifest = replaceData;
}
const manifestBytes = Buffer.from(JSON.stringify(manifest));
const manifestIntegrity = await calculateIntegrity(manifestBytes);
manifestDist.size = manifestBytes.length;
manifestDist.shasum = manifestIntegrity.shasum;
manifestDist.integrity = manifestIntegrity.integrity;
await this.distRepository.saveDist(manifestDist, manifestBytes);
// FIXME: should store new size, shasum and integrity to database
}
async _updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests) {
const modified = new Date();
if (fullManifests) {
fullManifests.time.modified = modified;
// same to dist
const fullManifestsDistBytes = Buffer.from(JSON.stringify(fullManifests));
const fullManifestsDistIntegrity = await calculateIntegrity(fullManifestsDistBytes);
if (pkg.manifestsDist?.distId) {
pkg.manifestsDist.size = fullManifestsDistBytes.length;
pkg.manifestsDist.shasum = fullManifestsDistIntegrity.shasum;
pkg.manifestsDist.integrity = fullManifestsDistIntegrity.integrity;
}
else {
pkg.manifestsDist = pkg.createFullManifests({
size: fullManifestsDistBytes.length,
shasum: fullManifestsDistIntegrity.shasum,
integrity: fullManifestsDistIntegrity.integrity,
});
}
await this.distRepository.saveDist(pkg.manifestsDist, fullManifestsDistBytes);
await this.packageRepository.savePackageDist(pkg, true);
}
if (abbreviatedManifests) {
abbreviatedManifests.modified = modified;
const abbreviatedManifestsDistBytes = Buffer.from(JSON.stringify(abbreviatedManifests));
const abbreviatedManifestsDistIntegrity = await calculateIntegrity(abbreviatedManifestsDistBytes);
if (pkg.abbreviatedsDist?.distId) {
pkg.abbreviatedsDist.size = abbreviatedManifestsDistBytes.length;
pkg.abbreviatedsDist.shasum = abbreviatedManifestsDistIntegrity.shasum;
pkg.abbreviatedsDist.integrity = abbreviatedManifestsDistIntegrity.integrity;
}
else {
pkg.abbreviatedsDist = pkg.createAbbreviatedManifests({
size: abbreviatedManifestsDistBytes.length,
shasum: abbreviatedManifestsDistIntegrity.shasum,
integrity: abbreviatedManifestsDistIntegrity.integrity,
});
}
await this.distRepository.saveDist(pkg.abbreviatedsDist, abbreviatedManifestsDistBytes);
await this.packageRepository.savePackageDist(pkg, false);
}
}
async _updatePackageManifestsToDistsWithBuilder(pkg, fullManifestsBuilder, abbreviatedManifestsBuilder) {
const modified = new Date();
if (fullManifestsBuilder) {
fullManifestsBuilder.setIn(['time', 'modified'], modified);
// same to dist
const fullManifestsDistBytes = fullManifestsBuilder.build();
const fullManifestsDistIntegrity = await calculateIntegrity(fullManifestsDistBytes);
if (pkg.manifestsDist?.distId) {
pkg.manifestsDist.size = fullManifestsDistBytes.length;
pkg.manifestsDist.shasum = fullManifestsDistIntegrity.shasum;
pkg.manifestsDist.integrity = fullManifestsDistIntegrity.integrity;
}
else {
pkg.manifestsDist = pkg.createFullManifests({
size: fullManifestsDistBytes.length,
shasum: fullManifestsDistIntegrity.shasum,
integrity: fullManifestsDistIntegrity.integrity,
});
}
await this.distRepository.saveDist(pkg.manifestsDist, fullManifestsDistBytes);
await this.packageRepository.savePackageDist(pkg, true);
}
if (abbreviatedManifestsBuilder) {
abbreviatedManifestsBuilder.setIn(['modified'], modified);
const abbreviatedManifestsDistBytes = abbreviatedManifestsBuilder.build();
const abbreviatedManifestsDistIntegrity = await calculateIntegrity(abbreviatedManifestsDistBytes);
if (pkg.abbreviatedsDist?.distId) {
pkg.abbreviatedsDist.size = abbreviatedManifestsDistBytes.length;
pkg.abbreviatedsDist.shasum = abbreviatedManifestsDistIntegrity.shasum;
pkg.abbreviatedsDist.integrity = abbreviatedManifestsDistIntegrity.integrity;
}
else {
pkg.abbreviatedsDist = pkg.createAbbreviatedManifests({
size: abbreviatedManifestsDistBytes.length,
shasum: abbreviatedManifestsDistIntegrity.shasum,
integrity: abbreviatedManifestsDistIntegrity.integrity,
});
}
await this.distRepository.saveDist(pkg.abbreviatedsDist, abbreviatedManifestsDistBytes);
await this.packageRepository.savePackageDist(pkg, false);
}
}
async _listPackageFullOrAbbreviatedManifests(scope, name, isFullManifests, isSync) {
let