cnpmcore
Version:
Private NPM Registry for Enterprise
924 lines โข 164 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
import { rm } from 'node:fs/promises';
import os from 'node:os';
import { setTimeout } from 'node:timers/promises';
import { Package as Packument } from '@cnpmjs/packument';
import { AccessLevel, Inject, SingletonProto } from 'egg';
import { Pointcut } from 'egg/aop';
import { BadRequestError } from 'egg/errors';
import { isEmpty, isEqual } from 'lodash-es';
import semver from 'semver';
import { AbstractService } from "../../common/AbstractService.js";
import { PresetRegistryName, SyncDeleteMode } from "../../common/constants.js";
import { TaskState, TaskType } from "../../common/enum/Task.js";
import { downloadToTempfile } from "../../common/FileUtil.js";
import { detectInstallScript, getScopeAndName } from "../../common/PackageUtil.js";
import { DistRepository } from "../../repository/DistRepository.js";
import { Task } from "../entity/Task.js";
import { EventCorkAdvice } from "./EventCorkerAdvice.js";
import { PackageVersionFileService, UNPKG_WHITE_LIST_URL } from "./PackageVersionFileService.js";
function isoNow() {
return new Date().toISOString();
}
export class RegistryNotMatchError extends BadRequestError {
}
let PackageSyncerService = class PackageSyncerService extends AbstractService {
async createTask(fullname, options) {
const [scope, name] = getScopeAndName(fullname);
const pkg = await this.packageRepository.findPackage(scope, name);
// sync task request registry is not same as package registry
if (pkg && pkg.registryId && options?.registryId && pkg.registryId !== options.registryId) {
throw new RegistryNotMatchError(`package ${fullname} is not in registry ${options.registryId}`);
}
return await this.taskService.createTask(Task.createSyncPackage(fullname, options), true);
}
async findTask(taskId) {
return await this.taskService.findTask(taskId);
}
async findTaskLog(task) {
return await this.taskService.findTaskLog(task);
}
async findExecuteTask() {
return (await this.taskService.findExecuteTask(TaskType.SyncPackage));
}
get allowSyncDownloadData() {
const config = this.config.cnpmcore;
if (config.enableSyncDownloadData && config.syncDownloadDataSourceRegistry && config.syncDownloadDataMaxDate) {
return true;
}
return false;
}
async syncDownloadData(task, pkg) {
if (!this.allowSyncDownloadData) {
return;
}
const fullname = pkg.fullname;
const start = '2011-01-01';
const end = this.config.cnpmcore.syncDownloadDataMaxDate;
const registry = this.config.cnpmcore.syncDownloadDataSourceRegistry;
const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
const logs = [];
let downloads;
logs.push(`[${isoNow()}][DownloadData] ๐ง๐ง๐ง๐ง๐ง Syncing "${fullname}" download data "${start}:${end}" on ${registry} ๐ง๐ง๐ง๐ง๐ง`);
const failEnd = 'โโโโโ ๐ฎ give up ๐ฎ โโโโโ';
try {
const { data, status, res } = await this.npmRegistry.getDownloadRanges(registry, fullname, start, end, {
remoteAuthToken,
});
downloads = data.downloads || [];
logs.push(`[${isoNow()}][DownloadData] ๐ง HTTP [${status}] timing: ${JSON.stringify(res.timing)}, downloads: ${downloads.length}`);
}
catch (err) {
const status = err.status || 'unknown';
logs.push(`[${isoNow()}][DownloadData] โ Get download data error: ${err}, status: ${status}`);
logs.push(`[${isoNow()}][DownloadData] ${failEnd}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
return;
}
const datas = new Map();
for (const item of downloads) {
// {
// "day": "2021-09-21",
// "downloads": 45
// },
const day = item.day;
const [year, month, date] = day.split('-');
const yearMonth = Number.parseInt(`${year}${month}`);
if (!datas.has(yearMonth)) {
datas.set(yearMonth, []);
}
const counters = datas.get(yearMonth);
// oxlint-disable-next-line typescript-eslint/no-non-null-assertion
counters.push([date, item.downloads]);
}
for (const [yearMonth, counters] of datas.entries()) {
await this.packageVersionDownloadRepository.saveSyncDataByMonth(pkg.packageId, yearMonth, counters);
logs.push(`[${isoNow()}][DownloadData] ๐ข ${yearMonth}: ${counters.length} days`);
}
logs.push(`[${isoNow()}][DownloadData] ๐ข๐ข๐ข๐ข๐ข ${registry}/${fullname} ๐ข๐ข๐ข๐ข๐ข`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
}
async syncUpstream(task) {
const registry = this.npmRegistry.registry;
const fullname = task.targetName;
const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
let logs = [];
let logId = '';
logs.push(`[${isoNow()}][UP] ๐ง๐ง๐ง๐ง๐ง Waiting sync "${fullname}" task on ${registry} ๐ง๐ง๐ง๐ง๐ง`);
const failEnd = `โโโโโ Sync ${registry}/${fullname} ๐ฎ give up ๐ฎ โโโโโ`;
try {
const { data, status, res } = await this.npmRegistry.createSyncTask(fullname, { remoteAuthToken });
logs.push(`[${isoNow()}][UP] ๐ง HTTP [${status}] timing: ${JSON.stringify(res.timing)}, data: ${JSON.stringify(data)}`);
logId = data.logId;
}
catch (err) {
const status = err.status || 'unknown';
// ๅฏ่ฝไผๆๅบ AggregateError ๅผๅธธ
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AggregateError
logs.push(`[${isoNow()}][UP] โ Sync ${fullname} fail, create sync task error: ${err}, status: ${status} ${err instanceof AggregateError ? err.errors : ''}`);
logs.push(`[${isoNow()}][UP] ${failEnd}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
return;
}
if (!logId) {
logs.push(`[${isoNow()}][UP] โ Sync ${fullname} fail, missing logId`);
logs.push(`[${isoNow()}][UP] ${failEnd}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
return;
}
const startTime = Date.now();
const maxTimeout = this.config.cnpmcore.sourceRegistrySyncTimeout;
let logUrl = '';
let syncError = '';
let offset = 0;
let useTime = Date.now() - startTime;
while (useTime < maxTimeout) {
// sleep 1s ~ 6s in random
const delay = process.env.NODE_ENV === 'test' ? 100 : 1000 + Math.random() * 5000;
await setTimeout(delay);
try {
const { data, status, url } = await this.npmRegistry.getSyncTask(fullname, logId, offset, { remoteAuthToken });
useTime = Date.now() - startTime;
logUrl = data?.logUrl ?? url;
syncError = data?.error ?? '';
const log = data?.log ?? '';
offset += log.length;
if (data?.syncDone) {
// error
if (syncError) {
logs.push(`[${isoNow()}][UP] โ Sync ${fullname} fail [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
logs.push(`[${isoNow()}][UP] โ upstream error: ${syncError}`);
logs.push(`[${isoNow()}][UP] ${failEnd}`);
}
else {
logs.push(`[${isoNow()}][UP] ๐ Sync ${fullname} success [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
logs.push(`[${isoNow()}][UP] ๐ ${registry}/${fullname}`);
}
await this.taskService.appendTaskLog(task, logs.join('\n'));
return;
}
logs.push(`[${isoNow()}][UP] ๐ง HTTP [${status}] [${useTime}ms], offset: ${offset}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
}
catch (err) {
useTime = Date.now() - startTime;
const status = err.status || 'unknown';
logs.push(`[${isoNow()}][UP] ๐ง HTTP [${status}] [${useTime}ms] error: ${err}`);
}
}
// timeout
logs.push(`[${isoNow()}][UP] โ Sync ${fullname} fail, timeout [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
if (syncError) {
logs.push(`[${isoNow()}][UP] โ upstream error: ${syncError}`);
}
logs.push(`[${isoNow()}][UP] ${failEnd}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
}
isRemovedInRemote(maintainers, time, versions) {
if (maintainers && maintainers.length > 0) {
return false;
}
// unpublished
// https://registry.npmjs.com/babel-plugin-autocss
// {
// "_id": "babel-plugin-autocss",
// "name": "babel-plugin-autocss",
// "time": {
// "created": "2021-10-29T08:21:56.032Z",
// "0.0.1": "2021-10-29T08:21:56.206Z",
// "modified": "2022-01-14T12:34:23.941Z",
// "unpublished": {
// "time": "2022-01-14T12:34:23.941Z",
// "versions": [
// "0.0.1"
// ]
// }
// }
// }
if (time?.unpublished) {
return true;
}
if (!versions) {
return false;
}
// security holder
// test/fixtures/registry.npmjs.org/security-holding-package.json
// {
// "_id": "xxx",
// "_rev": "9-a740a77bcd978abeec47d2d027bf688c",
// "name": "xxx",
// "time": {
// "modified": "2017-11-28T00:45:24.162Z",
// "created": "2013-09-20T23:25:18.122Z",
// "0.0.0": "2013-09-20T23:25:20.242Z",
// "1.0.0": "2016-06-22T00:07:41.958Z",
// "0.0.1-security": "2016-12-15T01:03:58.663Z",
// "unpublished": {
// "time": "2017-11-28T00:45:24.163Z",
// "versions": []
// }
// },
// "_attachments": {}
// }
let isSecurityHolder = true;
for (const versionInfo of Object.entries(versions)) {
const [v, info] = versionInfo;
// >=0.0.1-security <0.0.2-0
const isSecurityVersion = semver.satisfies(v, '^0.0.1-security');
const isNpmUser = info?._npmUser?.name === 'npm';
if (!isSecurityVersion || !isNpmUser) {
isSecurityHolder = false;
break;
}
}
return isSecurityHolder;
}
// sync deleted package, deps on the syncDeleteMode
// - ignore: do nothing, just finish the task
// - delete: remove the package from local registry
// - block: block the package, update the manifest.block, instead of delete versions
// ๆ นๆฎ syncDeleteMode ้
็ฝฎ๏ผๅค็ๅ ๅ
ๅบๆฏ
// - ignore: ไธๅไปปไฝๅค็๏ผ็ดๆฅ็ปๆไปปๅก
// - delete: ๅ ้คๅ
ๆฐๆฎ๏ผๅ
ๆฌ manifest ๅญๅจ
// - block: ่ฝฏๅ ้ค ๅฐๅ
ๆ ่ฎฐไธบ block๏ผ็จๆทๆ ๆณ็ดๆฅไฝฟ็จ
async syncDeletePkg({ task, pkg, logUrl, url, logs, data }) {
const fullname = task.targetName;
const failEnd = `โโโโโ ${url || fullname} โโโโโ`;
const syncDeleteMode = this.config.cnpmcore.syncDeleteMode;
const dataString = data.toString().substring(0, 1024);
logs.push(`[${isoNow()}] ๐ข Package "${fullname}" was removed in remote registry, response data: ${dataString}, config.syncDeleteMode = ${syncDeleteMode}`);
// pkg not exists in local registry
if (!pkg) {
task.error = `Package not exists, response data: ${dataString}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ${failEnd}`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
return;
}
if (syncDeleteMode === SyncDeleteMode.ignore) {
// ignore deleted package
logs.push(`[${isoNow()}] ๐ข Skip remove since config.syncDeleteMode = ignore`);
}
else if (syncDeleteMode === SyncDeleteMode.block) {
// block deleted package
await this.packageManagerService.blockPackage(pkg, 'Removed in remote registry');
logs.push(`[${isoNow()}] ๐ข Block the package since config.syncDeleteMode = block`);
}
else if (syncDeleteMode === SyncDeleteMode.delete) {
// delete package
await this.packageManagerService.unpublishPackage(pkg);
logs.push(`[${isoNow()}] ๐ข Delete the package since config.syncDeleteMode = delete`);
}
// update log
logs.push(`[${isoNow()}] ๐ Log URL: ${logUrl}`);
logs.push(`[${isoNow()}] ๐ ${url}`);
await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s', task.taskId, task.targetName);
}
// ๅๅงๅๅฏนๅบ็ Registry
// 1. ไผๅ
ไป pkg.registryId ่ทๅ (registryId ไธ็ป่ฎพ็ฝฎ ไธๅบๆนๅ)
// 1. ๅ
ถๆฌกไป task.data.registryId (ๅๅปบๅๅ
ๅๆญฅไปปๅกๆถไผ ๅ
ฅ)
// 2. ๆฅ็ๆ นๆฎ scope ่ฟ่ก่ฎก็ฎ (ไฝไธบๅญๅ
ไพ่ตๅๆญฅๆถๅ๏ผๆ registryId)
// 3. ๆๅ่ฟๅ default registryId (ๅฏ่ฝ default registry ไนไธๅญๅจ)
async initSpecRegistry(task, pkg = null, scope) {
const registryId = pkg?.registryId || task.data.registryId;
let targetHost = this.config.cnpmcore.sourceRegistry;
let registry = null;
// ๅฝๅไปปๅกไฝไธบ deps ๅผๅ
ฅๆถ๏ผไธไผ้
็ฝฎ registryId
// ๅๅฒ Task ๅฏ่ฝๆฒกๆ้
็ฝฎ registryId
if (registryId) {
registry = await this.registryManagerService.findByRegistryId(registryId);
}
else if (scope) {
const scopeModel = await this.scopeManagerService.findByName(scope);
if (scopeModel?.registryId) {
registry = await this.registryManagerService.findByRegistryId(scopeModel?.registryId);
}
}
// ้็จ้ป่ฎค็ registry
if (!registry) {
registry = await this.registryManagerService.ensureDefaultRegistry();
}
// ๆดๆฐ targetHost ๅฐๅ
// defaultRegistry ๅฏ่ฝ่ฟๆชๅๅปบ
if (registry.host) {
targetHost = registry.host;
}
this.npmRegistry.setRegistryHost(targetHost);
return registry;
}
// ็ฑไบ cnpmcore ๅฐ version ๅ tag ไฝไธบไธคไธช็ฌ็ซ็ changes ไบไปถๅๅ
// ๆฎ้็ๆฌๅๅธๆถ๏ผ็ญๆถ้ดๅ
ไผๆไธคๆก็ธๅ task ่ฟ่กๅๆญฅ
// ๅฐฝ้ไฟ่ฏ่ฏปๅๅๅๅ
ฅ้ฝ้ไฟ่ฏไปปๅกๅน็ญ๏ผ้่ฆ็กฎไฟ changes ๅจๅๆญฅไปปๅกๅฎๆๅๅ่งฆๅ
// ้่ฟ DB ๅฏไธ็ดขๅผๆฅไฟ่ฏไปปๅกๅน็ญ๏ผๆๅ
ฅๅคฑ่ดฅไธๅฝฑๅ pkg.manifests ๆดๆฐ
// ้่ฟ eventBus.cork/uncork ๆฅๆ็ผไบไปถ่งฆๅ
async executeTask(task) {
const fullname = task.targetName;
const [scope, name] = getScopeAndName(fullname);
const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, specificVersions, } = task.data;
let pkg = await this.packageRepository.findPackage(scope, name);
const registry = await this.initSpecRegistry(task, pkg, scope);
const registryHost = this.npmRegistry.registry;
const remoteAuthToken = registry.authToken;
let logs = [];
if (tips) {
logs.push(`[${isoNow()}] ๐๐๐๐๐ Tips: ${tips} ๐๐๐๐๐`);
}
const taskQueueLength = await this.taskService.getTaskQueueLength(task.type);
const taskQueueHighWaterSize = this.config.cnpmcore.taskQueueHighWaterSize;
const taskQueueInHighWaterState = taskQueueLength >= taskQueueHighWaterSize;
const skipDependencies = taskQueueInHighWaterState ? true : !!originSkipDependencies;
const syncUpstream = !!(!taskQueueInHighWaterState &&
this.config.cnpmcore.sourceRegistryIsCNpm &&
this.config.cnpmcore.syncUpstreamFirst &&
registry.name === PresetRegistryName.default);
const logUrl = `${this.config.cnpmcore.registry}/-/package/${fullname}/syncs/${task.taskId}/log`;
this.logger.info('[PackageSyncerService.executeTask:start] taskId: %s, targetName: %s, attempts: %s, taskQueue: %s/%s, syncUpstream: %s, log: %s', task.taskId, task.targetName, task.attempts, taskQueueLength, taskQueueHighWaterSize, syncUpstream, logUrl);
logs.push(`[${isoNow()}] ๐ง๐ง๐ง๐ง๐ง Syncing from ${registryHost}/${fullname}, \
skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, \
forceSyncHistory: ${!!forceSyncHistory}, attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", \
taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} ๐ง๐ง๐ง๐ง๐ง`);
if (specificVersions) {
logs.push(`[${isoNow()}] ๐ syncing specific versions: ${specificVersions.join(' | ')} ๐`);
}
logs.push(`[${isoNow()}] ๐ง log: ${logUrl}`);
if (registry.name === PresetRegistryName.self) {
logs.push(`[${isoNow()}] โโโโโ ${fullname} has been published to the self registry, skip sync โโโโโ`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, invalid registryId', task.taskId, task.targetName);
return;
}
if (pkg && pkg.registryId !== registry.registryId) {
if (pkg.registryId) {
logs.push(`[${isoNow()}] โโโโโ ${fullname} registry is ${pkg.registryId} not belong to ${registry.registryId}, skip sync โโโโโ`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, invalid registryId', task.taskId, task.targetName);
return;
}
// ๅคๅๆญฅๆบไนๅๆฒกๆ registryId
// publish() ็ๆฌไธๅๆถ๏ผไธไผๆดๆฐ registryId
// ๅจๅๆญฅๅ๏ผ่ฟ่กๆดๆฐๆไฝ
pkg.registryId = registry.registryId;
await this.packageRepository.savePackage(pkg);
}
if (syncDownloadData && pkg) {
await this.syncDownloadData(task, pkg);
logs.push(`[${isoNow()}] ๐ข log: ${logUrl}`);
logs.push(`[${isoNow()}] ๐ข๐ข๐ข๐ข๐ข Sync "${fullname}" download data success ๐ข๐ข๐ข๐ข๐ข`);
await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s', task.taskId, task.targetName);
return;
}
if (syncUpstream) {
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
// create sync task on sourceRegistry and skipDependencies = true
await this.syncUpstream(task);
}
if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) {
task.error = `stop sync by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-block-list] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
return;
}
if (await this.packageVersionFileService.isPackageBlockedToSync(scope, name)) {
task.error = `stop sync by block list, see ${UNPKG_WHITE_LIST_URL}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-package-blocked-to-sync] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
return;
}
let registryFetchResult;
try {
registryFetchResult = await this.npmRegistry.getFullManifestsBuffer(fullname, {
remoteAuthToken,
});
}
catch (err) {
const status = err.status || 'unknown';
task.error = `request manifests error: ${err}, status: ${status}`;
logs.push(`[${isoNow()}] โ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
this.logger.info('[PackageSyncerService.executeTask:fail-request-error] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.retryTask(task, logs.join('\n'));
return;
}
const { url: remoteUrl, data: remoteData, headers, res, status } = registryFetchResult;
if (status >= 500) {
// GET https://registry.npmjs.org/%40modern-js%2Fstyle-compiler?t=1683348626499&cache=0, status: 522
// registry will response status 522 and data will be null
// > TypeError: Cannot read properties of null (reading 'readme')
task.error = `request manifests response error, status: ${status}, data size: ${remoteData.length}, \
data sample: ${remoteData.subarray(0, 200).toString()}`;
logs.push(`[${isoNow()}] โ response headers: ${JSON.stringify(headers)}`);
logs.push(`[${isoNow()}] โ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
this.logger.info('[PackageSyncerService.executeTask:fail-request-error] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.retryTask(task, logs.join('\n'));
return;
}
if (status === 404) {
// ignore 404 status
// https://github.com/cnpm/cnpmcore/issues/739
task.error = `Package not found, status 404, data size: ${remoteData.length}, data sample: ${remoteData.subarray(0, 200).toString()}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
this.logger.info('[PackageSyncerService.executeTask:fail-request-error] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
return;
}
// deleted or blocked
if (status === 451) {
await this.syncDeletePkg({ task, pkg, logs, logUrl, url: remoteUrl, data: remoteData });
return;
}
logs.push(`[${isoNow()}] HTTP [${status}] body size: ${remoteData.length}, timing: ${JSON.stringify(res.timing)}`);
if (this.config.cnpmcore.experimental.syncPackageWithPackument && !this.config.cnpmcore.strictSyncSpecivicVersion) {
await this.syncPackageWithPackument({ task, remoteData, pkg, registry, logUrl, remoteUrl, logs });
return;
}
const startTime = Date.now();
let data;
try {
// @ts-expect-error JSON.parse accepts Buffer in Node.js, though TypeScript types don't reflect this
data = JSON.parse(remoteData);
}
catch (err) {
task.error = `parse manifest error: ${err}, data: ${remoteData.toString().substring(0, 1024)}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
this.logger.info('[PackageSyncerService.executeTask:fail-parse-manifest] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
return;
}
const useTime = Date.now() - startTime;
logs.push(`[${isoNow()}] ๐ Manifest parse use ${useTime}ms`);
let readme = data.readme || '';
if (typeof readme !== 'string') {
readme = JSON.stringify(readme);
}
// "time": {
// "created": "2021-03-27T12:30:23.891Z",
// "0.0.2": "2021-03-27T12:30:24.349Z",
// "modified": "2021-12-08T14:59:57.264Z",
const timeMap = (data.time ?? {});
const failEnd = `โโโโโ ${remoteUrl || fullname} โโโโโ`;
if (this.isRemovedInRemote(data.maintainers, timeMap, data.versions)) {
await this.syncDeletePkg({ task, pkg, logs, logUrl, url: remoteUrl, data: remoteData });
return;
}
const versionMap = (data.versions || {});
const distTags = data['dist-tags'] || {};
// show latest information
if (distTags.latest) {
logs.push(`[${isoNow()}] ๐ ${fullname} latest version: ${distTags.latest}, published time: ${timeMap[distTags.latest]}`);
}
// 1. save maintainers
// maintainers: [
// { name: 'foo', email: 'foo@gmail.com' },
// { name: 'bar', email: 'bar.laster.11@gmail.com' }
// ],
let maintainers = data.maintainers;
const maintainersMap = {};
const users = [];
let changedUserCount = 0;
if (!Array.isArray(maintainers) || maintainers.length === 0) {
// https://r.cnpmjs.org/webpack.js.org/sync/log/61dbc7c8ff747911a5701068
// https://registry.npmjs.org/webpack.js.org
// security holding package will not contains maintainers, auto set npm and npm@npmjs.com to maintainer
// "description": "security holding package",
// "repository": "npm/security-holder"
if (data.description === 'security holding package' || data.repository === 'npm/security-holder') {
data.maintainers = [{ name: 'npm', email: 'npm@npmjs.com' }];
maintainers = data.maintainers;
}
else {
// try to use latest tag version's maintainers instead
const latestPackageVersion = distTags.latest ? versionMap[distTags.latest] : undefined;
if (latestPackageVersion &&
Array.isArray(latestPackageVersion.maintainers) &&
latestPackageVersion.maintainers.length > 0) {
maintainers = latestPackageVersion.maintainers;
logs.push(`[${isoNow()}] ๐ Use the latest version(${latestPackageVersion.version}) maintainers instead`);
}
else if (latestPackageVersion?._npmUser?.name && latestPackageVersion._npmUser.email) {
// Fallback to _npmUser for OIDC-published packages (e.g., via GitHub Actions)
// These packages have empty maintainers but include _npmUser with publisher info
// https://github.com/cnpm/cnpm/pull/489
maintainers = [{ name: latestPackageVersion._npmUser.name, email: latestPackageVersion._npmUser.email }];
logs.push(`[${isoNow()}] ๐ Use _npmUser from version ${latestPackageVersion.version} as maintainer (${latestPackageVersion._npmUser.name})`);
}
}
}
if (Array.isArray(maintainers) && maintainers.length > 0) {
logs.push(`[${isoNow()}] ๐ง Syncing maintainers: ${JSON.stringify(maintainers)}`);
for (const maintainer of maintainers) {
if (maintainer.name && maintainer.email) {
maintainersMap[maintainer.name] = maintainer;
const { changed, user } = await this.userService.saveUser(maintainer.name, maintainer.email, registry.userPrefix);
users.push(user);
if (changed) {
changedUserCount++;
logs.push(`[${isoNow()}] ๐ข [${changedUserCount}] Synced ${maintainer.name} => ${user.name}(${user.userId})`);
}
}
}
}
if (users.length === 0) {
// check unpublished
// https://r.cnpmjs.org/-/package/babel-plugin-autocss/syncs/61e4be46c7cbfac94d2ec597/log
// {
// "name": "babel-plugin-autocss",
// "time": {
// "created": "2021-10-29T08:21:56.032Z",
// "0.0.1": "2021-10-29T08:21:56.206Z",
// "modified": "2022-01-14T12:34:23.941Z",
// "unpublished": {
// "time": "2022-01-14T12:34:23.941Z",
// "versions": [
// "0.0.1"
// ]
// }
// }
// }
// invalid maintainers, sync fail
task.error = `invalid maintainers: ${JSON.stringify(maintainers)}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ${failEnd}`);
this.logger.info('[PackageSyncerService.executeTask:fail-invalid-maintainers] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
return;
}
let lastErrorMessage = '';
const dependenciesSet = new Set();
const { data: existsData } = await this.packageManagerService.listPackageFullManifests(scope, name);
const { data: abbreviatedManifests } = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name);
const existsVersionMap = existsData?.versions ?? {};
const existsVersionCount = Object.keys(existsVersionMap).length;
const abbreviatedVersionMap = abbreviatedManifests?.versions ?? {};
// 2. save versions
if (specificVersions &&
!this.config.cnpmcore.strictSyncSpecivicVersion &&
!specificVersions.includes(distTags.latest)) {
logs.push(`[${isoNow()}] ๐ฆ Add latest tag version "${fullname}: ${distTags.latest}"`);
specificVersions.push(distTags.latest);
}
// Get the list of versions to sync this time
const versions = specificVersions
? Object.values(versionMap).filter((verItem) => specificVersions.includes(verItem.version))
: Object.values(versionMap);
// ๅ
จ้ๅๆญฅๆถ่ทณ่ฟๆๅบ
const sortedAvailableVersions = specificVersions ? versions.map((item) => item.version).sort(semver.rcompare) : [];
// ๅจstrictSyncSpecivicVersionๆจกๅผไธ๏ผไธๅๆญฅlatest๏ผไธๆๆไผ ๅ
ฅ็versionๅไธๅฏ็จ
if (specificVersions && sortedAvailableVersions.length === 0) {
logs.push(`[${isoNow()}] โ `);
task.error = 'There is no available specific versions, stop task.';
logs.push(`[${isoNow()}] ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
this.logger.info('[PackageSyncerService.executeTask:fail-empty-list] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
return;
}
if (specificVersions) {
// specific versions may not in manifest.
const notAvailableVersionList = specificVersions.filter((i) => !sortedAvailableVersions.includes(i));
logs.push(`[${isoNow()}] ๐ง Syncing specific versions: ${sortedAvailableVersions.join(' | ')}`);
if (notAvailableVersionList.length > 0) {
logs.push(`๐ง Some specific versions are not available: ๐ ${notAvailableVersionList.join(' | ')} ๐`);
}
}
else {
logs.push(`[${isoNow()}] ๐ง Syncing versions ${existsVersionCount} => ${versions.length}`);
}
const updateVersions = [];
const differentMetas = [];
let syncIndex = 0;
let largeVersionCount = 0;
for (const item of versions) {
const version = item.version;
// Skip empty versions, handle abnormal data
if (!version)
continue;
let existsItem = existsVersionMap[version];
let existsAbbreviatedItem = abbreviatedVersionMap[version];
const shouldDeleteReadme = !!(existsItem && 'readme' in existsItem);
if (pkg) {
if (existsItem && !existsAbbreviatedItem) {
// check item on AbbreviatedManifests
updateVersions.push(version);
logs.push(`[${isoNow()}] ๐ Remote version ${version} not exists on local abbreviated manifests, need to refresh`);
}
if (existsItem && forceSyncHistory === true) {
const pkgVer = await this.packageRepository.findPackageVersion(pkg.packageId, version);
if (pkgVer) {
logs.push(`[${isoNow()}] ๐ง [${syncIndex}] Remove version ${version} for force sync history`);
await this.packageManagerService.removePackageVersion(pkg, pkgVer, true);
existsItem = undefined;
existsAbbreviatedItem = undefined;
existsVersionMap[version] = undefined;
abbreviatedVersionMap[version] = undefined;
}
}
}
if (existsItem) {
// check metaDataKeys, if different value, override exists one
// https://github.com/cnpm/cnpmjs.org/issues/1667
// need libc field https://github.com/cnpm/cnpmcore/issues/187
// fix _npmUser field since https://github.com/cnpm/cnpmcore/issues/553
const metaDataKeys = [
'peerDependenciesMeta',
'os',
'cpu',
'libc',
'workspaces',
'hasInstallScript',
'deprecated',
'_npmUser',
'funding',
// https://github.com/cnpm/cnpmcore/issues/689
'acceptDependencies',
];
const ignoreInAbbreviated = new Set(['_npmUser']);
const diffMeta = {};
for (const key of metaDataKeys) {
let remoteItemValue = item[key];
// make sure hasInstallScript exists
if (key === 'hasInstallScript' && remoteItemValue === undefined && detectInstallScript(item)) {
remoteItemValue = true;
}
if (!isEqual(remoteItemValue, existsItem[key])) {
diffMeta[key] = remoteItemValue;
}
else if (!ignoreInAbbreviated.has(key) &&
existsAbbreviatedItem &&
!isEqual(remoteItemValue, existsAbbreviatedItem[key])) {
// should diff exists abbreviated item too
diffMeta[key] = remoteItemValue;
}
}
// should delete readme
if (shouldDeleteReadme) {
diffMeta.readme = undefined;
}
if (!isEmpty(diffMeta)) {
// Differences found, need to sync the changed metadata
differentMetas.push([existsItem, diffMeta]);
}
// Skip versions that have already been synced
// Avoid duplicate syncing
continue;
}
// New version found, start syncing
syncIndex++;
const description = item.description;
// "dist": {
// "shasum": "943e0ec03df00ebeb6273a5b94b916ba54b47581",
// "tarball": "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz"
// },
const dist = item.dist;
const tarball = dist?.tarball;
if (!tarball) {
lastErrorMessage = `missing tarball, dist: ${JSON.stringify(dist)}`;
logs.push(`[${isoNow()}] โ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
continue;
}
const size = dist.size ?? dist.unpackedSize;
if (size && size > this.config.cnpmcore.largePackageVersionSize) {
const allowed = await this.packageVersionFileService.isLargePackageVersionAllowed(scope, name, version);
const whiteListVersion = this.packageVersionFileService.unpkgWhiteListVersion;
if (!allowed) {
largeVersionCount++;
if (largeVersionCount > this.config.cnpmcore.largePackageVersionBlockThreshold) {
task.error = `Synced version ${version} fail, too many large versions (${largeVersionCount}), large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] โ ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] โโโโโ ${fullname} โโโโโ`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
return;
}
lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}, white list version: ${whiteListVersion}`;
logs.push(`[${isoNow()}] โ ๏ธ [${syncIndex}] Synced version ${version} skipped, ${lastErrorMessage}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
continue;
}
logs.push(`[${isoNow()}] ๐ง [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list, white list version: ${whiteListVersion}`);
}
const publishTimeISO = timeMap[version];
const publishTime = publishTimeISO ? new Date(publishTimeISO) : new Date();
const delay = Date.now() - publishTime.getTime();
logs.push(`[${isoNow()}] ๐ง [${syncIndex}] Syncing version ${version}, delay: ${delay}ms [${publishTimeISO}], tarball: ${tarball}, size: ${size}`);
let localFile;
try {
const { tmpfile, headers, timing } = await downloadToTempfile(this.httpClient, this.config.dataDir, tarball, {
remoteAuthToken,
});
localFile = tmpfile;
logs.push(`[${isoNow()}] ๐ง [${syncIndex}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)} => ${localFile}`);
}
catch (err) {
if (err.name === 'DownloadNotFoundError' || err.name === 'DownloadStatusInvalidError') {
this.logger.warn('Download tarball %s error: %s', tarball, err);
}
else {
this.logger.error('Download tarball %s error: %s', tarball, err);
}
lastErrorMessage = `download tarball error: ${err}`;
logs.push(`[${isoNow()}] โ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
continue;
}
if (!pkg) {
pkg = await this.packageRepository.findPackage(scope, name);
}
const publishCmd = {
scope,
name,
version,
description,
packageJson: item,
readme,
registryId: registry.registryId,
dist: {
localFile,
},
isPrivate: false,
publishTime,
skipRefreshPackageManifests: true,
};
try {
// ๅฝ version ่ฎฐๅฝๅทฒ็ปๅญๅจๆถ๏ผ่ฟ้่ฆๆ ก้ชไธไธ pkg.manifests ๆฏๅฆๅญๅจ
const publisher = users.find((user) => user.displayName === item._npmUser?.name) || users[0];
const pkgVersion = await this.packageManagerService.publish(publishCmd, publisher);
updateVersions.push(pkgVersion.version);
logs.push(`[${isoNow()}] ๐ [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
}
catch (err) {
if (err.name === 'ForbiddenError') {
logs.push(`[${isoNow()}] ๐ [${syncIndex}] Synced version ${version} already exists, skip publish, try to set in local manifest`);
// ๅฆๆ pkg.manifests ไธๅญๅจ๏ผ้่ฆ่กฅๅ
ไธไธ
updateVersions.push(version);
}
else {
err.taskId = task.taskId;
this.logger.error(err);
lastErrorMessage = `publish error: ${err}`;
logs.push(`[${isoNow()}] โ [${syncIndex}] Synced version ${version} error, ${lastErrorMessage}`);
if (err.name === 'BadRequestError') {
// ็ฑไบๅฝๅ็ๆฌ็ไพ่ตไธๆปก่ถณ๏ผๅฐ่ฏ้่ฏ
// ้ป่ฎคไผๅจๅฝๅ้ๅๆๅ้่ฏ
this.logger.info('[PackageSyncerService.executeTask:fail-validate-deps] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error);
await this.taskService.retryTask(task, logs.join('\n'));
return;
}
}
}
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
await rm(localFile, { force: true });
if (!skipDependencies) {
const dependencies = item.dependencies || {};
for (const dependencyName in dependencies) {
dependenciesSet.add(dependencyName);
}
const optionalDependencies = item.optionalDependencies || {};
for (const dependencyName in optionalDependencies) {
dependenciesSet.add(dependencyName);
}
}
}
// try to read package entity again after first sync
if (!pkg) {
pkg = await this.packageRepository.findPackage(scope, name);
}
if (!pkg || !pkg.id) {
// sync all versions fail in the first time
logs.push(`[${isoNow()}] โ All versions sync fail, package not exists, log: ${logUrl}`);
logs.push(`[${isoNow()}] ${failEnd}`);
task.error = lastErrorMessage;
this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, package not exists', task.taskId, task.targetName);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
return;
}
// 2.1 save differentMetas
for (const [existsItem, diffMeta] of differentMetas) {
const pkgVersion = await this.packageRepository.findPackageVersion(pkg.packageId, existsItem.version);
if (pkgVersion) {
await this.packageManagerService.savePackageVersionManifest(pkgVersion, diffMeta, diffMeta);
updateVersions.push(pkgVersion.version);
let diffMetaInfo = JSON.stringify(diffMeta);
if ('readme' in diffMeta) {
diffMetaInfo += ', delete exists readme';
}
logs.push(`[${isoNow()}] ๐ข Synced version ${existsItem.version} success, different meta: ${diffMetaInfo}`);
}
}
const removeVersions = [];
// 2.3 find out remove versions
for (const existsVersion in existsVersionMap) {
if (!(existsVersion in versionMap)) {
const pkgVersion = await this.packageRepository.findPackageVersion(pkg.packageId, existsVersion);
if (pkgVersion) {
await this.packageManagerService.removePackageVersion(pkg, pkgVersion, true);
logs.push(`[${isoNow()}] ๐ข Removed version ${existsVersion} success`);
}
removeVersions.push(existsVersion);
}
}
logs.push(`[${isoNow()}] ๐ข Synced updated ${updateVersions.length} versions, removed ${removeVersions.length} versions`);
if (updateVersions.length > 0 || removeVersions.length > 0) {
logs.push(`[${isoNow()}] ๐ง Refreshing manifests to dists ......`);
const start = Date.now();
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
await this.packageManagerService.refreshPackageChangeVersionsToDists(pkg, updateVersions, removeVersions);
logs.push(`[${isoNow()}] ๐ข Refresh use ${Date.now() - start}ms`);
}
// 3. update tags
// "dist-tags": {
// "latest": "0.0.7"
// },
const changedTags = [];
const existsDistTags = (existsData && existsData['dist-tags']) || {};
let shouldRefreshDistTags = false;
for (const tag in distTags) {
const version = distTags[tag];
const utf8mb3Regex = /[\u0020-\uD7FF\uE000-\uFFFD]/;
if (!utf8mb3Regex.test(tag)) {
logs.push(`[${isoNow()}] ๐ง invalid tag(${tag}: ${version}), tag name is out of utf8mb3, skip`);
continue;
}
// ๆฐ tag ๆๅ็็ๆฌๆขไธๅจๅญ้ๆฐๆฎ้๏ผไนไธๅจๆฌๆฌกๅๆญฅ็ๆฌๅ่กจ้
// ไพๅฆ latest ๅฏนๅบ็ version ๅๅ
ฅๅคฑ่ดฅ่ทณ่ฟ
if (!existsVersionMap[version] && !updateVersions.includes(version)) {
logs.push(`[${isoNow()}] ๐ง invalid tag(${tag}: ${version}), version is not exists, skip`);
continue;
}
const changed = await this.packageManagerService.savePackageTag(pkg, tag, version);
if (changed) {
changedTags.push({ action: 'change', tag, version });
shouldRefreshDistTags = false;
}
else if (version !== existsDistTags[tag]) {
shouldRefreshDistTags = true;
logs.push(`[${isoNow()}] ๐ง Remote tag(${tag}: ${version}) not exists in local dist-tags`);
}
}
// 3.1 find out remove tags
for (const tag in existsDistTags) {
if (!(tag in distTags)) {
const changed = await this.packageManagerService.removePackageTag(pkg, tag);
if (changed) {
changedTags.push({ action: 'remove', tag });
shouldRefreshDistTags = false;
}
}
}
// 3.2 should add latest tag
// ๅจๅๆญฅ specific version ๆถๅฆๆๆฒกๆๅๆญฅ latestTag ็็ๆฌไผๅบ็ฐ latestTag ไธขๅคฑๆๆๅ็ๆฌไธๆญฃ็กฎ็ๆ
ๅต
if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
// ไธๅ
่ฎธ่ชๅจๅๆญฅ latest ็ๆฌ๏ผไปๅทฒๅๆญฅ็ๆฌไธญ้ๅบ latest
let latestStableVersion = semver.maxSatisfying(sortedAvailableVersions, '*');
// ๆๆ็ๆฌ้ฝไธๆฏ็จณๅฎ็ๆฌๅๆๅ้็จณๅฎ็ๆฌไฟ่ฏ latest ๅญๅจ
if (!latestStableVersion) {
latestStableVersion = sortedAvailableVersions[0];
}
if (!existsDistTags.latest || semver.rcompare(existsDistTags.latest, latestStableVersion) === 1) {
logs.push(`[${isoNow()}] ๐ง patch latest tag from specific versions ๐ง`);
changedTags.push({
action: 'change',
tag: 'latest',
version: latestStableVersion,
});
await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);
}
}
if (changedTags.length > 0) {
logs.push(`[${isoNow()}] ๐ข Synced ${changedTags.length} tags: ${JSON.stringify(changedTags)}`);
}
if (shouldRefreshDistTags) {
await this.packageManagerService.refreshPackageDistTagsToDists(pkg);
logs.push(`[${isoNow()}] ๐ข Refresh dist-tags`);
}
// 4. add package maintainers
await this.packageManagerService.savePackageMaintainers(pkg, users);
// 4.1 find out remove maintainers
const removedMaintainers = [];
const existsMaintainers = (exists