sync-npm-packages
Version:
Sync released npm packages to mirror registries automatically.
462 lines (461 loc) • 15.4 kB
JavaScript
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import process from "node:process";
import { glob } from "tinyglobby";
import c from "tinyrainbow";
import { request } from "node:https";
import { createConfigLoader } from "unconfig";
//#region node_modules/.pnpm/@ntnyq+utils@0.13.3/node_modules/@ntnyq/utils/dist/index.js
/**
* Returns a new array with unique values.
* @param array - The array to process.
* @returns The new array.
* @example
*
* ```typescript
* import { unique } from '@ntnyq/utils'
*
* const result = unique([1, 1, 2, 3, 3])
* console.log(result) // => [1, 2, 3]
* ```
*
*/
function unique(array) {
return Array.from(new Set(array));
}
/**
* Converts a value to an array.
* @param array - The value to convert.
* @returns The array.
* @example
*
* ```typescript
* import { toArray } from '@ntnyq/utils'
*
* const result = toArray('hello')
* console.log(result) // => ['hello']
* ```
*
*/
function toArray(array) {
array ??= [];
return Array.isArray(array) ? array : [array];
}
//#endregion
//#region src/transport.ts
const DEFAULT_REQUEST_TIMEOUT = 1e4;
const DEFAULT_REGISTRY_HOST = "registry.npmmirror.com";
/**
* Check whether an IPv4 hostname belongs to private/local ranges.
* @param hostname - hostname in IPv4 format
* @returns true when host is private or local IPv4
*/
function isPrivateIpv4(hostname) {
const parts = hostname.split(".").map(Number);
if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) return false;
const [a, b] = parts;
return a === 0 || a === 10 || a === 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168;
}
/**
* Check whether an IPv6 hostname belongs to private/local ranges.
* @param hostname - hostname in IPv6 format
* @returns true when host is private or local IPv6
*/
function isPrivateIpv6(hostname) {
if (!hostname.includes(":")) return false;
const normalized = hostname.toLowerCase();
return normalized === "::1" || normalized.startsWith("fe80:") || normalized.startsWith("fc") || normalized.startsWith("fd");
}
/**
* Validate whether a host is unsafe for outbound registry requests.
* @param hostname - normalized hostname
* @returns true when hostname points to localhost/private network
*/
function isUnsafeHost(hostname) {
const normalized = hostname.toLowerCase();
return normalized === "localhost" || normalized.endsWith(".localhost") || isPrivateIpv4(normalized) || isPrivateIpv6(normalized);
}
/**
* Normalize and validate registry host for outbound requests.
* @param registry - registry host or HTTPS URL
*/
function normalizeRegistryHost(registry) {
const rawRegistry = (registry ?? DEFAULT_REGISTRY_HOST).trim();
if (!rawRegistry) throw new Error("Registry host cannot be empty");
const url = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//u.test(rawRegistry) ? new URL(rawRegistry) : new URL(`https://${rawRegistry}`);
if (url.protocol !== "https:") throw new Error("Registry must use https protocol");
if (url.username || url.password) throw new Error("Registry must not include credentials");
if (url.pathname && url.pathname !== "/") throw new Error("Registry must not include path");
if (url.search || url.hash) throw new Error("Registry must not include query or hash");
const host = url.hostname.toLowerCase();
if (!host) throw new Error("Registry host cannot be empty");
if (isUnsafeHost(host)) throw new Error("Registry host must be a public host");
return host;
}
/**
* Build the npmmirror sync path and safely encode package names.
* @param packageName - npm package name
*/
function buildSyncRequestPath(packageName) {
return `/-/package/${encodeURIComponent(packageName)}/syncs`;
}
/**
* Build sync path for custom target.
* @param packageName - npm package name
* @param syncPathTemplate - custom sync path template
* @returns sync path for custom registry
*/
function buildCustomSyncRequestPath(packageName, syncPathTemplate) {
if (!syncPathTemplate) throw new Error("syncPathTemplate is required when target is custom and must contain {packageName}");
if (!syncPathTemplate.includes("{packageName}")) throw new Error("syncPathTemplate must include {packageName} placeholder");
if (!syncPathTemplate.startsWith("/")) throw new Error("syncPathTemplate must start with /");
const encodedPackageName = encodeURIComponent(packageName);
return syncPathTemplate.replaceAll("{packageName}", encodedPackageName);
}
/**
* Resolve request method and path for the configured target.
* @param packageName - npm package name
* @param options - sync options
* @returns request method and path
*/
function resolveSyncRequest(packageName, options) {
const method = options.syncMethod ?? "PUT";
if (options.target === "custom") return {
method,
path: buildCustomSyncRequestPath(packageName, options.syncPathTemplate)
};
return {
method,
path: buildSyncRequestPath(packageName)
};
}
/**
* Sync package to npm mirror-compatible registry.
* @param packageName - npm package name
* @param options - sync options
*/
async function syncPackageToRegistry(packageName, options) {
const timeout = options.timeout ?? DEFAULT_REQUEST_TIMEOUT;
const registryHost = normalizeRegistryHost(options.registry);
const { method, path } = resolveSyncRequest(packageName, options);
return new Promise((resolve, reject) => {
const req = request({
method,
path,
host: registryHost,
protocol: "https:",
headers: { "content-length": 0 },
timeout
}, (res) => {
let responseBody = "";
res.on("data", (chunk) => {
responseBody += chunk;
});
res.on("end", () => {
if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) resolve();
else {
const errorMsg = responseBody ? `HTTP ${res.statusCode}: ${responseBody}` : `HTTP ${res.statusCode}`;
reject(/* @__PURE__ */ new Error(`Failed to sync ${packageName}: ${errorMsg}`));
}
});
});
req.on("error", (err) => {
reject(/* @__PURE__ */ new Error(`Failed to sync ${packageName}: ${err.message}`));
});
req.on("timeout", () => {
req.destroy();
reject(/* @__PURE__ */ new Error(`Failed to sync ${packageName}: Request timeout after ${timeout}ms`));
});
req.end();
});
}
//#endregion
//#region src/utils.ts
/**
* Check if given package json object is valid
*
* @param packageJson - package json object
* @returns true if is a valid package json, false otherwise
*/
function isValidPublicPackage(packageJson) {
if (packageJson.private) return false;
return Boolean(packageJson.name && packageJson.version);
}
const SUPPORTED_TARGETS = ["npmmirror", "custom"];
/**
* Assert given target is a valid sync target
*
* @param target - target site
*/
function assertSyncTarget(target) {
if (!target || !SUPPORTED_TARGETS.includes(target)) throw new Error(`Required option target to be one of ${SUPPORTED_TARGETS.join(", ")}`);
}
//#endregion
//#region src/core.ts
/**
* node_modules must be ignored
*/
const IGNORE_NODE_MODULES = "**/node_modules/**";
/**
* default patterns to be ignored
*/
const DEFAULT_IGNORE = [
IGNORE_NODE_MODULES,
"**/.git/**",
"**/docs/**",
"**/tests/**",
"**/examples/**",
"**/fixtures/**",
"**/playground/**"
];
/**
* glob pattern to match package.json files
*/
const GLOB_PACKAGE_JSON = "**/package.json";
/**
* Default retry count
*/
const DEFAULT_RETRY = 3;
/**
* Default retry delay in milliseconds
*/
const DEFAULT_RETRY_DELAY = 1e3;
/**
* Default concurrency limit
*/
const DEFAULT_CONCURRENCY = 5;
/**
* Delay for a specified amount of time
* @param ms - milliseconds to delay
*/
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Load cached package names
* @param cacheDir - cache directory
*/
async function loadSyncCache(cacheDir) {
try {
const content = await readFile(join(cacheDir, "synced-packages.json"), "utf8");
const data = JSON.parse(content);
return new Set(data.packages);
} catch {
return /* @__PURE__ */ new Set();
}
}
/**
* Save cached package names
* @param cacheDir - cache directory
* @param packages - package names to cache
*/
async function saveSyncCache(cacheDir, packages) {
try {
await mkdir(cacheDir, { recursive: true });
const cachePath = join(cacheDir, "synced-packages.json");
const data = {
packages: Array.from(packages),
timestamp: (/* @__PURE__ */ new Date()).toISOString()
};
await writeFile(cachePath, JSON.stringify(data, null, 2), "utf8");
} catch (error) {
if (error instanceof Error) console.warn(c.yellow(`Warning: Failed to save sync cache: ${error.message}`));
}
}
/**
* Sync package with retry mechanism
* @param packageName - package name
* @param options - sync options
*/
async function syncPackageWithRetry(packageName, options) {
const maxRetries = options.retry ?? DEFAULT_RETRY;
const retryDelay = options.retryDelay ?? DEFAULT_RETRY_DELAY;
const { verbose, silent, debug, beforeSync, afterSync } = options;
try {
if (beforeSync) await beforeSync(packageName);
} catch (error) {
if (afterSync) await afterSync(packageName, error instanceof Error ? error : new Error(String(error)));
throw error;
}
for (let attempt = 0; attempt <= maxRetries; attempt++) try {
if (debug && !silent) console.log(c.dim(` Attempting ${packageName} (${attempt + 1}/${maxRetries + 1})...`));
await syncPackageToRegistry(packageName, options);
if (verbose && !silent) console.log(c.green(` ✓ ${packageName}`));
if (afterSync) await afterSync(packageName);
return;
} catch (error) {
const isLastAttempt = attempt === maxRetries;
if (debug && !silent) console.log(c.yellow(` Attempt ${attempt + 1} failed for ${packageName}: ${error instanceof Error ? error.message : String(error)}`));
if (isLastAttempt) {
if (verbose && !silent) console.log(c.red(` ✗ ${packageName}: ${error instanceof Error ? error.message : String(error)}`));
if (afterSync) await afterSync(packageName, error instanceof Error ? error : new Error(String(error)));
throw error;
}
const waitTime = retryDelay * 2 ** attempt;
if (debug && !silent) console.log(c.dim(` Waiting ${waitTime}ms before retry...`));
await delay(waitTime);
}
}
/**
* Sync npm packages release to a mirror site
*
* @param input - package names
* @param options - sync options {@link SyncOptions}
* @returns a Promise that resolves when syncing is complete
*
* @example
*
* ```ts
* import { syncNpmPackages } from 'sync-npm-packages'
*
* // single package
* await syncNpmPackages('package-foobar', { target: 'npmmirror' })
*
* // multiple packages
* await syncNpmPackages(['package-foo', 'package-bar'], { target: 'npmmirror' })
* ```
*/
async function syncNpmPackages(input, options) {
let packages = unique(toArray(input));
const { silent, verbose, concurrency = DEFAULT_CONCURRENCY, cache, cacheDir = ".sync-cache" } = options;
assertSyncTarget(options.target);
let syncedPackages = /* @__PURE__ */ new Set();
if (cache) {
syncedPackages = await loadSyncCache(cacheDir);
packages = packages.filter((pkg) => !syncedPackages.has(pkg));
if (!silent && verbose && syncedPackages.size > 0) console.log(c.dim(`\nSkipping ${syncedPackages.size} already synced package(s)\n`));
}
if (packages.length === 0) {
if (!silent && verbose && cache) console.log(c.green("All packages have been synced already!"));
return;
}
if (!silent && verbose) console.log(c.dim(`\nSyncing ${packages.length} package(s) with concurrency ${concurrency}...\n`));
const errors = [];
const executing = [];
let completed = 0;
for (const pkg of packages) {
const promise = syncPackageWithRetry(pkg, options).then(() => {
completed++;
if (cache) syncedPackages.add(pkg);
if (!silent && !verbose) process.stdout.write(`\r${c.dim(`Progress: ${completed}/${packages.length}`)} ${c.green("✓".repeat(Math.floor(completed / packages.length * 20)))}`);
}).catch((error) => {
completed++;
errors.push({
package: pkg,
error
});
if (!silent && !verbose) process.stdout.write(`\r${c.dim(`Progress: ${completed}/${packages.length}`)} ${c.green("✓".repeat(Math.floor(completed / packages.length * 20)))}`);
}).finally(() => {
executing.splice(executing.indexOf(promise), 1);
});
executing.push(promise);
if (executing.length >= concurrency) await Promise.race(executing);
}
await Promise.all(executing);
if (cache && syncedPackages.size > 0) await saveSyncCache(cacheDir, syncedPackages);
if (!silent && !verbose && packages.length > 0) process.stdout.write("\n");
if (errors.length > 0) {
if (!silent) {
console.log(c.red(`\n${errors.length} package(s) failed to sync:`));
for (const { package: pkg, error } of errors) console.log(c.red(` - ${pkg}: ${error.message}`));
}
throw new Error(`Failed to sync ${errors.length} package(s)`);
}
}
/**
* Get valid package names from all package.json files
*
* @param options - detect options {@link DetectOptions}
* @returns a promise resolves valid package names
*/
async function getValidPackageNames(options = {}) {
const { cwd = process.cwd(), defaultIgnore: useDefaultIgnore = true, ignore: userIgnore = [], include = [], exclude = [], withOptional = false } = options;
const ignore = toArray(userIgnore);
if (useDefaultIgnore) ignore.push(...DEFAULT_IGNORE);
else ignore.push(IGNORE_NODE_MODULES);
const files = await glob(GLOB_PACKAGE_JSON, {
cwd,
ignore,
absolute: true,
onlyFiles: true
});
const packages = [...toArray(include)];
const fileContents = await Promise.all(files.map(async (file) => {
try {
const content = await readFile(file, "utf8");
return JSON.parse(content);
} catch (error) {
if (error instanceof Error) console.warn(c.yellow(`Warning: Failed to parse ${file}: ${error.message}`));
return null;
}
}));
const excludeSet = new Set(toArray(exclude));
for (const packageJson of fileContents) {
if (!packageJson) continue;
if (isValidPublicPackage(packageJson)) {
packages.push(packageJson.name);
if (withOptional) packages.push(...Object.keys(packageJson.optionalDependencies || {}));
}
}
return unique(packages.filter((pkg) => !excludeSet.has(pkg)));
}
/**
* Auto detect and sync npm packages release to a mirror site
* @param options - detect options {@link DetectOptions} and sync options {@link SyncOptions }
*
* @example
*
* ```ts
* import { syncNpmPackagesAuto } from 'sync-npm-packages'
*
* await syncNpmPackagesAuto({ target: 'npmmirror' })
* ```
*/
async function syncNpmPackagesAuto(options) {
assertSyncTarget(options.target);
return syncNpmPackages(await getValidPackageNames(options), options);
}
//#endregion
//#region src/config.ts
/**
* Define config for cli and NodeJS api
*
* @param config - user defined config
* @returns config
*/
function defineConfig(config = {}) {
return config;
}
/**
* Resolve user definedConfig based on cli config and config file
*
* @param cliConfig - cli config
* @returns merged config
*/
async function resolveConfig(cliConfig = {}) {
const { config = {} } = await createConfigLoader({
sources: [{
files: ["sync.config"],
extensions: [
"mts",
"cts",
"ts",
"mjs",
"cjs",
"js",
"json"
]
}, {
files: [".syncrc"],
extensions: ["json"]
}],
cwd: process.cwd(),
merge: false
}).load();
return {
...config,
...cliConfig
};
}
//#endregion
export { defineConfig, getValidPackageNames, resolveConfig, syncNpmPackages, syncNpmPackagesAuto };