@strapi/strapi
Version:
An open source headless CMS solution to create and manage your own API. It provides a powerful dashboard and features to make your life easier. Databases supported: MySQL, MariaDB, PostgreSQL, SQLite
601 lines (595 loc) • 26 kB
JavaScript
;
var chalk = require('chalk');
var path = require('node:path');
var CLITable = require('cli-table3');
var commander = require('commander');
var logger = require('@strapi/logger');
var core = require('@strapi/core');
var ora = require('ora');
var fp = require('lodash/fp');
var dataTransfer = require('@strapi/data-transfer');
var helpers = require('./helpers.js');
var commander$1 = require('./commander.js');
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
var chalk__default = /*#__PURE__*/_interopDefault(chalk);
var path__default = /*#__PURE__*/_interopDefault(path);
var CLITable__default = /*#__PURE__*/_interopDefault(CLITable);
var ora__default = /*#__PURE__*/_interopDefault(ora);
const { errors: { TransferEngineInitializationError } } = dataTransfer.engine;
const exitMessageText = (process1, error = false)=>{
const processCapitalized = process1[0].toUpperCase() + process1.slice(1);
if (!error) {
return chalk__default.default.bold(chalk__default.default.green(`${processCapitalized} process has been completed successfully!`));
}
return chalk__default.default.bold(chalk__default.default.red(`${processCapitalized} process failed.`));
};
const pad = (n)=>{
return (n < 10 ? '0' : '') + String(n);
};
const yyyymmddHHMMSS = ()=>{
const date = new Date();
return date.getFullYear() + pad(date.getMonth() + 1) + pad(date.getDate()) + pad(date.getHours()) + pad(date.getMinutes()) + pad(date.getSeconds());
};
const getDefaultExportName = ()=>{
return `export_${yyyymmddHHMMSS()}`;
};
const buildTransferTable = (resultData)=>{
if (!resultData) {
return;
}
// Build pretty table
const table = new CLITable__default.default({
head: [
'Type',
'Count',
'Size'
].map((text)=>chalk__default.default.bold.blue(text))
});
let totalBytes = 0;
let totalItems = 0;
Object.keys(resultData).forEach((stage)=>{
const item = resultData[stage];
if (!item) {
return;
}
table.push([
{
hAlign: 'left',
content: chalk__default.default.bold(stage)
},
{
hAlign: 'right',
content: item.count
},
{
hAlign: 'right',
content: `${helpers.readableBytes(item.bytes, 1, 11)} `
}
]);
totalBytes += item.bytes;
totalItems += item.count;
if (item.aggregates) {
Object.keys(item.aggregates).sort().forEach((subkey)=>{
if (!item.aggregates) {
return;
}
const subitem = item.aggregates[subkey];
table.push([
{
hAlign: 'left',
content: `-- ${chalk__default.default.bold.grey(subkey)}`
},
{
hAlign: 'right',
content: chalk__default.default.grey(subitem.count)
},
{
hAlign: 'right',
content: chalk__default.default.grey(`(${helpers.readableBytes(subitem.bytes, 1, 11)})`)
}
]);
});
}
});
table.push([
{
hAlign: 'left',
content: chalk__default.default.bold.green('Total')
},
{
hAlign: 'right',
content: chalk__default.default.bold.green(totalItems)
},
{
hAlign: 'right',
content: `${chalk__default.default.bold.green(helpers.readableBytes(totalBytes, 1, 11))} `
}
]);
return table;
};
const IGNORED_CONTENT_TYPE_PREFIXES = [
'admin::'
];
const IGNORED_CONTENT_TYPES = [
'plugin::content-releases.release',
'plugin::content-releases.release-action'
];
/** Media library content types — common target for `--exclude-content-types` (see issue #25008). */ const UPLOAD_CONTENT_TYPE_UIDS = [
'plugin::upload.file',
'plugin::upload.folder'
];
const isIgnoredContentType = (type)=>IGNORED_CONTENT_TYPE_PREFIXES.some((prefix)=>type.startsWith(prefix)) || IGNORED_CONTENT_TYPES.includes(type);
const abortTransfer = async ({ engine, strapi: strapi1 })=>{
try {
await engine.abortTransfer();
await strapi1.destroy();
} catch {
// ignore because there's not much else we can do
return false;
}
return true;
};
const setSignalHandler = async (handler, signals = [
'SIGINT',
'SIGTERM',
'SIGQUIT'
])=>{
signals.forEach((signal)=>{
// We specifically remove ALL listeners because we have to clear the one added in Strapi bootstrap that has a process.exit
// TODO: Ideally Strapi bootstrap would not add that listener, and then this could be more flexible and add/remove only what it needs to
process.removeAllListeners(signal);
process.on(signal, handler);
});
};
const createStrapiInstance = async (opts = {})=>{
try {
const appContext = await core.compileStrapi();
const app = core.createStrapi({
...opts,
...appContext
});
app.log.level = opts.logLevel || 'error';
return await app.load();
} catch (error) {
if (error instanceof Error && 'code' in error && error.code === 'ECONNREFUSED') {
throw new Error('Process failed. Check the database connection with your Strapi project.');
}
throw error;
}
};
const transferDataTypes = Object.keys(dataTransfer.engine.TransferGroupPresets);
const MEDIA_LIBRARY_PRESET = 'media-library';
const TRANSFER_FILTER_PRESET_DESCRIPTIONS = {
content: 'entities and links (incl. media library DB records)',
files: 'upload binaries in public/uploads (not media library DB records)',
config: 'core store and webhooks',
[MEDIA_LIBRARY_PRESET]: 'upload binaries and media library DB records (files + plugin::upload.file, plugin::upload.folder)'
};
const transferExcludePresetChoices = [
...transferDataTypes,
MEDIA_LIBRARY_PRESET
];
const formatTransferPresetHelp = (types)=>types.map((type)=>`${type} (${TRANSFER_FILTER_PRESET_DESCRIPTIONS[type]})`).join('; ');
const transferExcludePresetsHelp = formatTransferPresetHelp(transferExcludePresetChoices);
const transferOnlyPresetsHelp = formatTransferPresetHelp(transferDataTypes);
const throttleOption = new commander.Option('--throttle <delay after each entity>', `Add a delay in milliseconds between each transferred entity`).argParser(commander$1.parseInteger).hideHelp(); // This option is not publicly documented
const excludeOption = new commander.Option('--exclude <comma-separated data types>', `Exclude data: ${transferExcludePresetsHelp}`).argParser(commander$1.getParseListWithChoices(transferExcludePresetChoices, 'Invalid options for "exclude"'));
const onlyOption = new commander.Option('--only <command-separated data types>', `Include only these types (plus schemas): ${transferOnlyPresetsHelp}`).argParser(commander$1.getParseListWithChoices(transferDataTypes, 'Invalid options for "only"'));
const excludeContentTypesOption = new commander.Option('--exclude-content-types <comma-separated UIDs>', `Exclude content types from entities and links (e.g. ${UPLOAD_CONTENT_TYPE_UIDS.join(',')} to omit the media library; or use --exclude media-library to skip binaries and upload records — see issue #25008)`).argParser(commander$1.parseList);
const onlyContentTypesOption = new commander.Option('--only-content-types <comma-separated UIDs>', 'Transfer only these content types in entities and links (e.g. api::article.article)').argParser(commander$1.parseList);
const validateExcludeOnly = (command)=>{
const { exclude, only } = command.opts();
if (!only || !exclude) {
return;
}
const choicesInBoth = only.filter((n)=>{
return exclude.indexOf(n) !== -1;
});
if (choicesInBoth.length > 0) {
helpers.exitWith(1, `Data types may not be used in both "exclude" and "only" in the same command. Found in both: ${choicesInBoth.join(',')}`);
}
};
const validateContentTypeTransferOptions = (command)=>{
const { excludeContentTypes, onlyContentTypes } = command.opts();
if (!excludeContentTypes?.length || !onlyContentTypes?.length) {
return;
}
const overlap = excludeContentTypes.filter((uid)=>onlyContentTypes.includes(uid));
if (overlap.length > 0) {
helpers.exitWith(1, `Content types may not be used in both "--exclude-content-types" and "--only-content-types". Found in both: ${overlap.join(',')}`);
}
};
const assertKnownContentTypes = (uids, strapi1, flag)=>{
const known = new Set(Object.keys(strapi1.contentTypes));
const unknown = uids.filter((uid)=>!known.has(uid));
if (unknown.length > 0) {
helpers.exitWith(1, `Unknown content type(s) for ${flag}: ${unknown.join(', ')}`);
}
};
const validateContentTypeTransferOptionsForStrapi = (opts, strapi1)=>{
if (opts.excludeContentTypes?.length) {
assertKnownContentTypes(opts.excludeContentTypes, strapi1, '--exclude-content-types');
}
if (opts.onlyContentTypes?.length) {
assertKnownContentTypes(opts.onlyContentTypes, strapi1, '--only-content-types');
}
};
const shouldIncludeContentTypeInTransfer = (uid, opts)=>{
if (isIgnoredContentType(uid)) {
return false;
}
if (opts.excludeContentTypes?.includes(uid)) {
return false;
}
if (opts.onlyContentTypes?.length) {
return opts.onlyContentTypes.includes(uid);
}
return true;
};
const createEntityFilter = (opts)=>{
return (entity)=>shouldIncludeContentTypeInTransfer(entity.type, opts);
};
const createLinkFilter = (opts)=>{
return (link)=>shouldIncludeContentTypeInTransfer(link.left.type, opts) && shouldIncludeContentTypeInTransfer(link.right.type, opts);
};
const buildTransferTransforms = (opts)=>({
links: [
{
filter: createLinkFilter(opts)
}
],
entities: [
{
filter: createEntityFilter(opts)
}
]
});
const errorColors = {
fatal: chalk__default.default.red,
error: chalk__default.default.red,
silly: chalk__default.default.yellow
};
const formatDiagnostic = (operation, verbose)=>{
let logger$1;
let logFileBasename;
const getLogger = ()=>{
if (!logger$1) {
logFileBasename = `${operation}_${Date.now()}.log`;
const absoluteLogPath = path__default.default.resolve(process.cwd(), logFileBasename);
logger$1 = logger.createLogger(logger.configs.createOutputFileConfiguration(logFileBasename, {
level: 'info',
format: logger.formats?.detailedLogs
}, {
consoleLevel: verbose ? 'info' : 'warn'
}));
logger$1.info(`[${operation}] Diagnostic log file: ${absoluteLogPath} (info-level messages are written here even without --verbose)`);
}
return logger$1;
};
return ({ details, kind })=>{
try {
if (kind === 'error') {
const { message, severity = 'fatal' } = details;
const colorizeError = errorColors[severity];
const errorMessage = colorizeError(`[${severity.toUpperCase()}] ${message}`);
getLogger().error(errorMessage);
}
if (kind === 'info') {
const { message, params, origin } = details;
const msg = `[${origin ?? 'transfer'}] ${message}\n${params ? JSON.stringify(params, null, 2) : ''}`;
getLogger().info(msg);
}
if (kind === 'warning') {
const { origin, message } = details;
getLogger().warn(`(${origin ?? 'transfer'}) ${message}`);
}
} catch (err) {
getLogger().error(err);
}
};
};
/** Stages where throughput is dominated by DB work; items/s is more meaningful than JSON byte rate. */ const STAGES_WITH_ITEM_THROUGHPUT = new Set([
'entities',
'links'
]);
const MAX_ETA_MS = 86400000;
/**
* Linear ETA from completed amount vs total, using average rate so far (done / elapsedMs).
* Returns null when progress or totals are not usable yet.
*/ const estimateEtaMs = (elapsedMs, done, total)=>{
if (elapsedMs < 500 || done <= 0 || total <= 0 || done >= total) {
return null;
}
const ratePerMs = done / elapsedMs;
const remaining = total - done;
const etaMs = remaining / ratePerMs;
if (!Number.isFinite(etaMs) || etaMs <= 0 || etaMs >= MAX_ETA_MS) {
return null;
}
return etaMs;
};
const loadersFactory = (defaultLoaders = {})=>{
const loaders = defaultLoaders;
const updateLoader = (stage, data)=>{
if (!(stage in loaders)) {
createLoader(stage);
}
const stageData = data[stage];
const elapsedTime = stageData?.startTime ? (stageData?.endTime || Date.now()) - stageData.startTime : 0;
const bytes = stageData?.bytes ?? 0;
const count = stageData?.count ?? 0;
const totalBytes = stageData?.totalBytes;
const totalCount = stageData?.totalCount;
const countLabel = totalCount != null && totalCount > 0 ? `${count} / ${totalCount}` : String(count);
const sizeCompact = totalBytes != null && totalBytes > 0 ? `${helpers.readableBytes(bytes)} / ${helpers.readableBytes(totalBytes)}` : helpers.readableBytes(bytes);
const parts = [
`${stage}: ${countLabel} transferred`,
sizeCompact
];
if (elapsedTime > 0 && !stageData?.endTime) {
if (STAGES_WITH_ITEM_THROUGHPUT.has(stage)) {
const itemsPerSec = count * 1000 / elapsedTime;
parts.push(`${itemsPerSec.toFixed(1)} items/s`);
} else {
parts.push(`${helpers.readableBytes(bytes * 1000 / elapsedTime)}/s`);
}
}
let etaMs = null;
if (!stageData?.endTime) {
if (STAGES_WITH_ITEM_THROUGHPUT.has(stage) && totalCount != null) {
etaMs = estimateEtaMs(elapsedTime, count, totalCount);
} else if (totalBytes != null) {
etaMs = estimateEtaMs(elapsedTime, bytes, totalBytes);
}
}
parts.push(helpers.formatElapsedAndMaybeRemainingLabel(elapsedTime ?? 0, etaMs));
loaders[stage].text = parts.join(helpers.TRANSFER_PROGRESS_FIELD_SEP);
return loaders[stage];
};
const createLoader = (stage)=>{
Object.assign(loaders, {
[stage]: ora__default.default()
});
return loaders[stage];
};
const getLoader = (stage)=>{
return loaders[stage];
};
return {
updateLoader,
createLoader,
getLoader
};
};
/**
* Get the telemetry data to be sent for a didDEITSProcess* event from an initialized transfer engine object
*/ const getTransferTelemetryPayload = (engine)=>{
return {
eventProperties: {
source: engine?.sourceProvider?.name,
destination: engine?.destinationProvider?.name
}
};
};
/**
* Get a transfer engine schema diff handler that confirms with the user before bypassing a schema check
*/ const getDiffHandler = (engine, { force, action })=>{
return async (context, next)=>{
// if we abort here, we need to actually exit the process because of conflict with inquirer prompt
setSignalHandler(async ()=>{
await abortTransfer({
engine,
strapi: strapi
});
helpers.exitWith(1, exitMessageText(action, true));
});
let workflowsStatus;
const source = 'Schema Integrity';
Object.entries(context.diffs).forEach(([uid, diffs])=>{
for (const diff of diffs){
const path = [
uid
].concat(diff.path).join('.');
const endPath = diff.path[diff.path.length - 1];
// Catch known features
if (uid === 'plugin::review-workflows.workflow' || uid === 'plugin::review-workflows.workflow-stage' || endPath?.startsWith('strapi_stage') || endPath?.startsWith('strapi_assignee')) {
workflowsStatus = diff.kind;
} else if (diff.kind === 'added') {
engine.reportWarning(chalk__default.default.red(`${chalk__default.default.bold(path)} does not exist on source`), source);
} else if (diff.kind === 'deleted') {
engine.reportWarning(chalk__default.default.red(`${chalk__default.default.bold(path)} does not exist on destination`), source);
} else if (diff.kind === 'modified') {
engine.reportWarning(chalk__default.default.red(`${chalk__default.default.bold(path)} has a different data type`), source);
}
}
});
// output the known feature warnings
if (workflowsStatus === 'added') {
engine.reportWarning(chalk__default.default.red(`Review workflows feature does not exist on source`), source);
} else if (workflowsStatus === 'deleted') {
engine.reportWarning(chalk__default.default.red(`Review workflows feature does not exist on destination`), source);
} else if (workflowsStatus === 'modified') {
engine.panic(new TransferEngineInitializationError('Unresolved differences in schema [review workflows]'));
}
const confirmed = await commander$1.confirmMessage('There are differences in schema between the source and destination, and the data listed above will be lost. Are you sure you want to continue?', {
force
});
// reset handler back to normal
setSignalHandler(()=>abortTransfer({
engine,
strapi: strapi
}));
if (confirmed) {
context.ignoredDiffs = fp.merge(context.diffs, context.ignoredDiffs);
}
return next(context);
};
};
const getAssetsBackupHandler = (engine, { force, action })=>{
return async (context, next)=>{
// if we abort here, we need to actually exit the process because of conflict with inquirer prompt
setSignalHandler(async ()=>{
await abortTransfer({
engine,
strapi: strapi
});
helpers.exitWith(1, exitMessageText(action, true));
});
console.warn('The backup for the assets could not be created inside the public directory. Ensure Strapi has write permissions on the public directory.');
const confirmed = await commander$1.confirmMessage('Do you want to continue without backing up your public/uploads files?', {
force
});
if (confirmed) {
context.ignore = true;
}
// reset handler back to normal
setSignalHandler(()=>abortTransfer({
engine,
strapi: strapi
}));
return next(context);
};
};
const shouldSkipStage = (opts, dataKind)=>{
if (opts.exclude?.includes(dataKind)) {
return true;
}
if (opts.only) {
return !opts.only.includes(dataKind);
}
return false;
};
const areUploadContentTypesInTransferScope = (opts)=>UPLOAD_CONTENT_TYPE_UIDS.every((uid)=>shouldIncludeContentTypeInTransfer(uid, opts));
const areAllUploadContentTypesOutOfTransferScope = (opts)=>UPLOAD_CONTENT_TYPE_UIDS.every((uid)=>!shouldIncludeContentTypeInTransfer(uid, opts));
const isContentStageActive = (opts)=>!shouldSkipStage(opts, 'content');
const expandMediaLibraryPreset = (opts)=>{
if (!opts.exclude?.includes(MEDIA_LIBRARY_PRESET)) {
return;
}
const exclude = opts.exclude.filter((item)=>item !== MEDIA_LIBRARY_PRESET);
opts.exclude = exclude;
if (!opts.exclude.includes('files')) {
opts.exclude.push('files');
}
const excludeContentTypes = new Set(opts.excludeContentTypes ?? []);
for (const uid of UPLOAD_CONTENT_TYPE_UIDS){
excludeContentTypes.add(uid);
}
opts.excludeContentTypes = [
...excludeContentTypes
];
};
const autoExcludeFilesWhenUploadTypesOutOfScope = (opts)=>{
if (opts.filesAutoExcluded || !isContentStageActive(opts) || opts.only?.includes('files') || shouldSkipStage(opts, 'files') || !areAllUploadContentTypesOutOfTransferScope(opts)) {
return;
}
opts.exclude = [
...opts.exclude ?? [],
'files'
];
opts.filesAutoExcluded = true;
};
const normalizeTransferFilterOptions = (opts)=>{
expandMediaLibraryPreset(opts);
autoExcludeFilesWhenUploadTypesOutOfScope(opts);
return opts;
};
const normalizeTransferFilterOptionsHook = (command)=>{
normalizeTransferFilterOptions(command.opts());
};
const logTransferFilterSummary = (opts)=>{
const { exclude, only, excludeContentTypes, onlyContentTypes } = opts;
if (!exclude?.length && !only?.length && !excludeContentTypes?.length && !onlyContentTypes?.length) {
return;
}
const parts = [];
if (exclude?.length) {
parts.push(`excluding ${exclude.join(', ')}`);
}
if (only?.length) {
parts.push(`only ${only.join(', ')}`);
}
if (parts.length) {
console.log(chalk__default.default.dim(`Transfer filters: ${parts.join('; ')}.`));
}
const contentTypeParts = [];
if (excludeContentTypes?.length) {
contentTypeParts.push(`excluding ${excludeContentTypes.join(', ')}`);
}
if (onlyContentTypes?.length) {
contentTypeParts.push(`only ${onlyContentTypes.join(', ')}`);
}
if (contentTypeParts.length) {
console.log(chalk__default.default.dim(`Content type filters: ${contentTypeParts.join('; ')}.`));
}
if (opts.filesAutoExcluded) {
console.log(chalk__default.default.dim('Skipping files stage: upload content types are not in transfer scope (plugin::upload.file, plugin::upload.folder).'));
}
if (shouldSkipStage(opts, 'files') && !shouldSkipStage(opts, 'content') && areUploadContentTypesInTransferScope(opts) && !opts.filesAutoExcluded) {
console.log(chalk__default.default.dim('Note: Media library records (plugin::upload.file, plugin::upload.folder) are still transferred with the rest of your content (the entities stage). Sync upload binaries separately (e.g. rsync public/uploads).'));
}
};
// Based on exclude/only from options, create the restore object to match
const parseRestoreFromOptions = (opts, strapi1)=>{
const entitiesOptions = {
exclude: [
...Object.keys(strapi1.contentTypes).filter(isIgnoredContentType),
...IGNORED_CONTENT_TYPES,
...opts.excludeContentTypes ?? []
],
include: undefined
};
const contentInScope = !(opts.only && !opts.only.includes('content') || opts.exclude?.includes('content'));
if (!contentInScope) {
// Nothing from the entities stage is transferred; do not delete any records beforehand.
entitiesOptions.include = [];
} else if (opts.onlyContentTypes?.length) {
// Only wipe content types that are being replaced by this transfer.
entitiesOptions.include = opts.onlyContentTypes;
} else if (shouldSkipStage(opts, 'config')) {
// When config is excluded, scope pre-transfer deletion to user content types only.
// Internal models (e.g. strapi::core-store) must not be wiped via the entities path.
entitiesOptions.include = Object.keys(strapi1.contentTypes).filter((uid)=>!isIgnoredContentType(uid) && !opts.excludeContentTypes?.includes(uid));
}
const restoreConfig = {
entities: entitiesOptions,
assets: !shouldSkipStage(opts, 'files'),
configuration: {
webhook: !shouldSkipStage(opts, 'config'),
coreStore: !shouldSkipStage(opts, 'config')
}
};
return restoreConfig;
};
exports.UPLOAD_CONTENT_TYPE_UIDS = UPLOAD_CONTENT_TYPE_UIDS;
exports.abortTransfer = abortTransfer;
exports.areAllUploadContentTypesOutOfTransferScope = areAllUploadContentTypesOutOfTransferScope;
exports.areUploadContentTypesInTransferScope = areUploadContentTypesInTransferScope;
exports.buildTransferTable = buildTransferTable;
exports.buildTransferTransforms = buildTransferTransforms;
exports.createEntityFilter = createEntityFilter;
exports.createLinkFilter = createLinkFilter;
exports.createStrapiInstance = createStrapiInstance;
exports.excludeContentTypesOption = excludeContentTypesOption;
exports.excludeOption = excludeOption;
exports.exitMessageText = exitMessageText;
exports.formatDiagnostic = formatDiagnostic;
exports.getAssetsBackupHandler = getAssetsBackupHandler;
exports.getDefaultExportName = getDefaultExportName;
exports.getDiffHandler = getDiffHandler;
exports.getTransferTelemetryPayload = getTransferTelemetryPayload;
exports.isIgnoredContentType = isIgnoredContentType;
exports.loadersFactory = loadersFactory;
exports.logTransferFilterSummary = logTransferFilterSummary;
exports.normalizeTransferFilterOptions = normalizeTransferFilterOptions;
exports.normalizeTransferFilterOptionsHook = normalizeTransferFilterOptionsHook;
exports.onlyContentTypesOption = onlyContentTypesOption;
exports.onlyOption = onlyOption;
exports.parseRestoreFromOptions = parseRestoreFromOptions;
exports.setSignalHandler = setSignalHandler;
exports.shouldSkipStage = shouldSkipStage;
exports.throttleOption = throttleOption;
exports.validateContentTypeTransferOptions = validateContentTypeTransferOptions;
exports.validateContentTypeTransferOptionsForStrapi = validateContentTypeTransferOptionsForStrapi;
exports.validateExcludeOnly = validateExcludeOnly;
//# sourceMappingURL=data-transfer.js.map