sb-mig
Version:
CLI to rule the world. (and handle stuff related to Storyblok CMS)
615 lines (614 loc) • 25.4 kB
JavaScript
import path from "path";
import chalk from "chalk";
import { discoverMigrationConfig, discoverStories, LOOKUP_TYPE, SCOPE, } from "../../cli/utils/discover.js";
import storyblokConfig from "../../config/config.js";
import { createAndSaveToFile, getFileContentWithRequire, getFilesContentWithRequire, } from "../../utils/files.js";
import Logger from "../../utils/logger.js";
import { modifyOrCreateAppliedMigrationsFile } from "../../utils/migrations.js";
import { isObjectEmpty } from "../../utils/object-utils.js";
import { managementApi } from "../managementApi.js";
import { buildPreMigrationBackupBaseName, resolveOutputFileBaseName, shouldUseDatestampForArtifacts, } from "./file-naming.js";
import { extendMigrationMapperWithAliases, resolveMigrationComponentsToMigrate, } from "./migration-component-scope.js";
import { saveMigrationRunLog } from "./migration-run-log.js";
import { discoverMigrationValidatorForMigrationFile, MigrationValidationFailedError, runPreparedMigrationValidator, } from "./migration-validation.js";
import { summarizeMutationWriteResults } from "./write-summary.js";
export const normalizeMigrationConfigNames = (migrationConfig) => {
if (Array.isArray(migrationConfig)) {
return migrationConfig.filter((name) => Boolean(name));
}
if (typeof migrationConfig === "string" && migrationConfig.length > 0) {
return [migrationConfig];
}
return [];
};
function replaceComponentData({ parent, key, components, mapper, depth, maxDepth, sumOfReplacing, }) {
let currentMaxDepth = depth;
if (storyblokConfig.debug) {
Logger.warning(`Current max depth: ${depth}`);
}
if (typeof parent[key] === "object") {
if (parent[key]?.component &&
components.includes(parent[key].component)) {
const { data: dataToReplace, wasReplaced } = mapper[parent[key].component](parent[key]);
// Keep migration output authoritative so key removals stay removed.
parent[key] = dataToReplace;
if (storyblokConfig.debug) {
console.log(chalk.yellow(`______________ In __________________________________________`));
console.log(" ");
console.log(` Data from ${chalk.blue(dataToReplace.component)} component,\n with _uid: ${chalk.blue(dataToReplace._uid)} `);
console.log("Was it replaced? ", wasReplaced);
console.log(chalk.yellow(`____________________________________________________________`));
console.log(" ");
}
if (wasReplaced) {
sumOfReplacing[dataToReplace.component] = sumOfReplacing[dataToReplace.component]
? sumOfReplacing[dataToReplace.component] + 1
: 1;
if (storyblokConfig.debug) {
console.log("Sum of replacing: ");
console.log(sumOfReplacing);
}
}
}
if (Array.isArray(parent[key])) {
for (let i = 0; i < parent[key].length; i++) {
const childMaxDepth = replaceComponentData({
parent: parent[key],
key: i,
components,
mapper,
depth: depth + 1,
maxDepth,
sumOfReplacing,
});
currentMaxDepth = Math.max(currentMaxDepth, childMaxDepth);
}
}
else {
for (const subKey in parent[key]) {
const childMaxDepth = replaceComponentData({
parent: parent[key],
key: subKey,
components,
mapper,
depth: depth + 1,
maxDepth,
sumOfReplacing,
});
currentMaxDepth = Math.max(currentMaxDepth, childMaxDepth);
}
}
}
return currentMaxDepth;
}
export const prepareStoriesFromLocalFile = ({ from, fromFilePath, }) => {
if (fromFilePath) {
const resolvedFilePath = path.isAbsolute(fromFilePath)
? fromFilePath
: path.resolve(process.cwd(), fromFilePath);
const fileContent = getFileContentWithRequire({
file: resolvedFilePath,
});
if (!fileContent) {
throw new Error(`Couldn't receive data from provided stories path: ${chalk.red(fromFilePath)}`);
}
return fileContent;
}
if (!from) {
throw new Error("'from' is required for migrateFrom=file when fromFilePath is not provided.");
}
// Legacy discovery-based mode for story fixture names.
const allLocalStories = discoverStories({
scope: SCOPE.local,
type: LOOKUP_TYPE.fileName,
fileNames: [from],
});
const storiesFileContent = getFilesContentWithRequire({
files: allLocalStories,
})[0];
if (!storiesFileContent) {
throw new Error(`Couldn't receive data from provided stories filename: ${chalk.red(from)}`);
}
return storiesFileContent;
};
export const prepareMigrationConfigs = ({ migrationConfig, componentsToMigrate, migrationComponentAliases, migrationComponentOverrides, }) => {
const migrationConfigNames = normalizeMigrationConfigNames(migrationConfig);
if (migrationConfigNames.length === 0) {
throw new Error("Migration config is required. Pass at least one --migration value.");
}
const prepared = migrationConfigNames.map((migrationConfigName) => {
const migrationConfigFiles = discoverMigrationConfig({
scope: SCOPE.local,
type: LOOKUP_TYPE.fileName,
fileNames: [migrationConfigName],
});
const migrationConfigPath = migrationConfigFiles[0];
if (!migrationConfigPath) {
throw new Error(`Migration config '${migrationConfigName}' probably doesnt exist. Create one`);
}
const migrationConfigFileContent = getFileContentWithRequire({
file: migrationConfigPath,
});
if (isObjectEmpty(migrationConfigFileContent)) {
throw new Error(`Migration config file '${migrationConfigName}' is empty. Please provide default exported config object with components map to migrate`);
}
if (!migrationConfigFileContent) {
throw new Error(`Migration config '${migrationConfigName}' probably doesnt exist. Create one`);
}
const validator = discoverMigrationValidatorForMigrationFile({
migrationConfigName,
migrationConfigPath,
});
if (!validator) {
Logger.warning(`[VALIDATION] No co-located validator found for migration '${migrationConfigName}'. Expected a sibling '*.validation.*' file.`);
}
const aliasedMigrationConfigFileContent = extendMigrationMapperWithAliases(migrationConfigFileContent, migrationComponentAliases?.[migrationConfigName]);
const resolvedComponentsToMigrate = resolveMigrationComponentsToMigrate({
mapper: aliasedMigrationConfigFileContent,
migrationName: migrationConfigName,
globalComponentsToMigrate: componentsToMigrate,
perMigrationOverrides: migrationComponentOverrides,
});
return {
migrationConfigName,
migrationConfigPath,
migrationConfigFileContent: aliasedMigrationConfigFileContent,
componentsToMigrate: resolvedComponentsToMigrate,
validator,
};
});
Logger.success(`Migration config loaded.`);
return prepared;
};
export const prepareMigrationConfig = ({ migrationConfig, }) => {
const prepared = prepareMigrationConfigs({ migrationConfig });
const firstPrepared = prepared[0];
if (!firstPrepared) {
throw new Error("Migration config is required.");
}
return firstPrepared.migrationConfigFileContent;
};
const deepClone = (input) => JSON.parse(JSON.stringify(input));
const sumValues = (obj) => Object.values(obj).reduce((sum, value) => sum + value, 0);
const applySingleMigrationToItems = ({ itemType, itemsToMigrate, preparedMigrationConfig, }) => {
const arrayOfMaxDepths = [];
const replacementsByComponent = {};
let touchedItems = 0;
const updatedItems = itemsToMigrate.map((item, index) => {
const sumOfReplacing = {};
if (storyblokConfig.debug) {
Logger.success(`# ${index} #`);
}
let json = itemType === "story" ? item[itemType]?.content : item[itemType];
const rootWrapper = { root: json };
const maxDepth = replaceComponentData({
parent: rootWrapper,
key: "root",
components: preparedMigrationConfig.componentsToMigrate,
mapper: preparedMigrationConfig.migrationConfigFileContent,
depth: 0,
maxDepth: 0,
sumOfReplacing,
});
json = rootWrapper.root;
arrayOfMaxDepths.push(maxDepth);
const didReplace = Object.keys(sumOfReplacing).length > 0;
if (didReplace) {
touchedItems += 1;
Object.entries(sumOfReplacing).forEach(([component, count]) => {
replacementsByComponent[component] =
(replacementsByComponent[component] || 0) + count;
});
console.log(" ");
console.log(`Migration in ${chalk.magenta(itemType === "story"
? item[itemType]?.full_slug
: item[itemType]?.name)} page: `);
preparedMigrationConfig.componentsToMigrate.forEach((component) => {
if (sumOfReplacing[component]) {
console.log(`${chalk.blue(component)} component data was replaced: ${sumOfReplacing[component]} times.`);
}
});
return {
...item,
[itemType]: itemType === "story"
? {
...item[itemType],
content: json,
}
: {
...json,
},
};
}
return item;
});
const maxDepth = arrayOfMaxDepths.length > 0 ? Math.max(...arrayOfMaxDepths) : 0;
if (storyblokConfig.debug) {
console.log(" ");
if (maxDepth > 30) {
Logger.error(`Max depth: ${maxDepth}`);
}
else {
Logger.success(`Max depth: ${maxDepth}`);
}
console.log(" ");
}
return {
updatedItems,
stepReport: {
migrationConfig: preparedMigrationConfig.migrationConfigName,
touchedItems,
maxDepth,
replacementsByComponent,
totalComponentReplacements: sumValues(replacementsByComponent),
validation: null,
},
};
};
export const runMigrationPipelineInMemory = ({ itemType, itemsToMigrate, preparedMigrationConfigs, }) => {
let workingItems = deepClone(itemsToMigrate);
const stepReports = [];
for (const preparedMigrationConfig of preparedMigrationConfigs) {
const { updatedItems, stepReport } = applySingleMigrationToItems({
itemType,
itemsToMigrate: workingItems,
preparedMigrationConfig,
});
workingItems = updatedItems;
if (preparedMigrationConfig.validator) {
Logger.log(`[VALIDATION] Running '${preparedMigrationConfig.validator.id}' after migration '${preparedMigrationConfig.migrationConfigName}'...`);
const validationResult = runPreparedMigrationValidator({
validator: preparedMigrationConfig.validator,
data: workingItems,
isDebug: storyblokConfig.debug,
});
stepReport.validation = {
validatorId: preparedMigrationConfig.validator.id,
validatorName: preparedMigrationConfig.validator.name,
issueCount: validationResult.issueCount,
sourcePath: preparedMigrationConfig.validator.sourcePath,
};
if (!validationResult.ok) {
throw new MigrationValidationFailedError({
migrationConfig: preparedMigrationConfig.migrationConfigName,
validatorId: preparedMigrationConfig.validator.id,
validatorName: preparedMigrationConfig.validator.name,
issueCount: validationResult.issueCount,
issues: validationResult.issues,
});
}
Logger.success(`[VALIDATION] Passed '${preparedMigrationConfig.validator.id}' after migration '${preparedMigrationConfig.migrationConfigName}'.`);
}
stepReports.push(stepReport);
}
const changedItems = workingItems.filter((item, index) => {
const originalItem = itemsToMigrate[index];
if (!originalItem) {
return true;
}
return JSON.stringify(item) !== JSON.stringify(originalItem);
});
return {
changedItems,
finalItems: workingItems,
stepReports,
totalItems: workingItems.length,
};
};
const savePipelineSummary = async ({ artifactBaseName, useDatestamp, from, itemType, dryRun, publish, publishLanguages, migrateFrom, fromFilePath, pipelineResult, }, config) => {
await createAndSaveToFile({
datestamp: useDatestamp,
ext: "json",
filename: `${dryRun ? "dry-run--" : ""}${artifactBaseName}---${itemType}-migration-pipeline-summary`,
folder: "migrations",
res: {
itemType,
source: {
migrateFrom,
from,
fromFilePath: fromFilePath || null,
},
writeMode: itemType === "story" && publish ? "publish" : "save",
publishLanguages: itemType === "story" && publish
? {
requested: publishLanguages,
}
: null,
totalItems: pipelineResult.totalItems,
totalChangedItems: pipelineResult.changedItems.length,
steps: pipelineResult.stepReports,
},
}, config);
};
const saveDryRunDiffArtifacts = async ({ artifactBaseName, useDatestamp, itemType, dryRun, inputItems, finalItems, }, config) => {
if (!dryRun) {
return;
}
await createAndSaveToFile({
datestamp: useDatestamp,
ext: "json",
filename: `dry-run--${artifactBaseName}---${itemType}-input-full`,
folder: "migrations",
res: inputItems,
}, config);
await createAndSaveToFile({
datestamp: useDatestamp,
ext: "json",
filename: `dry-run--${artifactBaseName}---${itemType}-after-full`,
folder: "migrations",
res: finalItems,
}, config);
};
const loadItemsToMigrate = async ({ itemType, migrateFrom, from, filters, fromFilePath, }, config) => {
if (migrateFrom === "file") {
Logger.log("Migrating using file....");
const itemsFromFile = prepareStoriesFromLocalFile({
from,
fromFilePath,
});
const normalized = Array.isArray(itemsFromFile)
? itemsFromFile
: [itemsFromFile];
if (itemType === "story") {
return normalized.filter((it) => !(it?.story?.is_folder === true));
}
return normalized;
}
let itemsToMigrate = [];
if (itemType === "story") {
if (filters?.withSlug && filters.withSlug.length > 0) {
const results = await Promise.all(filters.withSlug.map((slug) => managementApi.stories.getStoryBySlug(slug, {
...config,
spaceId: from,
})));
itemsToMigrate = results.filter(Boolean).filter((it) => !(it?.story?.is_folder === true));
}
else if (filters?.startsWith) {
itemsToMigrate = await managementApi.stories.getAllStories({ options: { starts_with: filters.startsWith } }, {
...config,
spaceId: from,
});
itemsToMigrate = itemsToMigrate.filter((it) => !(it?.story?.is_folder === true));
}
else {
itemsToMigrate = await managementApi.stories.getAllStories({}, {
...config,
spaceId: from,
});
itemsToMigrate = itemsToMigrate.filter((it) => !(it?.story?.is_folder === true));
}
return itemsToMigrate;
}
return managementApi.presets.getAllPresets({
...config,
spaceId: from,
});
};
export const migrateAllComponentsDataInStories = async ({ itemType, migrationConfig, migrateFrom, from, to, filters, dryRun, publish, publishLanguages, fromFilePath, fileName, migrationComponentAliases, migrationComponentOverrides, }, config) => {
Logger.warning(`Trying to migrate all ${itemType} from ${migrateFrom}, ${from} to ${to}...`);
const preparedMigrationConfigs = prepareMigrationConfigs({
migrationConfig,
migrationComponentAliases,
migrationComponentOverrides,
});
if (storyblokConfig.debug) {
Logger.warning("_________ Components in stories to migrate ___________");
console.log(Array.from(new Set(preparedMigrationConfigs.flatMap((preparedMigrationConfig) => preparedMigrationConfig.componentsToMigrate))));
}
await migrateProvidedComponentsDataInStories({
itemType,
migrationConfig,
migrateFrom,
from,
to,
filters,
dryRun,
publish,
publishLanguages,
fromFilePath,
fileName,
preparedMigrationConfigs,
}, config);
};
export const doTheMigration = async ({ itemType = "story", from, itemsToMigrate, migrationConfig, migrationConfigs, to, dryRun, publish, publishLanguages, migrateFrom, fromFilePath, fileName, }, config) => {
const preparedMigrationConfigs = migrationConfigs ||
prepareMigrationConfigs({
migrationConfig: migrationConfig || [],
});
const artifactBaseName = resolveOutputFileBaseName({ from, fileName });
const useDatestamp = shouldUseDatestampForArtifacts(fileName);
let pipelineResult;
try {
pipelineResult = runMigrationPipelineInMemory({
itemType,
itemsToMigrate,
preparedMigrationConfigs,
});
}
catch (error) {
if (error instanceof MigrationValidationFailedError) {
await createAndSaveToFile({
datestamp: useDatestamp,
ext: "json",
filename: `${dryRun ? "dry-run--" : ""}${artifactBaseName}---${itemType}-validation-failed`,
folder: "migrations",
res: {
migrationConfig: error.migrationConfig,
validatorId: error.validatorId,
validatorName: error.validatorName,
issueCount: error.issueCount,
issues: error.issues,
},
}, config);
Logger.error(`[VALIDATION] Migration '${error.migrationConfig}' failed in step validator '${error.validatorId}' with ${error.issueCount} issue(s).`);
error.issues
.slice(0, 20)
.forEach((issue) => {
const uid = issue.uid ? ` (_uid: ${issue.uid})` : "";
console.log(` ${issue.componentPath} -> ${issue.component}${uid} ${issue.message}`);
});
if (error.issueCount > 20) {
Logger.warning("[VALIDATION] Showing first 20 issues only. Full report saved to migrations folder.");
}
}
throw error;
}
if (pipelineResult.changedItems.length === 0) {
console.log("# No Stories to update #");
}
else {
console.log(`${pipelineResult.changedItems.length} stories to migrate`);
}
await saveDryRunDiffArtifacts({
artifactBaseName,
useDatestamp,
itemType,
dryRun,
inputItems: itemsToMigrate,
finalItems: pipelineResult.finalItems,
}, config);
await createAndSaveToFile({
datestamp: useDatestamp,
ext: "json",
filename: `${dryRun ? "dry-run--" : ""}${artifactBaseName}---${itemType}-to-migrate`,
folder: "migrations",
res: pipelineResult.changedItems,
}, config);
await savePipelineSummary({
artifactBaseName,
useDatestamp,
from,
itemType,
dryRun,
publish,
publishLanguages,
migrateFrom,
fromFilePath,
pipelineResult,
}, config);
if (dryRun) {
console.log(" ");
Logger.success(`[DRY RUN] Migration preview complete. ${pipelineResult.changedItems.length} ${itemType}(s) would be affected.`);
Logger.success(`[DRY RUN] No API changes were made. Review the saved migration file for details.`);
return;
}
for (const preparedMigrationConfig of preparedMigrationConfigs) {
await modifyOrCreateAppliedMigrationsFile(preparedMigrationConfig.migrationConfigName, itemType);
}
if (pipelineResult.changedItems.length === 0) {
return;
}
let writeResults = [];
let resolvedPublishLanguages;
if (itemType === "story") {
if (publish && publishLanguages !== undefined) {
resolvedPublishLanguages =
await managementApi.stories.resolvePublishLanguageCodes(publishLanguages, {
...config,
spaceId: to,
});
}
writeResults = await managementApi.stories.updateStories({
stories: pipelineResult.changedItems,
spaceId: to,
options: {
publish: Boolean(publish),
publishLanguages: resolvedPublishLanguages,
preservePublishState: Boolean(publish),
},
}, config);
}
else if (itemType === "preset") {
writeResults = await managementApi.presets.updatePresets({
presets: pipelineResult.changedItems,
spaceId: to,
options: {},
}, config);
}
const writeSummary = summarizeMutationWriteResults(writeResults);
try {
await saveMigrationRunLog({
artifactBaseName,
useDatestamp,
from,
to,
itemType,
dryRun,
publish,
publishLanguages,
resolvedPublishLanguages,
migrateFrom,
fromFilePath,
pipelineResult,
writeResults,
writeSummary,
}, config);
}
catch (error) {
Logger.warning(`[MIGRATION] Could not write migration run log: ${error instanceof Error ? error.message : String(error)}`);
}
if (writeSummary.failed === 0) {
Logger.success(`[MIGRATION] Update complete. ${writeSummary.successful}/${writeSummary.total} ${itemType}(s) updated successfully.`);
return;
}
Logger.warning(`[MIGRATION] Update complete with partial failures. ${writeSummary.successful}/${writeSummary.total} ${itemType}(s) updated successfully, ${writeSummary.failed} failed.`);
writeSummary.failedItems.slice(0, 10).forEach((item) => {
const label = item.slug || item.name || item.id || "unknown";
Logger.error(`[MIGRATION] Failed ${itemType}: ${String(label)}`);
});
if (writeSummary.failedItems.length > 10) {
Logger.warning(`[MIGRATION] Showing first 10 failed ${itemType}(s) only.`);
}
};
const saveBackupToFile = async ({ itemType, res, folder, filename }, config) => {
await createAndSaveToFile({
ext: "json",
datestamp: true,
suffix: itemType === "story" ? ".sb.stories" : ".sb.presets",
filename,
folder,
res: res,
}, config);
};
export const migrateProvidedComponentsDataInStories = async ({ itemType, migrationConfig, migrateFrom, from, to, componentsToMigrate, filters, dryRun, publish, publishLanguages, fromFilePath, fileName, preparedMigrationConfigs, migrationComponentAliases, migrationComponentOverrides, }, config) => {
const resolvedMigrationConfigs = preparedMigrationConfigs ||
prepareMigrationConfigs({
migrationConfig,
componentsToMigrate,
migrationComponentAliases,
migrationComponentOverrides,
});
const itemsToMigrate = await loadItemsToMigrate({
itemType,
migrateFrom,
from,
filters,
fromFilePath,
}, config);
if (migrateFrom === "space" && !dryRun) {
const backupFolder = path.join("backup", itemType);
await saveBackupToFile({
itemType,
filename: buildPreMigrationBackupBaseName({
from,
fileName,
}),
folder: backupFolder,
res: itemsToMigrate,
}, config);
}
await doTheMigration({
itemType,
itemsToMigrate,
migrationConfigs: resolvedMigrationConfigs,
migrationConfig,
from,
to,
dryRun,
publish,
publishLanguages,
migrateFrom,
fromFilePath,
fileName,
}, config);
};