contentful-import
Version:
this tool allows you to import JSON dump exported by contentful-export
1,352 lines (1,327 loc) • 53.5 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// lib/index.ts
var index_exports = {};
__export(index_exports, {
default: () => index_default
});
module.exports = __toCommonJS(index_exports);
var import_cli_table3 = __toESM(require("cli-table3"));
var import_differenceInSeconds = __toESM(require("date-fns/differenceInSeconds"));
var import_formatDistance = __toESM(require("date-fns/formatDistance"));
var import_listr3 = __toESM(require("listr"));
var import_listr_update_renderer = __toESM(require("listr-update-renderer"));
var import_listr_verbose_renderer2 = __toESM(require("listr-verbose-renderer"));
var import_lodash2 = require("lodash");
var import_p_queue = __toESM(require("p-queue"));
var import_logging7 = require("contentful-batch-libs/dist/logging");
var import_listr4 = require("contentful-batch-libs/dist/listr");
// lib/tasks/init-client.ts
var import_contentful_management = require("contentful-management");
var import_logging = require("contentful-batch-libs/dist/logging");
function logHandler(level, data) {
import_logging.logEmitter.emit(level, data);
}
function initClient(opts) {
const defaultOpts = {
timeout: 3e4,
logHandler
};
const config = {
...defaultOpts,
...opts
};
return (0, import_contentful_management.createClient)(config);
}
// lib/tasks/get-destination-data.ts
var import_bluebird = __toESM(require("bluebird"));
var import_logging2 = require("contentful-batch-libs/dist/logging");
var BATCH_CHAR_LIMIT = 1990;
var BATCH_SIZE_LIMIT = 100;
var METHODS = {
contentTypes: { name: "content types", method: "getContentTypes" },
locales: { name: "locales", method: "getLocales" },
entries: { name: "entries", method: "getEntries" },
assets: { name: "assets", method: "getAssets" },
tags: { name: "tags", method: "getTags" }
};
async function batchedIdQuery({ environment, type, ids, requestQueue }) {
const method = METHODS[type].method;
const entityTypeName = METHODS[type].name;
const batches = getIdBatches(ids);
let totalFetched = 0;
const allPendingResponses = batches.map((idBatch) => {
return requestQueue.add(async () => {
const response = await environment[method]({
"sys.id[in]": idBatch,
limit: idBatch.split(",").length
});
totalFetched = totalFetched + response.items.length;
import_logging2.logEmitter.emit("info", `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
return response.items;
});
});
const responses = await import_bluebird.default.all(allPendingResponses);
return responses.flat();
}
async function batchedPageQuery({ environment, type, requestQueue }) {
const method = METHODS[type].method;
const entityTypeName = METHODS[type].name;
let totalFetched = 0;
const { items, total } = await requestQueue.add(async () => {
const response = await environment[method]({
skip: 0,
limit: BATCH_SIZE_LIMIT
});
totalFetched += response.items.length;
import_logging2.logEmitter.emit("info", `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
return { items: response.items, total: response.total };
});
const batches = getPagedBatches(totalFetched, total);
const remainingTotalResponses = batches.map(({ skip }) => {
return requestQueue.add(async () => {
const response = await environment[method]({
skip,
limit: BATCH_SIZE_LIMIT
});
totalFetched = totalFetched + response.items.length;
import_logging2.logEmitter.emit("info", `Fetched ${totalFetched} of ${response.total} ${entityTypeName}`);
return response.items;
});
});
const remainingResponses = await import_bluebird.default.all(remainingTotalResponses);
return items.concat(remainingResponses.flat());
}
function getIdBatches(ids) {
const batches = [];
let currentBatch = "";
let currentSize = 0;
while (ids.length > 0) {
const id = ids.splice(0, 1);
currentBatch += id;
currentSize = currentSize + 1;
if (currentSize === BATCH_SIZE_LIMIT || currentBatch.length > BATCH_CHAR_LIMIT || ids.length === 0) {
batches.push(currentBatch);
currentBatch = "";
currentSize = 0;
} else {
currentBatch += ",";
}
}
return batches;
}
function getPagedBatches(totalFetched, total) {
const batches = [];
if (totalFetched >= total) {
return batches;
}
let skip = totalFetched;
while (skip < total) {
batches.push({ skip });
skip += BATCH_SIZE_LIMIT;
}
return batches;
}
async function getDestinationData({
client,
spaceId,
environmentId,
sourceData,
contentModelOnly,
skipLocales,
skipContentModel,
requestQueue
}) {
const space = await client.getSpace(spaceId);
const environment = await space.getEnvironment(environmentId);
const result = {
contentTypes: [],
tags: [],
locales: [],
entries: [],
assets: []
};
sourceData = {
...result,
...sourceData
};
if (!skipContentModel) {
const contentTypeIds = sourceData.contentTypes?.map((e) => e.sys.id);
if (contentTypeIds) {
result.contentTypes = batchedIdQuery({
environment,
type: "contentTypes",
ids: contentTypeIds,
requestQueue
});
}
if (!skipLocales) {
const localeIds = sourceData.locales?.map((e) => e.sys.id);
if (localeIds && localeIds.length) {
result.locales = batchedPageQuery({
environment,
type: "locales",
requestQueue
});
}
}
}
try {
result.tags = await batchedPageQuery({ environment, type: "tags", requestQueue });
} catch (_) {
delete result.tags;
}
if (contentModelOnly) {
return import_bluebird.default.props(result);
}
const entryIds = sourceData.entries?.map((e) => e.sys.id);
const assetIds = sourceData.assets?.map((e) => e.sys.id);
if (entryIds) {
result.entries = batchedIdQuery({
environment,
type: "entries",
ids: entryIds,
requestQueue
});
}
if (assetIds) {
result.assets = batchedIdQuery({
environment,
type: "assets",
ids: assetIds,
requestQueue
});
}
result.webhooks = [];
return import_bluebird.default.props(result);
}
// lib/tasks/push-to-space/push-to-space.ts
var import_listr = __toESM(require("listr"));
var import_listr_verbose_renderer = __toESM(require("listr-verbose-renderer"));
var import_logging6 = require("contentful-batch-libs/dist/logging");
var import_listr2 = require("contentful-batch-libs/dist/listr");
// lib/tasks/push-to-space/assets.ts
var import_fs = __toESM(require("fs"));
var import_path = require("path");
var import_util = require("util");
var import_get_entity_name = __toESM(require("contentful-batch-libs/dist/get-entity-name"));
var import_logging3 = require("contentful-batch-libs/dist/logging");
// lib/utils/errors.ts
var ContentfulAssetError = class extends Error {
constructor(message, filePath) {
super(message);
this.filePath = filePath;
}
};
var ContentfulEntityError = class extends Error {
};
var ContentfulMultiError = class extends Error {
};
// lib/tasks/push-to-space/assets.ts
var stat = (0, import_util.promisify)(import_fs.default.stat);
async function getAssetStreamForURL(url, assetsDirectory) {
const [, assetPath] = url.split("//");
const filePath = (0, import_path.join)(assetsDirectory, assetPath);
try {
await stat(filePath);
return import_fs.default.createReadStream(filePath);
} catch (err) {
const error = new ContentfulAssetError(
"Cannot open asset from filesystem",
filePath
);
throw error;
}
}
async function processAssetForLocale(locale, asset, processingOptions) {
try {
return await asset.processForLocale(locale, processingOptions);
} catch (err) {
if (err instanceof ContentfulEntityError) {
err.entity = asset;
}
import_logging3.logEmitter.emit("error", err);
throw err;
}
}
async function lastResult(promises) {
if (!promises.length) throw new RangeError("No last result from no promises");
const results = [];
await Promise.all(
promises.map(
(p) => p.then((v) => {
results.push(v);
})
)
);
return results[results.length - 1];
}
async function processAssets({
assets: assets2,
timeout,
retryLimit,
requestQueue
}) {
const processingOptions = Object.assign(
{},
timeout && { processingCheckWait: timeout },
retryLimit && { processingCheckRetry: retryLimit }
);
const pendingProcessingAssets = assets2.map(async (asset) => {
import_logging3.logEmitter.emit("info", `Processing Asset ${(0, import_get_entity_name.default)(asset)}`);
const locales2 = Object.keys(asset.fields.file || {});
let latestAssetVersion = asset;
try {
latestAssetVersion = await lastResult(
locales2.map((locale) => {
return requestQueue.add(
() => processAssetForLocale(locale, asset, processingOptions)
);
})
);
} catch (err) {
return null;
}
return latestAssetVersion;
});
const potentiallyProcessedAssets = await Promise.all(pendingProcessingAssets);
return potentiallyProcessedAssets.filter((asset) => asset);
}
// lib/tasks/push-to-space/creation.ts
var import_collection = require("lodash/collection");
var import_object = require("lodash/object");
var import_get_entity_name2 = __toESM(require("contentful-batch-libs/dist/get-entity-name"));
var import_logging4 = require("contentful-batch-libs/dist/logging");
function createEntities({ context, entities, destinationEntitiesById, skipUpdates, requestQueue }) {
return createEntitiesWithConcurrency({ context, entities, destinationEntitiesById, skipUpdates, requestQueue });
}
function createLocales({ context, entities, destinationEntitiesById, requestQueue }) {
return createEntitiesInSequence({ context, entities, destinationEntitiesById, requestQueue });
}
async function createEntitiesWithConcurrency({ context, entities, destinationEntitiesById, skipUpdates, requestQueue }) {
const pendingCreatedEntities = entities.map((entity) => {
const destinationEntity = getDestinationEntityForSourceEntity(destinationEntitiesById, entity.transformed);
const updateOperation = skipUpdates ? "skip" : "update";
const operation = destinationEntity ? updateOperation : "create";
if (destinationEntity && skipUpdates) {
creationSuccessNotifier(operation, entity.transformed);
return;
}
return requestQueue.add(async () => {
try {
const createdEntity = await (destinationEntity ? updateDestinationWithSourceData(destinationEntity, entity.transformed) : createInDestination(context, entity.transformed));
creationSuccessNotifier(operation, createdEntity);
return createdEntity;
} catch (err) {
return handleCreationErrors(entity, err);
}
});
});
const createdEntities = await Promise.all(pendingCreatedEntities);
return createdEntities.filter((entity) => entity);
}
async function createEntitiesInSequence({ context, entities, destinationEntitiesById, requestQueue }) {
const createdEntities = [];
for (const entity of entities) {
const destinationEntity = getDestinationEntityForSourceEntity(destinationEntitiesById, entity.transformed);
const operation = destinationEntity ? "update" : "create";
try {
const createdEntity = await requestQueue.add(async () => {
const createdOrUpdatedEntity = await (destinationEntity ? updateDestinationWithSourceData(destinationEntity, entity.transformed) : createInDestination(context, entity.transformed));
return createdOrUpdatedEntity;
});
creationSuccessNotifier(operation, createdEntity);
createdEntities.push(createdEntity);
} catch (err) {
const maybeSubstituteEntity = handleCreationErrors(entity, err);
if (maybeSubstituteEntity) {
createdEntities.push(maybeSubstituteEntity);
}
}
}
return createdEntities;
}
async function createEntries({ context, entities, destinationEntitiesById, skipUpdates, requestQueue }) {
const createdEntries = await Promise.all(entities.map((entry) => {
return createEntry({ entry, target: context.target, skipContentModel: context.skipContentModel, destinationEntitiesById, skipUpdates, requestQueue });
}));
return createdEntries.filter((entry) => entry);
}
async function createEntry({ entry, target, skipContentModel, destinationEntitiesById, skipUpdates, requestQueue }) {
const contentTypeId = entry.original.sys.contentType.sys.id;
const destinationEntry = getDestinationEntityForSourceEntity(
destinationEntitiesById,
entry.transformed
);
const updateOperation = skipUpdates ? "skip" : "update";
const operation = destinationEntry ? updateOperation : "create";
if (destinationEntry && skipUpdates) {
creationSuccessNotifier(operation, entry.transformed);
return entry.transformed;
}
try {
const createdOrUpdatedEntry = await requestQueue.add(() => {
return destinationEntry ? updateDestinationWithSourceData(destinationEntry, entry.transformed) : createEntryInDestination(target, contentTypeId, entry.transformed);
});
creationSuccessNotifier(operation, createdOrUpdatedEntry);
return createdOrUpdatedEntry;
} catch (err) {
if (err instanceof Error) {
if (skipContentModel && err.name === "UnknownField") {
const errors = (0, import_object.get)(JSON.parse(err.message), "details.errors");
entry.transformed.fields = cleanupUnknownFields(entry.transformed.fields, errors);
return createEntry({ entry, target, skipContentModel, destinationEntitiesById, skipUpdates, requestQueue });
}
}
if (err instanceof ContentfulEntityError) {
err.entity = entry;
}
import_logging4.logEmitter.emit("error", err);
return null;
}
}
function updateDestinationWithSourceData(destinationEntity, sourceEntity) {
const plainData = getPlainData(sourceEntity);
(0, import_object.assign)(destinationEntity, plainData);
return destinationEntity.update();
}
function createInDestination(context, sourceEntity) {
const { type, target } = context;
if (type === "Tag") {
return createTagInDestination(context, sourceEntity);
}
const id = (0, import_object.get)(sourceEntity, "sys.id");
const plainData = getPlainData(sourceEntity);
return id ? target[`create${type}WithId`](id, plainData) : target[`create${type}`](plainData);
}
function createEntryInDestination(space, contentTypeId, sourceEntity) {
const id = sourceEntity.sys.id;
const plainData = getPlainData(sourceEntity);
return id ? space.createEntryWithId(contentTypeId, id, plainData) : space.createEntry(contentTypeId, plainData);
}
function createTagInDestination(context, sourceEntity) {
const id = sourceEntity.sys.id;
const visibility = sourceEntity.sys.visibility || "private";
const name = sourceEntity.name;
return context.target.createTag(id, name, visibility);
}
function handleCreationErrors(entity, err) {
if ((0, import_object.get)(err, "error.sys.id") === "ValidationFailed") {
const errors = (0, import_object.get)(err, "error.details.errors");
if (errors && errors.length > 0 && errors[0].name === "taken") {
return entity;
}
}
err.entity = entity.original;
import_logging4.logEmitter.emit("error", err);
return null;
}
function cleanupUnknownFields(fields, errors) {
return (0, import_object.omitBy)(fields, (field, fieldId) => {
return (0, import_collection.find)(errors, (error) => {
const [, errorFieldId] = error.path;
return error.name === "unknown" && errorFieldId === fieldId;
});
});
}
function getDestinationEntityForSourceEntity(destinationEntitiesById, sourceEntity) {
return destinationEntitiesById.get((0, import_object.get)(sourceEntity, "sys.id")) || null;
}
function creationSuccessNotifier(method, createdEntity) {
import_logging4.logEmitter.emit("info", `${method.toUpperCase()} ${createdEntity.sys.type} ${(0, import_get_entity_name2.default)(createdEntity)}`);
return createdEntity;
}
function getPlainData(entity) {
const data = entity.toPlainObject ? entity.toPlainObject() : entity;
return (0, import_object.omit)(data, "sys");
}
// lib/tasks/push-to-space/publishing.ts
var import_get_entity_name3 = __toESM(require("contentful-batch-libs/dist/get-entity-name"));
var import_logging5 = require("contentful-batch-libs/dist/logging");
async function publishEntities({ entities, requestQueue }) {
const entitiesToPublish = entities.filter((entity2) => {
if (!entity2 || !entity2.publish) {
import_logging5.logEmitter.emit("warning", `Unable to publish ${(0, import_get_entity_name3.default)(entity2)}`);
return false;
}
return true;
});
if (entitiesToPublish.length === 0) {
import_logging5.logEmitter.emit("info", "Skipping publishing since zero valid entities passed");
return [];
}
const entity = entities[0].original || entities[0];
const type = entity.sys.type || "unknown type";
import_logging5.logEmitter.emit("info", `Publishing ${entities.length} ${type}s`);
const result = await runQueue(entitiesToPublish, [], requestQueue);
import_logging5.logEmitter.emit("info", `Successfully published ${result.length} ${type}s`);
return result;
}
async function archiveEntities({ entities, requestQueue }) {
const entitiesToArchive = entities.filter((entity2) => {
if (!entity2 || !entity2.archive) {
import_logging5.logEmitter.emit("warning", `Unable to archive ${(0, import_get_entity_name3.default)(entity2)}`);
return false;
}
return true;
});
if (entitiesToArchive.length === 0) {
import_logging5.logEmitter.emit("info", "Skipping archiving since zero valid entities passed");
return [];
}
const entity = entities[0].original || entities[0];
const type = entity.sys.type || "unknown type";
import_logging5.logEmitter.emit("info", `Archiving ${entities.length} ${type}s`);
const pendingArchivedEntities = entitiesToArchive.map((entity2) => {
return requestQueue.add(async () => {
try {
const archivedEntity = await entity2.archive();
return archivedEntity;
} catch (err) {
if (err instanceof ContentfulEntityError) {
err.entity = entity2;
}
import_logging5.logEmitter.emit("error", err);
return null;
}
});
});
const allPossiblyArchivedEntities = await Promise.all(pendingArchivedEntities);
const allArchivedEntities = allPossiblyArchivedEntities.filter((entity2) => entity2);
import_logging5.logEmitter.emit("info", `Successfully archived ${allArchivedEntities.length} ${type}s`);
return allArchivedEntities;
}
async function runQueue(queue, result = [], requestQueue) {
const publishedEntities = [];
for (const entity of queue) {
import_logging5.logEmitter.emit("info", `Publishing ${entity.sys.type} ${(0, import_get_entity_name3.default)(entity)}`);
try {
const publishedEntity = await requestQueue.add(() => entity.publish());
publishedEntities.push(publishedEntity);
} catch (err) {
if (err instanceof ContentfulEntityError) {
err.entity = entity;
}
import_logging5.logEmitter.emit("error", err);
}
}
result = [
...result,
...publishedEntities
];
const publishedEntityIds = new Set(publishedEntities.map((entity) => entity.sys.id));
const unpublishedEntities = queue.filter((entity) => !publishedEntityIds.has(entity.sys.id));
if (unpublishedEntities.length > 0) {
if (queue.length === unpublishedEntities.length) {
const unpublishedEntityNames = unpublishedEntities.map(import_get_entity_name3.default).join(", ");
import_logging5.logEmitter.emit("error", `Could not publish the following entities: ${unpublishedEntityNames}`);
} else {
return runQueue(unpublishedEntities, result, requestQueue);
}
}
return result;
}
// lib/tasks/push-to-space/push-to-space.ts
var DEFAULT_CONTENT_STRUCTURE = {
entries: [],
assets: [],
contentTypes: [],
tags: [],
locales: [],
webhooks: [],
editorInterfaces: []
};
function pushToSpace({
sourceData,
destinationData = {},
client,
spaceId,
environmentId,
contentModelOnly,
skipContentModel,
skipContentUpdates,
skipLocales,
skipContentPublishing,
timeout,
retryLimit,
listrOptions,
uploadAssets,
skipAssetUpdates,
assetsDirectory,
requestQueue
}) {
sourceData = {
...DEFAULT_CONTENT_STRUCTURE,
...sourceData
};
destinationData = {
...DEFAULT_CONTENT_STRUCTURE,
...destinationData
};
listrOptions = listrOptions || {
renderer: import_listr_verbose_renderer.default
};
const destinationDataById = {};
for (const [entityType, entities] of Object.entries(destinationData)) {
const entitiesById = /* @__PURE__ */ new Map();
for (const entity of entities) {
entitiesById.set(entity.sys.id, entity);
}
destinationDataById[entityType] = entitiesById;
}
return new import_listr.default([
{
title: "Connecting to space",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const space = await client.getSpace(spaceId);
const environment = await space.getEnvironment(environmentId);
ctx.space = space;
ctx.environment = environment;
})
},
{
title: "Importing Locales",
task: (0, import_listr2.wrapTask)(async (ctx) => {
if (!destinationDataById.locales) {
return;
}
const locales2 = await createLocales({
context: { target: ctx.environment, type: "Locale" },
entities: sourceData.locales,
destinationEntitiesById: destinationDataById.locales,
requestQueue
});
ctx.data.locales = locales2;
}),
skip: () => skipContentModel || skipLocales
},
{
title: "Importing Content Types",
task: (0, import_listr2.wrapTask)(async (ctx) => {
if (!destinationDataById.contentTypes) {
return;
}
const contentTypes2 = await createEntities({
context: { target: ctx.environment, type: "ContentType" },
entities: sourceData.contentTypes,
destinationEntitiesById: destinationDataById.contentTypes,
skipUpdates: false,
requestQueue
});
ctx.data.contentTypes = contentTypes2;
}),
skip: () => skipContentModel
},
{
title: "Publishing Content Types",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const publishedContentTypes = await publishEntities2({
entities: ctx.data.contentTypes,
sourceEntities: sourceData.contentTypes,
requestQueue
});
ctx.data.contentTypes = publishedContentTypes;
}),
skip: () => skipContentModel
},
{
title: "Importing Tags",
task: (0, import_listr2.wrapTask)(async (ctx) => {
if (sourceData.tags && destinationDataById.tags) {
const tags2 = await createEntities({
context: { target: ctx.environment, type: "Tag" },
entities: sourceData.tags,
destinationEntitiesById: destinationDataById.tags,
skipUpdates: false,
requestQueue
});
ctx.data.tags = tags2;
}
}),
// we remove `tags` from destination data if an error was thrown trying to access them
// this means the user doesn't have access to this feature, skip importing tags
skip: () => !destinationDataById.tags
},
{
title: "Importing Editor Interfaces",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const allEditorInterfacesBeingFetched = ctx.data.contentTypes.map(async (contentType) => {
if (!sourceData.editorInterfaces) {
return;
}
const editorInterface = sourceData.editorInterfaces.find((editorInterface2) => {
return editorInterface2.sys.contentType.sys.id === contentType.sys.id;
});
if (!editorInterface) {
return;
}
try {
const ctEditorInterface = await requestQueue.add(() => ctx.environment.getEditorInterfaceForContentType(contentType.sys.id));
import_logging6.logEmitter.emit("info", `Fetched editor interface for ${contentType.name}`);
ctEditorInterface.controls = editorInterface.controls;
ctEditorInterface.groupControls = editorInterface.groupControls;
ctEditorInterface.editorLayout = editorInterface.editorLayout;
ctEditorInterface.sidebar = editorInterface.sidebar;
ctEditorInterface.editors = editorInterface.editors;
const updatedEditorInterface = await requestQueue.add(() => ctEditorInterface.update());
return updatedEditorInterface;
} catch (err) {
if (err instanceof ContentfulEntityError) {
err.entity = editorInterface;
}
throw err;
}
});
const allEditorInterfaces = await Promise.all(allEditorInterfacesBeingFetched);
const editorInterfaces = allEditorInterfaces.filter((editorInterface) => editorInterface);
ctx.data.editorInterfaces = editorInterfaces;
}),
skip: (ctx) => skipContentModel || ctx.data.contentTypes.length === 0
},
{
title: "Uploading Assets",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const allPendingUploads = [];
for (const asset of sourceData.assets) {
for (const file of Object.values(asset.transformed.fields.file)) {
allPendingUploads.push(requestQueue.add(async () => {
try {
import_logging6.logEmitter.emit("info", `Uploading Asset file ${file.upload}`);
const assetStream = await getAssetStreamForURL(file.upload, assetsDirectory);
const upload = await ctx.environment.createUpload({
fileName: asset.transformed.sys.id,
file: assetStream
});
delete file.upload;
file.uploadFrom = {
sys: {
type: "Link",
linkType: "Upload",
id: upload.sys.id
}
};
return upload;
} catch (err) {
import_logging6.logEmitter.emit("error", err);
}
}));
}
}
const uploads = await Promise.all(allPendingUploads);
ctx.data.uploadedAssetFiles = uploads;
}),
skip: () => !uploadAssets || !sourceData.assets.length
},
{
title: "Importing Assets",
task: (0, import_listr2.wrapTask)(async (ctx) => {
if (!destinationDataById.assets) {
return;
}
const assetsToProcess = await createEntities({
context: { target: ctx.environment, type: "Asset" },
entities: sourceData.assets,
destinationEntitiesById: destinationDataById.assets,
skipUpdates: skipAssetUpdates,
requestQueue
});
const processedAssets = await processAssets({
assets: assetsToProcess,
timeout,
retryLimit,
requestQueue
});
ctx.data.assets = processedAssets;
}),
skip: () => contentModelOnly
},
{
title: "Publishing Assets",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const publishedAssets = await publishEntities2({
entities: ctx.data.assets,
sourceEntities: sourceData.assets,
requestQueue
});
ctx.data.publishedAssets = publishedAssets;
}),
skip: () => contentModelOnly || skipContentPublishing
},
{
title: "Archiving Assets",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const archivedAssets = await archiveEntities2({
entities: ctx.data.assets,
sourceEntities: sourceData.assets,
requestQueue
});
ctx.data.archivedAssets = archivedAssets;
}),
skip: () => contentModelOnly || skipContentPublishing
},
{
title: "Importing Content Entries",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const entries2 = await createEntries({
context: { target: ctx.environment, skipContentModel },
entities: sourceData.entries,
destinationEntitiesById: destinationDataById.entries,
skipUpdates: skipContentUpdates,
requestQueue
});
ctx.data.entries = entries2;
}),
skip: () => contentModelOnly
},
{
title: "Publishing Content Entries",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const publishedEntries = await publishEntities2({
entities: ctx.data.entries,
sourceEntities: sourceData.entries,
requestQueue
});
ctx.data.publishedEntries = publishedEntries;
}),
skip: () => contentModelOnly || skipContentPublishing
},
{
title: "Archiving Entries",
task: (0, import_listr2.wrapTask)(async (ctx) => {
const archivedEntries = await archiveEntities2({
entities: ctx.data.entries,
sourceEntities: sourceData.entries,
requestQueue
});
ctx.data.archivedEntries = archivedEntries;
}),
skip: () => contentModelOnly || skipContentPublishing
},
{
title: "Creating Web Hooks",
task: (0, import_listr2.wrapTask)(async (ctx) => {
if (!sourceData.webhooks || !destinationDataById.webhooks) {
return;
}
const webhooks2 = await createEntities({
context: { target: ctx.space, type: "Webhook" },
entities: sourceData.webhooks,
destinationEntitiesById: destinationDataById.webhooks,
requestQueue
});
ctx.data.webhooks = webhooks2;
}),
skip: () => contentModelOnly || environmentId !== "master" && "Webhooks can only be imported in master environment"
}
], listrOptions);
}
function archiveEntities2({ entities, sourceEntities, requestQueue }) {
const entityIdsToArchive = sourceEntities.filter(({ original }) => original.sys.archivedVersion).map(({ original }) => original.sys.id);
const entitiesToArchive = entities.filter((entity) => entityIdsToArchive.indexOf(entity.sys.id) !== -1);
return archiveEntities({ entities: entitiesToArchive, requestQueue });
}
function publishEntities2({ entities, sourceEntities, requestQueue }) {
const entityIdsToPublish = sourceEntities.filter(({ original }) => original.sys.publishedVersion).map(({ original }) => original.sys.id);
const entitiesToPublish = entities.filter((entity) => entityIdsToPublish.indexOf(entity.sys.id) !== -1);
return publishEntities({ entities: entitiesToPublish, requestQueue });
}
// lib/transform/transform-space.ts
var import_object2 = require("lodash/object");
// lib/transform/transformers.ts
var transformers_exports = {};
__export(transformers_exports, {
assets: () => assets,
contentTypes: () => contentTypes,
entries: () => entries,
locales: () => locales,
tags: () => tags,
webhooks: () => webhooks
});
var import_lodash = require("lodash");
function contentTypes(contentType) {
return contentType;
}
function tags(tag) {
return tag;
}
function entries(entry, _, tagsEnabled = false) {
return removeMetadataTags(entry, tagsEnabled);
}
function webhooks(webhook) {
if (webhook.httpBasicUsername) {
delete webhook.httpBasicUsername;
}
if (webhook.headers) {
webhook.headers = webhook.headers.filter((header) => !header.secret);
}
return webhook;
}
function assets(asset, _, tagsEnabled = false) {
const transformedAsset = (0, import_lodash.omit)(asset, "sys");
transformedAsset.sys = (0, import_lodash.pick)(asset.sys, "id");
transformedAsset.fields = (0, import_lodash.pick)(asset.fields, "title", "description");
transformedAsset.fields.file = (0, import_lodash.reduce)(
asset.fields.file,
(newFile, localizedFile, locale) => {
newFile[locale] = (0, import_lodash.pick)(localizedFile, "contentType", "fileName");
if (!localizedFile.uploadFrom) {
const assetUrl = localizedFile.url || localizedFile.upload;
newFile[locale].upload = `${/^(http|https):\/\//i.test(assetUrl) ? "" : "https:"}${assetUrl}`;
} else {
newFile[locale].uploadFrom = localizedFile.uploadFrom;
}
return newFile;
},
{}
);
return removeMetadataTags(transformedAsset, tagsEnabled);
}
function locales(locale, destinationLocales) {
const transformedLocale = (0, import_lodash.pick)(locale, "code", "name", "contentManagementApi", "contentDeliveryApi", "fallbackCode", "optional");
const destinationLocale = (0, import_lodash.find)(destinationLocales, { code: locale.code });
if (destinationLocale) {
transformedLocale.sys = (0, import_lodash.pick)(destinationLocale.sys, "id");
}
return transformedLocale;
}
function removeMetadataTags(entity, tagsEnabled = false) {
if (!tagsEnabled) {
delete entity.metadata;
}
return entity;
}
// lib/utils/sort-entries.ts
var import_collection2 = require("lodash/collection");
var _o = __toESM(require("lodash/object"));
var import_array = require("lodash/array");
function sortEntries(entries2) {
const linkedEntries = getLinkedEntries(entries2);
const mergedLinkedEntries = mergeSort(linkedEntries, (a) => {
return hasLinkedIndexesInFront(a);
});
return (0, import_collection2.map)(mergedLinkedEntries, (linkInfo) => entries2[linkInfo.index]);
function hasLinkedIndexesInFront(item) {
if (hasLinkedIndexes(item)) {
return (0, import_collection2.some)(item.linkIndexes, (index) => index > item.index) ? 1 : -1;
}
return 0;
}
function hasLinkedIndexes(item) {
return item.linkIndexes.length > 0;
}
}
function getLinkedEntries(entries2) {
return (0, import_collection2.map)(entries2, function(entry) {
const entryIndex = entries2.indexOf(entry);
const rawLinks = (0, import_collection2.map)(entry.fields, (field) => {
field = _o.values(field)[0];
if (isEntryLink(field)) {
return getFieldEntriesIndex(field, entries2);
} else if (isEntityArray(field) && isEntryLink(field[0])) {
return (0, import_collection2.map)(field, (item) => getFieldEntriesIndex(item, entries2));
}
});
return {
index: entryIndex,
linkIndexes: (0, import_collection2.filter)((0, import_array.flatten)(rawLinks), (index) => index >= 0)
};
});
}
function getFieldEntriesIndex(field, entries2) {
const id = _o.get(field, "sys.id");
return entries2.findIndex((entry) => entry.sys.id === id);
}
function isEntryLink(item) {
return _o.get(item, "sys.type") === "Entry" || _o.get(item, "sys.linkType") === "Entry";
}
function isEntityArray(item) {
return Array.isArray(item) && item.length > 0 && _o.has(item[0], "sys");
}
function mergeSort(arr, compareFn) {
if (arr.length < 2) return arr;
if (compareFn == null) compareFn = defaultCompare;
const mid = ~~(arr.length / 2);
const left = mergeSort(arr.slice(0, mid), compareFn);
const right = mergeSort(arr.slice(mid, arr.length), compareFn);
return merge(left, right, compareFn);
}
function defaultCompare(a, b) {
return a < b ? -1 : a > b ? 1 : 0;
}
function merge(left, right, compareFn) {
const result = [];
while (left.length && right.length) {
if (compareFn(left[0], right[0]) <= 0) {
result.push(left.shift());
} else {
result.push(right.shift());
}
}
if (left.length) result.push(...left);
if (right.length) result.push(...right);
return result;
}
// lib/utils/sort-locales.ts
function sortLocales(locales2) {
const localeByFallback = {};
locales2.forEach((locale) => {
if (locale.fallbackCode === null) {
locale.fallbackCode = void 0;
}
if (!localeByFallback[locale.fallbackCode]) {
localeByFallback[locale.fallbackCode] = [];
}
localeByFallback[locale.fallbackCode].push(locale);
});
return sortByFallbackKey(localeByFallback);
}
function sortByFallbackKey(localeByFallback, key) {
if (!localeByFallback[`${key}`]) {
return [];
}
const sortedLocales = localeByFallback[`${key}`];
sortedLocales.forEach((locale) => {
sortByFallbackKey(localeByFallback, locale.code).forEach(
(x) => sortedLocales.push(x)
);
});
sortedLocales.forEach((locale) => {
if (!locale.fallbackCode) {
locale.fallbackCode = null;
}
});
return sortedLocales;
}
// lib/transform/transform-space.ts
var spaceEntities = [
"contentTypes",
"entries",
"assets",
"locales",
"webhooks",
"tags"
];
function transform_space_default(sourceData, destinationData, customTransformers, entities = spaceEntities) {
const transformers = (0, import_object2.defaults)(customTransformers, transformers_exports);
const baseSpaceData = (0, import_object2.omit)(sourceData, ...entities);
sourceData.locales = sortLocales(sourceData.locales);
const tagsEnabled = !!destinationData.tags;
return entities.reduce((transformedSpaceData, type) => {
const sortedEntities = type === "tags" ? sourceData[type] : sortEntries(sourceData[type]);
const transformedEntities = sortedEntities.map((entity) => ({
original: entity,
transformed: transformers[type](entity, destinationData[type], tagsEnabled)
}));
transformedSpaceData[type] = transformedEntities;
return transformedSpaceData;
}, baseSpaceData);
}
// lib/utils/schema.ts
var import_joi = __toESM(require("joi"));
var entrySchema = {
sys: import_joi.default.object(),
fields: import_joi.default.object()
};
var tagSchema = {
name: import_joi.default.string().required(),
sys: import_joi.default.object()
};
var contentTypeSchema = {
sys: import_joi.default.object(),
fields: import_joi.default.array().required().items(import_joi.default.object().keys({
id: import_joi.default.string().required(),
name: import_joi.default.string().required(),
type: import_joi.default.string().required().regex(/^Symbol|Text|Integer|Number|Date|Object|Boolean|Array|Link|Location$/),
validations: import_joi.default.array(),
disabled: import_joi.default.boolean(),
omitted: import_joi.default.boolean(),
required: import_joi.default.boolean(),
localized: import_joi.default.boolean(),
linkType: import_joi.default.string().when("type", { is: "Link", then: import_joi.default.string().regex(/^Asset|Entry$/), otherwise: import_joi.default.forbidden() })
}))
};
var assetSchema = {
sys: import_joi.default.object(),
fields: import_joi.default.object({
file: import_joi.default.object().pattern(/.+/, import_joi.default.object({
url: import_joi.default.string().required(),
details: import_joi.default.object({
size: import_joi.default.number(),
image: import_joi.default.object({
width: import_joi.default.number(),
height: import_joi.default.number()
})
}),
fileName: import_joi.default.string().required(),
contentType: import_joi.default.string().required()
}))
}).required()
};
var editorInterfaceSchema = {
sys: import_joi.default.object(),
controls: import_joi.default.array().items({ fieldId: import_joi.default.string(), widgetId: import_joi.default.string() })
};
var localeSchema = {
name: import_joi.default.string().required(),
internal_code: import_joi.default.string(),
code: import_joi.default.string().required(),
fallbackCode: import_joi.default.string().allow(null),
default: import_joi.default.boolean(),
contentManagementApi: import_joi.default.boolean(),
contentDeliveryApi: import_joi.default.boolean(),
optional: import_joi.default.boolean(),
sys: import_joi.default.object()
};
var webhookSchema = {
name: import_joi.default.string(),
url: import_joi.default.string().replace(/{[^}{]+?}/g, "x").regex(/^https?:\/\/[^ /}{][^ }{]*$/i).required(),
topics: import_joi.default.array().required(),
httpBasicUsername: import_joi.default.string().allow("", null)
};
var payloadSchema = import_joi.default.object({
entries: import_joi.default.array().items(entrySchema),
contentTypes: import_joi.default.array().items(contentTypeSchema),
tags: import_joi.default.array().items(tagSchema),
assets: import_joi.default.array().items(assetSchema),
locales: import_joi.default.array().items(localeSchema),
editorInterfaces: import_joi.default.array().items(editorInterfaceSchema),
webhooks: import_joi.default.array().items(webhookSchema)
});
// lib/utils/validations.ts
var import_get_entity_name4 = __toESM(require("contentful-batch-libs/dist/get-entity-name"));
var attachEntityName = (details, payload) => {
details.map((detail) => {
if (detail.path.length >= 2) {
detail.entity = (0, import_get_entity_name4.default)(payload[detail.path[0]][detail.path[1]]);
}
return detail;
});
};
var countInvalidEntities = (validationData) => {
const entityCount = validationData.reduce((entities, currentDetail) => {
if (!entities[currentDetail.path[0]]) {
entities[currentDetail.path[0]] = 1;
} else {
entities[currentDetail.path[0]]++;
}
return entities;
}, {});
return Object.keys(entityCount).map((key) => `${key}:${entityCount[key]}`);
};
var assertPayload = (payload) => {
const result = payloadSchema.validate(payload, { allowUnknown: true, abortEarly: false });
if (result.error) {
attachEntityName(result.error.details, payload);
const invalidEntityCount = countInvalidEntities(result.error.details).join(", ");
result.error.message = `${invalidEntityCount} - Get further details in the error log file`;
delete result.error._original;
throw result.error;
}
};
var assertDefaultLocale = (source, destination) => {
const sourceDefaultLocale = source.locales.find((locale) => locale.default === true);
const destinationDefaultLocale = destination.locales.find((locale) => locale.default === true);
if (!sourceDefaultLocale || !destinationDefaultLocale) {
return;
}
if (sourceDefaultLocale.code !== destinationDefaultLocale.code) {
throw new Error(`
Please make sure the destination space have the same default locale as the source
Default locale for source space : ${sourceDefaultLocale.code}
Default locale for destination space: ${destinationDefaultLocale.code}
`);
}
};
// lib/parseOptions.ts
var import_fs2 = __toESM(require("fs"));
var import_path2 = require("path");
var import_format = __toESM(require("date-fns/format"));
// package.json
var version = "0.0.0-determined-by-semantic-release";
// lib/utils/headers.ts
function getHeadersConfig(value) {
if (!value) {
return {};
}
const values2 = Array.isArray(value) ? value : [value];
return values2.reduce((headers, value2) => {
value2 = value2.trim();
const separatorIndex = value2.indexOf(":");
if (separatorIndex === -1) {
return headers;
}
const headerKey = value2.slice(0, separatorIndex).trim();
const headerValue = value2.slice(separatorIndex + 1).trim();
return {
...headers,
[headerKey]: headerValue
};
}, {});
}
// lib/parseOptions.ts
var import_proxy = require("contentful-batch-libs/dist/proxy");
var import_add_sequence_header = __toESM(require("contentful-batch-libs/dist/add-sequence-header"));
var import_json_ext = require("@discoveryjs/json-ext");
var SUPPORTED_ENTITY_TYPES = [
"contentTypes",
"tags",
"entries",
"assets",
"locales",
"webhooks",
"editorInterfaces"
];
async function parseOptions(params) {
const defaultOptions = {
skipContentModel: false,
skipLocales: false,
skipContentPublishing: false,
skipAssetUpdates: false,
skipContentUpdates: false,
useVerboseRenderer: false,
environmentId: "master",
rawProxy: false,
uploadAssets: false,
rateLimit: 7
};
const configFile = params.config ? require((0, import_path2.resolve)(process.cwd(), params.config)) : {};
const options = {
...defaultOptions,
...configFile,
...params,
headers: (0, import_add_sequence_header.default)(params.headers || getHeadersConfig(params.header))
};
if (!options.spaceId) {
throw new Error("The `spaceId` option is required.");
}
if (!options.managementToken) {
throw new Error("The `managementToken` option is required.");
}
if (!options.contentFile && !options.content) {
throw new Error("Either the `contentFile` or `content` option are required.");
}
if (options.contentModelOnly && options.skipContentModel) {
throw new Error("`contentModelOnly` and `skipContentModel` cannot be used together");
}
if (options.skipLocales && !options.contentModelOnly) {
throw new Error("`skipLocales` can only be used together with `contentModelOnly`");
}
const proxySimpleExp = /.+:\d+/;
const proxyAuthExp = /.+:.+@.+:\d+/;
if (typeof options.proxy === "string" && options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
throw new Error("Please provide the proxy config in the following format:\nhost:port or user:password@host:port");
}
options.startTime = /* @__PURE__ */ new Date();
if (!options.errorLogFile) {
options.errorLogFile = (0, import_path2.resolve)(process.cwd(), `contentful-import-error-log-${options.spaceId}-${(0, import_format.default)(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`);
} else {
options.errorLogFile = (0, import_path2.resolve)(process.cwd(), options.errorLogFile);
}
options.accessToken = options.managementToken;
if (!options.content) {
const fileStream = import_fs2.default.createReadStream(options.contentFile, { encoding: "utf8" });
options.content = await (0, import_json_ext.parseChunked)(fileStream);
}
Object.keys(options.content).forEach((type) => {
if (SUPPORTED_ENTITY_TYPES.indexOf(type) === -1) {
delete options.content[type];
}
});
SUPPORTED_ENTITY_TYPES.forEach((type) => {
options.content[type] = options.content[type] || [];
});
if (typeof options.proxy === "string") {
options.proxy = (0, import_proxy.proxyStringToObject)(options.proxy);
}
if (!options.rawProxy && options.proxy) {
options.httpsAgent = (0, import_proxy.agentFromProxy)(options.proxy);
delete options.proxy;
}
options.application = options.managementApplication || `contentful.import/${version}`;
options.feature = options.managementFeature || "library-import";
return options;
}
// lib/index.ts
var ONE_SECOND = 1e3;
var tableOptions = {
// remove ANSI color codes for better CI/CD compatibility
style: { head: [], border: [] }
};
function createListrOptions(options) {
if (options.useVerboseRenderer) {
return {
renderer: import_listr_verbose_renderer2.default
};
}
return {
renderer: import_listr_update_renderer.default,
collapse: false
};
}
async function runContentfulImport(params) {
const log = [];
const options = await parseOptions(params);
const listrOptions = createListrOptions(options);
const requestQueue = new import_p_queue.default({
interval: ONE_SECOND,
intervalCap: options.rateLimit,
carryoverConcurrencyCount: true
});
(0, import_logging7.setupLogging)(log);
const infoTable = new import_cli_table3.default(tableOptions);
infoTable.push([{ colSpan: 2, content: "The following entities are going to be imported:" }]);
Object.keys(options.content).forEach((type) => {
if (options.skipLocales && type === "locales") {
return;
}
if (options.skipContentModel && ["contentTypes", "editorInterfaces"].indexOf(type) >= 0) {
return;
}
if (options.contentModelOnly && !(["contentTypes", "editorInterfaces", "locales"].indexOf(type) >= 0)) {
return;
}
infoTable.push([(0, import_lodash2.startCase)(type), options.content[type].length]);
});
console.log(infoTable.toString());
const tasks = new import_listr3.default([
{
title: "Validating content-file",
task: () => {
assertPayload(options.content);
}
},
{
title: "Initialize client",
task: (0, import_listr4.wrapTask)(async (ctx) => {
ctx.client = initClient({ ...options, content: void 0 });
})
},
{
title: "Checking if destination space already has any content and retrieving it",
task: (0, import_listr4.wrapTask)(async (ctx) => {
const destinationData = await getDestinationDa