@agility/cli
Version:
Agility CLI for working with your content. (Public Beta)
416 lines • 27.8 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ContentBatchProcessor = void 0;
var batch_polling_1 = require("../batch-polling");
var ansi_colors_1 = __importDefault(require("ansi-colors"));
var model_mapper_1 = require("lib/mappers/model-mapper");
var container_mapper_1 = require("lib/mappers/container-mapper");
var asset_mapper_1 = require("lib/mappers/asset-mapper");
var find_content_in_other_locale_1 = require("./util/find-content-in-other-locale");
var state_1 = require("core/state");
/******
* USAGE PATTERN:
* 1. Filter content items BEFORE creating the batch processor using filterContentItemsForProcessing()
* 2. Create the batch processor with pre - filtered items
* 3. Call processBatches() with the filtered items
*
* This ensures consistent use of the new versioning logic and eliminates duplicate filtering.
*/
var ContentBatchProcessor = /** @class */ (function () {
function ContentBatchProcessor(config) {
this.config = __assign(__assign({}, config), { batchSize: config.batchSize || 250 });
}
/**
* Process content items in batches using saveContentItems API
* NOTE: Content items should already be filtered by the caller using filterContentItemsForProcessing()
*/
ContentBatchProcessor.prototype.processBatches = function (contentItems, logger, batchType) {
return __awaiter(this, void 0, void 0, function () {
var batchSize, contentBatches, totalSuccessCount, totalFailureCount, totalSkippedCount, allSuccessfulItems, allFailedItems, startTime, i, contentBatch, batchNumber, processedSoFar, elapsed, avgTimePerBatch, remainingBatches, etaMs, etaMinutes, progress, _a, contentPayloads, batchSkippedCount, batchIDResult, batchID, completedBatch, _b, successfulItems, failedItems, batchResult, callbackError_1, error_1, failedBatchItems;
var _this = this;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
batchSize = this.config.batchSize;
contentBatches = this.createContentBatches(contentItems, batchSize);
console.log("Processing ".concat(contentItems.length || 0, " content items in ").concat(contentBatches.length, " bulk ").concat(batchType || "", " batches"));
totalSuccessCount = 0;
totalFailureCount = 0;
totalSkippedCount = 0;
allSuccessfulItems = [];
allFailedItems = [];
startTime = Date.now();
i = 0;
_c.label = 1;
case 1:
if (!(i < contentBatches.length)) return [3 /*break*/, 14];
contentBatch = contentBatches[i];
batchNumber = i + 1;
processedSoFar = i * batchSize;
elapsed = Date.now() - startTime;
avgTimePerBatch = elapsed / batchNumber;
remainingBatches = contentBatches.length - batchNumber;
etaMs = remainingBatches * avgTimePerBatch;
etaMinutes = Math.round(etaMs / 60000);
progress = Math.round((batchNumber / contentBatches.length) * 100);
console.log("[".concat(progress, "%] Bulk batch ").concat(batchNumber, "/").concat(contentBatches.length, ": Processing ").concat(contentBatch.length, " content items (ETA: ").concat(etaMinutes, "m)..."));
_c.label = 2;
case 2:
_c.trys.push([2, 12, , 13]);
return [4 /*yield*/, this.prepareContentPayloads(contentBatch, this.config.sourceGuid, this.config.targetGuid)];
case 3:
_a = _c.sent(), contentPayloads = _a.payloads, batchSkippedCount = _a.skippedCount;
// Track skipped items from this batch
totalSkippedCount += batchSkippedCount;
return [4 /*yield*/, this.config.apiClient.contentMethods.saveContentItems(contentPayloads, this.config.targetGuid, this.config.locale, true // returnBatchID flag
)];
case 4:
batchIDResult = _c.sent();
batchID = Array.isArray(batchIDResult) ? batchIDResult[0] : batchIDResult;
return [4 /*yield*/, (0, batch_polling_1.pollBatchUntilComplete)(this.config.apiClient, batchID, this.config.targetGuid, contentPayloads, // Pass original payloads for FIFO error matching
300, // maxAttempts
2000, // intervalMs
batchType || "Content" // Use provided batch type or default to 'Content'
)];
case 5:
completedBatch = _c.sent();
_b = (0, batch_polling_1.extractBatchResults)(completedBatch, contentBatch), successfulItems = _b.successfulItems, failedItems = _b.failedItems;
batchResult = {
successCount: successfulItems.length,
failureCount: failedItems.length,
skippedCount: 0, // Individual batches don't track skipped items (handled at processBatches level)
successfulItems: successfulItems.map(function (item) { return ({
originalContent: item.originalItem,
newItem: item.newItem,
newContentId: item.newId,
}); }),
failedItems: failedItems.map(function (item) { return ({
originalContent: item.originalItem,
error: item.error,
}); }),
publishableIds: successfulItems.map(function (item) { return item.newId; }),
};
totalSuccessCount += batchResult.successCount;
totalFailureCount += batchResult.failureCount;
allSuccessfulItems.push.apply(allSuccessfulItems, batchResult.successfulItems);
allFailedItems.push.apply(allFailedItems, batchResult.failedItems);
// Update ID mappings for successful uploads
if (batchResult.successfulItems.length > 0) {
this.updateContentIdMappings(batchResult.successfulItems);
}
console.log("\n");
// Display individual item results for better visibility
if (batchResult.successfulItems.length > 0) {
batchResult.successfulItems.forEach(function (item) {
// const modelName = item.originalContent.properties.definitionName || "Unknown";
logger.content.created(item.originalContent, "created", _this.config.locale, state_1.state.targetGuid[0]);
});
}
if (batchResult.failedItems.length > 0) {
console.log("\u274C Batch ".concat(batchNumber, " failed items:"));
batchResult.failedItems.forEach(function (item) {
// const modelName = item.originalContent.properties.definitionName || "Unknown";
logger.content.error(item.originalContent, item.error, _this.config.locale, state_1.state.targetGuid[0]);
});
}
if (!this.config.onBatchComplete) return [3 /*break*/, 9];
_c.label = 6;
case 6:
_c.trys.push([6, 8, , 9]);
return [4 /*yield*/, this.config.onBatchComplete(batchResult, batchNumber)];
case 7:
_c.sent();
return [3 /*break*/, 9];
case 8:
callbackError_1 = _c.sent();
console.warn("\u26A0\uFE0F Batch completion callback failed for batch ".concat(batchNumber, ": ").concat(callbackError_1.message));
return [3 /*break*/, 9];
case 9:
if (!(i < contentBatches.length - 1)) return [3 /*break*/, 11];
return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 100); })];
case 10:
_c.sent();
_c.label = 11;
case 11: return [3 /*break*/, 13];
case 12:
error_1 = _c.sent();
console.error("\u274C Bulk batch ".concat(batchNumber, " failed:"), error_1.message);
failedBatchItems = contentBatch.map(function (item) { return ({
originalContent: item,
error: "Batch processing failed: ".concat(error_1.message),
}); });
totalFailureCount += failedBatchItems.length;
allFailedItems.push.apply(allFailedItems, failedBatchItems);
return [3 /*break*/, 13];
case 13:
i++;
return [3 /*break*/, 1];
case 14:
// console.log(`🎯 Content batch processing complete: ${totalSuccessCount} success, ${totalFailureCount} failed`);
return [2 /*return*/, {
successCount: totalSuccessCount,
failureCount: totalFailureCount,
skippedCount: totalSkippedCount,
successfulItems: allSuccessfulItems,
failedItems: allFailedItems,
publishableIds: allSuccessfulItems.map(function (item) { return item.newContentId; }),
}];
}
});
});
};
/**
* Create batches of content items for bulk processing
*/
ContentBatchProcessor.prototype.createContentBatches = function (contentItems, batchSize) {
var batches = [];
for (var i = 0; i < contentItems.length; i += batchSize) {
batches.push(contentItems.slice(i, i + batchSize));
}
return batches;
};
/**
* Prepare content payloads for bulk upload API
* Uses the same payload structure as individual content pusher
*/
ContentBatchProcessor.prototype.prepareContentPayloads = function (contentBatch, sourceGuid, targetGuid) {
return __awaiter(this, void 0, void 0, function () {
var payloads, skippedCount, modelMapper, containerMapper, assetMapper, _loop_1, this_1, _i, contentBatch_1, contentItem;
var _a, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
payloads = [];
skippedCount = 0;
modelMapper = new model_mapper_1.ModelMapper(sourceGuid, targetGuid);
containerMapper = new container_mapper_1.ContainerMapper(sourceGuid, targetGuid);
assetMapper = new asset_mapper_1.AssetMapper(sourceGuid, targetGuid);
_loop_1 = function (contentItem) {
var existingMapping, payload, modelMapping, sourceModel, errorDetails, model, containerMapping, targetContainer, existingMapping, existingTargetContentItem, existingContentID, ContentFieldMapper, fieldMapper, mappingResult, validatedFields, fieldNameMap_1, camelize_1, defaultSeo, defaultScripts, payload, error_2;
return __generator(this, function (_d) {
switch (_d.label) {
case 0:
if (!(contentItem.properties.definitionName.toLowerCase() === "richtextarea"
&& contentItem.fields.textblob)) return [3 /*break*/, 1];
existingMapping = this_1.config.referenceMapper.getContentItemMappingByContentID(contentItem.contentID, 'source');
payload = __assign(__assign({}, contentItem), { contentID: existingMapping ? existingMapping.targetContentID : -1 });
payloads.push(payload);
return [3 /*break*/, 7];
case 1:
modelMapping = modelMapper.getModelMappingByReferenceName(contentItem.properties.definitionName, 'source');
_d.label = 2;
case 2:
_d.trys.push([2, 6, , 7]);
sourceModel = null;
if (modelMapping)
sourceModel = modelMapper.getMappedEntity(modelMapping, 'source');
if (!sourceModel) {
errorDetails = [
"\uD83D\uDCCB Content Definition Not Found: \"".concat(contentItem.properties.definitionName, "\""),
"\uD83D\uDD0D Content Item: ".concat(contentItem.properties.referenceName),
"\uD83D\uDCA1 Common causes:",
" \u2022 Model was deleted from source instance",
" \u2022 Model(s) not included in sync elements"
].join("\n ");
throw new Error("Source model not found for content definition: ".concat(contentItem.properties.definitionName, "\n ").concat(errorDetails));
}
// STEP 2: Find target model using reference mapper (simplified)
if (!modelMapping) {
throw new Error("Target model mapping not found for: ".concat(sourceModel.referenceName, " (ID: ").concat(sourceModel.id, ")"));
}
model = {
id: modelMapping.targetID,
referenceName: sourceModel.referenceName,
fields: sourceModel.fields || []
};
containerMapping = containerMapper.getContainerMappingByReferenceName(contentItem.properties.referenceName, 'source');
if (!containerMapping) {
throw new Error("Container mapping not found: ".concat(contentItem.properties.referenceName));
}
targetContainer = containerMapper.getMappedEntity(containerMapping, 'target');
existingMapping = this_1.config.referenceMapper.getContentItemMappingByContentID(contentItem.contentID, 'source');
existingTargetContentItem = this_1.config.referenceMapper.getMappedEntity(existingMapping, 'target');
existingContentID = existingTargetContentItem ? existingTargetContentItem.contentID : -1;
if (!!existingTargetContentItem) return [3 /*break*/, 4];
return [4 /*yield*/, (0, find_content_in_other_locale_1.findContentInOtherLocale)({
sourceGuid: sourceGuid,
targetGuid: targetGuid,
sourceContentID: contentItem.contentID,
locale: this_1.config.locale
})];
case 3:
//see if this content item has been mapped in another locale
existingContentID = _d.sent();
_d.label = 4;
case 4: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require("../../content/content-field-mapper")); })];
case 5:
ContentFieldMapper = (_d.sent()).ContentFieldMapper;
fieldMapper = new ContentFieldMapper();
mappingResult = fieldMapper.mapContentFields(contentItem.fields || {}, {
referenceMapper: this_1.config.referenceMapper,
assetMapper: assetMapper,
apiClient: this_1.config.apiClient,
targetGuid: this_1.config.targetGuid,
});
// Only log field mapper issues if there are actual errors (not warnings)
if (mappingResult.validationErrors > 0) {
console.warn("\u26A0\uFE0F Field mapping errors for ".concat(contentItem.properties.referenceName, ": ").concat(mappingResult.validationErrors, " errors"));
}
validatedFields = __assign({}, mappingResult.mappedFields);
fieldNameMap_1 = new Map();
camelize_1 = function (str) {
return str
.replace(/(?:^\w|[A-Z]|\b\w)/g, function (word, index) {
return index === 0 ? word.toLowerCase() : word.toUpperCase();
})
.replace(/\s+/g, "");
};
if (model && model.fields) {
model.fields.forEach(function (fieldDef) {
var camelCaseFieldName = camelize_1(fieldDef.name);
fieldNameMap_1.set(camelCaseFieldName, fieldDef.name);
fieldNameMap_1.set(fieldDef.name.toLowerCase(), fieldDef.name);
});
}
defaultSeo = {
metaDescription: null,
metaKeywords: null,
metaHTML: null,
menuVisible: null,
sitemapVisible: null,
};
defaultScripts = { top: null, bottom: null };
payload = __assign(__assign({}, contentItem), { contentID: existingContentID, fields: validatedFields, properties: __assign(__assign({}, contentItem.properties), { referenceName: (targetContainer === null || targetContainer === void 0 ? void 0 : targetContainer.referenceName) || contentItem.properties.referenceName, itemOrder: existingTargetContentItem
? existingTargetContentItem.properties.itemOrder
: contentItem.properties.itemOrder }), seo: (_a = contentItem.seo) !== null && _a !== void 0 ? _a : defaultSeo, scripts: (_b = contentItem.scripts) !== null && _b !== void 0 ? _b : defaultScripts });
payloads.push(payload);
return [3 /*break*/, 7];
case 6:
error_2 = _d.sent();
console.error(ansi_colors_1.default.yellow("\u2717 Orphaned content item ".concat(contentItem.contentID, ", skipping - ").concat(error_2.message || 'payload preparation failed', ".")));
// Track skipped item and continue with the rest of the batch
skippedCount++;
return [2 /*return*/, "continue"];
case 7: return [2 /*return*/];
}
});
};
this_1 = this;
_i = 0, contentBatch_1 = contentBatch;
_c.label = 1;
case 1:
if (!(_i < contentBatch_1.length)) return [3 /*break*/, 4];
contentItem = contentBatch_1[_i];
return [5 /*yield**/, _loop_1(contentItem)];
case 2:
_c.sent();
_c.label = 3;
case 3:
_i++;
return [3 /*break*/, 1];
case 4: return [2 /*return*/, { payloads: payloads, skippedCount: skippedCount }];
}
});
});
};
/**
* Update content ID mappings in reference mapper
*/
ContentBatchProcessor.prototype.updateContentIdMappings = function (successfulItems) {
var _this = this;
successfulItems.forEach(function (item) {
var sourceContentItem = item.originalContent;
var targetContentItem = item.newItem;
var targetContentItemWithId = __assign(__assign({}, sourceContentItem), { contentID: targetContentItem.itemID, properties: {
versionID: targetContentItem.processedItemVersionID
} });
_this.config.referenceMapper.addMapping(sourceContentItem, targetContentItemWithId);
});
};
return ContentBatchProcessor;
}());
exports.ContentBatchProcessor = ContentBatchProcessor;
//# sourceMappingURL=content-batch-processor.js.map