@agility/cli
Version:
Agility CLI for working with your content. (Public Beta)
554 lines • 25.9 kB
JavaScript
;
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, "__esModule", { value: true });
var fs = require('fs');
var os = require('os');
var path = require('path');
var _a = require("proper-lockfile"), lockSync = _a.lockSync, unlockSync = _a.unlockSync, checkSync = _a.checkSync, check = _a.check;
var sleep_1 = require("../shared/sleep");
var _b = require('../../core/state'), getState = _b.getState, getLoggerForGuid = _b.getLoggerForGuid;
// RACE CONDITION FIX: Convert global stats to instance-specific stats
// Use rootPath as unique identifier for each concurrent download
var _instanceStats = new Map();
require("dotenv").config({
path: ".env.".concat(process.env.NODE_ENV),
});
/**
* Get the logger for the current operation
*/
function getLogger(options) {
// Extract GUID from options.rootPath or options.guid
var guid = (options === null || options === void 0 ? void 0 : options.guid) || (options === null || options === void 0 ? void 0 : options.sourceGuid) || extractGuidFromPath(options === null || options === void 0 ? void 0 : options.rootPath);
if (!guid)
return null;
return getLoggerForGuid(guid);
}
/**
* Extract GUID from rootPath (e.g., "agility-files/13a8b394-u/en-us/preview")
*/
function extractGuidFromPath(rootPath) {
if (!rootPath)
return null;
// Look for GUID pattern in path segments
var segments = rootPath.split('/');
for (var _i = 0, segments_1 = segments; _i < segments_1.length; _i++) {
var segment = segments_1[_i];
// Match GUID patterns like "13a8b394-u" or "af9a3c91-4ca0-42db-bdb9-cced53a818d6"
if (/^[a-f0-9]{8}-[a-f0-9-]{1,36}$/i.test(segment)) {
return segment;
}
}
return null;
}
/**
* Map Sync SDK itemType to ChangeDelta entity type
*/
function mapItemTypeToEntityType(itemType) {
var typeMap = {
'item': 'content-item',
'page': 'page'
};
return typeMap[itemType] || itemType;
}
/**
* Extract entity name from item content
*/
function extractEntityName(item, itemType) {
var _a;
if (itemType === 'page') {
return item.name || item.title || "Page ".concat(item.pageID);
}
if (itemType === 'item') {
return ((_a = item.properties) === null || _a === void 0 ? void 0 : _a.referenceName) || "Content ".concat(item.contentID);
}
return "".concat(itemType, " ").concat(item.id || 'Unknown');
}
/**
* Extract reference name from item content
*/
function extractReferenceName(item, itemType) {
var _a;
if (itemType === 'page') {
return item.name;
}
if (itemType === 'item') {
return (_a = item.properties) === null || _a === void 0 ? void 0 : _a.referenceName;
}
return undefined;
}
/**
* Get or create instance-specific stats for the given rootPath
*/
var getInstanceStats = function (rootPath) {
if (!_instanceStats.has(rootPath)) {
_instanceStats.set(rootPath, {
itemsSavedStats: [],
progressByType: {},
progressCallback: null,
syncStartTime: 0
});
}
return _instanceStats.get(rootPath);
};
/**
* Set a progress callback function that will be called whenever items are saved
* This allows the UI to get real-time updates during sync operations
*/
var setProgressCallback = function (callback, rootPath) {
if (rootPath) {
var instanceStats = getInstanceStats(rootPath);
instanceStats.progressCallback = callback;
}
else {
// Fallback: set for all instances if rootPath not specified
_instanceStats.forEach(function (stats) {
stats.progressCallback = callback;
});
}
};
/**
* Initialize progress tracking for a new sync operation
*/
var initializeProgress = function (rootPath) {
if (rootPath) {
var instanceStats = getInstanceStats(rootPath);
instanceStats.itemsSavedStats = [];
instanceStats.progressByType = {};
instanceStats.syncStartTime = Date.now();
}
else {
// Fallback: initialize all instances if rootPath not specified
_instanceStats.forEach(function (stats) {
stats.itemsSavedStats = [];
stats.progressByType = {};
stats.syncStartTime = Date.now();
});
}
};
/**
* Clean up old progress data to prevent memory bloat during long operations
*/
var cleanupProgressData = function (rootPath) {
var instanceStats = getInstanceStats(rootPath);
var MAX_STATS_HISTORY = 200; // Limit for memory management
if (instanceStats.itemsSavedStats.length > MAX_STATS_HISTORY) {
instanceStats.itemsSavedStats = instanceStats.itemsSavedStats.slice(-MAX_STATS_HISTORY);
}
};
/**
* Get current progress statistics without clearing the data
*/
var getProgressStats = function (rootPath) {
var instanceStats = getInstanceStats(rootPath);
var elapsedTime = Date.now() - instanceStats.syncStartTime;
var totalItems = instanceStats.itemsSavedStats.length;
return {
totalItems: totalItems,
itemsByType: __assign({}, instanceStats.progressByType),
elapsedTime: elapsedTime,
itemsPerSecond: totalItems > 0 ? (totalItems / (elapsedTime / 1000)) : 0,
recentActivity: instanceStats.itemsSavedStats.slice(-10).map(function (item) { return ({
itemType: item.itemType,
itemID: item.itemID,
timestamp: item.timestamp
}); })
};
};
/**
* Update progress and trigger callback if set
*/
var updateProgress = function (itemType, itemID, rootPath) {
var instanceStats = getInstanceStats(rootPath);
// Add to stats
instanceStats.itemsSavedStats.push({
itemType: itemType,
itemID: itemID,
languageCode: 'unknown', // Language not available at this level
timestamp: Date.now()
});
// Update type counts
instanceStats.progressByType[itemType] = (instanceStats.progressByType[itemType] || 0) + 1;
// Clean up old data periodically
if (instanceStats.itemsSavedStats.length % 50 === 0) {
cleanupProgressData(rootPath);
}
// Trigger callback if set
if (instanceStats.progressCallback) {
instanceStats.progressCallback(getProgressStats(rootPath));
}
};
/**
* The function to handle saving/updating an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
* @param {Object} params - The parameters object
* @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
* @param {String} params.options.rootPath - The path to store/access the content as JSON
* @param {Object} params.item - The object representing the Content Item, Page, Url Redirections, Sync State (state), or Sitemap that needs to be saved/updated
* @param {String} params.itemType - The type of item being saved/updated, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
* @param {String} params.languageCode - The locale code associated to the item being saved/updated
* @param {(String|Number)} params.itemID - The ID of the item being saved/updated - this could be a string or number depending on the itemType
* @returns {Void}
*/
var saveItem = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
var cwd, filePath, absoluteFilePath, dirPath, forceOverwrite, logger, json;
var options = _b.options, item = _b.item, itemType = _b.itemType, languageCode = _b.languageCode, itemID = _b.itemID;
return __generator(this, function (_c) {
// Null/undefined safety check - prevent crashes when SDK passes undefined items
if (item === null || item === undefined) {
console.warn("\u26A0\uFE0F Skipping save for ".concat(itemType, " (ID: ").concat(itemID, ") - item is ").concat(item));
return [2 /*return*/];
}
cwd = process.cwd();
filePath = getFilePath({ options: options, itemType: itemType, languageCode: languageCode, itemID: itemID });
absoluteFilePath = path.resolve(cwd, filePath);
dirPath = path.dirname(absoluteFilePath);
forceOverwrite = options.forceOverwrite;
logger = options.logger;
try {
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
if (!fs.existsSync(dirPath)) {
throw new Error("Failed to create directory: ".concat(dirPath));
}
}
json = JSON.stringify(item);
// Add specific debug logs around file write
// console.log(`[Debug saveItem] About to write: ${itemType} (ID: ${itemID}) to ${absoluteFilePath}`);
fs.writeFileSync(absoluteFilePath, json);
// console.log(`[Debug saveItem] Write successful for: ${absoluteFilePath}`);
// Use structured logging instead of basic console.log
if (logger) {
// if(itemType !== 'item' && itemType !== 'sitemap' && itemType !== 'list') { console.log('item', item); }
// Map itemType to appropriate logger method and include locale for content/pages
if (itemType === 'item') {
logger.content.downloaded(item, undefined, languageCode);
}
else if (itemType === 'page') {
logger.page.downloaded(item, undefined, languageCode);
}
else if (itemType === 'sitemap') {
logger.sitemap.downloaded({ name: 'sitemap.json' });
}
else {
// Fallback for other item types
// const entityName = extractEntityName(item, itemType);
// logger.info(`✓ Downloaded ${itemType}: ${entityName} [${languageCode}]`);
}
}
else {
// Fallback to basic logging if no logger available
// const state = getState();
// if (state.verbose) {
// console.log('✓ Downloaded',ansiColors.cyan(itemType), ansiColors.white(itemID));
// }
}
if (!fs.existsSync(absoluteFilePath)) {
throw new Error("File was not created: ".concat(absoluteFilePath));
}
// REMOVE direct log, PUSH to stats array
// console.log(`✓ Downloaded ${ansiColors.cyan(itemType)} (ID: ${itemID})`);
// updateProgress(itemType, itemID, options.rootPath);
}
catch (error) {
// Use structured error logging if available
if (logger) {
if (itemType === 'item') {
logger.contentitem.error(item, error, languageCode);
}
else if (itemType === 'page') {
logger.page.error(item, error, languageCode);
}
else {
logger.error("Failed to save ".concat(itemType, " (ID: ").concat(itemID, "): ").concat(error.message));
}
}
else {
console.error('Error in saveItem:', error);
}
console.error('Error details:', {
filePath: filePath,
absoluteFilePath: absoluteFilePath,
dirPath: dirPath,
cwd: cwd,
error: error.message,
stack: error.stack
});
throw error;
}
return [2 /*return*/];
});
}); };
/**
* The function to handle deleting an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
* @param {Object} params - The parameters object
* @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
* @param {String} params.options.rootPath - The path to store/access the content as JSON
* @param {String} params.itemType - The type of item being deleted, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
* @param {String} params.languageCode - The locale code associated to the item being saved/updated
* @param {(String|Number)} params.itemID - The ID of the item being deleted - this could be a string or number depending on the itemType
* @returns {Void}
*/
var deleteItem = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
var filePath;
var options = _b.options, itemType = _b.itemType, languageCode = _b.languageCode, itemID = _b.itemID;
return __generator(this, function (_c) {
filePath = getFilePath({ options: options, itemType: itemType, languageCode: languageCode, itemID: itemID });
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
return [2 /*return*/];
});
}); };
/**
* The function to handle updating and placing a Content Item into a "list" so that you can handle querying a collection of items.
* @param {Object} params - The parameters object
* @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
* @param {String} params.options.rootPath - The path to store/access the content as JSON
* @param {Object} params.item - The object representing the Content Item
* @param {String} params.languageCode - The locale code associated to the item being saved/updated
* @param {(String|Number)} params.itemID - The ID of the item being updated - this could be a string or number depending on the itemType
* @param {String} params.referenceName - The reference name of the Content List that this Content Item should be added to
* @param {String} params.definitionName - The Model name that the Content Item is based on
* @returns {Void}
*/
var mergeItemToList = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
var contentList, cIndex;
var options = _b.options, item = _b.item, languageCode = _b.languageCode, itemID = _b.itemID, referenceName = _b.referenceName, definitionName = _b.definitionName;
return __generator(this, function (_c) {
switch (_c.label) {
case 0: return [4 /*yield*/, getItem({ options: options, itemType: "list", languageCode: languageCode, itemID: referenceName })];
case 1:
contentList = _c.sent();
if (contentList == null) {
//initialize the list
contentList = [item];
}
else {
cIndex = contentList.findIndex(function (ci) {
return ci.contentID === itemID;
});
if (item.properties.state === 3) {
//*** deleted item (remove from the list) ***
if (cIndex >= 0) {
//remove the item
contentList.splice(cIndex, 1);
}
}
else {
//*** regular item (merge) ***
if (cIndex >= 0) {
//replace the existing item
contentList[cIndex] = item;
}
else {
//and it to the end of the
contentList.push(item);
}
}
}
return [4 /*yield*/, saveItem({ options: options, item: contentList, itemType: "list", languageCode: languageCode, itemID: referenceName })];
case 2:
_c.sent();
return [2 /*return*/];
}
});
}); };
/**
* The function to handle retrieving a Content Item, Page, Url Redirections, Sync State (state), or Sitemap
* @param {Object} params - The parameters object
* @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
* @param {String} params.options.rootPath - The path to store/access the content as JSON
* @param {String} params.itemType - The type of item being accessed, expected values are `item`, `list`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
* @param {String} params.languageCode - The locale code associated to the item being accessed
* @param {(String|Number)} params.itemID - The ID of the item being accessed - this could be a string or number depending on the itemType
* @returns {Object}
*/
var getItem = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
var filePath, json;
var options = _b.options, itemType = _b.itemType, languageCode = _b.languageCode, itemID = _b.itemID;
return __generator(this, function (_c) {
filePath = getFilePath({ options: options, itemType: itemType, languageCode: languageCode, itemID: itemID });
if (!fs.existsSync(filePath))
return [2 /*return*/, null];
json = fs.readFileSync(filePath, 'utf8');
return [2 /*return*/, JSON.parse(json)];
});
}); };
/**
* The function to handle clearing the cache of synchronized data from the CMS
* @param {Object} params - The parameters object
* @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
* @param {String} params.options.rootPath - The path to store/access the content as JSON
* @returns {Void}
*/
var clearItems = function (_a) { return __awaiter(void 0, [_a], void 0, function (_b) {
var options = _b.options;
return __generator(this, function (_c) {
fs.rmdirSync(options.rootPath, { recursive: true });
return [2 /*return*/];
});
}); };
/**
* The function to handle multi-threaded Syncs that may be happening at the same time. If you need to prevent a sync from happening and let it wait until another sync has finished use this.
* @returns {Promise}
*/
var mutexLock = function () { return __awaiter(void 0, void 0, void 0, function () {
var dir, lockFile, err_1, e2_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
dir = os.tmpdir();
lockFile = "".concat(dir, "/").concat("agility-sync", ".mutex");
if (!fs.existsSync(lockFile)) {
fs.writeFileSync(lockFile, "agility-sync");
}
//THE LOCK IS ALREADY HELD - WAIT UP!
return [4 /*yield*/, waitOnLock(lockFile)];
case 1:
//THE LOCK IS ALREADY HELD - WAIT UP!
_a.sent();
_a.label = 2;
case 2:
_a.trys.push([2, 3, , 12]);
return [2 /*return*/, lockSync(lockFile)];
case 3:
err_1 = _a.sent();
if (!("".concat(err_1).indexOf("Lock file is already being held") !== -1)) return [3 /*break*/, 11];
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
return [4 /*yield*/, (0, sleep_1.sleep)(100)];
case 4:
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
_a.sent();
return [4 /*yield*/, waitOnLock(lockFile)];
case 5:
_a.sent();
_a.label = 6;
case 6:
_a.trys.push([6, 7, , 11]);
return [2 /*return*/, lockSync(lockFile)];
case 7:
e2_1 = _a.sent();
if (!("".concat(err_1).indexOf("Lock file is already being held") !== -1)) return [3 /*break*/, 10];
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
return [4 /*yield*/, (0, sleep_1.sleep)(100)];
case 8:
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
_a.sent();
return [4 /*yield*/, waitOnLock(lockFile)];
case 9:
_a.sent();
return [2 /*return*/, lockSync(lockFile)];
case 10: return [3 /*break*/, 11];
case 11: throw Error("The mutex lock could not be obtained.");
case 12: return [2 /*return*/];
}
});
}); };
//private function to get a wait on a lock file
var waitOnLock = function (lockFile) { return __awaiter(void 0, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, check(lockFile)];
case 1:
if (!_a.sent()) return [3 /*break*/, 3];
return [4 /*yield*/, (0, sleep_1.sleep)(100)];
case 2:
_a.sent();
return [3 /*break*/, 0];
case 3: return [2 /*return*/];
}
});
}); };
//private function to get path of an item
var getFilePath = function (_a) {
var options = _a.options, itemType = _a.itemType, languageCode = _a.languageCode, itemID = _a.itemID;
if (typeof itemID === 'string' || itemID instanceof String) {
itemID = itemID.replace(/[`!@#$%^&*()+\=\[\]{};':"\\|,.<>\/?~]/g, "");
}
// Fix inconsistency: Convert "page" (singular) to "pages" (plural)
// to match where get-pages.ts expects to find them
// if (itemType === 'page') {
// itemType = 'pages';
// }
var fileName = "".concat(itemID, ".json");
return path.join(options.rootPath, itemType, fileName);
};
// Enhanced function to get and clear saved item stats with progress data
var getAndClearSavedItemStats = function (rootPath) {
var instanceStats = getInstanceStats(rootPath);
var stats = getProgressStats(rootPath);
// Prepare detailed summary
var summary = {
totalItems: stats.totalItems,
elapsedTime: stats.elapsedTime,
itemsPerSecond: stats.itemsPerSecond
};
// Clear stats for this instance
instanceStats.itemsSavedStats = [];
instanceStats.progressByType = {};
return {
summary: summary,
itemsByType: stats.itemsByType,
recentActivity: stats.recentActivity
};
};
module.exports = {
saveItem: saveItem,
deleteItem: deleteItem,
mergeItemToList: mergeItemToList,
getItem: getItem,
clearItems: clearItems,
mutexLock: mutexLock,
getAndClearSavedItemStats: getAndClearSavedItemStats, // RE-ADD Export
setProgressCallback: setProgressCallback,
initializeProgress: initializeProgress,
getCurrentProgress: getProgressStats, // Alias for getProgressStats
updateProgress: updateProgress,
cleanupProgressData: cleanupProgressData // NEW: Memory cleanup function
};
//# sourceMappingURL=store-interface-filesystem.js.map