@agility/cli
Version:
Agility CLI for working with your content. (Public Beta)
261 lines • 14.9 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadAllAssets = downloadAllAssets;
var fileOperations_1 = require("../../core/fileOperations");
var state_1 = require("../../core/state");
var fs_1 = __importDefault(require("fs"));
var path_1 = __importDefault(require("path"));
var asset_utils_1 = require("../assets/asset-utils");
function downloadAllAssets(guid) {
return __awaiter(this, void 0, void 0, function () {
// Note: Using shared getAssetFilePath utility for consistent filename handling
// This ensures URL decoding is consistent between download and processing phases
// Helper function to get local asset metadata
function getLocalAssetInfo(filePath) {
try {
if (!fs_1.default.existsSync(filePath)) {
return { exists: false };
}
var content = JSON.parse(fs_1.default.readFileSync(filePath, "utf8"));
return {
dateModified: content.dateModified,
exists: true,
};
}
catch (error) {
return { exists: false };
}
}
// Helper function to check if asset needs download based on dateModified
function shouldDownloadAsset(apiAsset, localInfo) {
if (state_1.state.update === false) {
return { shouldDownload: false, reason: "" };
}
if (!localInfo.exists) {
return { shouldDownload: true, reason: "new file" };
}
if (!localInfo.dateModified || !apiAsset.dateModified) {
return { shouldDownload: true, reason: "missing date info" };
}
var apiDate = new Date(apiAsset.dateModified);
var localDate = new Date(localInfo.dateModified);
if (apiDate > localDate) {
return { shouldDownload: true, reason: "content changed" };
}
return { shouldDownload: false, reason: "unchanged" };
}
// Helper function to format file size
function formatFileSize(bytes) {
if (bytes === 0)
return "0B";
var k = 1024;
var sizes = ["B", "KB", "MB", "GB"];
var i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + sizes[i];
}
var fileOps, update, apiClient, logger, pageSize, recordOffset, index, totalSuccessfullyDownloaded, totalSkippedAssets, totalAttemptedToProcess, totalRecords, startTime, unProcessedAssets, allAssets, initialRecords, iterations, iter, assetsPage, downloadableAssets, skippableAssets, _i, allAssets_1, asset, assetJsonPath, localInfo, downloadDecision, CONCURRENT_BATCH_SIZE, batches, i, batchIndex, batch, downloadPromises, results, _a, results_1, result, totalProcessed, unprocessedCount, error_1;
var _this = this;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
fileOps = new fileOperations_1.fileOperations(guid);
update = state_1.state.update;
apiClient = (0, state_1.getApiClient)();
logger = (0, state_1.getLoggerForGuid)(guid);
if (!logger) {
console.warn("\u26A0\uFE0F No logger found for GUID ".concat(guid, ", skipping asset logging"));
return [2 /*return*/];
}
logger.startTimer();
pageSize = 250;
recordOffset = 0;
index = 1;
totalSuccessfullyDownloaded = 0;
totalSkippedAssets = 0;
totalAttemptedToProcess = 0;
totalRecords = 0;
startTime = Date.now();
unProcessedAssets = {};
_b.label = 1;
case 1:
_b.trys.push([1, 11, , 12]);
allAssets = [];
return [4 /*yield*/, apiClient.assetMethods.getMediaList(pageSize, recordOffset, guid)];
case 2:
initialRecords = _b.sent();
totalRecords = initialRecords.totalCount;
fileOps.createFolder("assets/json");
// Export first page of JSON
fileOps.exportFiles("assets/json", index, initialRecords);
allAssets.push.apply(allAssets, initialRecords.assetMedias);
index++;
iterations = Math.ceil(totalRecords / pageSize);
if (iterations === 0 && totalRecords > 0)
iterations = 1;
else if (totalRecords === 0)
iterations = 0;
if (!(totalRecords > pageSize)) return [3 /*break*/, 6];
console.log("\uD83D\uDCCB Fetching ".concat(iterations - 1, " additional pages of asset metadata..."));
iter = 1;
_b.label = 3;
case 3:
if (!(iter < iterations)) return [3 /*break*/, 6];
recordOffset += pageSize;
return [4 /*yield*/, apiClient.assetMethods.getMediaList(pageSize, recordOffset, guid)];
case 4:
assetsPage = _b.sent();
fileOps.exportFiles("assets/json", index, assetsPage);
allAssets.push.apply(allAssets, assetsPage.assetMedias);
index++;
_b.label = 5;
case 5:
iter++;
return [3 /*break*/, 3];
case 6:
downloadableAssets = [];
skippableAssets = [];
for (_i = 0, allAssets_1 = allAssets; _i < allAssets_1.length; _i++) {
asset = allAssets_1[_i];
assetJsonPath = path_1.default.join(fileOps.getDataFolderPath("assets"), "".concat(asset.mediaID, ".json"));
localInfo = getLocalAssetInfo(assetJsonPath);
downloadDecision = shouldDownloadAsset(asset, localInfo);
if (downloadDecision.shouldDownload) {
downloadableAssets.push({ asset: asset, reason: downloadDecision.reason });
}
else {
logger.asset.skipped(asset);
skippableAssets.push({ asset: asset, reason: downloadDecision.reason });
}
}
if (skippableAssets.length > 0) {
logger.changeDetectionSummary("asset", downloadableAssets.length, skippableAssets.length);
}
// Phase 3: Download only the assets that need updating
if (downloadableAssets.length === 0) {
return [2 /*return*/];
}
CONCURRENT_BATCH_SIZE = 10;
batches = [];
for (i = 0; i < downloadableAssets.length; i += CONCURRENT_BATCH_SIZE) {
batches.push(downloadableAssets.slice(i, i + CONCURRENT_BATCH_SIZE));
}
batchIndex = 0;
_b.label = 7;
case 7:
if (!(batchIndex < batches.length)) return [3 /*break*/, 10];
batch = batches[batchIndex];
downloadPromises = batch.map(function (item) { return __awaiter(_this, void 0, void 0, function () {
var asset, reason, filePath, assetFilesPath, success, sizeDisplay, error_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
asset = item.asset, reason = item.reason;
_a.label = 1;
case 1:
_a.trys.push([1, 5, , 6]);
// Export asset JSON metadata
fileOps.exportFiles("assets", asset.mediaID.toString(), asset);
if (!asset.originUrl) return [3 /*break*/, 3];
filePath = (0, asset_utils_1.getAssetFilePath)(asset.originUrl);
assetFilesPath = path_1.default.join(fileOps.getDataFolderPath("assets"), filePath);
return [4 /*yield*/, fileOps.downloadFile(asset.originUrl, assetFilesPath)];
case 2:
success = _a.sent();
if (success) {
sizeDisplay = asset.size ? formatFileSize(asset.size) : "";
logger.asset.downloaded(asset);
return [2 /*return*/, { success: true, asset: asset }];
}
else {
logger.asset.error(asset, "Download failed");
throw new Error("Download failed");
}
return [3 /*break*/, 4];
case 3:
// Asset without downloadable file - just metadata
logger.warning("Asset without downloadable file", asset);
// logger.asset.downloaded(asset);
return [2 /*return*/, { success: true, asset: asset }];
case 4: return [3 /*break*/, 6];
case 5:
error_2 = _a.sent();
logger.asset.error(asset, error_2.message || "Unknown error");
unProcessedAssets[asset.mediaID] = asset.fileName;
return [2 /*return*/, { success: false, asset: asset, error: error_2 }];
case 6: return [2 /*return*/];
}
});
}); });
return [4 /*yield*/, Promise.all(downloadPromises)];
case 8:
results = _b.sent();
// Update counters
for (_a = 0, results_1 = results; _a < results_1.length; _a++) {
result = results_1[_a];
totalAttemptedToProcess++;
if (result.success) {
totalSuccessfullyDownloaded++;
}
totalProcessed = totalAttemptedToProcess + skippableAssets.length;
}
_b.label = 9;
case 9:
batchIndex++;
return [3 /*break*/, 7];
case 10:
// Final skipped assets processing for progress
totalSkippedAssets = skippableAssets.length;
// Performance and summary reporting
logger.endTimer();
unprocessedCount = Object.keys(unProcessedAssets).length;
logger.summary("pull", totalSuccessfullyDownloaded, totalSkippedAssets, unprocessedCount);
return [3 /*break*/, 12];
case 11:
error_1 = _b.sent();
console.error("Error in downloadAllAssets:", error_1);
throw error_1;
case 12: return [2 /*return*/];
}
});
});
}
//# sourceMappingURL=download-assets.js.map