mcdev
Version:
Accenture Salesforce Marketing Cloud DevTools
1,161 lines (1,092 loc) • 129 kB
JavaScript
'use strict';
import MetadataType from './MetadataType.js';
import { Util } from '../util/util.js';
import File from '../util/file.js';
import pLimit from 'p-limit';
import cliProgress from 'cli-progress';
import cache from '../util/cache.js';
import TriggeredSend from './TriggeredSend.js';
import Folder from './Folder.js';
import ReplaceCbReference from '../util/replaceContentBlockReference.js';
import toposort from 'toposort';
/**
* @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
* @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
* @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
* @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
* @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
* @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
* @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
*/
/**
* @typedef {import('../../types/mcdev.d.js').AssetSubType} AssetSubType
* @typedef {import('../../types/mcdev.d.js').AssetMap} AssetMap
* @typedef {import('../../types/mcdev.d.js').AssetItem} AssetItem
* @typedef {import('../../types/mcdev.d.js').AssetRequestParams} AssetRequestParams
* @typedef {import('../../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
*/
/**
* FileTransfer MetadataType
*
* @augments MetadataType
*/
class Asset extends MetadataType {
/**
* Retrieves Metadata of Asset
*
* @param {string} retrieveDir Directory where retrieved metadata directory will be saved
* @param {void | string[]} _ unused parameter
* @param {string[]} [subTypeArr] optionally limit to a single AssetSubType
* @param {string} [key] customer key
* @param {boolean} [loadShared] optionally retrieve assets from other BUs that were shared with the current
* @returns {Promise.<{metadata: AssetMap, type: string}>} Promise
*/
static async retrieve(retrieveDir, _, subTypeArr, key, loadShared = false) {
const items = [];
if (subTypeArr) {
// check if elements in subTypeArr exist in this.definition.subTypes
const invalidSubTypes = subTypeArr.filter(
(subType) => !this.definition.subTypes.includes(subType)
);
if (invalidSubTypes.length) {
throw new Error(`Invalid subType(s) found: ${invalidSubTypes.join(', ')}`);
}
}
subTypeArr ||= this._getSubTypes();
if (retrieveDir) {
await File.initPrettier();
}
if (retrieveDir && !cache.getCache()?.asset) {
// cache this for 3 reasons:
// 1) subtypes asset, message and template reference other content blocks by id/key/objectId as part of their views
// 2) subtype message references a template if it is template-based
// 3) all non-binary subtypes support ampscript / ssjs which can load otherc content blocks via ContentBlockyByX()
Util.logger.info(' - Caching dependent Metadata: asset');
const resultLocal = await this.retrieveForCache(
undefined,
this.definition.selflinkedSubTypes,
undefined,
false
);
cache.mergeMetadata('asset', resultLocal.metadata);
Util.logger.info(' - Caching dependent Metadata: shared asset');
const resultShared = await this.retrieveForCache(
undefined,
this.definition.selflinkedSubTypes,
undefined,
true
);
cache.mergeMetadata('asset', resultShared.metadata);
}
if (retrieveDir) {
if (key) {
// running post-delete tasks to clean up potential old files
await this.postDeleteTasks(key);
// retrieve by key/id/name
items.push(
...(await this.requestSubType(null, retrieveDir, key, null, loadShared))
);
} else {
// retrieve all
// loop through subtypes and return results of each subType for caching (saving is handled per subtype)
for (const subType of subTypeArr) {
// each subtype contains multiple different specific types (images contains jpg and png for example)
// we use await here to limit the risk of too many concurrent api requests at time
items.push(
...(await this.requestSubType(subType, retrieveDir, key, null, loadShared))
);
}
}
} else {
// caching
items.push(...(await this.requestSubType(subTypeArr, null, null, null, loadShared)));
}
const metadata = this.parseResponseBody({ items: items });
if (retrieveDir) {
Util.logger.info(
`Downloaded: ${this.definition.type} (${Object.keys(metadata).length})` +
Util.getKeysString(key)
);
}
return { metadata: metadata, type: this.definition.type };
}
/**
* Retrieves asset metadata for caching
*
* @param {void | string[]} [_] parameter not used
* @param {string[]} [subTypeArr] optionally limit to a single subtype
* @param {void | string} [__] parameter not used
* @param {boolean} [loadShared] optionally retrieve assets from other BUs that were shared with the current
* @returns {Promise.<{metadata: AssetMap, type: string}>} Promise
*/
static retrieveForCache(_, subTypeArr, __, loadShared = false) {
return this.retrieve(null, null, subTypeArr, undefined, loadShared);
}
/**
* Retrieves asset metadata for templating
*
* @deprecated Use `retrieve` followed by `build` instead. `retrieveAsTemplate` will be removed in a future version.
* @param {string} templateDir Directory where retrieved metadata directory will be saved
* @param {string} name name of the metadata file
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {AssetSubType} [selectedSubType] optionally limit to a single subtype
* @returns {Promise.<{metadata: AssetItem, type: string}>} Promise
*/
static async retrieveAsTemplate(templateDir, name, templateVariables, selectedSubType) {
Util.logDeprecated('retrieveAsTemplate', `'retrieve' followed by 'build'`);
const items = [];
const subTypes = selectedSubType ? [selectedSubType] : this._getSubTypes();
await File.initPrettier();
// loop through subtypes and return results of each subType for caching (saving is handled per subtype)
for (const subType of subTypes) {
// each subtype contains multiple different specific types (images contains jpg and png for example)
// we use await here to limit the risk of too many concurrent api requests at time
items.push(
...(await this.requestSubType(
subType,
templateDir,
'name:' + name,
templateVariables
))
);
}
const metadata = this.parseResponseBody({ items: items });
if (!Object.keys(metadata).length) {
Util.logger.error(`${this.definition.type} '${name}' not found on server.`);
}
Util.logger.info(`Downloaded: ${this.definition.type} (${Object.keys(metadata).length})`);
return { metadata: Object.values(metadata)[0], type: this.definition.type };
}
/**
* helper for {@link Asset.retrieve} + {@link Asset.retrieveAsTemplate}
*
* @private
* @returns {string[]} AssetSubType array
*/
static _getSubTypes() {
const selectedSubTypeArr = this.properties.metaDataTypes.retrieve.filter((type) =>
type.startsWith('asset-')
);
/* eslint-disable unicorn/prefer-ternary */
if (
this.properties.metaDataTypes.retrieve.includes('asset') ||
!selectedSubTypeArr.length
) {
// if "asset" is found in config assume to download the default subtypes only
return this.definition.typeRetrieveByDefault;
} else {
return selectedSubTypeArr.map((type) => type.replace('asset-', ''));
}
/* eslint-enable unicorn/prefer-ternary */
}
/**
* Returns Order in which metadata needs to be retrieved/deployed and skips components with missing components
*
* @param {AssetMap} metadataMap metadata thats about to be deployed
* @param {string} deployDir directory where deploy metadata are saved
* @returns {Promise.<AssetMap>} keyField => metadata map but sorted to ensure dependencies are deployed in correct order
*/
static async _getUpsertOrderAndSkipMissing(metadataMap, deployDir) {
/**
* one entry for each dependency with the first item being the key thats required by the second item
*
* @type {string[][]}
*/
const dependencies = [];
/** @type {ContentBlockConversionTypes[]} */
Util.OPTIONS.referenceFrom = ['key', 'name', 'id'];
/** @type {ContentBlockConversionTypes} */
Util.OPTIONS.referenceTo = 'key';
// loop through all metadata types which are being retrieved/deployed
for (const key in metadataMap) {
const errors = [];
const findAssetKeys = new Set();
// find asset references in metadata
const findAssetKeyMeta = Util.findLeafVals(metadataMap[key], 'r__asset_key');
for (const metaKey of findAssetKeyMeta) {
if (metadataMap[key] || cache.getByKey(this.definition.type, metaKey)) {
findAssetKeys.add(metaKey);
} else {
errors.push(
`content block ${metaKey} that is referenced via r__asset_key was not found on BU nor in deployment package`
);
}
}
try {
// find asset references in code
await this.replaceCbReference(metadataMap[key], deployDir, findAssetKeys);
} catch (ex) {
if (ex.code === 200) {
// no dependent keys found
// good from this perspective
} else {
errors.push(
`content block ${ex.message} that is referenced via ContentBlockByX was not found on BU nor in deployment package`
);
}
}
if (errors.length) {
Util.logger.error(
` ☇ skipping ${this.definition.type} ${key}:${errors.length > 1 ? '\n ·' : ''} ${errors.join('\n · ')}`
);
} else {
const dependentKeys = [...findAssetKeys];
if (dependentKeys.length > 0) {
dependencies.push(...dependentKeys.map((depKey) => [depKey, key]));
} else {
Util.logger.debug('Asset._getUpsertOrder: no dependent keys found for ' + key);
dependencies.push([undefined, key]);
}
}
}
// sort list & remove the undefined dependencies
const flatList = toposort(dependencies).filter((a) => !!a);
/** @type {AssetMap} */
const metadataTypeMapSorted = {};
// group subtypes per type
for (const key of flatList) {
if (metadataMap[key]) {
metadataTypeMapSorted[key] = metadataMap[key];
}
}
return metadataTypeMapSorted;
}
/**
* MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
*
* @param {AssetMap} metadataMap metadata mapped by their keyField
* @param {string} deployDir directory where deploy metadata are saved
* @returns {Promise.<AssetMap>} keyField => metadata map
*/
static async upsert(metadataMap, deployDir) {
// add already existing, cached items to our cache map to ensure we can find them
ReplaceCbReference.createCacheForMap(cache.getCache().asset);
// await ReplaceCbReference.createCache(this.properties, this.buObject, true);
// fill the cache map with our deployment package to ensure we can find
ReplaceCbReference.createCacheForMap(metadataMap);
// assets can link to other assets (via template, content block reference and SSJS/AMPscript) and deployment would fail if we did not sort this here
metadataMap = await this._getUpsertOrderAndSkipMissing(metadataMap, deployDir);
return super.upsert(metadataMap, deployDir, true);
}
/**
* helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP}
*
* @param {MetadataTypeItem} metadataEntry a single metadata Entry
* @param {object} apiResponse varies depending on the API call
* @returns {Promise.<object>} apiResponse, potentially modified
*/
static async postUpdateTasks(metadataEntry, apiResponse) {
// ensure we do not have any old files left aftert content blocks changed their type or got removed
await this.postDeleteTasks(metadataEntry[this.definition.keyField]);
return apiResponse;
}
/**
* Creates a single asset
*
* @param {AssetItem} metadata a single asset
* @returns {Promise} Promise
*/
static create(metadata) {
delete metadata.businessUnitAvailability;
const uri = '/asset/v1/content/assets/';
return super.createREST(metadata, uri);
}
/**
* helper for {@link MetadataType.createREST}
*
* @param {MetadataTypeItem} metadataEntry a single metadata Entry
* @param {object} apiResponse varies depending on the API call
* @returns {Promise.<object>} apiResponse, potentially modified
*/
static async postCreateTasks(metadataEntry, apiResponse) {
if (apiResponse[this.definition.idField]) {
// this also happens inside of MetadataType.upsert but we need it for createCacheMap()
// ensure we have the ID in the cache
metadataEntry[this.definition.idField] = apiResponse[this.definition.idField];
}
if (apiResponse.objectID) {
// ensure we have the asset objectId in the cache
metadataEntry.objectID = apiResponse.objectID;
}
// make this newly created item available in cache for other itmes that might reference it
/** @type {MetadataTypeMap} */
const newObject = {};
newObject[metadataEntry[this.definition.keyField]] = metadataEntry;
ReplaceCbReference.createCacheForMap(newObject);
return apiResponse;
}
/**
* Updates a single asset
*
* @param {AssetItem} metadata a single asset
* @returns {Promise} Promise
*/
static update(metadata) {
const uri = '/asset/v1/content/assets/' + metadata.id;
return super.updateREST(metadata, uri);
}
/**
* Retrieves Metadata of a specific asset type
*
* @param {string|string[]} subType group of similar assets to put in a folder (ie. images)
* @param {string} [retrieveDir] target directory for saving assets
* @param {string} [key] key/id/name to filter by
* @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
* @param {boolean} [loadShared] optionally retrieve assets from other BUs that were shared with the current
* @returns {Promise.<object[]>} Promise
*/
static async requestSubType(subType, retrieveDir, key, templateVariables, loadShared = false) {
const subTypeArr = Array.isArray(subType) ? subType : [subType];
if (retrieveDir) {
if (Array.isArray(subType)) {
throw new TypeError(
'requestSubType should not be called with multiple subtypes when retrieving to disk.'
);
} else if (!key) {
Util.logger.info(`- Retrieving Subtype: ${subType}`);
}
} else {
// in this mode we can accept arrays because we don't need to save this to a subtype folder but just want the records
Util.logSubtypes(subTypeArr);
}
/** @type {AssetSubType[]} */
const extendedSubTypeArr = key
? [null]
: subTypeArr.flatMap((subType) => this.definition.extendedSubTypes[subType]);
// the API can only assetType.ids at a time or else will throw: "DbUtility.GetPagedCollection passing through non-timeout DB error (30001)"
const subtypeIdsList = key
? [null]
: Util.chunk(
extendedSubTypeArr
?.map((subTypeItemName) => this.definition.typeMapping[subTypeItemName])
.sort((a, b) => a - b),
50
);
const uri = '/asset/v1/content/assets/query' + (loadShared ? '?scope=shared' : '');
/** @type {AssetRequestParams} */
const payload = {
page: {
page: 1,
pageSize: 50,
},
query: null,
fields: [
'category',
'createdDate',
'createdBy',
'modifiedDate',
'modifiedBy',
'objectID',
], // get folder to allow duplicate name check against cache
};
if (retrieveDir) {
payload.fields = [
'fileProperties',
'status',
'category',
'createdDate',
'createdBy',
'modifiedDate',
'modifiedBy',
'availableViews',
'data',
'tags',
'meta',
'content',
];
}
let items = [];
for (const subtypeIds of subtypeIdsList) {
if (key) {
if (key.startsWith('id:')) {
payload.query = {
property: this.definition.idField,
simpleOperator: 'equal',
value: key.slice(3),
};
} else if (key.startsWith('name:')) {
payload.query = {
property: this.definition.nameField,
simpleOperator: 'equal',
value: key.slice(5),
};
} else {
payload.query = {
property: this.definition.keyField,
simpleOperator: 'equal',
value: key,
};
}
} else {
payload.query = {
property: 'assetType.id',
simpleOperator: 'in',
value: subtypeIds,
};
// payload.sort = [{ property: 'id', direction: 'ASC' }];
}
// for caching we do not need these fields
let moreResults = false;
let lastPage = 0;
do {
payload.page.page = lastPage + 1;
const response = await this.client.rest.post(uri, payload);
if (response?.items?.length) {
// sometimes the api will return a payload without items
// --> ensure we only add proper items-arrays here
items = items.concat(response.items);
if (key && subType === null) {
subType = this.#getMainSubtype(response?.items[0].assetType.name);
}
}
// check if any more records
if (response?.message?.includes('all shards failed')) {
// When running certain filters, there is a limit of 10k on ElastiCache.
// Since we sort by ID, we can get the last ID then run new requests from there
payload.query = {
leftOperand: {
property: 'assetType.id',
simpleOperator: 'in',
value: subtypeIds,
},
logicalOperator: 'AND',
rightOperand: {
property: 'id',
simpleOperator: 'greaterThan',
value: items.at(-1).id,
},
};
lastPage = 0;
moreResults = true;
} else if (response.page * response.pageSize < response.count) {
moreResults = true;
lastPage = Number(response.page);
} else {
moreResults = false;
}
} while (moreResults);
}
// only when we save results do we need the complete metadata or files. caching can skip these
if (retrieveDir && !Array.isArray(subType)) {
if (items.length > 0) {
for (const item of items) {
if (item.customerKey.trim() !== item.customerKey) {
Util.logger.warn(
` - ${this.definition.type} ${item[this.definition.nameField]} (${
item[this.definition.keyField]
}) has leading or trailing spaces in customerKey. Please remove them in SFMC.`
);
}
}
// we have to wait on execution or it potentially causes memory reference issues when changing between BUs
await this.requestAndSaveExtended(items, subType, retrieveDir, templateVariables);
}
// always show the summary even if we already had the progress bar in the console
Util.logger.info(` Downloaded asset${subType ? '-' + subType : ''}: ${items.length}`);
}
return items;
}
/**
* Retrieves extended metadata (files or extended content) of asset
*
* @param {Array} items array of items to retrieve
* @param {string} subType group of similar assets to put in a folder (ie. images)
* @param {string} retrieveDir target directory for saving assets
* @param {TemplateMap} [templateVariables] variables to be replaced in the metadata
* @returns {Promise.<MetadataTypeMap>} Promise
*/
static async requestAndSaveExtended(items, subType, retrieveDir, templateVariables) {
// disable CLI logs other than error while retrieving subtype
const loggerLevelBak = Util.logger.level;
if (loggerLevelBak !== 'error') {
// disable logging to cli other than Errors
Util.setLoggingLevel({ silent: true });
}
const extendedBar = new cliProgress.SingleBar(
{
format:
' Downloading [{bar}] {percentage}% | {value}/{total} | asset-' +
subType,
},
cliProgress.Presets.shades_classic
);
const completed = [];
const failed = [];
const metadataMap = {};
// put in do loop to manage issues with connection timeout
do {
// use promise execution limiting to avoid rate limits on api, but speed up execution
// start the progress bar with a total value of 200 and start value of 0
extendedBar.start(items.length - completed.length, 0);
try {
const rateLimit = pLimit(5);
const promiseMap = await Promise.all(
items.map((item, index) =>
rateLimit(async () => {
const metadataMapSaveSingle = {};
// this is a file so extended is at another endpoint
if (item?.fileProperties?.extension && !completed.includes(item.id)) {
try {
// retrieving the extended file does not need to be awaited
await this._retrieveExtendedFile(item, subType, retrieveDir);
} catch (ex) {
failed.push({ item: item, error: ex });
}
// still return even if extended failed
metadataMap[item.customerKey] = item;
// even if the extended file failed, still save the metadata
metadataMapSaveSingle[item.customerKey] = item;
}
// this is a complex type which stores data in the asset itself
else if (!completed.includes(item.id)) {
try {
const extendedItem = await this.client.rest.get(
'asset/v1/content/assets/' + item.id
);
// only return the metadata if we have extended content
metadataMap[item.customerKey] = extendedItem;
// only save the metadata if we have extended content
metadataMapSaveSingle[item.customerKey] = extendedItem;
// overwrite the original item with the extended content to ensure retrieve() returns it
items[index] = extendedItem;
} catch (ex) {
failed.push({ item: item, error: ex });
}
}
completed.push(item.id);
if (metadataMapSaveSingle[item.customerKey]) {
await this.saveResults(
metadataMapSaveSingle,
retrieveDir,
'asset-' + subType,
templateVariables
);
}
// update the current value in your application..
extendedBar.increment();
})
)
);
// stop the progress bar
extendedBar.stop();
Asset._resetLogLevel(loggerLevelBak, failed);
return promiseMap;
} catch (ex) {
extendedBar.stop();
Asset._resetLogLevel(loggerLevelBak, failed);
// timeouts should be retried, others can be exited
if (ex.code !== 'ETIMEDOUT') {
throw ex;
}
}
} while (completed.length === items.length);
return metadataMap;
}
/**
* helper that reset the log level and prints errors
*
* @private
* @param {'info'|'verbose'|'debug'|'error'} loggerLevelBak original logger level
* @param {object[]} failed array of failed items
*/
static _resetLogLevel(loggerLevelBak, failed) {
// re-enable CLI logs
// reset logging level
let obj;
switch (loggerLevelBak) {
case 'info': {
obj = {};
break;
}
case 'verbose': {
obj = { verbose: true };
break;
}
case 'debug': {
obj = { debug: true };
break;
}
case 'error': {
obj = { silent: true };
}
}
Util.setLoggingLevel(obj);
if (failed.length) {
Util.logger.warn(
` - Failed to download ${failed.length} extended file${
failed.length > 1 ? 's' : ''
}:`
);
for (const fail of failed) {
Util.logger.warn(
` - "${fail.item.name}" (${fail.item.customerKey}): ${fail.error.message} (${
fail.error.code
})${
fail.error.endpoint
? Util.getGrayMsg(
' - ' +
fail.error.endpoint.split('rest.marketingcloudapis.com')[1]
)
: ''
}`
);
}
Util.logger.info(
' - You will still find a JSON file for each of these in the download directory.'
);
}
}
/**
* Some metadata types store their actual content as a separate file, e.g. images
* This method retrieves these and saves them alongside the metadata json
*
* @param {AssetItem} metadata a single asset
* @param {string} subType group of similar assets to put in a folder (ie. images)
* @param {string} retrieveDir target directory for saving assets
* @returns {Promise.<void>} -
*/
static async _retrieveExtendedFile(metadata, subType, retrieveDir) {
const file = await this.client.rest.get('asset/v1/content/assets/' + metadata.id + '/file');
// to handle uploaded files that bear the same name, SFMC engineers decided to add a number after the fileName
// however, their solution was not following standards: fileName="header.png (4) " and then extension="png (4) "
const fileExt = metadata.fileProperties.extension.split(' ')[0];
File.writeToFile(
[retrieveDir, this.definition.type, subType],
metadata.customerKey,
fileExt,
file,
'base64'
);
}
/**
* helper for {@link Asset.preDeployTasks}
* Some metadata types store their actual content as a separate file, e.g. images
* This method reads these from the local FS stores them in the metadata object allowing to deploy it
*
* @param {AssetItem} metadata a single asset
* @param {string} subType group of similar assets to put in a folder (ie. images)
* @param {string} deployDir directory of deploy files
* @param {boolean} [pathOnly] used by getFilesToCommit which does not need the binary file to be actually read
* @returns {Promise.<string>} if found will return the path of the binary file
*/
static async _readExtendedFileFromFS(metadata, subType, deployDir, pathOnly = false) {
// to handle uploaded files that bear the same name, SFMC engineers decided to add a number after the fileName
// however, their solution was not following standards: fileName="header.png (4) " and then extension="png (4) "
if (!metadata?.fileProperties?.extension) {
return;
}
const fileExt = metadata.fileProperties.extension.split(' ')[0];
const path = File.normalizePath([
deployDir,
this.definition.type,
subType,
`${metadata.customerKey}.${fileExt}`,
]);
if (await File.pathExists(path)) {
if (!pathOnly) {
metadata.file = await File.readFilteredFilename(
[deployDir, this.definition.type, subType],
metadata.customerKey,
fileExt,
'base64'
);
}
return path;
}
}
/**
* manages post retrieve steps
*
* @param {AssetItem} metadata a single asset
* @returns {CodeExtractItem} metadata
*/
static postRetrieveTasks(metadata) {
// folder
this.setFolderPath(metadata);
// template-based emails
if (metadata.assetType.name === 'templatebasedemail') {
// get template
try {
if (metadata.views?.html?.template?.id) {
metadata.views.html.template.r__asset_key = cache.searchForField(
'asset',
metadata.views?.html?.template?.id,
'id',
'customerKey'
);
delete metadata.views.html.template.id;
delete metadata.views.html.template.name;
delete metadata.views.html.template.assetType;
delete metadata.views.html.template.availableViews;
delete metadata.views.html.template.data;
delete metadata.views.html.template.modelVersion;
}
} catch {
Util.logger.warn(
` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
metadata[this.definition.keyField]
}): Could not find email template with id (${metadata.views.html.template.id})`
);
}
}
// get asset-asset
try {
if (metadata.meta?.thumbnailRefAssetId) {
// cloudpages', 'landingpage', 'microsite', 'interactivecontent'
metadata.meta.r__asset_key = cache.searchForField(
'asset',
metadata.meta?.thumbnailRefAssetId,
'id',
'customerKey'
);
delete metadata.meta.thumbnailRefAssetId;
}
} catch {
Util.logger.warn(
` - ${this.definition.type} '${metadata[this.definition.nameField]}' (${
metadata[this.definition.keyField]
}): Could not find associated cloudpage content with id (${metadata.meta.thumbnailRefAssetId})`
);
}
if (
['landingpage', 'microsite', 'interactivecontent'].includes(metadata.assetType.name) &&
typeof metadata.content === 'string'
) {
// cloudpages', 'landingpage', 'microsite', 'interactivecontent'
metadata.content = JSON.parse(metadata.content);
}
if (typeof metadata.data?.site?.content === 'string') {
// for all xx-coderesource types
metadata.data.site.content = JSON.parse(metadata.data.site.content);
}
if (metadata.meta?.cloudPages) {
metadata.meta.cloudPages.c__published = metadata.meta.cloudPages.publishDate
? true
: false;
}
// extract HTML for selected subtypes and convert payload for easier processing in MetadataType.saveResults()
return this._extractCode(metadata);
}
/**
* Gets executed after deployment of metadata type
*
* @param {MetadataTypeMap} metadata metadata mapped by their keyField
* @param {MetadataTypeMap} _ originalMetadata to be updated (contains additioanl fields)
* @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
* @returns {Promise.<void>} -
*/
static async postDeployTasks(metadata, _, createdUpdated) {
if (Util.OPTIONS.refresh) {
if (createdUpdated.updated) {
// only run this if assets were updated. for created assets we do not expect
await this._refreshTriggeredSend(metadata);
} else {
Util.logger.warn(
'You set the --refresh flag but no updated assets found. Skipping refresh of triggeredSendDefinitions.'
);
}
}
}
/**
* helper for {@link Asset.postDeployTasks}. triggers a refresh of active triggerredSendDefinitions associated with the updated asset-message items. Gets executed if refresh option has been set.
*
* @private
* @param {MetadataTypeMap} metadata metadata mapped by their keyField
* @returns {Promise.<void>} -
*/
static async _refreshTriggeredSend(metadata) {
// get legacyData.legacyId from assets to compare to TSD's metadata.Email.ID to
const legacyIdArr = Object.keys(metadata)
.map((key) => metadata[key]?.legacyData?.legacyId)
.filter(Boolean);
if (!legacyIdArr.length) {
Util.logger.warn(
'No legacyId found in updated emails. Skipping refresh of triggeredSendDefinitions.'
);
return;
}
// prep triggeredSendDefinition class
TriggeredSend.properties = this.properties;
TriggeredSend.buObject = this.buObject;
TriggeredSend.client = this.client;
try {
// find refreshable TSDs
const tsdObj = (await TriggeredSend.findRefreshableItems(true))?.metadata;
const tsdCountInitial = Object.keys(tsdObj).length;
const emailCount = legacyIdArr.length;
// filter TSDs by legacyId
for (const key in tsdObj) {
if (!legacyIdArr.includes(tsdObj[key].Email.ID)) {
delete tsdObj[key];
}
}
const tsdCountFiltered = Object.keys(tsdObj).length;
Util.logger.info(
`Found ${tsdCountFiltered} out of ${tsdCountInitial} total triggeredSendDefinitions for ${emailCount} deployed emails. Commencing validation...`
);
// get keys of TSDs to refresh
const keyArr = await TriggeredSend.getKeysForValidTSDs(tsdObj);
await TriggeredSend.refresh(keyArr);
} catch {
Util.logger.warn('Failed to refresh triggeredSendDefinition');
}
}
/**
* prepares an asset definition for deployment
*
* @param {AssetItem} metadata a single asset
* @param {string} deployDir directory of deploy files
* @returns {Promise.<AssetItem>} Promise
*/
static async preDeployTasks(metadata, deployDir) {
// additonalattributes fail where the value is "" so we need to remove them from deploy
if (metadata?.data?.email?.attributes?.length > 0) {
metadata.data.email.attributes = metadata.data.email.attributes.filter(
(attr) => attr.value
);
}
// folder
this.setFolderId(metadata);
if (!cache.getByKey(this.definition.type, metadata[this.definition.keyField])) {
if (metadata.assetType.name === 'webpage') {
// we are attempting to CREATE a cloudpage asset which needs to be prevented.
throw new Error(
'CloudPage content cannot be created via mcdev. Please create it via the UI first, then you can update the content or key.'
);
}
if (
this.definition.extendedSubTypes.cloudpage.includes(metadata.assetType.name) ||
this.definition.extendedSubTypes.coderesource.includes(metadata.assetType.name)
) {
// if we do create those, we get the error "Asset <id> not found" because the internal endpoint internal/v2/cloudpages/sites won't find it.
throw new Error(
'CloudPage assets cannot be created via mcdev. Please create it via the UI first, then you can update the content or key.'
);
}
}
// template-based emails
if (
metadata.assetType.name === 'templatebasedemail' &&
metadata.views?.html?.template?.r__asset_key
) {
// template
metadata.views.html.template.id = cache.searchForField(
'asset',
metadata.views.html.template.r__asset_key,
'customerKey',
'id'
);
metadata.views.html.template.name = cache.searchForField(
'asset',
metadata.views.html.template.r__asset_key,
'customerKey',
'name'
);
metadata.views.html.template.assetType = {
id: 4,
name: 'template',
displayName: 'Template',
};
metadata.views.html.template.data = {
email: { options: { generateFrom: null } },
};
metadata.views.html.template.availableViews = [];
metadata.views.html.template.modelVersion = 2;
delete metadata.views.html.template.r__asset_key;
}
if (metadata.meta?.r__asset_key) {
// cloudpages: 'landingpage', 'microsite', 'interactivecontent'
// get asset-asset
metadata.meta.thumbnailRefAssetId = cache.searchForField(
'asset',
metadata.meta.r__asset_key,
'customerKey',
'id'
);
delete metadata.meta.r__asset_key;
}
if (metadata.meta?.cloudPages?.c__published) {
// TODO: removal should be done via asset.definition instead
delete metadata.meta.cloudPages.c__published;
}
if (
['landingpage', 'microsite', 'interactivecontent'].includes(metadata.assetType.name) &&
typeof metadata.content === 'object'
) {
// cloudpages: 'landingpage', 'microsite', 'interactivecontent'
metadata.content = JSON.stringify(metadata.content);
}
if (typeof metadata.data?.site?.content === 'object') {
// coderesource: for all xx-coderesource types
metadata.data.site.content = JSON.stringify(metadata.data.site.content);
}
// restore asset type id which is needed for deploy
metadata.assetType.id = this.definition.typeMapping[metadata.assetType.name];
// define asset's subtype
const subType = this._getSubtype(metadata);
// #0 format blocks in slots for deployment
await this._preDeployTasksBocks(metadata);
// #1 get text extracts back into the JSON
await this._mergeCode(metadata, deployDir, subType);
// #2 get file from local disk and insert as base64
await this._readExtendedFileFromFS(metadata, subType, deployDir);
// only execute #3 if we are deploying / copying from one BU to another, not while using mcdev as a developer tool
if (
Util.OPTIONS.autoMidSuffix &&
this.buObject.mid &&
metadata.memberId != this.buObject.mid && // soft comparison to accomodate for string-version of mid
!metadata[this.definition.keyField].endsWith(this.buObject.mid)
) {
// #3 make sure customer key is unique by suffixing it with target MID (unless we are deploying to the same MID)
// check if this suffixed with the source MID
const suffix = '-' + this.buObject.mid;
// for customer key max is 36 chars
metadata[this.definition.keyField] =
metadata[this.definition.keyField].slice(0, Math.max(0, 36 - suffix.length)) +
suffix;
}
if (!Util.OPTIONS.matchName) {
// #4 make sure the name is unique
const assetCache = cache.getCache()[this.definition.type];
const namesInFolder = Object.keys(assetCache)
.filter((key) => assetCache[key].category.id === metadata.category.id)
.map((key) => ({
type: this.#getMainSubtype(assetCache[key].assetType.name),
key: key,
name: assetCache[key].name,
}));
// if the name is already in the folder for a different key, add a number to the end
metadata[this.definition.nameField] = this.findUniqueName(
metadata[this.definition.keyField],
metadata[this.definition.nameField],
namesInFolder,
this.#getMainSubtype(metadata.assetType.name)
);
}
return metadata;
}
/**
* find the subType matching the extendedSubType
*
* @param {string} extendedSubType webpage, htmlblock, etc
* @returns {string} subType: block, message, other, etc
*/
static #getMainSubtype(extendedSubType) {
return Object.keys(this.definition.extendedSubTypes).find((subType) =>
this.definition.extendedSubTypes[subType].includes(extendedSubType)
);
}
/**
* determines the subtype of the current asset
*
* @private
* @param {AssetItem} metadata a single asset
* @returns {string} subtype
*/
static _getSubtype(metadata) {
for (const sub in this.definition.extendedSubTypes) {
if (this.definition.extendedSubTypes[sub].includes(metadata.assetType.name)) {
return sub;
}
}
return;
}
/**
* helper for {@link MetadataType.buildDefinition}
* handles extracted code if any are found for complex types
*
* @param {string} templateDir Directory where metadata templates are stored
* @param {string} targetDir Directory where built definitions will be saved
* @param {AssetItem} metadata main JSON file that was read from file system
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {string} templateName name of the template to be built
* @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
*/
static buildDefinitionForNested(
templateDir,
targetDir,
metadata,
templateVariables,
templateName
) {
return this._buildForNested(
templateDir,
targetDir,
metadata,
templateVariables,
templateName,
'definition'
);
}
/**
* helper for {@link MetadataType.buildTemplate}
* handles extracted code if any are found for complex types
*
* @example assets of type codesnippetblock will result in 1 json and 1 amp/html file. both files need to be run through templating
* @param {string} templateDir Directory where metadata templates are stored
* @param {string|string[]} targetDir (List of) Directory where built definitions will be saved
* @param {AssetItem} metadata main JSON file that was read from file system
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {string} templateName name of the template to be built
* @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
*/
static buildTemplateForNested(
templateDir,
targetDir,
metadata,
templateVariables,
templateName
) {
return this._buildForNested(
templateDir,
targetDir,
metadata,
templateVariables,
templateName,
'template'
);
}
/**
* helper for {@link MetadataType.buildDefinition}
* handles extracted code if any are found for complex types
*
* @param {string} templateDir Directory where metadata templates are stored
* @param {string|string[]} targetDir (List of) Directory where built definitions will be saved
* @param {AssetItem} metadata main JSON file that was read from file system
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {string} templateName name of the template to be built
* @param {'definition'|'template'} mode defines what we use this helper for
* @returns {Promise.<string[][]>} list of extracted files with path-parts provided as an array
*/
static async _buildForNested(
templateDir,
targetDir,
metadata,
templateVariables,
templateName,
mode
) {
// * because asset's _mergeCode() is overwriting 'metadata', clone it to ensure the main file is not modified by what we do in here
metadata = structuredClone(metadata);
// #1 text extracts
// define asset's subtype
const subType = this._getSubtype(metadata);
// get HTML from filesystem
const fileList = await this._mergeCode(metadata, templateDir, subType, templateName);
// replace template variables with their values
for (const extractedFile of fileList) {
try {
if (mode === 'definition') {
// replace template variables with their values
extractedFile.content = this.applyTemplateValues(
extractedFile.content,
templateVariables
);
} else if (mode === 'template') {
// replace template values with corresponding variable names
extractedFile.content = this.applyTemplateNames(
extractedFile.content,
templateVariables
);