mcdev
Version:
Accenture Salesforce Marketing Cloud DevTools
1,066 lines (1,008 loc) • 136 kB
JavaScript
'use strict';
/* eslint no-unused-vars:off */
/*
* README no-unused-vars is reduced to WARNING here as this file is a template
* for all metadata types and methods often define params that are not used
* in the generic version of the method
*/
import { Util } from '../util/util.js';
import File from '../util/file.js';
import cache from '../util/cache.js';
import deepEqual from 'deep-equal';
import pLimit from 'p-limit';
import Mustache from 'mustache';
import MetadataTypeInfo from '../MetadataTypeInfo.js';
import validationsRules from '../util/validations.js';
/**
* @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
* @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
* @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
* @typedef {import('../../types/mcdev.d.js').Mcdevrc} Mcdevrc
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
* @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
* @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
* @typedef {import('../../types/mcdev.d.js').MultiMetadataTypeList} MultiMetadataTypeList
* @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
* @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
* @typedef {import('../../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
* @typedef {import('sfmc-sdk').default} SDK
* @typedef {import('../../types/mcdev.d.js').SDKError} SDKError
* @typedef {import('../../types/mcdev.d.js').SOAPError} SOAPError
* @typedef {import('../../types/mcdev.d.js').RestError} RestError
* @typedef {import('../../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
*/
/**
* ensure that Mustache does not escape any characters
*
* @param {string} text -
* @returns {string} text
*/
Mustache.escape = function (text) {
return text;
};
/**
* MetadataType class that gets extended by their specific metadata type class.
* Provides default functionality that can be overwritten by child metadata type classes
*
*/
class MetadataType {
/**
* Returns file contents mapped to their filename without '.json' ending
*
* @param {string} dir directory with json files, e.g. /retrieve/cred/bu/event, /deploy/cred/bu/event, /template/event
* @param {boolean} [listBadKeys] do not print errors, used for badKeys()
* @param {string[]} [selectedSubType] asset, message, ...
* @returns {Promise.<MetadataTypeMap>} fileName => fileContent map
*/
static async getJsonFromFS(dir, listBadKeys, selectedSubType) {
const fileName2FileContent = {};
try {
const files = await File.readdir(dir);
for (const fileName of files) {
try {
if (fileName.endsWith('.json')) {
const fileContent = await File.readJSONFile(dir, fileName, false);
// ! convert numbers to string to allow numeric keys to be checked properly
const key = Number.isInteger(fileContent[this.definition.keyField])
? fileContent[this.definition.keyField].toString()
: fileContent[this.definition.keyField];
// ensure filename includes extended metadata extension
const regex = new RegExp(/\.(\w|-)+-meta.json/);
const errorDir = dir.split('\\').join('/');
if (regex.test(fileName)) {
const fileNameWithoutEnding = File.reverseFilterIllegalFilenames(
fileName.split(regex)[0]
);
// We always store the filename using the External Key (CustomerKey or key) to avoid duplicate names.
// to ensure any changes are done to both the filename and external key do a check here
if (key === fileNameWithoutEnding || listBadKeys) {
fileName2FileContent[fileNameWithoutEnding] = fileContent;
} else {
Util.logger.error(
` ☇ skipping ${this.definition.type} ${key}: Name of the metadata file and the JSON-key (${this.definition.keyField}) must match. Expected: ${key}.${this.definition.type}-meta.json. Actual: ` +
Util.getGrayMsg(`${errorDir}/`) +
fileName
);
}
} else {
Util.logger.error(
` ☇ skipping ${this.definition.type} ${key}: Name of the metadata file must end on the extended metadata suffix. Expected: ${key}.${this.definition.type}-meta.json. Actual: ` +
Util.getGrayMsg(`${errorDir}/`) +
fileName
);
}
}
} catch (ex) {
// by catching this in the loop we gracefully handle the issue and move on to the next file
Util.metadataLogger('debug', this.definition.type, 'getJsonFromFS', ex);
}
}
} catch (ex) {
// this will catch issues with readdirSync
Util.metadataLogger('debug', this.definition.type, 'getJsonFromFS', ex);
throw new Error(ex);
}
return fileName2FileContent;
}
/**
* Returns fieldnames of Metadata Type. 'this.definition.fields' variable only set in child classes.
*
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @param {boolean} [isCaching] if true, then check if field should be skipped for caching
* @returns {string[]} Fieldnames
*/
static getFieldNamesToRetrieve(additionalFields, isCaching) {
const fieldNames = [];
for (const fieldName in this.definition.fields) {
if (
additionalFields?.includes(fieldName) ||
(this.definition.fields[fieldName].retrieving &&
!(isCaching && this.definition.fields[fieldName].skipCache))
) {
fieldNames.push(fieldName);
}
}
if (!fieldNames.includes(this.definition.idField)) {
// Always retrieve the ID because it may be used in references
fieldNames.push(this.definition.idField);
}
return fieldNames;
}
/**
* Deploys metadata
*
* @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
* @param {string} deployDir directory where deploy metadata are saved, ending on cred/bu
* @param {string} retrieveDir directory where metadata after deploy should be saved, ending on cred/bu
* @returns {Promise.<MetadataTypeMap>} Promise of keyField => metadata map
*/
static async deploy(metadataMap, deployDir, retrieveDir) {
const upsertedMetadataMap = await this.upsert(metadataMap, deployDir);
if (retrieveDir) {
// deploy can be run with retrieveDir set to null for deploying auto-created foldes - these should not be saved to the retrieve-folder while everything else should
const savedMetadataMap = await this.saveResults(upsertedMetadataMap, retrieveDir, null);
if (
this.properties.metaDataTypes.documentOnRetrieve.includes(this.definition.type) &&
!this.definition.documentInOneFile
) {
// * do not await here as this might take a while and has no impact on the deploy
// * this should only be run if documentation is on a per metadata record level. Types that document an overview into a single file will need a full retrieve to work instead
await this.document(savedMetadataMap, true);
}
}
return upsertedMetadataMap;
}
/**
* Gets executed after deployment of metadata type
*
* @param {MetadataTypeMap} upsertResults metadata mapped by their keyField as returned by update/create
* @param {MetadataTypeMap} originalMetadata metadata to be updated (contains additioanl fields)
* @param {{created: number, updated: number}} createdUpdated counter representing successful creates/updates
* @returns {Promise.<void>} -
*/
static async postDeployTasks(upsertResults, originalMetadata, createdUpdated) {}
/**
* Gets executed before deleting a list of keys for the current type
*
* @param {string} keyArr metadata keys
* @returns {Promise.<void>} -
*/
static async preDeleteTasks(keyArr) {}
/**
* helper for {@link MetadataType.createREST}
*
* @param {MetadataTypeItem} metadataEntry a single metadata Entry
* @param {object} apiResponse varies depending on the API call
* @param {MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
* @returns {Promise.<object>} apiResponse, potentially modified
*/
static async postCreateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {
return apiResponse;
}
/**
* helper for {@link MetadataType.updateREST} and {@link MetadataType.updateSOAP}
*
* @param {MetadataTypeItem} metadataEntry a single metadata Entry
* @param {object} apiResponse varies depending on the API call
* @param {MetadataTypeItem} metadataEntryWithAllFields like metadataEntry but before non-creatable fields were stripped
* @returns {Promise.<object>} apiResponse, potentially modified
*/
static postUpdateTasks(metadataEntry, apiResponse, metadataEntryWithAllFields) {
return apiResponse;
}
/**
* helper for {@link MetadataType.createREST} when legacy API endpoints as these do not return the created item but only their new id
*
* @param {MetadataTypeItem} metadataEntry a single metadata Entry
* @param {object} apiResponse varies depending on the API call
* @returns {Promise.<void>} -
*/
static async postDeployTasks_legacyApi(metadataEntry, apiResponse) {
if (!apiResponse?.[this.definition.idField] && !metadataEntry?.[this.definition.idField]) {
return;
}
const id =
apiResponse?.[this.definition.idField] || metadataEntry?.[this.definition.idField];
// re-retrieve created items because the API does not return any info for them except the new id (api key)
try {
const { metadata } = await this.retrieveForCache(null, null, 'id:' + id);
const item = Object.values(metadata)[0];
// ensure the "created item" cli log entry has the new auto-generated value
metadataEntry[this.definition.keyField] = item[this.definition.keyField];
// ensure postRetrieveTasks has the complete object in "apiResponse"
Object.assign(apiResponse, item);
// postRetrieveTasks will be run automatically on this via super.saveResult
} catch (ex) {
throw new Error(
`Could not get details for new ${this.definition.type} ${id} from server (${ex.message})`
);
}
}
/**
* Gets executed after retreive of metadata type
*
* @param {MetadataTypeItem} metadata a single item
* @param {string} targetDir folder where retrieves should be saved
* @param {boolean} [isTemplating] signals that we are retrieving templates
* @returns {MetadataTypeItem} cloned metadata
*/
static postRetrieveTasks(metadata, targetDir, isTemplating) {
return structuredClone(metadata);
}
/**
* generic script that retrieves the folder path from cache and updates the given metadata with it after retrieve
*
* @param {MetadataTypeItem} metadata a single item
*/
static setFolderPath(metadata) {
if (!this.definition.folderIdField) {
return;
}
try {
metadata.r__folder_Path = cache.searchForField(
'folder',
metadata[this.definition.folderIdField],
'ID',
'Path'
);
delete metadata[this.definition.folderIdField];
} catch (ex) {
Util.logger.warn(
Util.getMsgPrefix(this.definition, metadata) +
`: Could not find folder (${ex.message})`
);
}
}
/**
* generic script that retrieves the folder ID from cache and updates the given metadata with it before deploy
*
* @param {MetadataTypeItem} metadata a single item
*/
static setFolderId(metadata) {
if (!this.definition.folderIdField) {
return;
}
if (!metadata.r__folder_Path) {
throw new Error(
`Dependent folder could not be found because r__folder_Path is not set`
);
}
metadata[this.definition.folderIdField] = cache.getFolderId(metadata.r__folder_Path);
delete metadata.r__folder_Path;
}
/**
* Gets metadata from Marketing Cloud
*
* @param {string} retrieveDir Directory where retrieved metadata directory will be saved
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @param {string[]} [subTypeArr] optionally limit to a single subtype
* @param {string} [key] customer key of single item to retrieve
* @returns {Promise.<MetadataTypeMapObj>} metadata
*/
static async retrieve(retrieveDir, additionalFields, subTypeArr, key) {
Util.logNotSupported(this.definition, 'retrieve');
return { metadata: {}, type: this.definition.type };
}
/**
* Gets metadata from Marketing Cloud
*
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @param {string[]} [subTypeArr] optionally limit to a single subtype
* @returns {Promise.<MetadataTypeMapObj>} metadata
*/
static retrieveChangelog(additionalFields, subTypeArr) {
return this.retrieveForCache(additionalFields, subTypeArr);
}
/**
* Gets metadata cache with limited fields and does not store value to disk
*
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @param {string[]} [subTypeArr] optionally limit to a single subtype
* @param {string} [key] customer key of single item to retrieve
* @returns {Promise.<MetadataTypeMapObj>} metadata
*/
static async retrieveForCache(additionalFields, subTypeArr, key) {
return this.retrieve(null, additionalFields, subTypeArr, key);
}
/**
* Gets metadata cache with limited fields and does not store value to disk
*
* @deprecated Use `retrieve` followed by `build` instead. `retrieveAsTemplate` will be removed in a future version.
* @param {string} templateDir Directory where retrieved metadata directory will be saved
* @param {string} name name of the metadata file
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {string} [subType] optionally limit to a single subtype
* @returns {Promise.<MetadataTypeItemObj>} metadata
*/
static async retrieveAsTemplate(templateDir, name, templateVariables, subType) {
Util.logNotSupported(this.definition, 'retrieveAsTemplate');
Util.logDeprecated('retrieveAsTemplate', `'retrieve' followed by 'build'`);
return { metadata: null, type: this.definition.type };
}
/**
* Retrieve a specific Script by Name
*
* @param {string} templateDir Directory where retrieved metadata directory will be saved
* @param {string} uri rest endpoint for GET
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @param {string} name name (not key) of the metadata item
* @returns {Promise.<{metadata: MetadataTypeItem, type: string}>} Promise
*/
static async retrieveTemplateREST(templateDir, uri, templateVariables, name) {
return this.retrieveREST(templateDir, uri, templateVariables, name);
}
/**
* Gets metadata cache with limited fields and does not store value to disk
*
* @param {string} retrieveDir Directory where retrieved metadata directory will be saved
* @param {string} templateDir (List of) Directory where built definitions will be saved
* @param {string} key name of the metadata file
* @param {TemplateMap} templateVariables variables to be replaced in the metadata
* @returns {Promise.<MetadataTypeItemObj>} single metadata
*/
static async buildTemplate(retrieveDir, templateDir, key, templateVariables) {
// retrieve metadata template
let metadataStr;
const typeDirArr = [this.definition.type];
const subType = await this.findSubType(retrieveDir, key);
if (subType) {
typeDirArr.push(subType);
}
const suffix = subType ? `-${subType}-meta` : '-meta';
const fileName = key + '.' + this.definition.type + suffix;
try {
// ! do not load via readJSONFile to ensure we get a string, not parsed JSON
// templated files might contain illegal json before the conversion back to the file that shall be saved
metadataStr = await File.readFilteredFilename(
[retrieveDir, ...typeDirArr],
fileName,
'json'
);
if (!metadataStr) {
throw new Error('File not found');
}
} catch (ex) {
try {
metadataStr = await this.readSecondaryFolder(
retrieveDir,
typeDirArr,
key,
fileName,
ex
);
} catch {
// only happening for types that use readSecondaryFolder (e.g. asset)
// if we still have no metadataStr then we have to skip this metadata for all types and hence handle it outside of this catch
}
if (!metadataStr) {
Util.logger.warn(
Util.getGrayMsg(` ☇ skipped ${this.definition.type} ${key}: not found`)
);
return;
}
}
if (this.definition.stringifyFieldsBeforeTemplate) {
// numeric fields are returned as numbers by the SDK/API. If we try to replace them in buildTemplate it would break the JSON format - but not if we stringify them first because then the {{{var}}} is in quotes
const metadataTemp = JSON.parse(metadataStr);
for (const field of this.definition.stringifyFieldsBeforeTemplate) {
if (metadataTemp[field]) {
if (Array.isArray(metadataTemp[field])) {
for (let i = 0; i < metadataTemp[field].length; i++) {
metadataTemp[field][i] = metadataTemp[field][i].toString();
}
} else if ('object' === typeof metadataTemp[field]) {
for (const subField in metadataTemp[field]) {
metadataTemp[field][subField] =
metadataTemp[field][subField].toString();
}
} else {
metadataTemp[field] = metadataTemp[field].toString();
}
}
}
metadataStr = JSON.stringify(metadataTemp);
}
// handle extracted code
// templating to extracted content is applied inside of buildTemplateForNested()
await this.buildTemplateForNested(
retrieveDir,
templateDir,
JSON.parse(metadataStr),
templateVariables,
key
);
const metadata = JSON.parse(Util.replaceByObject(metadataStr, templateVariables));
this.keepTemplateFields(metadata);
try {
// write to file
await File.writeJSONToFile([templateDir, ...typeDirArr], fileName, metadata);
Util.logger.info(
` - templated ${this.definition.type}: ${key} (${
metadata[this.definition.nameField]
})`
);
return { metadata: metadata, type: this.definition.type };
} catch (ex) {
throw new Error(`${this.definition.type}:: ${ex.message}`);
}
}
/**
* Gets executed before deploying metadata
*
* @param {MetadataTypeItem} metadata a single metadata item
* @param {string} deployDir folder where files for deployment are stored
* @returns {Promise.<MetadataTypeItem>} Promise of a single metadata item
*/
static async preDeployTasks(metadata, deployDir) {
return metadata;
}
/**
* helper to find a new unique name during item creation
*
* @param {string} key key of the item
* @param {string} name name of the item
* @param {{ type: string; key: string; name: any; }[]} namesInFolder names of the items in the same folder
* @param {string} [subtype] itemType-name
* @returns {string} new name
*/
static findUniqueName(key, name, namesInFolder, subtype) {
let newName = name;
let suffix;
let i = 1;
while (
namesInFolder.find(
(item) =>
item.name === newName && item.key !== key && (!subtype || item.type === subtype)
)
) {
suffix = ' (' + i + ')';
// for customer key max is 100 chars
newName = name.slice(0, Math.max(0, 100 - suffix.length)) + suffix;
i++;
}
return newName;
}
/**
* Abstract create method that needs to be implemented in child metadata type
*
* @param {MetadataTypeItem} metadata single metadata entry
* @param {string} deployDir directory where deploy metadata are saved
* @returns {Promise.<object> | null} Promise of API response or null in case of an error
*/
static async create(metadata, deployDir) {
Util.logNotSupported(this.definition, 'create', metadata);
return;
}
/**
* Abstract update method that needs to be implemented in child metadata type
*
* @param {MetadataTypeItem} metadata single metadata entry
* @param {MetadataTypeItem} [metadataBefore] metadata mapped by their keyField
* @returns {Promise.<object> | null} Promise of API response or null in case of an error
*/
static async update(metadata, metadataBefore) {
Util.logNotSupported(this.definition, 'update', metadata);
return;
}
/**
* Abstract refresh method that needs to be implemented in child metadata type
*
* @param {string[]} [keyArr] metadata keys
* @param {boolean} [checkKey] whether to check if the key is valid
* @param {MetadataTypeMap} [upsertResults] metadata mapped by their keyField as returned by update/create; needs to be refreshed after publish
* @returns {Promise.<string[]>} Returns list of keys that were refreshed
*/
static async refresh(keyArr, checkKey = true, upsertResults) {
Util.logNotSupported(this.definition, 'refresh');
return [];
}
/**
*
* @param {string[]} keyArr limit retrieval to given metadata type
* @param {string} retrieveDir retrieve dir including cred and bu
* @param {Set.<string>} [findAssetKeys] list of keys that were found referenced via ContentBlockByX; if set, method only gets keys and runs no updates
* @returns {Promise.<Set.<string>>} found asset keys
*/
static async getCbReferenceKeys(keyArr, retrieveDir, findAssetKeys) {
if (!Object.prototype.hasOwnProperty.call(this, 'replaceCbReference')) {
// only types that have a replaceCbReference method actually have ampscript/ssjs
return;
}
// get all metadata of the current type; then filter by keys in selectedTypes
const metadataMap = Util.filterObjByKeys(
await this.getJsonFromFS(File.normalizePath([retrieveDir, this.definition.type])),
keyArr
);
await this.replaceCbReferenceLoop(metadataMap, retrieveDir, findAssetKeys);
return findAssetKeys;
}
/**
* this iterates over all items found in the retrieve folder and executes the type-specific method for replacing references
*
* @param {MetadataTypeMap} metadataMap list of metadata (keyField => metadata)
* @param {string} retrieveDir retrieve dir including cred and bu
* @param {Set.<string>} [findAssetKeys] list of keys that were found referenced via ContentBlockByX; if set, method only gets keys and runs no updates
* @returns {Promise.<string[]>} Returns list of keys for which references were replaced
*/
static async replaceCbReferenceLoop(metadataMap, retrieveDir, findAssetKeys) {
const keysForDeploy = [];
if (!metadataMap) {
// if a type was skipped e.g. because it shall only be looked at on the parent then we would expect metadataMap to be undefined
return keysForDeploy;
}
const fromDescription = Util.OPTIONS.referenceFrom
.map((from) => 'ContentBlockBy' + Util.capitalizeFirstLetter(from))
.join(' and ');
if (Object.keys(metadataMap).length) {
Util.logger.debug(` - Searching in ${this.definition.type} `);
const baseDir = [retrieveDir, ...this.definition.type.split('-')];
const deployMap = {};
for (const key in metadataMap) {
const item = metadataMap[key];
if (this.isFiltered(item, true) || this.isFiltered(item, false)) {
// we would not have saved these items to disk but they exist in the cache and hence need to be skipped here
continue;
}
try {
// add key but make sure to turn it into string or else numeric keys will be filtered later
deployMap[key] = await this.replaceCbReference(
item,
retrieveDir,
findAssetKeys
);
keysForDeploy.push(key + '');
if (!findAssetKeys) {
await this.saveToDisk(deployMap, key, baseDir);
Util.logger.info(
` - added ${this.definition.type} to update queue: ${key}`
);
}
} catch (ex) {
if (ex.code !== 200) {
// dont print error if we simply did not find relevant content blocks
Util.logger.errorStack(
ex,
'issue with ' + this.definition.type + ' ' + key
);
}
if (!findAssetKeys) {
Util.logger.info(
Util.getGrayMsg(
` ☇ skipping ${Util.getTypeKeyName(this.definition, item)}: no ${fromDescription} found`
)
);
}
}
}
if (!findAssetKeys) {
Util.logger.info(
`Found ${keysForDeploy.length} ${this.definition.type}${keysForDeploy.length === 1 ? '' : 's'} to update`
);
}
}
return keysForDeploy;
}
/**
* Abstract execute method that needs to be implemented in child metadata type
*
* @param {MetadataTypeItem} item single metadata item
* @param {string} [retrieveDir] directory where metadata is saved
* @param {Set.<string>} [findAssetKeys] list of keys that were found referenced via ContentBlockByX; if set, method only gets keys and runs no updates
* @returns {Promise.<MetadataTypeItem | CodeExtractItem>} key of the item that was updated
*/
static async replaceCbReference(item, retrieveDir, findAssetKeys) {
Util.logNotSupported(this.definition, 'replaceCbReference');
return [];
}
/**
* Abstract execute method that needs to be implemented in child metadata type
*
* @param {string[]} keyArr customerkey of the metadata
* @param {MetadataTypeMapObj} [cache] metadata cache used by refresh to avoid recaching
* @returns {Promise.<string[]>} Returns list of keys that were executed
*/
static async execute(keyArr, cache) {
Util.logNotSupported(this.definition, 'execute');
return [];
}
/**
* Abstract schedule method that needs to be implemented in child metadata type
*
* @param {string[]} keyArr customerkey of the metadata
* @param {MetadataTypeMapObj} [cache] metadata cache used by refresh to avoid recaching
* @returns {Promise.<string[]>} Returns list of keys that were executed
*/
static async schedule(keyArr, cache) {
Util.logNotSupported(this.definition, 'schedule');
return [];
}
/**
* Abstract pause method that needs to be implemented in child metadata type
*
* @param {string[]} keyArr customerkey of the metadata
* @param {MetadataTypeMapObj} [cache] metadata cache used by refresh to avoid recaching
* @returns {Promise.<string[]>} Returns list of keys that were paused
*/
static async pause(keyArr, cache) {
Util.logNotSupported(this.definition, 'pause');
return [];
}
/**
* Abstract stop method that needs to be implemented in child metadata type
*
* @param {string[]} keyArr customerkey of the metadata
* @returns {Promise.<string[]>} Returns list of keys that were stopped
*/
static async stop(keyArr) {
Util.logNotSupported(this.definition, 'stop');
return [];
}
/**
* test if metadata was actually changed or not to potentially skip it during deployment
*
* @param {MetadataTypeItem} cachedVersion cached version from the server
* @param {MetadataTypeItem} metadata item to upload
* @param {string} [fieldName] optional field name to use for identifying the record in logs
* @returns {boolean} true if metadata was changed
*/
static hasChanged(cachedVersion, metadata, fieldName) {
// should be set up type by type but the *_generic version is likely a good start for many types
return true;
}
/**
* test if metadata was actually changed or not to potentially skip it during deployment
*
* @param {MetadataTypeItem} cachedVersion cached version from the server
* @param {MetadataTypeItem} metadataItem item to upload
* @param {string} [fieldName] optional field name to use for identifying the record in logs
* @param {boolean} [silent] optionally suppress logging
* @returns {boolean} true on first identified deviation or false if none are found
*/
static hasChangedGeneric(cachedVersion, metadataItem, fieldName, silent) {
if (!cachedVersion) {
return true;
}
// we do need the full set in other places and hence need to work with a clone here
const clonedMetada = structuredClone(metadataItem);
// keep copy of identifier in case it is among the non-updateable fields
const identifier = clonedMetada[fieldName || this.definition.keyField];
this.removeNotUpdateableFields(clonedMetada);
// iterate over what we want to upload rather than what we cached to avoid false positives
for (const prop in clonedMetada) {
if (this.definition.ignoreFieldsForUpdateCheck?.includes(prop)) {
continue;
}
if (
clonedMetada[prop] === null ||
['string', 'number', 'boolean'].includes(typeof clonedMetada[prop])
) {
// check simple variables directly
// check should ignore types to bypass string/number auto-conversions caused by SFMC-SDK
if (clonedMetada[prop] != cachedVersion[prop]) {
Util.logger.debug(
`${this.definition.type}:: ${
identifier
}.${prop} changed: '${cachedVersion[prop]}' to '${clonedMetada[prop]}'`
);
return true;
}
} else if (deepEqual(clonedMetada[prop], cachedVersion[prop])) {
// test complex objects here
Util.logger.debug(
`${this.definition.type}:: ${
identifier
}.${prop} changed: '${cachedVersion[prop]}' to '${clonedMetada[prop]}'`
);
return true;
}
}
if (!silent) {
Util.logger.verbose(
` ☇ skipping ${this.definition.type} ${identifier} / ${
clonedMetada[this.definition.nameField] || ''
}: no change detected`
);
}
return false;
}
/**
* MetadataType upsert, after retrieving from target and comparing to check if create or update operation is needed.
*
* @param {MetadataTypeMap} metadataMap metadata mapped by their keyField
* @param {string} deployDir directory where deploy metadata are saved
* @param {boolean} [runUpsertSequentially] when a type has self-dependencies creates need to run one at a time and created/changed keys need to be cached to ensure following creates/updates have thoses keys available
* @returns {Promise.<MetadataTypeMap>} keyField => metadata map
*/
static async upsert(metadataMap, deployDir, runUpsertSequentially = false) {
const orignalMetadataMap = structuredClone(metadataMap);
const metadataToUpdate = [];
const metadataToCreate = [];
let createResults = [];
let updateResults = [];
let filteredByPreDeploy = 0;
for (const metadataKey in metadataMap) {
let hasError = false;
try {
// preDeployTasks parsing
let deployableMetadata;
try {
metadataMap[metadataKey] = await this.validation(
'deploy',
metadataMap[metadataKey],
deployDir
);
if (metadataMap[metadataKey]) {
// only run unless we encountered a situation in our validation that made us want to filter this record
deployableMetadata = await this.preDeployTasks(
metadataMap[metadataKey],
deployDir
);
}
} catch (ex) {
// do this in case something went wrong during pre-deploy steps to ensure the total counter is correct
hasError = true;
deployableMetadata = metadataMap[metadataKey];
if (deployableMetadata) {
// * include ": ${ex.message}" in the error if this is ever turned back into Util.logger.error()
Util.logger.errorStack(
ex,
` ☇ skipping ${this.definition.type} ${
deployableMetadata[this.definition.keyField]
} / ${deployableMetadata[this.definition.nameField]}`
);
} else {
Util.logger.errorStack(
ex,
` ☇ skipping ${this.definition.type} ${metadataKey}`
);
}
}
// if preDeploy returns nothing then it cannot be deployed so skip deployment
if (deployableMetadata) {
metadataMap[metadataKey] = deployableMetadata;
// create normalizedKey off of whats in the json rather than from "metadataKey" because preDeployTasks might have altered something (type asset)
const action = await this.createOrUpdate(
metadataMap,
metadataKey,
hasError,
metadataToUpdate,
metadataToCreate
);
if (runUpsertSequentially) {
if (action === 'create') {
// handle creates sequentially here becasue we might have interdepencies
const result = await this.create(metadataMap[metadataKey], deployDir);
if (result) {
createResults.push(result);
if (result[this.definition.idField]) {
// ensure we have the ID in the cache
metadataMap[metadataKey][this.definition.idField] =
result[this.definition.idField];
}
// make this newly created item available in cache for other itmes that might reference it
/** @type {MetadataTypeMap} */
const newObject = {};
newObject[metadataKey] = metadataMap[metadataKey];
cache.mergeMetadata(this.definition.type, newObject);
}
} else if (action === 'update' && !Util.OPTIONS.noUpdate) {
const metadataEntry = metadataToUpdate.find(
(el) =>
el !== null &&
el.after[this.definition.keyField] ===
metadataMap[metadataKey][this.definition.keyField]
);
if (!metadataEntry) {
Util.logger.error(
` - ${this.definition.type} ${metadataKey} / ${metadataMap[metadataKey][this.definition.keyField]} not found in update list`
);
continue;
}
// handle updates sequentially here becasue we might have interdepencies
// this is especially important when we use features like --matchName which are updates but change the key
const result = await this.update(
metadataEntry.after,
metadataEntry.before
);
if (result) {
updateResults.push(result);
// make this newly created item available in cache for other itmes that might reference it
const newObject = {};
newObject[metadataKey] = structuredClone(metadataMap[metadataKey]);
if (result.objectID) {
// required for assets
newObject[metadataKey].objectID = result.objectID;
}
cache.mergeMetadata(this.definition.type, newObject);
}
}
}
} else {
filteredByPreDeploy++;
}
} catch (ex) {
if (
metadataMap[metadataKey] &&
metadataMap[metadataKey][this.definition.nameField]
) {
Util.logger.error(
` ☇ skipping ${this.definition.type} ${metadataKey} / ${metadataMap[metadataKey][this.definition.nameField]}: ${ex.message}`
);
} else {
Util.logger.errorStack(ex, `Upserting ${this.definition.type} failed`);
}
}
}
if (!runUpsertSequentially) {
// create
const createLimit = pLimit(10);
createResults = (
await Promise.all(
metadataToCreate
.filter((r) => r !== undefined && r !== null)
.map((metadataEntry) =>
createLimit(() => this.create(metadataEntry, deployDir))
)
)
).filter((r) => r !== undefined && r !== null);
// update
if (Util.OPTIONS.noUpdate && metadataToUpdate.length) {
Util.logger.info(
` ☇ skipping update of ${metadataToUpdate.length} ${this.definition.type}${metadataToUpdate.length == 1 ? '' : 's'}: --noUpdate flag is set`
);
} else if (metadataToUpdate.length) {
const updateLimit = pLimit(10);
updateResults = (
await Promise.all(
metadataToUpdate
.filter((r) => r !== undefined && r !== null)
.map((metadataEntry) =>
updateLimit(() =>
this.update(metadataEntry.after, metadataEntry.before)
)
)
)
).filter((r) => r !== undefined && r !== null);
}
}
// Logging
Util.logger.info(
`${this.definition.type} upsert: ${createResults.length} of ${metadataToCreate.length} created / ${updateResults.length} of ${metadataToUpdate.length} updated` +
(filteredByPreDeploy > 0 ? ` / ${filteredByPreDeploy} filtered` : '')
);
let upsertResults;
if (this.definition.bodyIteratorField === 'Results') {
// if Results then parse as SOAP
// put in Retrieve Format for parsing
// todo add handling when response does not contain items.
const metadataResults = createResults
.concat(updateResults)
// TODO remove Object.keys check after create/update SOAP methods stop returning empty objects instead of null
.filter((r) => r !== undefined && r !== null && Object.keys(r).length !== 0)
.flatMap((r) => r.Results)
.map((r) => r.Object);
upsertResults = this.parseResponseBody({ Results: metadataResults });
} else {
// likely comming from one of the many REST APIs
// put in Retrieve Format for parsing
// todo add handling when response does not contain items.
const metadataResults = createResults.concat(updateResults).filter(Boolean);
upsertResults = this.parseResponseBody(metadataResults);
}
await this.postDeployTasks(upsertResults, orignalMetadataMap, {
created: createResults.length,
updated: updateResults.length,
});
return upsertResults;
}
/**
* helper for {@link MetadataType.upsert}
*
* @param {MetadataTypeMap} metadataMap list of metadata
* @param {string} metadataKey key of item we are looking at
* @param {boolean} hasError error flag from previous code
* @param {MetadataTypeItemDiff[]} metadataToUpdate list of items to update
* @param {MetadataTypeItem[]} metadataToCreate list of items to create
* @returns {Promise.<'create' | 'update' | 'skip'>} action to take
*/
static async createOrUpdate(
metadataMap,
metadataKey,
hasError,
metadataToUpdate,
metadataToCreate
) {
let normalizedKey = File.reverseFilterIllegalFilenames(
metadataMap[metadataKey][this.definition.keyField]
);
// Update if it already exists; Create it if not
const maxKeyLength = this.definition.maxKeyLength || 36;
// make sure keySuffix always has a string value
Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : '';
if (Util.OPTIONS.keySuffix && !cache.getByKey(this.definition.type, normalizedKey)) {
// to ensure we go into update mode like we would with appending the MID to asset-keys, recreated the normalized key here
const newKey = this.getNewKey(
this.definition.keyField,
metadataMap[metadataKey],
maxKeyLength
);
normalizedKey = File.reverseFilterIllegalFilenames(newKey);
}
const cacheMatchedByKey = cache.getByKey(this.definition.type, normalizedKey);
const cacheMatchedByName = cacheMatchedByKey
? null
: this.getCacheMatchedByName(metadataMap[metadataKey]);
if (
Util.logger.level === 'debug' &&
metadataMap[metadataKey][this.definition.idField] &&
this.definition.idField !== this.definition.keyField
) {
// TODO: re-evaluate in future releases if & when we managed to solve folder dependencies once and for all
// only used if resource is excluded from cache and we still want to update it
// needed e.g. to rewire lost folders
Util.logger.warn(
' - Hotfix for non-cachable resource found in deploy folder. Trying update:'
);
Util.logger.warn(JSON.stringify(metadataMap[metadataKey]));
if (hasError) {
metadataToUpdate.push(null);
return 'skip';
} else {
metadataToUpdate.push({
before: {},
after: metadataMap[metadataKey],
});
return 'update';
}
} else if (cacheMatchedByKey || cacheMatchedByName) {
// normal way of processing update files
const cachedVersion = cacheMatchedByKey || cacheMatchedByName;
if (!cacheMatchedByKey && cacheMatchedByName) {
Util.matchedByName[this.definition.type] ||= {};
Util.matchedByName[this.definition.type][metadataKey] =
cacheMatchedByName[this.definition.keyField];
}
if (!this.hasChanged(cachedVersion, metadataMap[metadataKey])) {
hasError = true;
}
if (Util.OPTIONS.changeKeyField) {
if (this.definition.keyField == this.definition.idField) {
Util.logger.error(
` - --changeKeyField cannot be used for types that use their ID as key. Skipping change.`
);
hasError = true;
} else if (this.definition.keyIsFixed) {
Util.logger.error(
` - type ${this.definition.type} does not support --changeKeyField and --changeKeyValue. Skipping change.`
);
hasError = true;
} else if (!metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
Util.logger.error(
` - --changeKeyField is set to ${Util.OPTIONS.changeKeyField} but no value was found in the metadata. Skipping change.`
);
hasError = true;
} else if (Util.OPTIONS.changeKeyField === this.definition.keyField) {
// simple issue, used WARN to avoid signaling a problem to ci/cd and don't fail deploy
Util.logger.warn(
` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.`
);
} else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
const newKey = this.getNewKey(
Util.OPTIONS.changeKeyField,
metadataMap[metadataKey],
maxKeyLength
);
if (
metadataMap[metadataKey][Util.OPTIONS.changeKeyField] +
'' +
Util.OPTIONS.keySuffix >
maxKeyLength
) {
Util.logger.warn(
`${this.definition.type} ${this.definition.keyField} may not exceed ${maxKeyLength} characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}`
);
}
if (metadataKey == newKey) {
Util.log