mcdev
Version:
Accenture Salesforce Marketing Cloud DevTools
1,272 lines (1,189 loc) • 94.8 kB
JavaScript
'use strict';
import { Util } from './util/util.js';
import auth from './util/auth.js';
import File from './util/file.js';
import config from './util/config.js';
import Init from './util/init.js';
import InitGit from './util/init.git.js';
import Cli from './util/cli.js';
import DevOps from './util/devops.js';
import BuHelper from './util/businessUnit.js';
import Builder from './Builder.js';
import Deployer from './Deployer.js';
import MetadataTypeInfo from './MetadataTypeInfo.js';
import MetadataTypeDefinitions from './MetadataTypeDefinitions.js';
import Retriever from './Retriever.js';
import cache from './util/cache.js';
import ReplaceContentBlockReference from './util/replaceContentBlockReference.js';
import pLimit from 'p-limit';
import path from 'node:path';
import { confirm } from '@inquirer/prompts';
/**
* @typedef {import('../types/mcdev.d.js').BuObject} BuObject
* @typedef {import('../types/mcdev.d.js').CodeExtract} CodeExtract
* @typedef {import('../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
* @typedef {import('../types/mcdev.d.js').DeltaPkgItem} DeltaPkgItem
* @typedef {import('../types/mcdev.d.js').Mcdevrc} Mcdevrc
* @typedef {import('../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
* @typedef {import('../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
* @typedef {import('../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
* @typedef {import('../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
* @typedef {import('../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
* @typedef {import('../types/mcdev.d.js').MultiMetadataTypeList} MultiMetadataTypeList
* @typedef {import('../types/mcdev.d.js').MultiMetadataTypeMap} MultiMetadataTypeMap
* @typedef {import('../types/mcdev.d.js').SkipInteraction} SkipInteraction
* @typedef {import('../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
* @typedef {import('../types/mcdev.d.js').TemplateMap} TemplateMap
* @typedef {import('../types/mcdev.d.js').TypeKeyCombo} TypeKeyCombo
* @typedef {import('../types/mcdev.d.js').ExplainType} ExplainType
* @typedef {import('../types/mcdev.d.js').ContentBlockConversionTypes} ContentBlockConversionTypes
*/
/**
* main class
*/
class Mcdev {
/**
* @returns {string} current version of mcdev
*/
static version() {
console.log('mcdev v' + Util.packageJsonMcdev.version); // eslint-disable-line no-console
return Util.packageJsonMcdev.version;
}
/**
* helper method to use unattended mode when including mcdev as a package
*
* @param {SkipInteraction} [skipInteraction] signals what to insert automatically for things usually asked via wizard
* @returns {void}
*/
static setSkipInteraction(skipInteraction) {
Util.skipInteraction = skipInteraction;
}
/**
* configures what is displayed in the console
*
* @param {object} argv list of command line parameters given by user
* @param {boolean} [argv.silent] only errors printed to CLI
* @param {boolean} [argv.verbose] chatty user CLI output
* @param {boolean} [argv.debug] enables developer output & features
* @returns {void}
*/
static setLoggingLevel(argv) {
Util.setLoggingLevel(argv);
}
static knownOptions = [
'_runningTest',
'_welcomeMessageShown',
'api',
'autoMidSuffix',
'changeKeyField',
'changeKeyValue',
'commitHistory',
'dependencies',
'errorLog',
'execute',
'filter',
'fix',
'fixShared',
'format',
'fromRetrieve',
'ignoreFolder',
'ignoreSfFields',
'json',
'keySuffix',
'like',
'matchName',
'noLogColors',
'noLogFile',
'noUpdate',
'publish',
'purge',
'range',
'referenceFrom',
'referenceTo',
'refresh',
'retrieve',
'schedule',
'skipDeploy',
'skipInteraction',
'skipRetrieve',
'skipStatusCheck',
'skipValidation',
'validate',
];
/**
* allows setting system wide / command related options
*
* @param {object} argv list of command line parameters given by user
* @returns {void}
*/
static setOptions(argv) {
for (const option of this.knownOptions) {
if (argv[option] !== undefined) {
Util.OPTIONS[option] = argv[option];
}
}
// set logging level
const loggingOptions = ['silent', 'verbose', 'debug'];
for (const option of loggingOptions) {
if (argv[option] !== undefined) {
this.setLoggingLevel(argv);
break;
}
}
// set skip interaction
if (argv.skipInteraction !== undefined) {
this.setSkipInteraction(argv.skipInteraction);
}
}
/**
* handler for 'mcdev createDeltaPkg
*
* @param {object} argv yargs parameters
* @param {string} [argv.commitrange] git commit range via positional
* @param {string} [argv.range] git commit range via option
* @param {string} [argv.filter] filter file paths that start with any
* @param {number} [argv.commitHistory] filter file paths that start with any
* @param {DeltaPkgItem[]} [argv.diffArr] list of files to include in delta package (skips git diff when provided)
* @returns {Promise.<DeltaPkgItem[]>} list of changed items
*/
static async createDeltaPkg(argv) {
Util.startLogger();
Util.logger.info('Create Delta Package ::');
const properties = await config.getProperties();
if (!properties) {
return;
}
if (argv.commitrange) {
Util.logger.warn(
`Depecation Notice: Please start using --range to define the commit range or target branch. The positional argument will be removed in the next major release.`
);
}
const range = argv.commitrange || Util.OPTIONS.range;
try {
return await (argv.filter
? // get source market and source BU from config
DevOps.getDeltaList(properties, range, true, argv.filter, argv.commitHistory)
: // If no custom filter was provided, use deployment marketLists & templating
DevOps.buildDeltaDefinitions(
properties,
range,
argv.diffArr,
argv.commitHistory
));
} catch (ex) {
Util.logger.error(ex.message);
}
}
/**
* @returns {Promise} .
*/
static async selectTypes() {
Util.startLogger();
const properties = await config.getProperties();
if (!properties) {
return;
}
await Cli.selectTypes(properties);
}
/**
* @returns {ExplainType[]} list of supported types with their apiNames
*/
static explainTypes() {
return Cli.explainTypes();
}
/**
* @returns {Promise.<boolean>} success flag
*/
static async upgrade() {
Util.startLogger();
const properties = await config.getProperties(false, true);
if (!properties) {
return;
}
if ((await InitGit.initGitRepo()).status === 'error') {
return false;
}
return Init.upgradeProject(properties, false);
}
/**
* helper to show an off-the-logs message to users
*/
static #welcomeMessage() {
if (Util.OPTIONS._welcomeMessageShown) {
// ensure we don't spam the user in case methods are called multiple times
return;
}
Util.OPTIONS._welcomeMessageShown = true;
const color = Util.color;
/* eslint-disable no-console */
if (process.env['USERDNSDOMAIN'] === 'DIR.SVC.ACCENTURE.COM') {
// Accenture internal message
console.log(
`\n` +
` Thank you for using Accenture SFMC DevTools on your Accenture laptop!\n` +
` We are trying to understand who is using mcdev across the globe and would therefore appreciate it if you left a message\n` +
` in our Accenture Teams channel ${color.bgWhite}telling us about your journey with mcdev${color.reset}: ${color.fgBlue}https://go.accenture.com/mcdevTeams${color.reset}.\n` +
`\n` +
` For any questions or concerns, please feel free to create a ticket in GitHub: ${color.fgBlue}https://bit.ly/mcdev-support${color.reset}.\n`
);
} else {
// external message
console.log(
`\n` +
` Thank you for using Accenture SFMC DevTools!\n` +
`\n` +
` For any questions or concerns, please feel free to create a ticket in GitHub: ${color.fgBlue}https://bit.ly/mcdev-support${color.reset}.\n`
);
}
/* eslint-enable no-console */
}
/**
* Retrieve all metadata from the specified business unit into the local file system.
*
* @param {string} businessUnit references credentials from properties.json
* @param {string[] | TypeKeyCombo} [selectedTypesArr] limit retrieval to given metadata type
* @param {string[]} [keys] limit retrieval to given metadata key
* @param {boolean} [changelogOnly] skip saving, only create json in memory
* @returns {Promise.<object>} -
*/
static async retrieve(businessUnit, selectedTypesArr, keys, changelogOnly) {
this.#welcomeMessage();
console.time('Time'); // eslint-disable-line no-console
Util.startLogger();
Util.logger.info('mcdev:: Retrieve');
const properties = await config.getProperties();
if (!properties) {
return;
}
// assume a list was passed in and check each entry's validity
if (selectedTypesArr) {
for (const selectedType of Array.isArray(selectedTypesArr)
? selectedTypesArr
: Object.keys(selectedTypesArr)) {
if (!Util._isValidType(selectedType)) {
return;
}
}
}
const resultsObj = {};
if (businessUnit === '*') {
Util.logger.info(':: Retrieving all BUs for all credentials');
let counter_credTotal = 0;
for (const cred in properties.credentials) {
Util.logger.info(`:: Retrieving all BUs for ${cred}`);
let counter_credBu = 0;
for (const bu in properties.credentials[cred].businessUnits) {
resultsObj[`${cred}/${bu}`] = await this.#retrieveBU(
cred,
bu,
selectedTypesArr,
keys
);
counter_credBu++;
Util.startLogger(true);
}
counter_credTotal += counter_credBu;
Util.logger.info(`:: ${counter_credBu} BUs of ${cred}\n`);
}
const credentialCount = Object.keys(properties.credentials).length;
Util.logger.info(
`:: Done for ${counter_credTotal} BUs of ${credentialCount} credential${
credentialCount === 1 ? '' : 's'
} in total\n`
);
} else {
let [cred, bu] = businessUnit ? businessUnit.split('/') : [null, null];
// to allow all-BU via user selection we need to run this here already
if (
properties.credentials &&
(!properties.credentials[cred] ||
(bu !== '*' && !properties.credentials[cred].businessUnits[bu]))
) {
const buObject = await Cli.getCredentialObject(
properties,
cred === null ? null : cred + '/' + bu,
null,
true
);
if (buObject === null) {
return;
} else {
cred = buObject.credential;
bu = buObject.businessUnit;
}
}
if (bu === '*' && properties.credentials && properties.credentials[cred]) {
Util.logger.info(`:: Retrieving all BUs for ${cred}`);
let counter_credBu = 0;
for (const bu in properties.credentials[cred].businessUnits) {
resultsObj[`${cred}/${bu}`] = await this.#retrieveBU(
cred,
bu,
selectedTypesArr,
keys
);
counter_credBu++;
Util.startLogger(true);
}
Util.logger.info(`:: Done for ${counter_credBu} BUs of ${cred}\n`);
} else {
// retrieve a single BU; return
const retrieveChangelog = await this.#retrieveBU(
cred,
bu,
selectedTypesArr,
keys,
changelogOnly
);
if (changelogOnly) {
console.timeEnd('Time'); // eslint-disable-line no-console
return retrieveChangelog;
} else {
resultsObj[`${cred}/${bu}`] = retrieveChangelog;
}
Util.logger.info(`:: Done\n`);
}
}
// merge all results into one object
for (const credBu in resultsObj) {
for (const type in resultsObj[credBu]) {
const base = resultsObj[credBu][type][0];
for (let i = 1; i < resultsObj[credBu][type].length; i++) {
// merge all items into the first array
Object.assign(base, resultsObj[credBu][type][i]);
}
resultsObj[credBu][type] = resultsObj[credBu][type][0];
}
}
console.timeEnd('Time'); // eslint-disable-line no-console
return resultsObj;
}
/**
* helper for {@link Mcdev.retrieve}
*
* @param {string} cred name of Credential
* @param {string} bu name of BU
* @param {string[] | TypeKeyCombo} [selectedTypesArr] limit retrieval to given metadata type/subtype
* @param {string[]} [keys] limit retrieval to given metadata key
* @param {boolean} [changelogOnly] skip saving, only create json in memory
* @returns {Promise.<object>} ensure that BUs are worked on sequentially
*/
static async #retrieveBU(cred, bu, selectedTypesArr, keys, changelogOnly) {
// ensure changes to the selectedTypesArr on one BU do not affect other BUs called in the same go
selectedTypesArr = structuredClone(selectedTypesArr);
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(
properties,
cred === null ? null : cred + '/' + bu,
null,
true
);
if (buObject !== null) {
cache.initCache(buObject);
cred = buObject.credential;
bu = buObject.businessUnit;
Util.logger.info('');
Util.logger.info(`:: Retrieving ${cred}/${bu}`);
const retrieveTypesArr = [];
if (selectedTypesArr) {
for (const selectedType of Array.isArray(selectedTypesArr)
? selectedTypesArr
: Object.keys(selectedTypesArr)) {
const { type, subType } = Util.getTypeAndSubType(selectedType);
const removePathArr = [properties.directories.retrieve, cred, bu, type];
if (
type &&
subType &&
MetadataTypeInfo[type] &&
MetadataTypeDefinitions[type].subTypes.includes(subType)
) {
// Clear output folder structure for selected sub-type
removePathArr.push(subType);
retrieveTypesArr.push(selectedType);
} else if (type && MetadataTypeInfo[type]) {
// Clear output folder structure for selected type
retrieveTypesArr.push(type);
}
const areKeySet = Array.isArray(selectedTypesArr)
? !!keys
: selectedTypesArr[selectedType] !== null;
if (!areKeySet) {
// dont delete directories if we are just re-retrieving a single file
await File.remove(File.normalizePath(removePathArr));
}
}
}
if (!retrieveTypesArr.length) {
// assume no type was given and config settings are used instead:
// Clear output folder structure
await File.remove(File.normalizePath([properties.directories.retrieve, cred, bu]));
// removes subtypes and removes duplicates
retrieveTypesArr.push(
...new Set(properties.metaDataTypes.retrieve.map((type) => type.split('-')[0]))
);
for (const selectedType of retrieveTypesArr) {
const test = Util._isValidType(selectedType);
if (!test) {
Util.logger.error(
`Please remove the type ${selectedType} from your ${Util.configFileName}`
);
return;
}
}
}
const retriever = new Retriever(properties, buObject);
try {
// await is required or the calls end up conflicting
const retrieveChangelog = await retriever.retrieve(
retrieveTypesArr,
Array.isArray(selectedTypesArr) ? keys : selectedTypesArr,
null,
changelogOnly
);
return retrieveChangelog;
} catch (ex) {
Util.logger.errorStack(ex, 'mcdev.retrieve failed');
}
}
}
/**
* Deploys all metadata located in the 'deploy' directory to the specified business unit
*
* @param {string} businessUnit references credentials from properties.json
* @param {string[] | TypeKeyCombo} [selectedTypesArr] limit deployment to given metadata type
* @param {string[]} [keyArr] limit deployment to given metadata keys
* @returns {Promise.<Object.<string, MultiMetadataTypeMap>>} deployed metadata per BU (first key: bu name, second key: metadata type)
*/
static async deploy(businessUnit, selectedTypesArr, keyArr) {
this.#welcomeMessage();
console.time('Time'); // eslint-disable-line no-console
Util.startLogger();
const deployResult = await Deployer.deploy(businessUnit, selectedTypesArr, keyArr);
console.timeEnd('Time'); // eslint-disable-line no-console
return deployResult;
}
/**
* Creates template file for properties.json
*
* @param {string} [credentialsName] identifying name of the installed package / project
* @returns {Promise.<void>} -
*/
static async initProject(credentialsName) {
Util.startLogger();
Util.logger.info('mcdev:: Setting up project');
const properties = await config.getProperties(!!credentialsName, true);
try {
await Init.initProject(properties, credentialsName);
} catch (ex) {
Util.logger.error(ex.message);
}
}
/**
* Clones an existing project from git repository and installs it
*
* @returns {Promise.<void>} -
*/
static async joinProject() {
Util.startLogger();
Util.logger.info('mcdev:: Joining an existing project');
try {
await Init.joinProject();
} catch (ex) {
Util.logger.error(ex.message);
}
}
/**
* Refreshes BU names and ID's from MC instance
*
* @param {string} credentialsName identifying name of the installed package / project
* @returns {Promise.<void>} -
*/
static async findBUs(credentialsName) {
Util.startLogger();
Util.logger.info('mcdev:: Load BUs');
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, credentialsName, true);
if (buObject !== null) {
BuHelper.refreshBUProperties(properties, buObject.credential);
}
}
/**
* Creates docs for supported metadata types in Markdown and/or HTML format
*
* @param {string} businessUnit references credentials from properties.json
* @param {string} type metadata type
* @returns {Promise.<void>} -
*/
static async document(businessUnit, type) {
Util.startLogger();
Util.logger.info('mcdev:: Document');
const properties = await config.getProperties();
if (!properties) {
return;
}
if (type && !MetadataTypeInfo[type]) {
Util.logger.error(`:: '${type}' is not a valid metadata type`);
return;
}
try {
const parentBUOnlyTypes = ['user', 'role'];
const buObject = await Cli.getCredentialObject(
properties,
parentBUOnlyTypes.includes(type) ? businessUnit.split('/')[0] : businessUnit,
parentBUOnlyTypes.includes(type) ? Util.parentBuName : null
);
if (buObject !== null) {
MetadataTypeInfo[type].properties = properties;
MetadataTypeInfo[type].buObject = buObject;
MetadataTypeInfo[type].document();
}
} catch (ex) {
Util.logger.error('mcdev.document ' + ex.message);
Util.logger.debug(ex.stack);
Util.logger.info(
'If the directoy does not exist, you may need to retrieve this BU first.'
);
}
}
/**
* deletes metadata from MC instance by key
*
* @param {string} businessUnit references credentials from properties.json
* @param {string | TypeKeyCombo} selectedTypes supported metadata type (single) or complex object
* @param {string[] | string} [keys] Identifier of metadata
* @returns {Promise.<boolean>} true if successful, false otherwise
*/
static async deleteByKey(businessUnit, selectedTypes, keys) {
Util.startLogger();
Util.logger.info('mcdev:: delete');
/** @typedef {string[]} */
let selectedTypesArr;
/** @typedef {TypeKeyCombo} */
let selectedTypesObj;
let keyArr;
keyArr = 'string' === typeof keys ? [keys] : keys;
if ('string' === typeof selectedTypes) {
selectedTypesArr = [selectedTypes];
} else {
selectedTypesObj = selectedTypes;
// reset keys array because it will be overriden by values from selectedTypesObj
keyArr = null;
}
// check if types are valid
for (const selectedType of selectedTypesArr || Object.keys(selectedTypesObj)) {
if (!Util._isValidType(selectedType)) {
return;
}
}
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (!buObject) {
return;
}
let client;
try {
client = auth.getSDK(buObject);
} catch (ex) {
Util.logger.error(ex.message);
return;
}
let status = true;
for (const type of selectedTypesArr || Object.keys(selectedTypesObj)) {
keyArr = selectedTypesArr ? keyArr : selectedTypesObj[type];
if (!keyArr) {
Util.logger.error(`No keys set for ${type}`);
return;
}
MetadataTypeInfo[type].client = client;
MetadataTypeInfo[type].properties = properties;
MetadataTypeInfo[type].buObject = buObject;
await MetadataTypeInfo[type].preDeleteTasks(keyArr);
const deleteLimit = pLimit(
MetadataTypeInfo[type].definition.deleteSynchronously ? 1 : 20
);
await Promise.allSettled(
keyArr.map((key) =>
deleteLimit(async () => {
try {
const result = await MetadataTypeInfo[type].deleteByKey(key);
status &&= result;
} catch (ex) {
Util.logger.errorStack(
ex,
` - Deleting ${type} ${key} on BU ${businessUnit} failed`
);
status = false;
}
return status;
})
)
);
}
return status;
}
/**
* get name & key for provided id
*
* @param {string} businessUnit references credentials from properties.json
* @param {string} type supported metadata type
* @param {string} id Identifier of metadata
* @returns {Promise.<{key:string, name:string, path:string}>} key, name and path of metadata; null if not found
*/
static async resolveId(businessUnit, type, id) {
Util.startLogger();
if (!Util.OPTIONS.json) {
Util.logger.info('mcdev:: resolveId');
}
if (!Util._isValidType(type)) {
return;
}
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (buObject !== null) {
try {
MetadataTypeInfo[type].client = auth.getSDK(buObject);
} catch (ex) {
Util.logger.error(ex.message);
return;
}
if (!Util.OPTIONS.json) {
Util.logger.info(
Util.getGrayMsg(` - Searching ${type} with id ${id} on BU ${businessUnit}`)
);
}
try {
MetadataTypeInfo[type].properties = properties;
MetadataTypeInfo[type].buObject = buObject;
return await MetadataTypeInfo[type].resolveId(id);
} catch (ex) {
Util.logger.errorStack(ex, ` - Could not resolve ID of ${type} ${id}`);
}
}
}
/**
* ensures triggered sends are restarted to ensure they pick up on changes of the underlying emails
*
* @param {string} businessUnit references credentials from properties.json
* @param {string[] | TypeKeyCombo} selectedTypes limit to given metadata types
* @param {string[]} [keys] customerkey of the metadata
* @returns {Promise.<Object.<string, Object.<string, string[]>>>} key: business unit name, key2: type, value: list of affected item keys
*/
static async refresh(businessUnit, selectedTypes, keys) {
return this.#runMethod('refresh', businessUnit, selectedTypes, keys);
}
/**
* method for contributors to get details on SOAP objects
*
* @param {string} type references credentials from properties.json
* @param {string} [businessUnit] defaults to first credential's ParentBU
* @returns {Promise.<void>} -
*/
static async describeSoap(type, businessUnit) {
Util.startLogger();
Util.logger.info('mcdev:: describe SOAP');
const properties = await config.getProperties();
if (!properties) {
return;
}
const credential = Object.keys(properties.credentials)[0];
businessUnit ||=
credential + '/' + Object.keys(properties.credentials[credential].businessUnits)[0];
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (!buObject) {
return;
}
try {
const client = auth.getSDK(buObject);
const response = await client.soap.describe(type);
if (response?.ObjectDefinition?.Properties) {
Util.logger.info(
`Properties for SOAP object ${response.ObjectDefinition.ObjectType}:`
);
const properties = response.ObjectDefinition.Properties.map((prop) => {
delete prop.PartnerKey;
delete prop.ObjectID;
return prop;
});
if (Util.OPTIONS.json) {
console.log(JSON.stringify(properties, null, 2)); // eslint-disable-line no-console
} else {
console.table(properties); // eslint-disable-line no-console
}
return properties;
} else {
throw new Error(
`Soap object ${type} not found. Please check the spelling and retry`
);
}
} catch (ex) {
Util.logger.error(ex.message);
}
}
/**
* Converts metadata to legacy format. Output is saved in 'converted' directory
*
* @param {string} businessUnit references credentials from properties.json
* @returns {Promise.<void>} -
*/
static async badKeys(businessUnit) {
Util.startLogger();
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (buObject !== null) {
Util.logger.info('Gathering list of Name<>External Key mismatches (bad keys)');
const retrieveDir = File.filterIllegalPathChars(
File.normalizePath([
properties.directories.retrieve,
buObject.credential,
buObject.businessUnit,
])
);
const docPath = File.normalizePath([
properties.directories.docs,
'badKeys',
buObject.credential,
]);
const filename = File.normalizePath([
docPath,
File.filterIllegalFilenames(buObject.businessUnit) + '.badKeys.md',
]);
await File.ensureDir(docPath);
if (await File.pathExists(filename)) {
await File.remove(filename);
}
const regex = new RegExp(String.raw`(\w+-){4}\w+`);
await File.ensureDir(retrieveDir);
const metadata = await Deployer.readBUMetadata(retrieveDir, null, true);
let output = '# List of Metadata with Name-Key mismatches\n';
for (const metadataType in metadata) {
let listEntries = '';
for (const entry in metadata[metadataType]) {
const metadataEntry = metadata[metadataType][entry];
if (regex.test(entry)) {
if (metadataType === 'query' && metadataEntry.Status === 'Inactive') {
continue;
}
listEntries +=
'- ' +
entry +
(metadataEntry.name || metadataEntry.Name
? ' => ' + (metadataEntry.name || metadataEntry.Name)
: '') +
'\n';
}
}
if (listEntries !== '') {
output += '\n## ' + metadataType + '\n\n' + listEntries;
}
}
await File.writeToFile(
docPath,
File.filterIllegalFilenames(buObject.businessUnit) + '.badKeys',
'md',
output
);
Util.logger.info('Bad keys documented in ' + filename);
}
}
/**
* Retrieve a specific metadata file and templatise.
*
* @deprecated Use `retrieve` followed by `build` instead. `retrieveAsTemplate` will be removed in a future version.
* @param {string} businessUnit references credentials from properties.json
* @param {string} selectedType supported metadata type
* @param {string[]} name name of the metadata
* @param {string} market market which should be used to revert template
* @returns {Promise.<MultiMetadataTypeList>} -
*/
static async retrieveAsTemplate(businessUnit, selectedType, name, market) {
Util.startLogger();
Util.logDeprecated('retrieveAsTemplate', `'retrieve' followed by 'build'`);
const properties = await config.getProperties();
if (!properties) {
return;
}
if (!Util._isValidType(selectedType)) {
return;
}
const { type, subType } = Util.getTypeAndSubType(selectedType);
let retrieveTypesArr;
if (
type &&
subType &&
MetadataTypeInfo[type] &&
MetadataTypeDefinitions[type].subTypes.includes(subType)
) {
retrieveTypesArr = [selectedType];
} else if (type && MetadataTypeInfo[type]) {
retrieveTypesArr = [type];
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (buObject !== null) {
cache.initCache(buObject);
const retriever = new Retriever(properties, buObject);
if (Util.checkMarket(market, properties)) {
return retriever.retrieve(retrieveTypesArr, name, properties.markets[market]);
}
}
}
/**
* @param {string} businessUnit references credentials from properties.json
* @param {TypeKeyCombo} typeKeyList limit retrieval to given metadata type
* @returns {Promise.<TypeKeyCombo>} selected types including dependencies
*/
static async addDependentCbReferences(businessUnit, typeKeyList) {
if (!Util.OPTIONS.dependencies) {
return;
}
const initialAssetNumber = typeKeyList['asset']?.length || 0;
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
Util.logger.info(
'Searching for additional dependencies that were linked via ContentBlockByKey, ContentBlockByName and ContentBlockById'
);
await ReplaceContentBlockReference.createCache(properties, buObject, true);
// because we re-use the replaceReference logic here we need to manually set this value
/** @type {ContentBlockConversionTypes[]} */
Util.OPTIONS.referenceFrom = ['key', 'name', 'id'];
/** @type {ContentBlockConversionTypes} */
Util.OPTIONS.referenceTo = 'key';
/** @type {Set.<string>} */
const assetDependencies = new Set();
const retrieveDir = File.filterIllegalPathChars(
File.normalizePath([
properties.directories.retrieve,
buObject.credential,
buObject.businessUnit,
])
);
// check all non-asset types for dependencies
for (const depType in typeKeyList) {
if (
!Object.prototype.hasOwnProperty.call(
MetadataTypeInfo[depType],
'replaceCbReference'
) ||
depType === 'asset'
) {
continue;
}
MetadataTypeInfo[depType].properties = properties;
MetadataTypeInfo[depType].buObject = buObject;
await MetadataTypeInfo[depType].getCbReferenceKeys(
typeKeyList[depType],
retrieveDir,
assetDependencies
);
}
// add dependencies to selectedTypes
if (assetDependencies.size) {
const depType = 'asset';
if (typeKeyList[depType]) {
typeKeyList[depType].push(...assetDependencies);
} else {
typeKeyList[depType] = [...assetDependencies];
}
// remove duplicates in main object after adding dependencies
typeKeyList[depType] = [...new Set(typeKeyList[depType])];
}
// check all assets for dependencies recursively
if (typeKeyList.asset?.length) {
const depType = 'asset';
const Asset = MetadataTypeInfo[depType];
Asset.properties = properties;
Asset.buObject = buObject;
const additionalAssetDependencies = [
...(await Asset.getCbReferenceKeys(
typeKeyList[depType],
retrieveDir,
new Set(typeKeyList[depType])
)),
];
if (additionalAssetDependencies.length) {
Util.logger.info(
`Found ${additionalAssetDependencies.length - initialAssetNumber} additional assets linked via ContentBlockByX.`
);
}
// reset cache in case this is used progammatically somehow
Asset.getJsonFromFSCache = null;
// remove duplicates in main object after adding dependencies
typeKeyList[depType] = [...new Set(typeKeyList[depType])];
}
return typeKeyList;
}
/**
*
* @param {string} businessUnit references credentials from properties.json
* @param {TypeKeyCombo} typeKeyList limit retrieval to given metadata type
* @returns {Promise.<TypeKeyCombo>} dependencies
*/
static async addDependencies(businessUnit, typeKeyList) {
if (!Util.OPTIONS.dependencies) {
return;
}
Util.logger.info(
'You might see warnings about items not being found if you have not re-retrieved everything lately.'
);
// try re-retrieve without passing selectedTypes to ensure we find all dependencies
await this._reRetrieve(businessUnit, true, null, typeKeyList);
Util.logger.info(
'Searching for selected items and their dependencies in your project folder'
);
/** @type {TypeKeyCombo} */
const dependencies = {};
/** @type {TypeKeyCombo} */
const notFoundList = {};
const initiallySelectedTypesArr = Object.keys(typeKeyList);
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
for (const type of initiallySelectedTypesArr) {
MetadataTypeInfo[type].properties = properties;
MetadataTypeInfo[type].buObject = buObject;
await MetadataTypeInfo[type].getDependentFiles(
typeKeyList[type],
dependencies,
notFoundList,
true
);
}
if (Util.getTypeKeyCount(notFoundList)) {
// if we have missing items, we need to retrieve them
Util.logger.warn(
`We recommend you retrieve the missing items with the following command and then re-run buildDefinition:`
);
Util.logger.warn(
` mcdev retrieve ${businessUnit} ${Util.convertTypeKeyToCli(notFoundList)}`
);
}
// remove duplicates & empty types
for (const type in dependencies) {
if (dependencies[type].length) {
dependencies[type] = [...new Set(dependencies[type])];
} else {
delete dependencies[type];
}
}
// add dependencies to selectedTypes
if (Object.keys(dependencies).length) {
Util.logger.info(
`Found ${Util.getTypeKeyCount(dependencies)} items across ${Object.keys(dependencies).length} types.`
);
for (const type in dependencies) {
if (typeKeyList[type]) {
typeKeyList[type].push(...dependencies[type]);
} else {
typeKeyList[type] = dependencies[type];
}
// remove duplicates in main object after adding dependencies
typeKeyList[type] = [...new Set(typeKeyList[type])];
}
}
return dependencies;
}
/**
* Build a template based on a list of metadata files in the retrieve folder.
*
* @param {string} businessUnitTemplate references credentials from properties.json
* @param {string} businessUnitDefinition references credentials from properties.json
* @param {TypeKeyCombo} typeKeyCombo limit retrieval to given metadata type
* @returns {Promise.<MultiMetadataTypeList | object>} response from buildDefinition
*/
static async clone(businessUnitTemplate, businessUnitDefinition, typeKeyCombo) {
return this.build(
businessUnitTemplate,
businessUnitDefinition,
typeKeyCombo,
['__clone__'],
['__clone__']
);
}
/**
* Build a template based on a list of metadata files in the retrieve folder.
*
* @param {string} businessUnitTemplate references credentials from properties.json
* @param {string} businessUnitDefinition references credentials from properties.json
* @param {TypeKeyCombo} typeKeyCombo limit retrieval to given metadata type
* @param {string[]} marketTemplate market localizations
* @param {string[]} marketDefinition market localizations
* @param {boolean} [bulk] runs buildDefinitionBulk instead of buildDefinition; requires marketList to be defined and given via marketDefinition
* @returns {Promise.<MultiMetadataTypeList | object>} response from buildDefinition
*/
static async build(
businessUnitTemplate,
businessUnitDefinition,
typeKeyCombo,
marketTemplate,
marketDefinition,
bulk
) {
if (!bulk && !businessUnitDefinition) {
Util.logger.error(
'Please provide a business unit to deploy to via --buTo or activate --bulk'
);
return;
}
// check if types are valid
for (const type of Object.keys(typeKeyCombo)) {
if (!Util._isValidType(type)) {
return;
}
if (!Array.isArray(typeKeyCombo[type]) || typeKeyCombo[type].length === 0) {
Util.logger.error('You need to define keys, not just types to run build');
// we need an array of keys here
return;
}
}
// redirect templates to temporary folder when executed via build()
const properties = await config.getProperties();
if (!properties) {
return;
}
const templateDirBackup = properties.directories.template;
properties.directories.template = '.mcdev/template/';
Util.logger.info('mcdev:: Build Template & Build Definition');
const templates = await this.buildTemplate(
businessUnitTemplate,
typeKeyCombo,
null,
marketTemplate
);
// check if any templates were found
if (!Object.keys(templates).length || !Util.getTypeKeyCount(typeKeyCombo)) {
Util.logger.error('No templates created. Aborting build');
properties.directories.template = templateDirBackup;
return;
}
if (typeof Util.OPTIONS.purge !== 'boolean') {
// deploy folder is in targets for definition creation
// recommend to purge their content first
Util.OPTIONS.purge = await confirm({
message: `Do you want to empty relevant BU sub-folders in /${properties.directories.deploy} (ensures no files from previous deployments remain)?`,
default: true,
});
}
const response = bulk
? await this.buildDefinitionBulk(marketDefinition[0], typeKeyCombo, null)
: await this.buildDefinition(
businessUnitDefinition,
typeKeyCombo,
null,
marketDefinition
);
// reset temporary template folder
try {
await File.remove(properties.directories.template);
} catch {
// sometimes the first attempt is not successful for some operating system reason. Trying again mostly solves this
await File.remove(properties.directories.template);
}
properties.directories.template = templateDirBackup;
return response;
}
/**
* Build a template based on a list of metadata files in the retrieve folder.
*
* @param {string} businessUnit references credentials from properties.json
* @param {string | TypeKeyCombo} selectedTypes limit retrieval to given metadata type
* @param {string[] | undefined} keyArr customerkey of the metadata
* @param {string[]} marketArr market localizations
* @returns {Promise.<MultiMetadataTypeList>} -
*/
static async buildTemplate(businessUnit, selectedTypes, keyArr, marketArr) {
this.#welcomeMessage();
Util.startLogger();
Util.logger.info('mcdev:: Build Template from retrieved files');
const properties = await config.getProperties();
if (!properties) {
return;
}
const buObject = await Cli.getCredentialObject(properties, businessUnit);
if (!Util.checkMarketList(marketArr, properties)) {
return;
}
const typeKeyList = Util.checkAndPrepareTypeKeyCombo(
selectedTypes,
keyArr,
'buildTemplate'
);
if (!typeKeyList) {
return;
}
if (!Util.OPTIONS.dependencies) {
await this._reRetrieve(businessUnit, false, typeKeyList);
}
// convert names to keys
const retrieveDir = File.normalizePath([
properties.directories.retrieve,
buObject.credential,
buObject.businessUnit,
]);
for (const type of Object.keys(typeKeyList)) {
const keyArr = typeKeyList[type];
if (keyArr.some((key) => key.startsWith('name:'))) {
// at least one key was provided as a name -> load all files from disk to try and find that key
const builTemplateCache = Object.values(
await MetadataTypeInfo[type].getJsonFromFS(retrieveDir + path.sep + type)
);
typeKeyList[type] = keyArr
.map((key) => {
if (key.startsWith('name:')) {
// key was defined by name. try and find matching item on disk
const name = key.slice(5);
const foundKeysByName = builTemplateCache
.filter(
(item) =>
name == item[MetadataTypeInfo[type].definition.nameField]
)
.map((item) => item[MetadataTypeInfo[type].definition.keyField]);
if (foundKeysByName.length === 1) {
key = foundKeysByName[0];
Util.logger.debug(
`- found ${type} key '${key}' for name '${name}'`
);
return key;
} else if (foundKeysByName.length > 1) {
Util.logger.error(
`Found multiple keys (${foundKeysByName.join(', ')}) for name: ${key}`
);
return;
} else {
Util.logger.error(`Could not find any keys for name: ${name}`);
return;
}
} else {
return key;
}
})
.filter(Boolean);
}
}
// if dependencies are enabled, we need to search for them and add them to our
await this.addDependencies(businessUnit, typeKeyList);
await this.addDependentCbReferences(businessUnit, typeKeyList);
/** @type {MultiMetadataTypeList} */
const returnObj = {};
for (const type of Object.keys(typeKeyList).sort()) {
// ensure keys are sorted again, after finding dependencies, to enhance log readability
typeKeyList[type].sort();
const result = await B