mcdev
Version:
Accenture Salesforce Marketing Cloud DevTools
568 lines (527 loc) • 24.2 kB
JavaScript
;
import MetadataType from './MetadataType.js';
import { Util } from '../util/util.js';
import DataExtension from './DataExtension.js';
import { confirm } from '@inquirer/prompts';
import auth from '../util/auth.js';
/**
* @typedef {import('../../types/mcdev.d.js').BuObject} BuObject
* @typedef {import('../../types/mcdev.d.js').CodeExtract} CodeExtract
* @typedef {import('../../types/mcdev.d.js').CodeExtractItem} CodeExtractItem
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItem} MetadataTypeItem
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItemDiff} MetadataTypeItemDiff
* @typedef {import('../../types/mcdev.d.js').MetadataTypeItemObj} MetadataTypeItemObj
* @typedef {import('../../types/mcdev.d.js').MetadataTypeMap} MetadataTypeMap
* @typedef {import('../../types/mcdev.d.js').MetadataTypeMapObj} MetadataTypeMapObj
* @typedef {import('../../types/mcdev.d.js').SoapRequestParams} SoapRequestParams
* @typedef {import('../../types/mcdev.d.js').TemplateMap} TemplateMap
*/
/**
* @typedef {import('../../types/mcdev.d.js').DataExtensionFieldMap} DataExtensionFieldMap
* @typedef {import('../../types/mcdev.d.js').DataExtensionFieldItem} DataExtensionFieldItem
*/
/**
* DataExtensionField MetadataType
*
* @augments MetadataType
*/
class DataExtensionField extends MetadataType {
static fixShared_fields;
/**
* Retrieves all records and saves it to disk
*
* @param {string} retrieveDir Directory where retrieved metadata directory will be saved
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @returns {Promise.<{metadata: DataExtensionFieldMap, type: string}>} Promise of items
*/
static async retrieve(retrieveDir, additionalFields) {
return super.retrieveSOAP(retrieveDir, null, null, additionalFields);
}
/**
* Retrieves all records and saves it to disk
*
* @returns {Promise.<MetadataTypeMapObj>} Promise of items
*/
static async retrieveForCache() {
const cachedDEs = cache.getCache().dataExtension;
if (cachedDEs) {
await DataExtension.attachFields(cachedDEs);
}
return;
}
/**
* helps retrieving fields for a single DE. Use when needing field data for one DE only.
*
* @param {string} customerKey external key of single DE
* @returns {Promise.<object>} Promise of items
*/
static async retrieveFieldsForSingleDe(customerKey) {
/** @type {SoapRequestParams} */
const fieldOptions = {
filter: {
leftOperand: 'DataExtension.CustomerKey',
operator: 'equals',
rightOperand: customerKey,
},
};
const response = await DataExtensionField.retrieveForCacheDE(fieldOptions);
return DataExtensionField.convertToSortedArray(response.metadata).reduce((acc, field) => {
acc[field.CustomerKey] = field;
return acc;
}, {});
}
/**
* determines if a dataExtension is shared (from parent BU) by checking its folder's ContentType in the parent BU folder cache
*
* @param {string} customerKey external key of single DE
* @returns {boolean} true if the DE is shared (stored on parent BU)
*/
static isSharedDe(customerKey) {
if (this.buObject.eid === this.buObject.mid) {
// already on parent BU, no shared DEs possible
return false;
}
const categoryId = cache.getCache().dataExtension?.[customerKey]?.CategoryID;
if (!categoryId) {
return false;
}
try {
const contentType = cache.searchForField(
'folder',
categoryId,
'ID',
'ContentType',
this.buObject.eid
);
return MetadataTypeDefinitions.folder.folderTypesFromParent.includes(contentType);
} catch {
// folder not found in parent BU cache → local DE
return false;
}
}
/**
* helps retrieving fields for a single DE from parent BU.
* Use when the DE is shared (stored on parent BU) and its fields are not in the current BU's cache.
*
* @param {string} customerKey external key of single DE
* @returns {Promise.<object>} Promise of items
*/
static async retrieveFieldsForSingleSharedDe(customerKey) {
if (this.buObject.eid === this.buObject.mid) {
// already on parent BU, shared retrieval is not needed - all DEs are local here
Util.logger.debug(
`DataExtensionField.retrieveFieldsForSingleSharedDe: already on parent BU, skipping for ${customerKey}`
);
return {};
}
if (!this.properties.credentials[this.buObject.credential]) {
Util.logger.error(
`DataExtensionField.retrieveFieldsForSingleSharedDe: credential '${this.buObject.credential}' not found in properties`
);
return {};
}
const buObjectBak = this.buObject;
const clientBak = this.client;
/** @type {BuObject} */
const buObjectParentBu = {
eid: this.properties.credentials[this.buObject.credential].eid,
mid: this.properties.credentials[this.buObject.credential].eid,
businessUnit: Util.parentBuName,
credential: this.buObject.credential,
};
try {
this.buObject = buObjectParentBu;
this.client = auth.getSDK(buObjectParentBu);
} catch (ex) {
Util.logger.error(ex.message);
this.buObject = buObjectBak;
this.client = clientBak;
return {};
}
const fields = await this.retrieveFieldsForSingleDe(customerKey);
// revert client to current default
this.buObject = buObjectBak;
this.client = clientBak;
return fields;
}
/**
* Routes field retrieval to the correct method depending on whether the DE is local or shared.
* Use this instead of duplicating the isSharedDe() check at each call site.
*
* @param {string} customerKey external key of single DE
* @returns {Promise.<object>} Promise of items
*/
static async retrieveFieldsForSingleDeAuto(customerKey) {
return this.isSharedDe(customerKey)
? this.retrieveFieldsForSingleSharedDe(customerKey)
: this.retrieveFieldsForSingleDe(customerKey);
}
/**
* Retrieves all records for caching
*
* @param {SoapRequestParams} [requestParams] required for the specific request (filter for example)
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @returns {Promise.<{metadata: DataExtensionFieldMap, type: string}>} Promise of items
*/
static async retrieveForCacheDE(requestParams, additionalFields) {
let response;
response = await super.retrieveSOAP(null, requestParams, null, additionalFields);
if (!response) {
// try again but without filters as a workaround for the "String or binary data would be truncated." issue
response = await super.retrieveSOAP(null, {}, null, additionalFields);
}
return response;
}
/**
* helper for DataExtension.retrieveFieldsForSingleDe that sorts the fields into an array
*
* @param {DataExtensionFieldMap} fieldsObj customerKey-based list of fields for one dataExtension
* @returns {DataExtensionFieldItem[]} sorted array of field objects
*/
static convertToSortedArray(fieldsObj) {
return (
Object.keys(fieldsObj)
.map((field) => fieldsObj[field])
// the API returns the fields not sorted
.toSorted(this.sortDeFields)
);
}
/**
* sorting method to ensure `Ordinal` is respected
*
* @param {DataExtensionFieldItem} a -
* @param {DataExtensionFieldItem} b -
* @returns {number} sorting based on Ordinal
*/
static sortDeFields(a, b) {
return a.Ordinal - b.Ordinal;
}
/**
* manages post retrieve steps; only used by DataExtension class
*
* @param {DataExtensionFieldItem} metadata a single item
* @returns {DataExtensionFieldItem} metadata
*/
static postRetrieveTasksDE(metadata) {
// remove fields according to definition
this.keepRetrieveFields(metadata);
// remove fields that we do not need after association to a DE
delete metadata.CustomerKey;
delete metadata.DataExtension;
delete metadata.Ordinal;
if (metadata.FieldType !== 'Decimal') {
// remove scale - it's only used for "Decimal" to define the digits behind the decimal
delete metadata.Scale;
}
return metadata;
}
/**
* Mofifies passed deployColumns for update by mapping ObjectID to their target column's values.
* Removes FieldType field if its the same in deploy and target column, because it results in an error even if its of the same type
*
* @param {DataExtensionFieldItem[]} deployColumns Columns of data extension that will be deployed
* @param {string} deKey external/customer key of Data Extension
* @returns {Promise.<DataExtensionFieldMap>} existing fields by their original name to allow re-adding FieldType after update
*/
static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
// create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic
this.fixShared_fields ||= {};
this.fixShared_fields[deKey] ||= {};
// get row count to know which field restrictions apply
let hasData = false;
try {
const rowset = await this.client.rest.get(
`/data/v1/customobjectdata/key/${deKey}/rowset?$page=1&$pagesize=1`
);
const rowCount = rowset.count;
hasData = rowCount > 0;
Util.logger.debug(`dataExtension ${deKey} row count: ${rowCount}`);
} catch (ex) {
Util.logger.debug(`Could not retrieve rowcount for ${deKey}: ${ex.message}`);
}
// retrieve existing fields to enable updating them
const response = await this.retrieveForCacheDE(
{
filter: {
leftOperand: 'DataExtension.CustomerKey',
operator: 'equals',
rightOperand: deKey,
},
},
['Name', 'ObjectID']
);
const fieldsObj = response.metadata;
// ensure fields can be updated properly by their adding ObjectId based on Name-matching
/** @type {DataExtensionFieldMap} */
const existingFieldByName = {};
for (const key of Object.keys(fieldsObj)) {
// make sure we stringify the name in case it looked numeric and then lowercase it for easy comparison as the server is comparing field names case-insensitive
existingFieldByName[(fieldsObj[key].Name + '').toLowerCase()] = fieldsObj[key];
}
// capture field names from deploy payload before the loop modifies them (for renamed fields, the original name is used)
const deployColumnNames = new Set(
deployColumns.map((item) => (item.Name + '').toLowerCase())
);
for (let i = deployColumns.length - 1; i >= 0; i--) {
const item = deployColumns[i];
// make sure we stringify the name in case it looked numeric and then lowercase it for easy comparison as the server is comparing field names case-insensitive
const itemOld = existingFieldByName[(item.Name + '').toLowerCase()];
if (itemOld) {
// field is getting updated ---
// Updating to a new FieldType will result in an error; warn & afterwards remove it
if (itemOld.FieldType !== item.FieldType) {
// applicable: with or without data but simply ignored by API
Util.logger.error(
` - The Field Type of an existing field cannot be changed. [${deKey}].[${item.Name}]: '${itemOld.FieldType}'`
);
}
if (item.FieldType !== 'Decimal') {
// remove scale - it's only used for "Decimal" to define the digits behind the decimal
delete item.Scale;
}
delete item.FieldType;
if (itemOld.MaxLength > item.MaxLength) {
// applicable: with or without data (Code 310007)
Util.logger.error(
` - The length of an existing field cannot be decreased. [${deKey}].[${item.Name}]: '${itemOld.MaxLength}'`
);
}
if (Util.isFalse(itemOld.IsRequired) && Util.isTrue(item.IsRequired)) {
// applicable: with or without data (Code 310007)
Util.logger.error(
` - A field cannot be changed to be required on update after it was created to allow nulls. [${deKey}].[${item.Name}]`
);
}
// enable renaming
if (item.Name_new) {
Util.logger.info(
` - Found Name_new='${item.Name_new}' for field ${deKey}.${item.Name} - trying to rename.`
);
item.Name = item.Name_new;
delete item.Name_new;
}
// check if any changes were found
let changeFound = false;
for (const key of Object.keys(item)) {
if (item[key] !== itemOld[key]) {
changeFound = true;
}
}
// share fields with fixShared logic
this.fixShared_fields[deKey][item.Name] = structuredClone(item);
this.fixShared_fields[deKey][item.Name].FieldType = itemOld.FieldType;
if (!changeFound) {
deployColumns.splice(i, 1);
Util.logger.verbose(`no change - removed field [${deKey}].[${item.Name}]`);
continue;
}
// set the ObjectId for clear identification during update
item.ObjectID = itemOld.ObjectID;
} else {
// field is getting added ---
if (hasData && Util.isTrue(item.IsRequired) && item.DefaultValue === '') {
// applicable: with data only
if (Util.isFalse(item.IsPrimaryKey)) {
Util.logger.warn(
` - Adding new fields to an existing table requires that these fields are either not-required (nullable) or have a default value set. Changing [${deKey}].[${item.Name}] to be not-required`
);
item.IsRequired = false;
} else {
Util.logger.error(
`- You cannot add a new primary key field to an existing table that has data. [${deKey}].[${item.Name}]`
);
}
}
if (item.Name_new) {
Util.logger.info(
` - Found Name_new='${item.Name_new}' for field ${deKey}.${item.Name} but could not find a corresponding DE field on the server - adding new field instead of updating.`
);
delete item.Name_new;
}
// Field doesn't exist in target, therefore Remove ObjectID if present
delete item.ObjectID;
this.fixShared_fields[deKey][item.Name] = structuredClone(item);
}
if (Util.isTrue(item.IsPrimaryKey) && Util.isFalse(item.IsRequired)) {
// applicable: with or without data
Util.logger.warn(
`- Primary Key field [${deKey}].[${item.Name}] cannot be not-required (nullable). Changing field to be required!`
);
item.IsRequired = true;
}
// filter bad manual changes to the json
if (!Util.isTrue(item.IsRequired) && !Util.isFalse(item.IsRequired)) {
Util.logger.error(
`- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'.`
);
}
if (!Util.isTrue(item.IsPrimaryKey) && !Util.isFalse(item.IsPrimaryKey)) {
Util.logger.error(
`- Invalid value for 'IsPrimaryKey' of [${deKey}].[${item.Name}]. Found '${item.IsPrimaryKey}' instead of 'true'/'false'.`
);
}
}
Util.logger.info(
Util.getGrayMsg(
` - Found ${deployColumns.length} added/updated Fields for ${deKey}${
deployColumns.length ? ': ' : ''
}` + deployColumns.map((item) => item.Name).join(', ')
)
);
// warn about fields that exist on the server but are not in the deploy payload
// these fields are NOT auto-deleted to prevent unintended data loss
const removedFieldNames = Object.values(existingFieldByName)
.filter((field) => !deployColumnNames.has((field.Name + '').toLowerCase()))
.map((field) => field.Name)
.toSorted();
if (removedFieldNames.length > 0) {
Util.logger.warn(
` - dataExtension ${deKey}: found ${removedFieldNames.length} field${removedFieldNames.length > 1 ? 's' : ''} on the server but not in the deploy payload. Fields cannot be auto-deleted due to potential data loss. To delete them, please run:`
);
Util.logger.info(
Util.getGrayMsg(
`mcdev delete ${this.buObject.credential}/${this.buObject.businessUnit} -m ` +
removedFieldNames
.map((name) => `dataExtensionField:"${deKey}.${name}"`)
.join(' ')
)
);
}
return existingFieldByName;
}
/**
* Delete a metadata item from the specified business unit
*
* @param {string} customerKey Identifier of data extension
* @returns {Promise.<boolean>} deletion success status
*/
static async deleteByKey(customerKey) {
return this.deleteByKeySOAP(customerKey);
}
/**
* Delete a data extension from the specified business unit
*
* @param {string} customerKey Identifier of metadata
* @param {string} [fieldId] for programmatic deletes only one can pass in the ID directly
* @returns {Promise.<boolean>} deletion success flag
*/
static async deleteByKeySOAP(customerKey, fieldId) {
const [deKey, fieldName] = customerKey.split('.');
customerKey = `[${deKey}].[${fieldName}]`;
let fieldObjectID = fieldId;
let deletionQueue;
// get the object id
if (!fieldObjectID) {
if (deKey === '*') {
const response = await this.retrieveForCacheDE(
{
filter: {
leftOperand: 'Name',
operator: 'equals',
rightOperand: fieldName,
},
},
['Name', 'ObjectID', 'DataExtension.CustomerKey']
);
deletionQueue = Object.values(response.metadata).map((item) => ({
fieldObjectID: item.ObjectID,
fieldName: fieldName,
deKey: item.DataExtension.CustomerKey,
}));
Util.logger.info(
` - Found ${deletionQueue.length} Data Extensions with field ${fieldName} in your BU:\n - ${deletionQueue
.map((item) => item.deKey)
.toSorted()
.join('\n - ')}`
);
if (deletionQueue.length > 0 && !Util.skipInteraction) {
const massDelete = await confirm({
message: `Do you really want to delete that field from all of the above Data Extensions?`,
default: false,
});
if (!massDelete) {
Util.logger.info(
` ☇ skipping deletion of ${fieldName} based on user-choice.`
);
return false;
}
}
} else {
const response = await this.retrieveForCacheDE(
{
filter: {
leftOperand: 'CustomerKey',
operator: 'equals',
rightOperand: customerKey,
},
},
['Name', 'ObjectID']
);
fieldObjectID = response.metadata[customerKey]?.ObjectID;
}
}
if (!fieldObjectID && !deletionQueue) {
Util.logger.error(`Could not find ${customerKey} on your BU`);
return false;
} else if (!deletionQueue) {
deletionQueue = [
{
fieldObjectID: fieldObjectID,
fieldName: fieldName,
deKey: deKey,
},
];
}
let success = true;
const fieldDeletedFromDEKeys = [];
for (const item of deletionQueue) {
// normal code
const keyObj = {
CustomerKey: item.deKey,
Fields: {
Field: {
ObjectID: item.fieldObjectID,
},
},
};
try {
// ! we really do need to delete from DataExtension not DataExtensionField here!
await this.client.soap.delete('DataExtension', keyObj, null);
if (!fieldId) {
Util.logger.info(
` - deleted ${this.definition.type}: ${item.deKey}.${item.fieldName}`
);
}
fieldDeletedFromDEKeys.push(item.deKey);
// return true;
} catch (ex) {
const errorMsg = ex?.results?.length
? `${ex.results[0].StatusMessage} (Code ${ex.results[0].ErrorCode})`
: ex?.json?.Results?.length
? `${ex.json.Results[0].StatusMessage} (Code ${ex.json.Results[0].ErrorCode})`
: ex.message;
Util.logger.error(
`- error deleting ${this.definition.type} ${item.deKey}.${item.fieldName}: ${errorMsg}`
);
success = false;
}
}
const uniqueDEKeys = [...new Set(fieldDeletedFromDEKeys)];
if (uniqueDEKeys.length > 0) {
Util.logger.info(
Util.getGrayMsg(
`To refresh your local files, run mcdev r ${this.buObject.credential}/${this.buObject.businessUnit} -m ${uniqueDEKeys
.map((key) => 'dataExtension:"' + key + '"')
.toSorted()
.join(' ')}`
)
);
}
return success;
}
}
// Assign definition to static attributes
import MetadataTypeDefinitions from '../MetadataTypeDefinitions.js';
import cache from '../util/cache.js';
DataExtensionField.definition = MetadataTypeDefinitions.dataExtensionField;
export default DataExtensionField;