@salesforce/source-tracking
Version:
API for tracking local and remote Salesforce metadata changes
428 lines (427 loc) • 22.3 kB
JavaScript
;
/*
* Copyright 2025, Salesforce, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.remoteChangeElementToChangeResult = exports.RemoteSourceTrackingService = void 0;
const node_path_1 = __importDefault(require("node:path"));
const node_fs_1 = __importDefault(require("node:fs"));
const node_os_1 = require("node:os");
const ts_retry_promise_1 = require("ts-retry-promise");
const core_1 = require("@salesforce/core");
const kit_1 = require("@salesforce/kit");
const ts_types_1 = require("@salesforce/ts-types");
const metadataKeys_1 = require("../metadataKeys");
const functions_1 = require("../functions");
const expectedSourceMembers_1 = require("./expectedSourceMembers");
const fileOperations_1 = require("./fileOperations");
const orgQueries_1 = require("./orgQueries");
/*
* after some results have returned, how many times should we poll for missing sourcemembers
* even when there is a longer timeout remaining (because the deployment is very large)
*/
const POLLING_DELAY_MS = 1000;
const CONSECUTIVE_EMPTY_POLLING_RESULT_LIMIT = (core_1.envVars.getNumber('SF_SOURCE_MEMBER_POLLING_TIMEOUT') ?? 120) / kit_1.Duration.milliseconds(POLLING_DELAY_MS).seconds;
/** if a cached instance is older than this, it will be purged */
const MAX_INSTANCE_CACHE_TTL = 1000 * 60 * 60 * 1; // 1 hour
class RemoteSourceTrackingService {
/** map of constructed, init'ed instances; key is orgId. It's like a singleton at the org level */
static instanceMap = new Map();
filePath;
logger;
serverMaxRevisionCounter = 0;
sourceMembers = new Map();
org;
// A short term cache (within the same process) of query results based on a revision.
// Useful for source:pull, which makes 3 of the same queries; during status, building manifests, after pull success.
queryCache = new Map();
userQueryCache = new Map();
/**
* Initializes the service with existing remote source tracking data, or sets
* the state to begin source tracking of metadata changes in the org.
*/
constructor(options) {
this.org = options.org;
this.filePath = node_path_1.default.join(options.projectPath, '.sf', 'orgs', this.org.getOrgId(), fileOperations_1.FILENAME);
}
/**
* Get the singleton instance for a given user.
*
* @param {RemoteSourceTrackingService.Options} options that contain the org
* @returns {Promise<RemoteSourceTrackingService>} the remoteSourceTrackingService object for the given username
*/
static async getInstance(options) {
const orgId = options.org.getOrgId();
const service = this.instanceMap.get(orgId)?.service ?? (await new RemoteSourceTrackingService(options).init());
this.instanceMap.set(orgId, { service, lastUsed: Date.now() });
// when we get an instance, we make sure old ones are not accumulating. Important in multitenant environments
purgeOldInstances(this.instanceMap);
// even if there was already an instance around, its queries might no longer be accurate (ex: missing new changes but queryFrom would return stale results)
service.queryCache.clear();
service.userQueryCache.clear();
service.org = options.org;
return service;
}
/**
* Delete the RemoteSourceTracking for a given org.
*
* @param orgId
* @returns the path of the deleted source tracking file
*/
static async delete(orgId) {
const fileToDelete = (0, fileOperations_1.getFilePath)(orgId);
// the file might not exist, in which case we don't need to delete it
if (node_fs_1.default.existsSync(fileToDelete)) {
await node_fs_1.default.promises.unlink(fileToDelete);
}
return node_path_1.default.isAbsolute(fileToDelete) ? fileToDelete : node_path_1.default.join(process.cwd(), fileToDelete);
}
/**
* pass in a series of SDR FilResponses .\
* it sets their last retrieved revision to the current revision counter from the server.
*/
async syncSpecifiedElements(elements) {
if (elements.length === 0) {
return;
}
const quietLogger = elements.length > 100 ? this.logger.silent.bind(this.logger) : this.logger.debug.bind(this.logger);
quietLogger(`Syncing ${elements.length} Revisions by key`);
// this can be super-repetitive on a large ExperienceBundle where there is an element for each file but only one Revision for the entire bundle
// any item in an aura/LWC bundle needs to represent the top (bundle) level and the file itself
// so we de-dupe via a set
Array.from(new Set(elements.flatMap((element) => (0, metadataKeys_1.getMetadataKeyFromFileResponse)(element)))).map((metadataKey) => {
const revision = this.sourceMembers.get(metadataKey) ?? this.sourceMembers.get(decodeURI(metadataKey));
if (!revision) {
this.logger.warn(`found no matching revision for ${metadataKey}`);
}
else if (doesNotMatchServer(revision)) {
quietLogger(`Syncing ${metadataKey} revision from ${revision.lastRetrievedFromServer ?? 'null'} to ${revision.RevisionCounter}`);
this.setMemberRevision(metadataKey, {
...revision,
lastRetrievedFromServer: revision.RevisionCounter,
});
}
});
await (0, fileOperations_1.writeTrackingFile)({
filePath: this.filePath,
maxCounter: this.serverMaxRevisionCounter,
members: this.sourceMembers,
});
}
/**
* Resets source tracking state by first clearing all tracked data, then
* queries and synchronizes SourceMembers from the associated org.
*
* If a toRevision is passed, it will query for all `SourceMembers` with
* a `RevisionCounter` less than or equal to the provided revision number.
*
* When no toRevision is passed, it will query and sync all `SourceMembers`.
*
* @param toRevision The `RevisionCounter` number to sync to.
*/
async reset(toRevision) {
// Called during a source:tracking:reset
this.serverMaxRevisionCounter = 0;
this.sourceMembers = new Map();
const members = toRevision !== undefined && toRevision !== null
? await (0, orgQueries_1.querySourceMembersTo)(this.org.getConnection(), toRevision)
: await (0, orgQueries_1.querySourceMembersFrom)({
fromRevision: 0,
logger: this.logger,
userQueryCache: this.userQueryCache,
queryCache: this.queryCache,
conn: this.org.getConnection(),
});
await this.trackSourceMembers(members, true);
return members.map((member) => (0, functions_1.getMetadataKey)(member.MemberType, member.MemberName));
}
/**
* Queries the org for any new, updated, or deleted metadata and updates
* source tracking state. All `ChangeElements` not in sync with the org
* are returned.
*/
// Internal implementation of the public `retrieveUpdates` function that adds the ability
// to sync the retrieved SourceMembers; meaning it will update the lastRetrievedFromServer
// field to the SourceMember's RevisionCounter, and update the serverMaxRevisionCounter
// to the highest RevisionCounter.
async retrieveUpdates() {
// Always track new SourceMember data, or update tracking when we sync.
const queriedSourceMembers = await (0, orgQueries_1.querySourceMembersFrom)({
fromRevision: this.serverMaxRevisionCounter,
logger: this.logger,
userQueryCache: this.userQueryCache,
queryCache: this.queryCache,
conn: this.org.getConnection(),
});
await this.trackSourceMembers(queriedSourceMembers);
// Look for any changed that haven't been synced. I.e, the lastRetrievedFromServer
// does not match the serverRevisionCounter.
const returnElements = Array.from(this.sourceMembers.values())
.filter(doesNotMatchServer)
.map(fileOperations_1.revisionToRemoteChangeElement);
this.logger.debug(returnElements.length
? `Found ${returnElements.length} elements not synced with org`
: 'Remote source tracking is up to date');
return returnElements;
}
/**
* Polls the org for SourceMember objects matching the provided metadata member names,
* stopping when all members have been matched or the polling timeout is met or exceeded.
* NOTE: This can be removed when the Team Dependency (TD-0085369) for W-7737094 is delivered.
*
* @param expectedMemberNames Array of metadata names to poll
* @param pollingTimeout maximum amount of time in seconds to poll for SourceMembers
*/
async pollForSourceTracking(expectedMembers) {
if (core_1.envVars.getBoolean('SF_DISABLE_SOURCE_MEMBER_POLLING')) {
return this.logger.warn('Not polling for SourceMembers since SF_DISABLE_SOURCE_MEMBER_POLLING = true.');
}
if (expectedMembers.length === 0) {
return;
}
const outstandingSourceMembers = (0, expectedSourceMembers_1.calculateExpectedSourceMembers)(expectedMembers);
const originalOutstandingSize = outstandingSourceMembers.size;
// this will be the absolute timeout from the start of the poll. We can also exit early if it doesn't look like more results are coming in
let highestRevisionSoFar = this.serverMaxRevisionCounter;
const pollingTimeout = (0, orgQueries_1.calculateTimeout)(this.logger)(outstandingSourceMembers.size);
let pollAttempts = 0;
let consecutiveEmptyResults = 0;
let someResultsReturned = false;
/** we weren't expecting these SourceMembers, based on the deployment results */
const bonusTypes = new Set();
this.logger.debug(`Polling for ${outstandingSourceMembers.size} SourceMembers from revision ${highestRevisionSoFar} with timeout of ${pollingTimeout.seconds}s`);
const poll = async () => {
pollAttempts += 1; // not used to stop polling, but for debug logging
// get sourceMembers added since our most recent max
// use the "new highest" revision from the last poll that returned results
const queriedMembers = await (0, orgQueries_1.querySourceMembersFrom)({
conn: this.org.getConnection(),
fromRevision: highestRevisionSoFar,
logger: pollAttempts > 1 ? undefined : this.logger,
});
if (queriedMembers.length) {
queriedMembers.map((member) => {
// remove anything returned from the query list
const metadataKey = (0, functions_1.getMetadataKey)(member.MemberType, member.MemberName);
const deleted = outstandingSourceMembers.delete(metadataKey);
if (!deleted) {
bonusTypes.add(metadataKey);
}
highestRevisionSoFar = Math.max(highestRevisionSoFar, member.RevisionCounter);
});
consecutiveEmptyResults = 0;
// flips on the first batch of results
someResultsReturned = true;
}
else {
consecutiveEmptyResults++;
}
await core_1.Lifecycle.getInstance().emit('sourceMemberPollingEvent', {
original: originalOutstandingSize,
remaining: outstandingSourceMembers.size,
attempts: pollAttempts,
consecutiveEmptyResults,
});
this.logger.debug(`[${pollAttempts}] Found ${originalOutstandingSize - outstandingSourceMembers.size} of ${originalOutstandingSize} expected SourceMembers`);
// update but don't sync
await this.trackSourceMembers(queriedMembers, false);
// exit if all have returned
if (outstandingSourceMembers.size === 0) {
return;
}
if (someResultsReturned && consecutiveEmptyResults >= CONSECUTIVE_EMPTY_POLLING_RESULT_LIMIT) {
throw new ts_retry_promise_1.NotRetryableError(`Polling found no results for ${consecutiveEmptyResults} consecutive attempts`);
}
this.logger.debug(outstandingSourceMembers.size < 20
? `Still looking for SourceMembers: ${Array.from(outstandingSourceMembers.keys()).join(',')}`
: `Still looking for ${outstandingSourceMembers.size} Source Members`);
throw new Error();
};
const pollingFunction = (0, ts_retry_promise_1.retryDecorator)(poll, {
timeout: pollingTimeout.milliseconds,
delay: POLLING_DELAY_MS,
retries: 'INFINITELY',
});
const lc = core_1.Lifecycle.getInstance();
try {
await pollingFunction();
this.logger.debug(`Retrieved all SourceMember data after ${pollAttempts} attempts`);
// find places where the expectedSourceMembers might be too pruning too aggressively
if (bonusTypes.size) {
void lc.emitTelemetry({
eventName: 'sourceMemberBonusTypes',
library: 'SourceTracking',
deploymentSize: expectedMembers.length,
bonusTypes: Array.from(bonusTypes).sort().join(','),
});
}
}
catch {
await Promise.all([
lc.emitWarning(`Polling for ${outstandingSourceMembers.size} SourceMembers timed out after ${pollAttempts} attempts (last ${consecutiveEmptyResults} were empty).
Missing SourceMembers:
${formatSourceMemberWarnings(outstandingSourceMembers)}`),
lc.emitTelemetry({
eventName: 'sourceMemberPollingTimeout',
library: 'SourceTracking',
timeoutSeconds: pollingTimeout.seconds,
attempts: pollAttempts,
consecutiveEmptyResults,
missingQuantity: outstandingSourceMembers.size,
deploymentSize: expectedMembers.length,
bonusTypes: Array.from(bonusTypes).sort().join(','),
types: [...new Set(Array.from(outstandingSourceMembers.values()).map((member) => member.type))]
.sort()
.join(','),
members: Array.from(outstandingSourceMembers.keys()).join(','),
}),
]);
}
}
/**
* Adds the given SourceMembers to the list of tracked MemberRevisions, optionally updating
* the lastRetrievedFromServer field (sync), and persists the changes to maxRevision.json.
*/
async trackSourceMembers(sourceMembers, sync = false) {
if (sourceMembers.length === 0) {
return;
}
const quietLogger = sourceMembers.length > 100 ? this.logger.silent.bind(this.logger) : this.logger.debug.bind(this.logger);
quietLogger(`Upserting ${sourceMembers.length} SourceMembers to maxRevision.json`);
// Update the serverMaxRevisionCounter to the highest RevisionCounter
this.serverMaxRevisionCounter = Math.max(this.serverMaxRevisionCounter, ...sourceMembers.map((m) => m.RevisionCounter));
this.logger.debug(`Updating serverMaxRevisionCounter to ${this.serverMaxRevisionCounter}`);
sourceMembers.map((change) => {
// try accessing the sourceMembers object at the index of the change's name
// if it exists, we'll update the fields - if it doesn't, we'll create and insert it
const key = (0, functions_1.getMetadataKey)(change.MemberType, change.MemberName);
const sourceMemberFromTracking = this.getSourceMember(key);
quietLogger(`${sourceMemberFromTracking ? `Updating ${key} to` : `Inserting ${key} with`} RevisionCounter: ${change.RevisionCounter}${sync ? ' and syncing' : ''}`);
this.setMemberRevision(key, {
...change,
// If we are syncing changes then we need to update the lastRetrievedFromServer field to
// match the RevisionCounter from the SourceMember.
lastRetrievedFromServer: sync ? change.RevisionCounter : sourceMemberFromTracking?.lastRetrievedFromServer,
});
});
await (0, fileOperations_1.writeTrackingFile)({
filePath: this.filePath,
maxCounter: this.serverMaxRevisionCounter,
members: this.sourceMembers,
});
}
/** reads the tracking file and inits the logger and contents */
async init() {
if (!(await this.org.supportsSourceTracking())) {
;
const messages = new core_1.Messages('@salesforce/source-tracking', 'source', new Map([["NonSourceTrackedOrgError", "This org does not have source tracking."]]));
throw new core_1.SfError(messages.getMessage('NonSourceTrackedOrgError'), 'NonSourceTrackedOrgError');
}
this.logger = core_1.Logger.getRawRootLogger().child({ name: this.constructor.name });
if (node_fs_1.default.existsSync(this.filePath)) {
// read the file contents and turn it into the map
const rawContents = await (0, fileOperations_1.readFileContents)(this.filePath);
if (rawContents.serverMaxRevisionCounter && rawContents.sourceMembers) {
this.serverMaxRevisionCounter = rawContents.serverMaxRevisionCounter;
this.sourceMembers = new Map(Object.entries(rawContents.sourceMembers ?? {}));
}
}
else {
// we need to init the file
await (0, fileOperations_1.writeTrackingFile)({
filePath: this.filePath,
maxCounter: this.serverMaxRevisionCounter,
members: this.sourceMembers,
});
}
return this;
}
/** Return a tracked element as MemberRevision data.*/
getSourceMember(key) {
return (this.sourceMembers.get(key) ??
this.sourceMembers.get(getDecodedKeyIfSourceMembersHas({ sourceMembers: this.sourceMembers, key, logger: this.logger })));
}
setMemberRevision(key, sourceMember) {
const sourceMembers = this.sourceMembers;
const matchingKey = sourceMembers.get(key)
? key
: getDecodedKeyIfSourceMembersHas({ sourceMembers, key, logger: this.logger });
this.sourceMembers.set(matchingKey, { ...sourceMember, MemberName: decodeURIComponent(sourceMember.MemberName) });
}
}
exports.RemoteSourceTrackingService = RemoteSourceTrackingService;
/**
* pass in an RCE, and this will return a pullable ChangeResult.
* Useful for correcing bundle types where the files show change results with types but aren't resolvable
*/
const remoteChangeElementToChangeResult = (rce) => ({
...rce,
...(metadataKeys_1.mappingsForSourceMemberTypesToMetadataType.has(rce.type)
? {
// SNOWFLAKE: EmailTemplateFolder is treated as an alias for EmailFolder so it has a mapping.
// The name must be handled differently than with bundle types.
name: rce.type === 'EmailTemplateFolder' ? rce.name : rce.name.split('/')[0],
type: metadataKeys_1.mappingsForSourceMemberTypesToMetadataType.get(rce.type),
}
: {}),
origin: 'remote', // we know they're remote
});
exports.remoteChangeElementToChangeResult = remoteChangeElementToChangeResult;
/**
*
* iterate SourceMember keys and compare their decoded value with the decoded key.
* if there's a match, return the matching decoded key, otherwise, return the original key
*/
const getDecodedKeyIfSourceMembersHas = ({ key, sourceMembers, logger, }) => {
try {
const originalKeyDecoded = decodeURIComponent(key);
const match = Array.from(sourceMembers.keys()).find((memberKey) => decodeURIComponent(memberKey) === originalKeyDecoded);
if (match) {
logger.debug(`${match} matches already tracked member: ${key}`);
return match;
}
}
catch (e) {
// Log the error and the key
const errMsg = e instanceof Error ? e.message : (0, ts_types_1.isString)(e) ? e : 'unknown';
logger.debug(`Could not decode metadata key: ${key} due to: ${errMsg}`);
}
return key;
};
/** organize by type and format for warning output */
const formatSourceMemberWarnings = (outstandingSourceMembers) => {
// TODO: use Map.groupBy when we node22 is minimum
// ex: CustomObject : [Foo__c, Bar__c]
const mapByType = Array.from(outstandingSourceMembers.values()).reduce((acc, value) => {
acc.set(value.type, [...(acc.get(value.type) ?? []), value.fullName]);
return acc;
}, new Map());
return Array.from(mapByType.entries())
.map(([type, names]) => ` - ${type}: ${names.join(', ')}`)
.join(node_os_1.EOL);
};
const doesNotMatchServer = (member) => member.RevisionCounter !== member.lastRetrievedFromServer;
const purgeOldInstances = (instances) => {
const now = Date.now();
Array.from(instances.entries())
.filter(([, { lastUsed }]) => now - lastUsed > MAX_INSTANCE_CACHE_TTL)
.map(([orgId]) => {
instances.delete(orgId);
});
};
//# sourceMappingURL=remoteSourceTrackingService.js.map