@teambit/workspace
Version:
1,024 lines (1,000 loc) • 41.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.WorkspaceComponentLoader = void 0;
function _pMap() {
const data = _interopRequireDefault(require("p-map"));
_pMap = function () {
return data;
};
return data;
}
function _legacy() {
const data = require("@teambit/legacy.utils");
_legacy = function () {
return data;
};
return data;
}
function _toolboxPromise() {
const data = require("@teambit/toolbox.promise.map-pool");
_toolboxPromise = function () {
return data;
};
return data;
}
function _harmonyModules() {
const data = require("@teambit/harmony.modules.concurrency");
_harmonyModules = function () {
return data;
};
return data;
}
function _component() {
const data = require("@teambit/component");
_component = function () {
return data;
};
return data;
}
function _componentId() {
const data = require("@teambit/component-id");
_componentId = function () {
return data;
};
return data;
}
function _pMapSeries() {
const data = _interopRequireDefault(require("p-map-series"));
_pMapSeries = function () {
return data;
};
return data;
}
function _lodash() {
const data = require("lodash");
_lodash = function () {
return data;
};
return data;
}
function _legacy2() {
const data = require("@teambit/legacy.consumer-component");
_legacy2 = function () {
return data;
};
return data;
}
function _legacy3() {
const data = require("@teambit/legacy.bit-map");
_legacy3 = function () {
return data;
};
return data;
}
function _componentIssues() {
const data = require("@teambit/component-issues");
_componentIssues = function () {
return data;
};
return data;
}
function _legacy4() {
const data = require("@teambit/legacy.scope");
_legacy4 = function () {
return data;
};
return data;
}
function _dependencyResolver() {
const data = require("@teambit/dependency-resolver");
_dependencyResolver = function () {
return data;
};
return data;
}
function _envs() {
const data = require("@teambit/envs");
_envs = function () {
return data;
};
return data;
}
function _legacy5() {
const data = require("@teambit/legacy.extension-data");
_legacy5 = function () {
return data;
};
return data;
}
function _harmonyModules2() {
const data = require("@teambit/harmony.modules.in-memory-cache");
_harmonyModules2 = function () {
return data;
};
return data;
}
function _workspaceComponent() {
const data = require("./workspace-component");
_workspaceComponent = function () {
return data;
};
return data;
}
function _mergeConfigConflict() {
const data = require("../exceptions/merge-config-conflict");
_mergeConfigConflict = function () {
return data;
};
return data;
}
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == typeof i ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != typeof t || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != typeof i) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
class WorkspaceComponentLoader {
// cache loaded components
constructor(workspace, logger, dependencyResolver, envs, aspectLoader) {
this.workspace = workspace;
this.logger = logger;
this.dependencyResolver = dependencyResolver;
this.envs = envs;
this.aspectLoader = aspectLoader;
_defineProperty(this, "componentsCache", void 0);
// cache loaded components
/**
* Cache components that loaded from scope (especially for get many for perf improvements)
*/
_defineProperty(this, "scopeComponentsCache", void 0);
/**
* Cache extension list for components. used by get many for perf improvements.
* And to make sure we load extensions first.
*/
_defineProperty(this, "componentsExtensionsCache", void 0);
_defineProperty(this, "componentLoadedSelfAsAspects", void 0);
this.componentsCache = (0, _harmonyModules2().createInMemoryCache)({
maxSize: (0, _harmonyModules2().getMaxSizeForComponents)()
});
this.scopeComponentsCache = (0, _harmonyModules2().createInMemoryCache)({
maxSize: (0, _harmonyModules2().getMaxSizeForComponents)()
});
this.componentsExtensionsCache = (0, _harmonyModules2().createInMemoryCache)({
maxSize: (0, _harmonyModules2().getMaxSizeForComponents)()
});
this.componentLoadedSelfAsAspects = (0, _harmonyModules2().createInMemoryCache)({
maxSize: (0, _harmonyModules2().getMaxSizeForComponents)()
});
}
async getMany(ids, loadOpts, throwOnFailure = true) {
const idsWithoutEmpty = (0, _lodash().compact)(ids);
if (!idsWithoutEmpty.length) {
return {
components: [],
invalidComponents: []
};
}
const callId = Math.floor(Math.random() * 1000); // generate a random callId to be able to identify the call from the logs
this.logger.profileTrace(`getMany-${callId}`);
this.logger.setStatusLine(`loading ${ids.length} component(s)`);
const loadOptsWithDefaults = Object.assign(
// We don't want to load extension or execute the load slot at this step
// we will do it later
// this important for better performance
// We don't want to resolveExtensionsVersions as with get many we call aspect merger merge before update dependencies
// so we will have the correct versions for extensions already and update them after will resolve wrong versions
// in some cases
{
loadExtensions: false,
executeLoadSlot: false,
loadSeedersAsAspects: true,
resolveExtensionsVersions: false
}, loadOpts || {});
const loadOrCached = {
idsToLoad: [],
fromCache: []
};
idsWithoutEmpty.forEach(id => {
const componentFromCache = this.getFromCache(id, loadOptsWithDefaults);
if (componentFromCache) {
loadOrCached.fromCache.push(componentFromCache);
} else {
loadOrCached.idsToLoad.push(id);
}
}, loadOrCached);
const {
components: loadedComponents,
invalidComponents
} = await this.getAndLoadSlotOrdered(loadOrCached.idsToLoad || [], loadOptsWithDefaults, callId);
invalidComponents.forEach(({
err
}) => {
if (throwOnFailure) throw err;
});
const components = (0, _lodash().uniqBy)([...loadedComponents, ...loadOrCached.fromCache], comp => {
return comp.id.toString();
});
// this.logger.clearStatusLine();
components.forEach(comp => {
this.saveInCache(comp, {
loadExtensions: true,
executeLoadSlot: true
});
});
const idsWithEmptyStrs = ids.map(id => id.toString());
const requestedComponents = components.filter(comp => idsWithEmptyStrs.includes(comp.id.toString()) || idsWithEmptyStrs.includes(comp.id.toStringWithoutVersion()));
this.logger.profileTrace(`getMany-${callId}`);
this.logger.clearStatusLine();
return {
components: requestedComponents,
invalidComponents
};
}
async getAndLoadSlotOrdered(ids, loadOpts, callId = 0) {
if (!ids?.length) return {
components: [],
invalidComponents: []
};
const workspaceScopeIdsMap = await this.groupAndUpdateIds(ids);
this.logger.profileTrace('buildLoadGroups');
const groupsToHandle = await this.buildLoadGroups(workspaceScopeIdsMap);
this.logger.profileTrace('buildLoadGroups');
// prefix your command with "BIT_LOG=*" to see the detailed groups
if (process.env.BIT_LOG) {
printGroupsToHandle(groupsToHandle, this.logger);
}
const groupsRes = (0, _lodash().compact)(await (0, _pMapSeries().default)(groupsToHandle, async (group, index) => {
const {
scopeIds,
workspaceIds,
aspects,
core,
seeders,
envs
} = group;
const groupDesc = `getMany-${callId} group ${index + 1}/${groupsToHandle.length} - ${loadGroupToStr(group)}`;
this.logger.profileTrace(groupDesc);
if (!workspaceIds.length && !scopeIds.length) {
throw new Error('getAndLoadSlotOrdered - group has no ids to load');
}
const res = await this.getAndLoadSlot(workspaceIds, scopeIds, _objectSpread(_objectSpread({}, loadOpts), {}, {
core,
seeders,
aspects,
envs
}));
this.logger.profileTrace(groupDesc);
// We don't want to return components that were not asked originally (we do want to load them)
if (!group.seeders) return undefined;
return res;
}));
const finalRes = groupsRes.reduce((acc, curr) => {
return {
components: [...acc.components, ...curr.components],
invalidComponents: [...acc.invalidComponents, ...curr.invalidComponents]
};
}, {
components: [],
invalidComponents: []
});
return finalRes;
}
async buildLoadGroups(workspaceScopeIdsMap) {
const wsIds = Array.from(workspaceScopeIdsMap.workspaceIds.values());
const scopeIds = Array.from(workspaceScopeIdsMap.scopeIds.values());
const allIds = [...wsIds, ...scopeIds];
const groupedByIsCoreEnvs = (0, _lodash().groupBy)(allIds, id => {
return this.envs.isCoreEnv(id.toStringWithoutVersion());
});
const nonCoreEnvs = groupedByIsCoreEnvs.false || [];
await this.populateScopeAndExtensionsCache(nonCoreEnvs, workspaceScopeIdsMap);
const allExtIds = new Map();
nonCoreEnvs.forEach(id => {
const idStr = id.toString();
const fromCache = this.componentsExtensionsCache.get(idStr);
if (!fromCache || !fromCache.extensions) {
return;
}
fromCache.extensions.forEach(ext => {
if (!allExtIds.has(ext.stringId) && ext.newExtensionId) {
allExtIds.set(ext.stringId, ext.newExtensionId);
}
});
});
const allExtCompIds = Array.from(allExtIds.values());
await this.populateScopeAndExtensionsCache(allExtCompIds || [], workspaceScopeIdsMap);
// const allExtIdsStr = allExtCompIds.map((id) => id.toString());
const envsIdsOfWsComps = new Set();
wsIds.forEach(id => {
const idStr = id.toString();
const fromCache = this.componentsExtensionsCache.get(idStr);
if (!fromCache || !fromCache.envId) {
return;
}
const envId = fromCache.envId;
if (envId) {
envsIdsOfWsComps.add(envId);
}
});
const groupedByIsEnvOfWsComps = (0, _lodash().groupBy)(allExtCompIds, id => {
const idStr = id.toString();
const withoutVersion = idStr.split('@')[0];
return envsIdsOfWsComps.has(idStr) || envsIdsOfWsComps.has(withoutVersion);
});
const notEnvOfWsCompsStrs = (groupedByIsEnvOfWsComps.false || []).map(id => id.toString());
const groupedByIsExtOfAnother = (0, _lodash().groupBy)(nonCoreEnvs, id => {
return notEnvOfWsCompsStrs.includes(id.toString());
});
const extIdsFromTheList = (groupedByIsExtOfAnother.true || []).map(id => id.toString());
const extsNotFromTheList = [];
for (const [, id] of allExtIds.entries()) {
if (!extIdsFromTheList.includes(id.toString())) {
extsNotFromTheList.push(id);
}
}
await this.groupAndUpdateIds(extsNotFromTheList, workspaceScopeIdsMap);
const layeredExtFromTheList = this.regroupExtIdsFromTheList(groupedByIsExtOfAnother.true);
const layeredExtGroups = layeredExtFromTheList.map(ids => {
return {
ids,
core: false,
aspects: true,
seeders: true,
envs: false
};
});
const layeredEnvsFromTheList = this.regroupEnvsIdsFromTheList(groupedByIsEnvOfWsComps.true, envsIdsOfWsComps);
const layeredEnvsGroups = layeredEnvsFromTheList.map(ids => {
return {
ids,
core: false,
aspects: true,
seeders: true,
envs: true
};
});
const groupsToHandle = [
// Always load first core envs
{
ids: groupedByIsCoreEnvs.true || [],
core: true,
aspects: true,
seeders: true,
envs: true
},
// { ids: groupedByIsEnvOfWsComps.true || [], core: false, aspects: true, seeders: false, envs: true },
...layeredEnvsGroups, {
ids: extsNotFromTheList || [],
core: false,
aspects: true,
seeders: false,
envs: false
}, ...layeredExtGroups, {
ids: groupedByIsExtOfAnother.false || [],
core: false,
aspects: false,
seeders: true,
envs: false
}];
// This is a special use case mostly for the bit core repo
const envsOfCoreAspectEnv = ['teambit.harmony/envs/core-aspect-env', 'teambit.harmony/envs/core-aspect-env-jest'];
const coreAspectEnvGroup = {
ids: [],
core: true,
aspects: true,
seeders: true,
envs: true
};
layeredEnvsGroups.forEach(group => {
const filteredIds = group.ids.filter(id => envsOfCoreAspectEnv.includes(id.toStringWithoutVersion()));
if (filteredIds.length) {
// @ts-ignore
coreAspectEnvGroup.ids.push(...filteredIds);
}
});
if (coreAspectEnvGroup.ids.length) {
// enter first in the list
groupsToHandle.unshift(coreAspectEnvGroup);
}
// END of bit repo special use case
const groupsByWsScope = groupsToHandle.map(group => {
if (!group.ids?.length) return undefined;
const groupedByWsScope = (0, _lodash().groupBy)(group.ids, id => {
return workspaceScopeIdsMap.workspaceIds.has(id.toString());
});
return {
workspaceIds: groupedByWsScope.true || [],
scopeIds: groupedByWsScope.false || [],
core: group.core,
aspects: group.aspects,
seeders: group.seeders,
envs: group.envs
};
});
return (0, _lodash().compact)(groupsByWsScope);
}
/**
* This function will get a list of envs ids and will regroup them into two groups:
* 1. envs that are envs of envs from the group
* 2. other envs (envs which are just envs of regular components of the workspace)
* For Example:
* envsIds: [ReactEnv, NodeEnv, BitEnv]
* The env of ReactEnv and NodeEnv is BitEnv
* The result will be:
* [ [BitEnv], [ReactEnv, NodeEnv] ]
*
* At the moment this function is not recursive, in the future we might want to make it recursive
* @param envIds
* @param envsIdsOfWsComps
* @returns
*/
regroupEnvsIdsFromTheList(envIds = [], envsIdsOfWsComps) {
const envsOfEnvs = new Set();
envIds.forEach(envId => {
const idStr = envId.toString();
const fromCache = this.componentsExtensionsCache.get(idStr);
if (!fromCache || !fromCache.extensions) {
return;
}
const envOfEnvId = fromCache.envId;
if (envOfEnvId && !envsIdsOfWsComps.has(idStr)) {
envsOfEnvs.add(envOfEnvId);
}
});
const existingEnvsOfEnvs = envIds.filter(id => envsOfEnvs.has(id.toString()) || envsOfEnvs.has(id.toStringWithoutVersion()));
const notExistingEnvsOfEnvs = envIds.filter(id => !envsOfEnvs.has(id.toString()) && !envsOfEnvs.has(id.toStringWithoutVersion()));
return [existingEnvsOfEnvs, notExistingEnvsOfEnvs];
}
regroupExtIdsFromTheList(ids) {
// TODO: implement this function
// this should handle a case when you have:
// compA that has extA and that extA has extB
// in that case we now get the following group:
// ids: [extA, extB]
// while we need extB to be in a different group before extA
return [ids];
}
async getAndLoadSlot(workspaceIds, scopeIds, loadOpts) {
const {
workspaceComponents,
scopeComponents,
invalidComponents
} = await this.getComponentsWithoutLoadExtensions(workspaceIds, scopeIds, loadOpts);
// If we are here it means we are on workspace, in that case we don't want to load
// aspects of scope components as aspects only aspects of workspace components
// const components = workspaceComponents.concat(scopeComponents);
const allExtensions = workspaceComponents.map(component => {
return component.state._consumer.extensions;
});
// Ensure we won't load the same extension many times
// We don't want to ignore version here, as we do want to load different extensions with same id but different versions here
const mergedExtensions = _legacy5().ExtensionDataList.mergeConfigs(allExtensions, false);
const filteredMergeExtensions = mergedExtensions.filter(ext => {
return !loadOpts.idsToNotLoadAsAspects?.includes(ext.stringId);
});
if (loadOpts.loadExtensions) {
this.logger.profileTrace('loadComponentsExtensions');
await this.workspace.loadComponentsExtensions(filteredMergeExtensions);
this.logger.profileTrace('loadComponentsExtensions');
}
let wsComponentsWithAspects = workspaceComponents;
// if (loadOpts.seeders) {
this.logger.profileTrace('executeLoadSlot');
wsComponentsWithAspects = await (0, _toolboxPromise().pMapPool)(workspaceComponents, component => this.executeLoadSlot(component), {
concurrency: (0, _harmonyModules().concurrentComponentsLimit)()
});
this.logger.profileTrace('executeLoadSlot');
await this.warnAboutMisconfiguredEnvs(wsComponentsWithAspects);
// }
const withAspects = wsComponentsWithAspects.concat(scopeComponents);
// It's important to load the workspace components as aspects here
// otherwise the envs from the workspace won't be loaded at time
// so we will get wrong dependencies from component who uses envs from the workspace
this.logger.profileTrace('loadCompsAsAspects');
if (loadOpts.loadSeedersAsAspects || loadOpts.core && loadOpts.aspects) {
await this.loadCompsAsAspects(workspaceComponents.concat(scopeComponents), {
loadApps: true,
loadEnvs: true,
loadAspects: loadOpts.aspects,
core: loadOpts.core,
seeders: loadOpts.seeders,
idsToNotLoadAsAspects: loadOpts.idsToNotLoadAsAspects
});
}
this.logger.profileTrace('loadCompsAsAspects');
return {
components: withAspects,
invalidComponents
};
}
// TODO: this is similar to scope.main.runtime loadCompAspects func, we should merge them.
async loadCompsAsAspects(components, opts = {
loadApps: true,
loadEnvs: true,
loadAspects: true
}) {
const aspectIds = [];
components.forEach(component => {
const firstTimeToLoad = this.componentLoadedSelfAsAspects.get(component.id.toString()) === undefined;
const excluded = opts.idsToNotLoadAsAspects?.includes(component.id.toString());
const isCore = this.aspectLoader.isCoreAspect(component.id.toStringWithoutVersion());
const alreadyLoaded = this.aspectLoader.isAspectLoaded(component.id.toString());
const skipLoading = excluded || isCore || alreadyLoaded || !firstTimeToLoad;
if (skipLoading) {
return;
}
const idStr = component.id.toString();
const appData = component.state.aspects.get('teambit.harmony/application');
if (opts.loadApps && appData?.data?.appName) {
aspectIds.push(idStr);
this.componentLoadedSelfAsAspects.set(idStr, true);
}
const envsData = component.state.aspects.get(_envs().EnvsAspect.id);
if (opts.loadEnvs && (envsData?.data?.services || envsData?.data?.self || envsData?.data?.type === 'env')) {
aspectIds.push(idStr);
this.componentLoadedSelfAsAspects.set(idStr, true);
}
if (opts.loadAspects && envsData?.data?.type === 'aspect') {
aspectIds.push(idStr);
this.componentLoadedSelfAsAspects.set(idStr, true);
}
});
if (!aspectIds.length) return;
try {
await this.workspace.loadAspects(aspectIds, true, 'self loading aspects', {
useScopeAspectsCapsule: true
});
} catch (err) {
this.logger.warn(`failed loading components as aspects for components ${aspectIds.join(', ')}`, err);
// we ignore that errors at the moment
}
}
async populateScopeAndExtensionsCache(ids, workspaceScopeIdsMap) {
return (0, _pMapSeries().default)(ids, async id => {
const idStr = id.toString();
let componentFromScope;
if (!this.scopeComponentsCache.has(idStr)) {
try {
// Do not import automatically if it's missing, it will throw an error later
componentFromScope = await this.workspace.scope.get(id, undefined, false);
if (componentFromScope) {
this.scopeComponentsCache.set(idStr, componentFromScope);
}
// This is fine here, as it will be handled later in the process
} catch (err) {
const wsAspectLoader = this.workspace.getWorkspaceAspectsLoader();
wsAspectLoader.throwWsJsoncAspectNotFoundError(err);
this.logger.warn(`populateScopeAndExtensionsCache - failed loading component ${idStr} from scope`, err);
}
}
if (!this.componentsExtensionsCache.has(idStr) && workspaceScopeIdsMap.workspaceIds.has(idStr)) {
componentFromScope = componentFromScope || this.scopeComponentsCache.get(idStr);
const {
extensions,
errors,
envId
} = await this.workspace.componentExtensions(id, componentFromScope, undefined, {
loadExtensions: false
});
this.componentsExtensionsCache.set(idStr, {
extensions,
errors,
envId
});
}
});
}
async warnAboutMisconfiguredEnvs(components) {
const allIds = (0, _lodash().uniq)(components.map(component => this.envs.getEnvId(component)));
return Promise.all(allIds.map(envId => this.workspace.warnAboutMisconfiguredEnv(envId)));
}
async groupAndUpdateIds(ids, existingGroups) {
const result = existingGroups || {
scopeIds: new Map(),
workspaceIds: new Map()
};
await Promise.all(ids.map(async componentId => {
const inWs = await this.isInWsIncludeDeleted(componentId);
if (!inWs) {
result.scopeIds.set(componentId.toString(), componentId);
return undefined;
}
const resolvedVersions = this.resolveVersion(componentId);
result.workspaceIds.set(resolvedVersions.toString(), resolvedVersions);
return undefined;
}));
return result;
}
async isInWsIncludeDeleted(componentId) {
const nonDeletedWsIds = this.workspace.listIds();
const deletedWsIds = await this.workspace.locallyDeletedIds();
const allWsIds = nonDeletedWsIds.concat(deletedWsIds);
const inWs = allWsIds.find(id => id.isEqual(componentId, {
ignoreVersion: !componentId.hasVersion()
}));
return !!inWs;
}
async getComponentsWithoutLoadExtensions(workspaceIds, scopeIds, loadOpts) {
const invalidComponents = [];
const errors = [];
const loadOptsWithDefaults = Object.assign(
// We don't want to load extension or execute the load slot at this step
// we will do it later
// this important for better performance
// We don't want to store deps in fs cache, as at this point extensions are not loaded yet
// so it might save a wrong deps into the cache
{
loadExtensions: false,
executeLoadSlot: false
}, loadOpts || {});
const idsIndex = {};
workspaceIds.forEach(id => {
idsIndex[id.toString()] = id;
});
this.logger.profileTrace('consumer.loadComponents');
const {
components: legacyComponents,
invalidComponents: legacyInvalidComponents,
removedComponents
} = await this.workspace.consumer.loadComponents(_componentId().ComponentIdList.fromArray(workspaceIds), false, loadOptsWithDefaults);
this.logger.profileTrace('consumer.loadComponents');
const allLegacyComponents = legacyComponents.concat(removedComponents);
legacyInvalidComponents.forEach(invalidComponent => {
const entry = {
id: idsIndex[invalidComponent.id.toString()],
err: invalidComponent.error
};
if (_legacy2().ConsumerComponent.isComponentInvalidByErrorType(invalidComponent.error)) {
invalidComponents.push(entry);
return;
}
if (this.isComponentNotExistsError(invalidComponent.error) || invalidComponent.error instanceof _legacy2().ComponentNotFoundInPath) {
errors.push(entry);
}
});
const getWithCatch = (id, legacyComponent) => {
return this.get(id, legacyComponent, undefined, undefined, loadOptsWithDefaults).catch(err => {
if (_legacy2().ConsumerComponent.isComponentInvalidByErrorType(err)) {
invalidComponents.push({
id,
err
});
return undefined;
}
if (this.isComponentNotExistsError(err) || err instanceof _legacy2().ComponentNotFoundInPath) {
errors.push({
id,
err
});
return undefined;
}
throw err;
});
};
// await this.getConsumerComponent(id, loadOpts)
const componentsP = (0, _pMap().default)(allLegacyComponents, legacyComponent => {
// const componentsP = Promise.all(
// allLegacyComponents.map(async (legacyComponent) => {
let id = idsIndex[legacyComponent.id.toString()];
if (!id) {
const withoutVersion = idsIndex[legacyComponent.id.toStringWithoutVersion()] || legacyComponent.id;
if (withoutVersion) {
id = withoutVersion.changeVersion(legacyComponent.id.version);
idsIndex[legacyComponent.id.toString()] = id;
}
}
return getWithCatch(id, legacyComponent);
}, {
concurrency: (0, _harmonyModules().concurrentComponentsLimit)()
});
errors.forEach(err => {
this.logger.console(`failed loading component ${err.id.toString()}, see full error in debug.log file`);
this.logger.warn(`failed loading component ${err.id.toString()}`, err.err);
});
const components = (0, _lodash().compact)(await componentsP);
// Here we need to load many, otherwise we will get wrong overrides dependencies data
// as when loading the next batch of components (next group) we won't have the envs loaded
try {
const scopeComponents = await this.workspace.scope.getMany(scopeIds);
// We don't want to load envs as part of this step, they will be loaded later
// const scopeComponents = await this.workspace.scope.loadMany(scopeIds, undefined, {
// loadApps: false,
// loadEnvs: true,
// loadCompAspects: false,
// });
return {
workspaceComponents: components,
scopeComponents,
invalidComponents
};
} catch (err) {
const wsAspectLoader = this.workspace.getWorkspaceAspectsLoader();
wsAspectLoader.throwWsJsoncAspectNotFoundError(err);
throw err;
}
}
async getInvalid(ids) {
const idsWithoutEmpty = (0, _lodash().compact)(ids);
const errors = [];
const longProcessLogger = this.logger.createLongProcessLogger('loading components', ids.length);
await (0, _pMapSeries().default)(idsWithoutEmpty, async id => {
longProcessLogger.logProgress(id.toString());
try {
await this.workspace.consumer.loadComponent(id);
} catch (err) {
if (_legacy2().ConsumerComponent.isComponentInvalidByErrorType(err)) {
errors.push({
id,
err
});
return;
}
throw err;
}
});
return errors;
}
async get(componentId, legacyComponent, useCache = true, storeInCache = true, loadOpts, getOpts = {
resolveIdVersion: true
}) {
const loadOptsWithDefaults = Object.assign({
loadExtensions: true,
executeLoadSlot: true
}, loadOpts || {});
const id = getOpts?.resolveIdVersion ? this.resolveVersion(componentId) : componentId;
const fromCache = this.getFromCache(id, loadOptsWithDefaults);
if (fromCache && useCache) {
return fromCache;
}
let consumerComponent = legacyComponent;
const inWs = await this.isInWsIncludeDeleted(id);
if (inWs && !consumerComponent) {
consumerComponent = await this.getConsumerComponent(id, loadOptsWithDefaults);
}
// in case of out-of-sync, the id may changed during the load process
const updatedId = consumerComponent ? consumerComponent.id : id;
const component = await this.loadOne(updatedId, consumerComponent, loadOptsWithDefaults);
if (storeInCache) {
this.addMultipleEnvsIssueIfNeeded(component); // it's in storeInCache block, otherwise, it wasn't fully loaded
this.saveInCache(component, loadOptsWithDefaults);
}
return component;
}
async getIfExist(componentId) {
try {
return await this.get(componentId);
} catch (err) {
if (this.isComponentNotExistsError(err)) {
return undefined;
}
throw err;
}
}
resolveVersion(componentId) {
const bitIdWithVersion = (0, _legacy().getLatestVersionNumber)(this.workspace.consumer.bitmapIdsFromCurrentLaneIncludeRemoved, componentId);
const id = bitIdWithVersion.version ? componentId.changeVersion(bitIdWithVersion.version) : componentId;
return id;
}
addMultipleEnvsIssueIfNeeded(component) {
const envs = this.envs.getAllEnvsConfiguredOnComponent(component);
const envIds = (0, _lodash().uniq)(envs.map(env => env.id));
if (envIds.length < 2) {
return;
}
component.state.issues.getOrCreate(_componentIssues().IssuesClasses.MultipleEnvs).data = envIds;
}
clearCache() {
this.componentsCache.deleteAll();
this.scopeComponentsCache.deleteAll();
this.componentsExtensionsCache.deleteAll();
this.componentLoadedSelfAsAspects.deleteAll();
}
clearComponentCache(id) {
const idStr = id.toString();
const cachesToClear = [this.componentsCache, this.scopeComponentsCache, this.componentsExtensionsCache, this.componentLoadedSelfAsAspects];
cachesToClear.forEach(cache => {
for (const cacheKey of cache.keys()) {
if (cacheKey === idStr || cacheKey.startsWith(`${idStr}:`)) {
cache.delete(cacheKey);
}
}
});
}
async loadOne(id, consumerComponent, loadOpts) {
const idStr = id.toString();
const componentFromScope = this.scopeComponentsCache.has(idStr) ? this.scopeComponentsCache.get(idStr) : await this.workspace.scope.get(id);
if (!consumerComponent) {
if (!componentFromScope) throw new (_legacy3().MissingBitMapComponent)(id.toString());
return componentFromScope;
}
const extErrorsFromCache = this.componentsExtensionsCache.has(idStr) ? this.componentsExtensionsCache.get(idStr) : undefined;
const {
extensions,
errors
} = extErrorsFromCache || (await this.workspace.componentExtensions(id, componentFromScope, undefined, {
loadExtensions: loadOpts?.loadExtensions
}));
if (errors?.some(err => err instanceof _mergeConfigConflict().MergeConfigConflict)) {
consumerComponent.issues.getOrCreate(_componentIssues().IssuesClasses.MergeConfigHasConflict).data = true;
}
// temporarily mutate consumer component extensions until we remove all direct access from legacy to extensions data
// TODO: remove this once we remove all direct access from legacy code to extensions data
consumerComponent.extensions = extensions;
const state = new (_component().State)(new (_component().Config)(consumerComponent), await this.workspace.createAspectList(extensions), _component().ComponentFS.fromVinyls(consumerComponent.files), consumerComponent.dependencies, consumerComponent);
if (componentFromScope) {
// Removed by @gilad. do not mutate the component from the scope
// componentFromScope.state = state;
// const workspaceComponent = WorkspaceComponent.fromComponent(componentFromScope, this.workspace);
const workspaceComponent = new (_workspaceComponent().WorkspaceComponent)(componentFromScope.id, componentFromScope.head, state, componentFromScope.tags, this.workspace);
if (loadOpts?.executeLoadSlot) {
return this.executeLoadSlot(workspaceComponent, loadOpts);
}
// const updatedComp = await this.executeLoadSlot(workspaceComponent, loadOpts);
return workspaceComponent;
}
const newComponent = this.newComponentFromState(id, state);
if (!loadOpts?.executeLoadSlot) {
return newComponent;
}
return this.executeLoadSlot(newComponent, loadOpts);
}
saveInCache(component, loadOpts) {
const cacheKey = createComponentCacheKey(component.id, loadOpts);
this.componentsCache.set(cacheKey, component);
}
/**
* make sure that not only the id-str match, but also the legacy-id.
* this is needed because the ComponentID.toString() is the same whether or not the legacy-id has
* scope-name, as it includes the defaultScope if the scope is empty.
* as a result, when out-of-sync is happening and the id is changed to include scope-name in the
* legacy-id, the component is the cache has the old id.
*/
getFromCache(componentId, loadOpts) {
const bitIdWithVersion = this.resolveVersion(componentId);
const id = bitIdWithVersion.version ? componentId.changeVersion(bitIdWithVersion.version) : componentId;
const cacheKey = createComponentCacheKey(id, loadOpts);
// If we try to look for the cache without load extensions/ without execute load slot
// but there is an entry after the load, we want to use it as well.
// as we want the component, so if we already loaded it with everything, it's fine.
// this sometime relevant for cases with tiny cache size (during tag)
const cacheKeyWithTrueLoadOpts = createComponentCacheKey(id, {
loadExtensions: true,
executeLoadSlot: true
});
const fromCache = this.componentsCache.get(cacheKey) || this.componentsCache.get(cacheKeyWithTrueLoadOpts);
if (fromCache && fromCache.id.isEqual(id)) {
return fromCache;
}
return undefined;
}
async getConsumerComponent(id, loadOpts = {}) {
loadOpts.originatedFromHarmony = true;
try {
const {
components,
removedComponents
} = await this.workspace.consumer.loadComponents(_componentId().ComponentIdList.fromArray([id]), true, loadOpts);
return components?.[0] || removedComponents?.[0];
} catch (err) {
// don't return undefined for any error. otherwise, if the component is invalid (e.g. main
// file is missing) it returns the model component later unexpectedly, or if it's new, it
// shows MissingBitMapComponent error incorrectly.
if (this.isComponentNotExistsError(err)) {
this.logger.debug(`failed loading component "${id.toString()}" from the workspace due to "${err.name}" error\n${err.message}`);
return undefined;
}
throw err;
}
}
isComponentNotExistsError(err) {
return err instanceof _legacy4().ComponentNotFound || err instanceof _legacy3().MissingBitMapComponent;
}
async executeLoadSlot(component, loadOpts) {
if (component.state._consumer.removed) {
// if it was soft-removed now, the component is not in the FS. loading aspects such as composition ends up with
// errors as they try to read component files from the filesystem.
return component;
}
// Special load events which runs from the workspace but should run from the correct aspect
// TODO: remove this once those extensions dependent on workspace
const envsData = await this.envs.calcDescriptor(component, {
skipWarnings: !!this.workspace.inInstallContext
});
const wsDeps = component.state._consumer.dependencies.dependencies || [];
const modelDeps = component.state._consumer.componentFromModel?.dependencies.dependencies || [];
const merged = _legacy2().Dependencies.merge([wsDeps, modelDeps]);
const envExtendsDeps = merged.get();
// Move to deps resolver main runtime once we switch ws<> deps resolver direction
const policy = await this.dependencyResolver.mergeVariantPolicies(component.config.extensions, component.id, component.state._consumer.files, envExtendsDeps);
const dependenciesList = await this.dependencyResolver.extractDepsFromLegacy(component, policy);
const resolvedEnvJsonc = await this.envs.calculateEnvManifest(component, component.state._consumer.files, envExtendsDeps);
if (resolvedEnvJsonc) {
// @ts-ignore
envsData.resolvedEnvJsonc = resolvedEnvJsonc;
}
const depResolverData = {
packageName: this.dependencyResolver.calcPackageName(component),
dependencies: dependenciesList.serialize(),
policy: policy.serialize(),
componentRangePrefix: this.dependencyResolver.calcComponentRangePrefixByConsumerComponent(component.state._consumer)
};
// Make sure we are adding the envs / deps data first because other on load events might depend on it
await Promise.all([this.upsertExtensionData(component, _envs().EnvsAspect.id, envsData), this.upsertExtensionData(component, _dependencyResolver().DependencyResolverAspect.id, depResolverData)]);
// We are updating the component state with the envs and deps data here, so in case we have other slots that depend on this data
// they will be able to get it, as it's very common use case that during on load someone want to access to the component env for example
const aspectListWithEnvsAndDeps = await this.workspace.createAspectList(component.state.config.extensions);
component.state.aspects = aspectListWithEnvsAndDeps;
const entries = this.workspace.onComponentLoadSlot.toArray();
await (0, _pMapSeries().default)(entries, async ([extension, onLoad]) => {
const data = await onLoad(component, loadOpts);
await this.upsertExtensionData(component, extension, data);
// Update the aspect list to have changes happened during the on load slot (new data added above)
component.state.aspects.upsertEntry(await this.workspace.resolveComponentId(extension), data);
});
return component;
}
newComponentFromState(id, state) {
return new (_workspaceComponent().WorkspaceComponent)(id, null, state, new (_component().TagMap)(), this.workspace);
}
async upsertExtensionData(component, extension, data) {
if (!data) return;
const existingExtension = component.state.config.extensions.findExtension(extension);
if (existingExtension) {
// Only merge top level of extension data
Object.assign(existingExtension.data, data);
return;
}
component.state.config.extensions.push(await this.getDataEntry(extension, data));
}
async getDataEntry(extension, data) {
// TODO: @gilad we need to refactor the extension data entry api.
return new (_legacy5().ExtensionDataEntry)(undefined, undefined, extension, undefined, data);
}
}
exports.WorkspaceComponentLoader = WorkspaceComponentLoader;
function createComponentCacheKey(id, loadOpts) {
const relevantOpts = (0, _lodash().pick)(loadOpts, ['loadExtensions', 'executeLoadSlot', 'loadDocs', 'loadCompositions']);
return `${id.toString()}:${JSON.stringify(sortKeys(relevantOpts ?? {}))}`;
}
function sortKeys(obj) {
return (0, _lodash().fromPairs)(Object.entries(obj).sort(([k1], [k2]) => k1.localeCompare(k2)));
}
function printGroupsToHandle(groupsToHandle, logger) {
groupsToHandle.forEach(group => {
const {
scopeIds,
workspaceIds,
aspects,
core,
seeders,
envs
} = group;
logger.console(`workspace-component-loader ~ groupsToHandle ${JSON.stringify({
scopeIds: scopeIds.map(id => id.toString()),
workspaceIds: workspaceIds.map(id => id.toString()),
aspects,
core,
seeders,
envs
}, null, 2)}`);
});
}
function loadGroupToStr(loadGroup) {
const {
scopeIds,
workspaceIds,
aspects,
core,
seeders,
envs
} = loadGroup;
const attr = [];
if (aspects) attr.push('aspects');
if (core) attr.push('core');
if (seeders) attr.push('seeders');
if (envs) attr.push('envs');
return `workspaceIds: ${workspaceIds.length}, scopeIds: ${scopeIds.length}, (${attr.join('+')})`;
}
//# sourceMappingURL=workspace-component-loader.js.map