@teambit/export
Version:
370 lines (365 loc) • 17 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.exportManyBareScope = exportManyBareScope;
exports.mergeObjects = mergeObjects;
exports.persistRemotes = persistRemotes;
exports.removePendingDirs = removePendingDirs;
exports.resumeExport = resumeExport;
exports.saveObjects = saveObjects;
exports.validateRemotes = validateRemotes;
function _pMapSeries() {
const data = _interopRequireDefault(require("p-map-series"));
_pMapSeries = function () {
return data;
};
return data;
}
function _lodash() {
const data = require("lodash");
_lodash = function () {
return data;
};
return data;
}
function _componentId() {
const data = require("@teambit/component-id");
_componentId = function () {
return data;
};
return data;
}
function _legacy() {
const data = require("@teambit/legacy.logger");
_legacy = function () {
return data;
};
return data;
}
function _scope() {
const data = require("@teambit/scope.remotes");
_scope = function () {
return data;
};
return data;
}
function _legacy2() {
const data = require("@teambit/legacy.scope");
_legacy2 = function () {
return data;
};
return data;
}
function _objects() {
const data = require("@teambit/objects");
_objects = function () {
return data;
};
return data;
}
function _scope2() {
const data = require("@teambit/scope.remote-actions");
_scope2 = function () {
return data;
};
return data;
}
function _legacy3() {
const data = require("@teambit/legacy.loader");
_legacy3 = function () {
return data;
};
return data;
}
function _toolboxPromise() {
const data = require("@teambit/toolbox.promise.map-pool");
_toolboxPromise = function () {
return data;
};
return data;
}
function _harmonyModules() {
const data = require("@teambit/harmony.modules.concurrency");
_harmonyModules = function () {
return data;
};
return data;
}
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
/**
* ** Legacy and "bit sign" Only **
*
* @TODO there is no real difference between bare scope and a working directory scope - let's adjust terminology to avoid confusions in the future
* saves a component into the objects directory of the remote scope, then, resolves its
* dependencies, saves them as well. Finally runs the build process if needed on an isolated
* environment.
*/
async function exportManyBareScope(scope, objectList) {
_legacy().logger.debugAndAddBreadCrumb('exportManyBareScope', `started with ${objectList.objects.length} objects`);
const mergedIds = await saveObjects(scope, objectList);
_legacy().logger.debugAndAddBreadCrumb('exportManyBareScope', 'will try to importMany in case there are missing dependencies');
const scopeComponentsImporter = scope.scopeImporter;
await scopeComponentsImporter.importManyFromOriginalScopes(mergedIds); // resolve dependencies
_legacy().logger.debugAndAddBreadCrumb('exportManyBareScope', 'successfully ran importMany');
return mergedIds;
}
/**
* save objects into the scope.
*/
async function saveObjects(scope, objectList) {
const bitObjectList = await objectList.toBitObjects();
const objectsNotRequireMerge = bitObjectList.getObjectsNotRequireMerge();
// components and lanes can't be just added, they need to be carefully merged.
const {
mergedIds,
mergedComponentsResults,
mergedLanes,
mergedLaneHistories
} = await mergeObjects(scope, bitObjectList);
const mergedComponents = mergedComponentsResults.map(_ => _.mergedComponent);
const versionObjects = objectsNotRequireMerge.filter(o => o instanceof _objects().Version);
const versionsHistory = await updateVersionHistory(scope, mergedComponents, versionObjects);
const allObjects = [...mergedComponents, ...mergedLanes, ...objectsNotRequireMerge, ...versionsHistory, ...mergedLaneHistories];
scope.objects.validateObjects(true, allObjects);
await scope.objects.writeObjectsToTheFS(allObjects);
_legacy().logger.debug(`export-scope-components.saveObjects, ${allObjects.length} objects were written successfully to the filesystem`);
return mergedIds;
}
/**
* Previously, the VersionHistory was populated during fetch. However, we want the fetch operation to be more efficient
* so we move this logic to the export operation.
* Before version 0.2.22, the Version object didn't have any info about the component-id, so we do update only for
* rebase. For versions that tagged by > 0.2.22, we have the "origin.id" and we know to what component this version
* belongs to.
*/
async function updateVersionHistory(scope, mergedComponents, versionObjects) {
if (!mergedComponents.length) {
// this is important for bit-sign to not try to update VersionHistory and then error due to missing components
_legacy().logger.debug('updateVersionHistory, no components were merged, no need to update VersionHistory');
return [];
}
_legacy().logger.debug(`updateVersionHistory, total components: ${mergedComponents.length}, total versions: ${versionObjects.length}`);
const versionsWithComponentId = versionObjects.filter(obj => obj.origin?.id);
const [versionsWithOrigin, versionWithoutOrigin] = (0, _lodash().partition)(versionsWithComponentId, v => v.origin?.id);
const versionHistoryOfVersionsWithOrigin = await _updateVersionHistoryForVersionsWithOrigin(scope, mergedComponents, versionsWithOrigin);
const versionHistoryOfVersionsWithoutOrigin = await _updateVersionHistoryForVersionsWithoutOrigin(scope, mergedComponents, versionWithoutOrigin);
return [...versionHistoryOfVersionsWithOrigin, ...versionHistoryOfVersionsWithoutOrigin];
}
/**
* In case of rebase (squash / unrelated) where the version history is changed, make the necessary changes in the
* VersionHistory.
* Because previously (bit-version < 0.2.22) we only knew about this from the Version object, and the Version object
* didn't have any info about what the component-id is, we have to iterate all model-components, grab their
* version-history and check whether the version-hash is inside their VersionHistory.
* it's not ideal performance wise. however, in most cases, this rebase is about squashing, and when squashing, it's
* done for the entire lane, so all components need to be updated regardless.
*/
async function _updateVersionHistoryForVersionsWithoutOrigin(scope, mergedComponents, versionWithoutOrigin) {
const mutatedVersionObjects = versionWithoutOrigin.filter(v => v.squashed || v.unrelated);
if (!mutatedVersionObjects.length) return [];
_legacy().logger.debug(`_updateVersionHistoryForVersionsWithoutOrigin, found ${mutatedVersionObjects.length} mutated version`);
const versionsHistory = await Promise.all(mergedComponents.map(async modelComp => modelComp.updateRebasedVersionHistory(scope.objects, mutatedVersionObjects)));
const versionsHistoryNoNull = (0, _lodash().compact)(versionsHistory);
_legacy().logger.debug(`_updateVersionHistoryForVersionsWithoutOrigin, found ${versionsHistoryNoNull.length} versionsHistory to update
${versionsHistoryNoNull.map(v => v.compId.toString()).join(', ')}`);
return versionsHistoryNoNull;
}
async function _updateVersionHistoryForVersionsWithOrigin(scope, mergedComponents, versionObjects) {
if (!versionObjects.length) return [];
_legacy().logger.debug(`_updateVersionHistoryForVersionsWithOrigin, found ${versionObjects.length} versions with origin`);
const componentVersionMap = new Map();
versionObjects.forEach(version => {
const component = mergedComponents.find(c => c.scope === version.origin?.id.scope && c.name === version.origin?.id.name);
if (!component) {
_legacy().logger.error(`updateVersionHistoryIfNeeded, unable to find component for version ${version.hash().toString()}`);
return;
}
const versions = componentVersionMap.get(component) || [];
componentVersionMap.set(component, [...versions, version]);
});
const versionsHistory = await (0, _toolboxPromise().pMapPool)(mergedComponents, async modelComp => {
const versions = componentVersionMap.get(modelComp);
if (!versions || !versions.length) return undefined;
return modelComp.updateVersionHistory(scope.objects, versions);
}, {
concurrency: (0, _harmonyModules().concurrentComponentsLimit)()
});
return (0, _lodash().compact)(versionsHistory);
}
/**
* merge components into the scope.
*
* a component might have multiple versions that some where merged and some were not.
* the BitIds returned here includes the versions that were merged. so it could contain multiple
* ids of the same component with different versions
*/
async function mergeObjects(scope, bitObjectList, throwForMissingDeps = false) {
const components = bitObjectList.getComponents();
const lanesObjects = bitObjectList.getLanes();
const versions = bitObjectList.getVersions();
const lanesHistory = bitObjectList.getLaneHistories();
_legacy().logger.debugAndAddBreadCrumb('export-scope-components.mergeObjects', `Going to merge ${components.length} components, ${lanesObjects.length} lanes`);
const {
mergeResults,
errors
} = lanesObjects.length ? {
mergeResults: [],
errors: []
} // for lanes, no need to merge component objects, the lane is merged later.
: await scope.sources.mergeComponents(components, versions);
const mergeAllLanesResults = await (0, _pMapSeries().default)(lanesObjects, laneObject => scope.sources.mergeLane(laneObject, false, versions, components));
const lanesErrors = mergeAllLanesResults.map(r => r.mergeErrors).flat();
const componentsNeedUpdate = [...errors.filter(result => result instanceof _legacy2().ComponentNeedsUpdate), ...lanesErrors];
const componentsWithConflicts = errors.filter(result => result instanceof _legacy2().MergeConflict);
if (componentsWithConflicts.length || componentsNeedUpdate.length) {
const idsAndVersions = componentsWithConflicts.map(c => ({
id: c.id,
versions: c.versions,
isDeleted: c.isDeleted
}));
const idsAndVersionsWithConflicts = (0, _lodash().sortBy)(idsAndVersions, (0, _lodash().property)('id'));
const idsOfNeedUpdateComps = (0, _lodash().sortBy)(componentsNeedUpdate.map(c => ({
id: c.id,
lane: c.lane,
isDeleted: c.isDeleted
})), (0, _lodash().property)('id'));
scope.objects.clearObjectsFromCache(); // just in case this error is caught. we don't want to persist anything by mistake.
throw new (_legacy2().MergeConflictOnRemote)(idsAndVersionsWithConflicts, idsOfNeedUpdateComps);
}
if (throwForMissingDeps) await throwForMissingLocalDependencies(scope, versions, components, lanesObjects);
const mergedComponents = mergeResults.filter(({
mergedVersions
}) => mergedVersions.length);
const mergedLanesComponents = mergeAllLanesResults.map(r => r.mergeResults).flat().filter(({
mergedVersions
}) => mergedVersions.length);
const mergedComponentsResults = [...mergedComponents, ...mergedLanesComponents];
const getMergedIds = ({
mergedComponent,
mergedVersions
}) => mergedVersions.map(version => mergedComponent.toBitId().changeVersion(version));
const mergedIds = _componentId().ComponentIdList.uniqFromArray(mergedComponentsResults.map(getMergedIds).flat());
const mergedLanes = mergeAllLanesResults.map(r => r.mergeLane);
const mergedLaneHistories = await (0, _pMapSeries().default)(lanesHistory, async laneHistory => {
const existingLaneHistory = await scope.objects.load(laneHistory.hash());
if (existingLaneHistory) {
existingLaneHistory.merge(laneHistory);
return existingLaneHistory;
}
return laneHistory;
});
return {
mergedIds,
mergedComponentsResults,
mergedLanes,
mergedLaneHistories
};
}
/**
* make sure that all local objects were actually transferred into the remote.
* this gets called as part of the export-validate step. it doesn't check for dependencies from
* other scopes, as they'll be retrieved later by the fetch-missing-deps step.
* we can't wait for that step to validate local dependencies because it happens after persisting,
* and we don't want to persist when local dependencies were not exported.
*/
async function throwForMissingLocalDependencies(scope, versions, components, lanes) {
const compsWithHeads = lanes.length ? lanes.map(lane => lane.toBitIds()).flat() : components.map(c => c.toComponentIdWithHead());
await Promise.all(versions.map(async version => {
const originComp = compsWithHeads.find(id => version.hash().toString() === id.version);
if (!originComp) {
// coz if an older version has a missing dep, then, it's fine. (it can easily happen when exporting lane, which
// all old versions are exported)
return;
}
const getOriginCompWithVer = () => {
const compObj = components.find(c => c.toComponentId().isEqualWithoutVersion(originComp));
if (!compObj) return originComp;
const tag = compObj.getTagOfRefIfExists(_objects().Ref.from(originComp.version));
if (tag) return originComp.changeVersion(tag);
return originComp;
};
const depsIds = version.getAllFlattenedDependencies();
await Promise.all(depsIds.map(async depId => {
if (depId.scope !== scope.name) return;
const existingModelComponent = (await scope.getModelComponentIfExist(depId)) || components.find(c => c.toComponentId().isEqualWithoutVersion(depId));
if (!existingModelComponent) {
scope.objects.clearObjectsFromCache(); // just in case this error is caught. we don't want to persist anything by mistake.
throw new (_legacy2().ComponentNotFound)(depId.toString(), getOriginCompWithVer().toString());
}
const versionRef = existingModelComponent.getRef(depId.version);
if (!versionRef) throw new Error(`unable to find Ref/Hash of ${depId.toString()}`);
const objectExist = scope.objects.getCache(versionRef) || (await scope.objects.has(versionRef)) || versions.find(v => v.hash().isEqual(versionRef));
if (!objectExist) {
scope.objects.clearObjectsFromCache(); // just in case this error is caught. we don't want to persist anything by mistake.
throw new (_legacy2().ComponentNotFound)(depId.toString(), getOriginCompWithVer().toString());
}
}));
}));
}
async function validateRemotes(remotes, clientId, isResumingExport = true) {
_legacy3().loader.start('verifying that objects can be merged on the remotes...');
try {
await Promise.all(remotes.map(remote => remote.action(_scope2().ExportValidate.name, {
clientId,
isResumingExport: true
})));
} catch (err) {
_legacy().logger.errorAndAddBreadCrumb('validateRemotes', 'failed validating remotes', {}, err);
if (!isResumingExport) {
// when resuming export, we don't want to delete the pending-objects because some scopes
// have them persisted and some not. we want to persist to all failing scopes.
await removePendingDirs(remotes, clientId);
}
throw err;
}
}
async function persistRemotes(manyObjectsPerRemote, clientId) {
const persistedRemotes = [];
await (0, _pMapSeries().default)(manyObjectsPerRemote, async objectsPerRemote => {
const {
remote
} = objectsPerRemote;
_legacy3().loader.start(`persisting data on the remote "${remote.name}"...`);
const maxRetries = 3;
let succeed = false;
let lastErrMsg = '';
for (let i = 0; i < maxRetries; i += 1) {
try {
// eslint-disable-next-line no-await-in-loop
const exportedIds = await remote.action(_scope2().ExportPersist.name, {
clientId
});
objectsPerRemote.exportedIds = exportedIds;
succeed = true;
break;
} catch (err) {
lastErrMsg = err.message;
_legacy().logger.errorAndAddBreadCrumb('persistRemotes', `failed on remote ${remote.name}, attempt ${i + 1} out of ${maxRetries}`, {}, err);
}
}
if (!succeed) {
throw new (_legacy2().PersistFailed)([remote.name], {
[remote.name]: lastErrMsg
});
}
_legacy().logger.debugAndAddBreadCrumb('persistRemotes', `successfully pushed all ids to the bare-scope ${remote.name}`);
persistedRemotes.push(remote.name);
});
}
async function resumeExport(scope, exportId, remotes) {
const scopeRemotes = await (0, _scope().getScopeRemotes)(scope);
const remotesObj = await Promise.all(remotes.map(r => scopeRemotes.resolve(r)));
const remotesForPersist = remotesObj.map(remote => ({
remote
}));
await validateRemotes(remotesObj, exportId);
await persistRemotes(remotesForPersist, exportId);
return (0, _lodash().compact)(remotesForPersist.map(r => r.exportedIds).flat());
}
async function removePendingDirs(pushedRemotes, clientId) {
await Promise.all(pushedRemotes.map(remote => remote.action(_scope2().RemovePendingDir.name, {
clientId
})));
}
//# sourceMappingURL=export-scope-components.js.map