@itwin/core-backend
Version:
iTwin.js backend components
321 lines • 17.9 kB
JavaScript
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
/** @packageDocumentation
* @module iModels
*/
// cspell:ignore BLOCKCACHE
import * as path from "path";
import { NativeLoggerCategory } from "@bentley/imodeljs-native";
import { BeEvent, ChangeSetStatus, Guid, IModelStatus, Logger, LogLevel, OpenMode, StopWatch } from "@itwin/core-bentley";
import { BriefcaseIdValue, IModelError, IModelVersion, } from "@itwin/core-common";
import { BackendLoggerCategory } from "./BackendLoggerCategory";
import { BriefcaseManager } from "./BriefcaseManager";
import { CloudSqlite } from "./CloudSqlite";
import { IModelHost } from "./IModelHost";
import { IModelJsFs } from "./IModelJsFs";
import { SnapshotDb } from "./IModelDb";
import { IModelNative } from "./internal/NativePlatform";
import { _hubAccess, _mockCheckpoint, _nativeDb } from "./internal/Symbols";
const loggerCategory = BackendLoggerCategory.IModelDb;
/** Return value from [[ProgressFunction]].
* @public
*/
export var ProgressStatus;
(function (ProgressStatus) {
/** Continue download. */
ProgressStatus[ProgressStatus["Continue"] = 0] = "Continue";
/** Abort download. */
ProgressStatus[ProgressStatus["Abort"] = 1] = "Abort";
})(ProgressStatus || (ProgressStatus = {}));
/** @internal */
export class Downloads {
static _active = new Map();
static async process(job, fn) {
const jobName = job.request.localFile; // save this, it can change inside call to `fn`!
this._active.set(jobName, job);
try {
return await fn(job);
}
finally {
this._active.delete(jobName);
}
}
static isInProgress(pathName) {
return this._active.get(pathName);
}
static async download(request, downloadFn) {
const pathName = request.localFile;
let job = this.isInProgress(pathName);
if (undefined !== job)
return job.promise;
IModelJsFs.recursiveMkDirSync(path.dirname(pathName));
job = { request };
return job.promise = this.process(job, downloadFn);
}
}
/**
* Utility class for opening V2 checkpoints from cloud containers, and also for downloading them.
* @internal
*/
export class V2CheckpointManager {
static cloudCacheName = "Checkpoints";
static _cloudCache;
static containers = new Map();
/** used by HubMock
* @internal
*/
static [_mockCheckpoint];
static getFolder() {
const cloudCachePath = path.join(BriefcaseManager.cacheDir, V2CheckpointManager.cloudCacheName);
if (!(IModelJsFs.existsSync(cloudCachePath))) {
IModelJsFs.recursiveMkDirSync(cloudCachePath);
}
return cloudCachePath;
}
/* only used by tests that reset the state of the v2CheckpointManager. all dbs should be closed before calling this function. */
static cleanup() {
for (const [_, value] of this.containers.entries()) {
if (value.isConnected)
value.disconnect({ detach: true });
}
CloudSqlite.CloudCaches.dropCache(this.cloudCacheName)?.destroy();
this._cloudCache = undefined;
this.containers.clear();
}
static get cloudCache() {
if (!this._cloudCache) {
let cacheDir = process.env.CHECKPOINT_CACHE_DIR ?? this.getFolder();
// See if there is a daemon running, otherwise use profile directory for cloudCache
if (!(IModelJsFs.existsSync(path.join(cacheDir, "portnumber.bcv"))))
cacheDir = undefined; // no daemon running, use profile directory
this._cloudCache = CloudSqlite.CloudCaches.getCache({ cacheName: this.cloudCacheName, cacheDir, cacheSize: "50G" });
}
return this._cloudCache;
}
/** Member names differ slightly between the V2Checkpoint api and the CloudSqlite api. Add aliases `accessName` for `accountName` and `accessToken` for `sasToken` */
static toCloudContainerProps(from) {
return { ...from, baseUri: `https://${from.accountName}.blob.core.windows.net`, accessToken: from.sasToken, storageType: "azure" };
}
static getContainer(v2Props, checkpoint) {
let container = this.containers.get(v2Props.containerId);
if (undefined === container) {
let tokenFn;
let tokenRefreshSeconds = -1;
// from Rpc, the accessToken in the checkpoint request is from the current user. It is used to request the sasToken for the container and
// the sasToken is checked for refresh (before it expires) on every Rpc request using that user's accessToken. For Ipc, the
// accessToken in the checkpoint request is undefined, and the sasToken is requested by IModelHost.getAccessToken(). It is refreshed on a timer.
if (undefined === checkpoint.accessToken) {
tokenFn = async () => (await IModelHost[_hubAccess].queryV2Checkpoint(checkpoint))?.sasToken ?? "";
tokenRefreshSeconds = undefined;
}
container = CloudSqlite.createCloudContainer({ ...this.toCloudContainerProps(v2Props), tokenRefreshSeconds, logId: process.env.POD_NAME, tokenFn });
this.containers.set(v2Props.containerId, container);
}
return container;
}
static async attach(checkpoint) {
if (this[_mockCheckpoint]) // used by HubMock
return { dbName: this[_mockCheckpoint].mockAttach(checkpoint), container: undefined };
let v2props;
try {
v2props = await IModelHost[_hubAccess].queryV2Checkpoint(checkpoint);
if (!v2props)
throw new Error("no checkpoint");
}
catch (err) {
throw new IModelError(IModelStatus.NotFound, `V2 checkpoint not found: err: ${err.message}`);
}
try {
const container = this.getContainer(v2props, checkpoint);
const dbName = v2props.dbName;
// Use the new token from the recently queried v2 checkpoint just incase the one we currently have is expired.
container.accessToken = v2props.sasToken;
if (!container.isConnected)
container.connect(this.cloudCache);
container.checkForChanges();
const dbStats = container.queryDatabase(dbName);
if (IModelHost.appWorkspace.settings.getBoolean("Checkpoints/prefetch", false)) {
const getPrefetchConfig = (name, defaultVal) => IModelHost.appWorkspace.settings.getNumber(`Checkpoints/prefetch/${name}`, defaultVal);
const minRequests = getPrefetchConfig("minRequests", 3);
const maxRequests = getPrefetchConfig("maxRequests", 6);
const timeout = getPrefetchConfig("timeout", 100);
const maxBlocks = getPrefetchConfig("maxBlocks", 500); // default size of 2GB. Assumes a checkpoint block size of 4MB.
if (dbStats?.totalBlocks !== undefined && dbStats.totalBlocks <= maxBlocks && dbStats.nPrefetch === 0) {
const logPrefetch = async (prefetch) => {
const stopwatch = new StopWatch(`[${container.containerId}/${dbName}]`, true);
Logger.logInfo(loggerCategory, `Starting prefetch of ${stopwatch.description}`, { minRequests, maxRequests, timeout });
const done = await prefetch.promise;
Logger.logInfo(loggerCategory, `Prefetch of ${stopwatch.description} complete=${done} (${stopwatch.elapsedSeconds} seconds)`, { minRequests, maxRequests, timeout });
};
// eslint-disable-next-line @typescript-eslint/no-floating-promises
logPrefetch(CloudSqlite.startCloudPrefetch(container, dbName, { minRequests, nRequests: maxRequests, timeout }));
}
else {
Logger.logInfo(loggerCategory, `Skipping prefetch due to size limits or ongoing prefetch.`, { maxBlocks, numPrefetches: dbStats?.nPrefetch, totalBlocksInDb: dbStats?.totalBlocks, v2props });
}
}
return { dbName, container };
}
catch (e) {
const error = `Cloud cache connect failed: ${e.message}`;
if (checkpoint.expectV2)
Logger.logError(loggerCategory, error);
throw new IModelError(e.errorNumber, error);
}
}
/** @internal */
static async performDownload(job) {
const request = job.request;
if (this[_mockCheckpoint])
this[_mockCheckpoint].mockDownload(request);
else {
const v2props = await IModelHost[_hubAccess].queryV2Checkpoint({ ...request.checkpoint, allowPreceding: true });
if (!v2props)
throw new IModelError(IModelStatus.NotFound, "V2 checkpoint not found");
CheckpointManager.onDownloadV2.raiseEvent(job);
const container = CloudSqlite.createCloudContainer(this.toCloudContainerProps(v2props));
await CloudSqlite.transferDb("download", container, { dbName: v2props.dbName, localFileName: request.localFile, onProgress: request.onProgress });
}
return request.checkpoint.changeset.id;
}
/** Fully download a V2 checkpoint to a local file that can be used to create a briefcase or to work offline.
* @returns a Promise that is resolved when the download completes with the changesetId of the downloaded checkpoint (which will
* be the same as the requested changesetId or the most recent checkpoint before it.)
*/
static async downloadCheckpoint(request) {
return Downloads.download(request, async (job) => this.performDownload(job));
}
}
/** @internal */
export class CheckpointManager {
static onDownloadV2 = new BeEvent();
static getKey(checkpoint) { return `${checkpoint.iModelId}:${checkpoint.changeset.id}`; }
static async doDownload(request) {
// first see if there's a V2 checkpoint available.
const stopwatch = new StopWatch(`[${request.checkpoint.changeset.id}]`, true);
Logger.logInfo(loggerCategory, `Starting download of V2 checkpoint with id ${stopwatch.description}`);
const changesetId = await V2CheckpointManager.downloadCheckpoint(request);
Logger.logInfo(loggerCategory, `Downloaded V2 checkpoint with id ${stopwatch.description} (${stopwatch.elapsedSeconds} seconds)`);
if (changesetId !== request.checkpoint.changeset.id)
Logger.logInfo(loggerCategory, `Downloaded previous v2 checkpoint because requested checkpoint not found.`, { requestedChangesetId: request.checkpoint.changeset.id, iModelId: request.checkpoint.iModelId, changesetId, iTwinId: request.checkpoint.iTwinId });
else
Logger.logInfo(loggerCategory, `Downloaded v2 checkpoint.`, { iModelId: request.checkpoint.iModelId, changesetId: request.checkpoint.changeset.id, iTwinId: request.checkpoint.iTwinId });
return changesetId;
}
static async updateToRequestedVersion(request) {
const checkpoint = request.checkpoint;
const targetFile = request.localFile;
const traceInfo = { iTwinId: checkpoint.iTwinId, iModelId: checkpoint.iModelId, changeset: checkpoint.changeset };
try {
// Open checkpoint for write
const prevLogLevel = Logger.getLevel(NativeLoggerCategory.SQLite) ?? LogLevel.Error; // Get log level before we set it to None.
Logger.setLevel(NativeLoggerCategory.SQLite, LogLevel.None); // Ignores noisy error messages when applying changesets.
const db = SnapshotDb.openForApplyChangesets(targetFile);
const nativeDb = db[_nativeDb];
try {
if (nativeDb.hasPendingTxns()) {
Logger.logWarning(loggerCategory, "Checkpoint with Txns found - deleting them", () => traceInfo);
nativeDb.deleteAllTxns();
}
if (nativeDb.getBriefcaseId() !== BriefcaseIdValue.Unassigned)
nativeDb.resetBriefcaseId(BriefcaseIdValue.Unassigned);
CheckpointManager.validateCheckpointGuids(checkpoint, db);
// Apply change sets if necessary
const currentChangeset = nativeDb.getCurrentChangeset();
if (currentChangeset.id !== checkpoint.changeset.id) {
const accessToken = checkpoint.accessToken;
const toIndex = checkpoint.changeset.index ??
(await IModelHost[_hubAccess].getChangesetFromVersion({ accessToken, iModelId: checkpoint.iModelId, version: IModelVersion.asOfChangeSet(checkpoint.changeset.id) })).index;
await BriefcaseManager.pullAndApplyChangesets(db, { accessToken, toIndex });
}
else {
// make sure the parent changeset index is saved in the file - old versions didn't have it.
currentChangeset.index = checkpoint.changeset.index; // eslint-disable-line @typescript-eslint/no-non-null-assertion
nativeDb.saveLocalValue("parentChangeSet", JSON.stringify(currentChangeset));
}
}
finally {
Logger.setLevel(NativeLoggerCategory.SQLite, prevLogLevel); // Set logging to what it was before we started applying changesets.
db.saveChanges();
db.close();
}
}
catch (error) {
Logger.logError(loggerCategory, "Error downloading checkpoint - deleting it", () => traceInfo);
IModelJsFs.removeSync(targetFile);
if (error.errorNumber === ChangeSetStatus.CorruptedChangeStream || error.errorNumber === ChangeSetStatus.InvalidId || error.errorNumber === ChangeSetStatus.InvalidVersion) {
Logger.logError(loggerCategory, "Detected potential corruption of change sets. Deleting them to enable retries", () => traceInfo);
BriefcaseManager.deleteChangeSetsFromLocalDisk(checkpoint.iModelId);
}
throw error;
}
}
/** Download a checkpoint file from iModelHub into a local file specified in the request parameters. */
static async downloadCheckpoint(request) {
if (this.verifyCheckpoint(request.checkpoint, request.localFile))
return;
if (request.aliasFiles) {
for (const alias of request.aliasFiles) {
if (this.verifyCheckpoint(request.checkpoint, alias)) {
request.localFile = alias;
return;
}
}
}
await this.doDownload(request);
return this.updateToRequestedVersion(request);
}
/** checks a file's dbGuid & iTwinId for consistency, and updates the dbGuid when possible */
static validateCheckpointGuids(checkpoint, snapshotDb) {
const traceInfo = { iTwinId: checkpoint.iTwinId, iModelId: checkpoint.iModelId };
const nativeDb = snapshotDb[_nativeDb];
const dbChangeset = nativeDb.getCurrentChangeset();
const iModelId = Guid.normalize(nativeDb.getIModelId());
if (iModelId !== Guid.normalize(checkpoint.iModelId)) {
if (nativeDb.isReadonly())
throw new IModelError(IModelStatus.ValidationFailed, "iModelId is not properly set up in the checkpoint");
Logger.logWarning(loggerCategory, "iModelId is not properly set up in the checkpoint. Updated checkpoint to the correct iModelId.", () => ({ ...traceInfo, dbGuid: iModelId }));
const iModelIdNormalized = Guid.normalize(checkpoint.iModelId);
nativeDb.setIModelId(iModelIdNormalized);
snapshotDb._iModelId = iModelIdNormalized;
// Required to reset the ChangeSetId because setDbGuid clears the value.
nativeDb.saveLocalValue("ParentChangeSetId", dbChangeset.id);
if (undefined !== dbChangeset.index)
nativeDb.saveLocalValue("parentChangeSet", JSON.stringify(dbChangeset));
}
const iTwinId = Guid.normalize(nativeDb.getITwinId());
if (iTwinId !== Guid.normalize(checkpoint.iTwinId))
throw new IModelError(IModelStatus.ValidationFailed, "iTwinId was not properly set up in the checkpoint");
}
/** @returns true if the file is the checkpoint requested */
static verifyCheckpoint(checkpoint, fileName) {
if (!IModelJsFs.existsSync(fileName))
return false;
const nativeDb = new IModelNative.platform.DgnDb();
try {
nativeDb.openIModel(fileName, OpenMode.Readonly);
}
catch {
return false;
}
const isValid = checkpoint.iModelId === nativeDb.getIModelId() && checkpoint.changeset.id === nativeDb.getCurrentChangeset().id;
nativeDb.closeFile();
if (!isValid)
IModelJsFs.removeSync(fileName);
return isValid;
}
static async toCheckpointProps(args) {
const changeset = args.changeset ?? await IModelHost[_hubAccess].getLatestChangeset({ ...args, accessToken: await IModelHost.getAccessToken() });
return {
iModelId: args.iModelId,
iTwinId: args.iTwinId,
changeset: {
index: changeset.index,
id: changeset.id ?? (await IModelHost[_hubAccess].queryChangeset({ ...args, changeset, accessToken: await IModelHost.getAccessToken() })).id,
},
};
}
}
//# sourceMappingURL=CheckpointManager.js.map