@itwin/core-backend
Version:
iTwin.js backend components
230 lines • 9.22 kB
JavaScript
/*---------------------------------------------------------------------------------------------
* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
* See LICENSE.md in the project root for license terms and full copyright notice.
*--------------------------------------------------------------------------------------------*/
/** @packageDocumentation
* @module ECDb
*/
import { DbResult, Guid, OpenMode } from "@itwin/core-bentley";
import { Base64EncodedString } from "@itwin/core-common";
import { SQLiteDb } from "./SQLiteDb";
/** @beta */
export var ChangeUnifierCache;
(function (ChangeUnifierCache) {
/**
* Creates an in-memory cache backed by a `Map`.
* Fast, but may exhaust memory for very large changesets.
* @returns An [[ChangeCache]] backed by an in-memory `Map`.
* @beta
*/
function createInMemoryCache() {
return new InMemoryCache();
}
ChangeUnifierCache.createInMemoryCache = createInMemoryCache;
/**
* Creates a SQLite-backed cache stored in a temporary SQlite database.
* Slower than in-memory but useful in handling large changesets.
* Temporary SQlite database is first created in memory and
* parts of a temporary database might be flushed to disk if the database becomes large or
* if SQLite comes under memory pressure.
* @param bufferedReadInstanceSizeInBytes Read-batch size in bytes (default 10 MB).
* @returns An [[ChangeCache]] backed by a SQLite temp table.
* @beta
*/
function createSqliteBackedCache(bufferedReadInstanceSizeInBytes = 1024 * 1024 * 10) {
return new SqliteBackedCache(bufferedReadInstanceSizeInBytes);
}
ChangeUnifierCache.createSqliteBackedCache = createSqliteBackedCache;
})(ChangeUnifierCache || (ChangeUnifierCache = {}));
// ---------------------------------------------------------------------------
// Private: InMemoryCache
// ---------------------------------------------------------------------------
class InMemoryCache {
_cache = new Map();
get(key) {
return this._cache.get(key);
}
set(key, value) {
// Remove undefined meta keys to keep serialised form compact.
const meta = value.$meta;
if (meta) {
Object.keys(meta).forEach((k) => meta[k] === undefined && delete meta[k]);
}
this._cache.set(key, value);
}
*all() {
for (const key of Array.from(this._cache.keys()).sort()) {
const instance = this._cache.get(key);
if (instance)
yield instance;
}
}
count() {
return this._cache.size;
}
[Symbol.dispose]() {
this._cache.clear();
}
}
// ---------------------------------------------------------------------------
// Private: NativeSqliteBackedInstanceCache
// ---------------------------------------------------------------------------
class SqliteBackedCache {
bufferedReadInstanceSizeInBytes;
_cacheTable = `[${Guid.createValue()}]`;
static defaultBufferSize = 1024 * 1024 * 10; // 10 MB
_db;
constructor(bufferedReadInstanceSizeInBytes = SqliteBackedCache.defaultBufferSize) {
this.bufferedReadInstanceSizeInBytes = bufferedReadInstanceSizeInBytes;
this._db = new SQLiteDb();
this._db.openDb("", { skipFileCheck: true, rawSQLite: true, openMode: OpenMode.ReadWrite }); // creating temp sqlite db https://sqlite.org/inmemorydb.html#:~:text=Temporary%20Databases,under%20the%20default%20SQLite%20configuration.
if (bufferedReadInstanceSizeInBytes <= 0)
throw new Error("bufferedReadInstanceSizeInBytes must be greater than 0");
this.createTempTable();
}
createTempTable() {
this._db.withSqliteStatement(`CREATE TABLE ${this._cacheTable} ([key] text primary key, [value] text)`, (stmt) => {
if (DbResult.BE_SQLITE_DONE !== stmt.step())
throw new Error("unable to create temp cache table");
});
}
get(key) {
return this._db.withPreparedSqliteStatement(`SELECT [value] FROM ${this._cacheTable} WHERE [key]=?`, (stmt) => {
stmt.reset();
stmt.clearBindings();
stmt.bindString(1, key);
if (stmt.step() === DbResult.BE_SQLITE_ROW)
return JSON.parse(stmt.getValueString(0), Base64EncodedString.reviver);
return undefined;
});
}
set(key, value) {
const shallowCopy = Object.assign({}, value);
this._db.withPreparedSqliteStatement(`INSERT INTO ${this._cacheTable} ([key], [value]) VALUES (?, ?) ON CONFLICT ([key]) DO UPDATE SET [value] = [excluded].[value]`, (stmt) => {
stmt.reset();
stmt.clearBindings();
stmt.bindString(1, key);
stmt.bindString(2, JSON.stringify(shallowCopy, Base64EncodedString.replacer));
stmt.step();
});
}
*all() {
const sql = `
SELECT JSON_GROUP_ARRAY(JSON([value]))
FROM (
SELECT [value],
SUM(LENGTH([value])) OVER (ORDER BY [key] ROWS UNBOUNDED PRECEDING) / ${this.bufferedReadInstanceSizeInBytes} AS [bucket]
FROM ${this._cacheTable}
)
GROUP BY [bucket]`;
const stmt = this._db.prepareSqliteStatement(sql);
try {
while (stmt.step() === DbResult.BE_SQLITE_ROW) {
const bucket = JSON.parse(stmt.getValueString(0), Base64EncodedString.reviver);
for (const instance of bucket)
yield instance;
}
}
finally {
stmt[Symbol.dispose]();
}
}
count() {
return this._db.withPreparedSqliteStatement(`SELECT COUNT(*) FROM ${this._cacheTable}`, (stmt) => {
stmt.reset();
if (stmt.step() === DbResult.BE_SQLITE_ROW)
return stmt.getValue(0).getInteger();
return 0;
});
}
[Symbol.dispose]() {
this._db.closeDb();
}
}
// ---------------------------------------------------------------------------
// PartialChangeUnifier
// ---------------------------------------------------------------------------
/**
* Combines partial EC change instances (one per SQLite table row) into complete
* instances that span all tables mapping to a single EC entity.
*
* The merge key is derived from the `instanceKey` and `stage` stored in `$meta.instanceKey` and `$meta.stage`.
*
* **Usage:**
* ```ts
* using reader = ChangesetReader.openFile({ fileName, db });
* using unifier = new PartialChangeUnifier();
* while (reader.step()) {
* unifier.appendFrom(reader);
* }
* for (const instance of unifier.instances) { ... }
* ```
* @beta
*/
export class PartialChangeUnifier {
_cache;
constructor(_cache = new InMemoryCache()) {
this._cache = _cache;
}
/** Releases the underlying cache. */
[Symbol.dispose]() {
this._cache[Symbol.dispose]();
}
/** Number of complete (merged) instances currently accumulated. */
get instanceCount() {
return this._cache.count();
}
/**
* Append partial changes from the current reader row and merge them into the cache.
*
* @param source Any [ChangeSource]($backend) positioned on a valid row.
* @beta
*/
appendFrom(source) {
if (source.op === "Updated") {
if (source.inserted)
this.combine(source.inserted);
if (source.deleted)
this.combine(source.deleted);
}
else if (source.op === "Inserted" && source.inserted) {
this.combine(source.inserted);
}
else if (source.op === "Deleted" && source.deleted) {
this.combine(source.deleted);
}
}
/**
* Iterator over all fully-merged EC change instances.
* @beta
*/
get instances() {
return this._cache.all();
}
// ---------------------------------------------------------------------------
// Private helpers
// ---------------------------------------------------------------------------
buildKey(instance) {
const { instanceKey, stage } = instance.$meta;
return `${instanceKey}-${stage}`.toLowerCase();
}
combine(rhs) {
const key = this.buildKey(rhs);
const lhs = this._cache.get(key);
if (lhs) {
// Merge data fields — rhs wins for any overlapping columns.
const { $meta: _rhsMeta, ...rhsData } = rhs;
Object.assign(lhs, rhsData);
// Accumulate per-table metadata lists.
lhs.$meta.tables = [...lhs.$meta.tables, ...rhs.$meta.tables];
lhs.$meta.changeIndexes = [...lhs.$meta.changeIndexes, ...rhs.$meta.changeIndexes];
// ECInstanceId will be part of changeset fetchedProps for every table, so we should not include multiple of those in the final list
lhs.$meta.changeFetchedPropNames = [...new Set([...lhs.$meta.changeFetchedPropNames, ...rhs.$meta.changeFetchedPropNames])];
this._cache.set(key, lhs);
}
else {
this._cache.set(key, rhs);
}
}
}
//# sourceMappingURL=PartialChangeUnifier.js.map