@solid/community-server
Version:
Community Solid Server: an open and modular implementation of the Solid specifications
546 lines • 26.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WrappedIndexedStorage = void 0;
const node_crypto_1 = require("node:crypto");
const global_logger_factory_1 = require("global-logger-factory");
const InternalServerError_1 = require("../../util/errors/InternalServerError");
const NotFoundHttpError_1 = require("../../util/errors/NotFoundHttpError");
const NotImplementedHttpError_1 = require("../../util/errors/NotImplementedHttpError");
const IndexedStorage_1 = require("./IndexedStorage");
/**
* An {@link IndexedStorage} that makes use of 2 {@link KeyValueStorage}s to implement the interface.
* Due to being limited by key/value storages, there are some restrictions on the allowed type definitions:
*
* * There needs to be exactly 1 type with no references to other types.
* * All other types need to have exactly 1 reference to another type.
* * Types can't reference each other to create a cycle of references.
*
* All of the above to create a tree-like structure of references.
* Such a tree is then stored in one of the storages.
* The other storage is used to store all indexes that are used to find the correct tree object when solving queries.
*/
class WrappedIndexedStorage {
logger = (0, global_logger_factory_1.getLoggerFor)(this);
valueStorage;
indexStorage;
/**
* For every type, the keys on which an index tracks the values and which root object they are contained in.
* All types for which a `defineType` call was made will have a key in this object.
* For all types that are not the root, there will always be an index on their ID value.
*/
indexes;
/**
* Keeps track of type validation.
* If true the defined types create a valid structure that can be used.
*/
validDefinition = false;
/**
* The variable in which the root type is stored.
* A separate getter is used to always return the value
* so the potential `undefined` does not have to be taken into account.
*/
rootTypeVar;
/**
* All parent/child relations between all types in the storage,
* including the keys in both types that are used to reference each other.
*/
relations;
constructor(valueStorage, indexStorage) {
this.valueStorage = valueStorage;
this.indexStorage = indexStorage;
this.indexes = {};
this.relations = [];
}
async defineType(type, description) {
if (this.rootTypeVar) {
this.logger.error(`Trying to define new type "${type}" after types were already validated.`);
throw new InternalServerError_1.InternalServerError(`Trying to define new type "${type}" after types were already validated.`);
}
this.validDefinition = false;
let hasParentKey = false;
for (const [key, desc] of Object.entries(description)) {
if (desc.startsWith(`${IndexedStorage_1.INDEX_ID_KEY}:`)) {
if (hasParentKey) {
this.logger.error(`Type definition of ${type} has multiple references, only 1 is allowed.`);
throw new InternalServerError_1.InternalServerError(`Type definition of ${type} has multiple references, only 1 is allowed.`);
}
if (desc.endsWith('[]')) {
this.logger.error(`Type definition of ${type} has array references, which is not allowed.`);
throw new InternalServerError_1.InternalServerError(`Type definition of ${type} has array references, which is not allowed.`);
}
if (desc.endsWith('?')) {
this.logger.error(`Type definition of ${type} has optional references, which is not allowed.`);
throw new InternalServerError_1.InternalServerError(`Type definition of ${type} has optional references, which is not allowed.`);
}
hasParentKey = true;
this.relations.push({
parent: { type: desc.slice(IndexedStorage_1.INDEX_ID_KEY.length + 1), key: `**${type}**` },
child: { type, key },
});
}
}
this.indexes[type] = new Set([IndexedStorage_1.INDEX_ID_KEY]);
}
async createIndex(type, key) {
// An index on the key targeting the parent is the same as having an index on the identifier of that parent.
// Such an index gets created automatically when the type is defined so this can now be ignored.
if (key === this.getParentRelation(type)?.child.key) {
return;
}
const typeIndexes = this.indexes[type];
if (!typeIndexes) {
this.logger.error(`Trying to create index on key ${key} of undefined type ${type}`);
throw new InternalServerError_1.InternalServerError(`Trying to create index on key ${key} of undefined type ${type}`);
}
typeIndexes.add(key);
}
async has(type, id) {
this.validateDefinition(type);
if (type === this.rootType) {
return this.valueStorage.has(id);
}
const result = await this.find(type, { [IndexedStorage_1.INDEX_ID_KEY]: id });
return result.length > 0;
}
async get(type, id) {
this.validateDefinition(type);
const result = await this.find(type, { [IndexedStorage_1.INDEX_ID_KEY]: id });
if (result.length === 0) {
return;
}
return result[0];
}
async create(type, value) {
this.validateDefinition(type);
const id = (0, node_crypto_1.randomUUID)();
const newObj = { ...value, [IndexedStorage_1.INDEX_ID_KEY]: id };
// Add the virtual keys
for (const relation of this.getChildRelations(type)) {
newObj[relation.parent.key] = {};
}
const relation = this.getParentRelation(type);
// No parent relation implies that this is the root type
if (!relation) {
await this.valueStorage.set(id, newObj);
await this.updateTypeIndex(type, id, undefined, newObj);
return this.toTypeObject(type, newObj);
}
// We know this will be a string due to the typing requirements and how the relations object is built
const parentId = newObj[relation.child.key];
const root = await this.getRoot(relation.parent.type, parentId);
if (!root) {
throw new NotFoundHttpError_1.NotFoundHttpError(`Unknown object of type ${relation.parent.type} with ID ${parentId}`);
}
const parentObj = relation.parent.type === this.rootType ?
root :
this.getContainingRecord(root, relation.parent.type, parentId)[parentId];
// Parent relation key could be undefined if the index structure changed
if (!parentObj[relation.parent.key]) {
parentObj[relation.parent.key] = {};
}
parentObj[relation.parent.key][id] = newObj;
await this.valueStorage.set(root[IndexedStorage_1.INDEX_ID_KEY], root);
await this.updateTypeIndex(type, root[IndexedStorage_1.INDEX_ID_KEY], undefined, newObj);
return this.toTypeObject(type, newObj);
}
async set(type, value) {
this.validateDefinition(type);
return this.updateValue(type, value, true);
}
async setField(type, id, key, value) {
this.validateDefinition(type);
return this.updateValue(type, { [IndexedStorage_1.INDEX_ID_KEY]: id, [key]: value }, false);
}
async delete(type, id) {
this.validateDefinition(type);
const root = await this.getRoot(type, id);
if (!root) {
return;
}
let oldObj;
if (type === this.rootType) {
oldObj = root;
await this.valueStorage.delete(id);
}
else {
const objs = this.getContainingRecord(root, type, id);
oldObj = objs[id];
delete objs[id];
await this.valueStorage.set(root[IndexedStorage_1.INDEX_ID_KEY], root);
}
// Updating index of removed type and all children as those are also gone
await this.updateDeepTypeIndex(type, root[IndexedStorage_1.INDEX_ID_KEY], oldObj);
}
async findIds(type, query) {
this.validateDefinition(type);
if (type === this.rootType) {
// Root IDs are the only ones we can get more efficiently.
// For all other types we have to find the full objects anyway.
const indexedRoots = await this.findIndexedRoots(type, query);
if (!Array.isArray(indexedRoots)) {
this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`);
throw new InternalServerError_1.InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`);
}
return indexedRoots;
}
return (await this.solveQuery(type, query)).map((result) => result.id);
}
async find(type, query) {
this.validateDefinition(type);
return (await this.solveQuery(type, query)).map((result) => this.toTypeObject(type, result));
}
async *entries(type) {
this.validateDefinition(type);
const path = this.getPathToType(type);
for await (const [, root] of this.valueStorage.entries()) {
const children = this.getChildObjects(root, path);
yield* children.map((child) => this.toTypeObject(type, child));
}
}
// --------------------------------- OUTPUT HELPERS ---------------------------------
/**
* Converts a {@link VirtualObject} into a {@link TypeObject}.
* To be used when outputting results.
*/
toTypeObject(type, obj) {
const result = { ...obj };
for (const relation of this.getChildRelations(type)) {
delete result[relation.parent.key];
}
return result;
}
// --------------------------------- ROOT HELPERS ---------------------------------
/**
* The root type for this storage.
* Use this instead of rootTypeVar to prevent having to check for `undefined`.
* This value will always be defined if the type definitions have been validated.
*/
get rootType() {
return this.rootTypeVar;
}
/**
* Finds the root object that contains the requested type/id combination.
*/
async getRoot(type, id) {
let rootId;
if (type === this.rootType) {
rootId = id;
}
else {
// We know there always is an index on the identifier key
const indexKey = this.getIndexKey(type, IndexedStorage_1.INDEX_ID_KEY, id);
const indexResult = await this.indexStorage.get(indexKey);
if (!indexResult || indexResult.length !== 1) {
return;
}
rootId = indexResult[0];
}
return this.valueStorage.get(rootId);
}
// --------------------------------- PATH HELPERS ---------------------------------
/**
* Returns the sequence of virtual keys that need to be accessed to reach the given type, starting from the root.
*/
getPathToType(type) {
const result = [];
let relation = this.getParentRelation(type);
while (relation) {
result.unshift(relation.parent.key);
relation = this.getParentRelation(relation.parent.type);
}
return result;
}
/**
* Finds all records that can be found in the given object by following the given path of virtual keys.
*/
getPathRecords(obj, path) {
const record = obj[path[0]];
if (!record) {
// Can occur if virtual keys are missing due to the index structure changing
return [];
}
if (path.length === 1) {
return [record];
}
const subPath = path.slice(1);
return Object.values(record)
.flatMap((child) => this.getPathRecords(child, subPath));
}
/**
* Finds all objects in the provided object that can be found by following the provided path of virtual keys.
*/
getChildObjects(obj, path) {
if (path.length === 0) {
return [obj];
}
return this.getPathRecords(obj, path).flatMap((record) => Object.values(record));
}
/**
* Finds the record in the given object that contains the given type/id combination.
* This function assumes it was already verified through an index that this object contains the given combination.
*/
getContainingRecord(rootObj, type, id) {
const path = this.getPathToType(type);
const records = this.getPathRecords(rootObj, path);
const match = records.find((record) => Boolean(record[id]));
if (!match) {
this.logger.error(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`);
throw new InternalServerError_1.InternalServerError(`Could not find ${type} ${id} in ${this.rootType} ${rootObj.id}`);
}
return match;
}
async updateValue(type, partial, replace) {
const id = partial[IndexedStorage_1.INDEX_ID_KEY];
let root = await this.getRoot(type, id);
if (!root) {
throw new NotFoundHttpError_1.NotFoundHttpError(`Unknown object of type ${type} with ID ${id}`);
}
let oldObj;
let newObj;
const relation = this.getParentRelation(type);
if (relation) {
const objs = this.getContainingRecord(root, type, id);
if (partial[relation.child.key] && objs[id][relation.child.key] !== partial[relation.child.key]) {
this.logger.error(`Trying to modify reference key ${objs[id][relation.child.key]} on "${type}" ${id}`);
throw new NotImplementedHttpError_1.NotImplementedHttpError('Changing reference keys of existing objects is not supported.');
}
oldObj = objs[id];
newObj = (replace ? { ...partial } : { ...oldObj, ...partial });
objs[id] = newObj;
}
else {
oldObj = root;
newObj = (replace ? { ...partial } : { ...oldObj, ...partial });
root = newObj;
}
// Copy over the child relations
for (const childRelation of this.getChildRelations(type)) {
newObj[childRelation.parent.key] = oldObj[childRelation.parent.key];
}
await this.valueStorage.set(root[IndexedStorage_1.INDEX_ID_KEY], root);
await this.updateTypeIndex(type, root[IndexedStorage_1.INDEX_ID_KEY], oldObj, newObj);
}
// --------------------------------- TYPE HELPERS ---------------------------------
/**
* Returns all relations where the given type is the parent.
*/
getChildRelations(type) {
return this.relations.filter((relation) => relation.parent.type === type);
}
/**
* Returns the relation where the given type is the child.
* Will return `undefined` for the root type as that one doesn't have a parent.
*/
getParentRelation(type) {
return this.relations.find((relation) => relation.child.type === type);
}
/**
* Makes sure the defined types fulfill all the requirements necessary for types on this storage.
* Will throw an error if this is not the case.
* This should be called before doing any data interactions.
* Stores success in a variable so future calls are instantaneous.
*/
validateDefinition(type) {
// We can't know if all types are already defined.
// This prevents issues even if the other types together are valid.
if (!this.indexes[type]) {
const msg = `Type "${type}" was not defined. The defineType functions needs to be called before accessing data.`;
this.logger.error(msg);
throw new InternalServerError_1.InternalServerError(msg);
}
if (this.validDefinition) {
return;
}
const rootTypes = new Set();
// `this.indexes` will contain a key for each type as we always have an index on the identifier of a type
for (let indexType of Object.keys(this.indexes)) {
const foundTypes = new Set([indexType]);
// Find path to root from this type, thereby ensuring that there is no cycle.
let relation = this.getParentRelation(indexType);
while (relation) {
indexType = relation.parent.type;
if (foundTypes.has(indexType)) {
const msg = `The following types cyclically reference each other: ${[...foundTypes].join(', ')}`;
this.logger.error(msg);
throw new InternalServerError_1.InternalServerError(msg);
}
foundTypes.add(indexType);
relation = this.getParentRelation(indexType);
}
rootTypes.add(indexType);
}
if (rootTypes.size > 1) {
const msg = `Only one type definition with no references is allowed. Found ${[...rootTypes].join(', ')}`;
this.logger.error(msg);
throw new InternalServerError_1.InternalServerError(msg);
}
this.rootTypeVar = [...rootTypes.values()][0];
// Remove the root index as we don't need it, and it can cause confusion when resolving queries
this.indexes[this.rootTypeVar]?.delete(IndexedStorage_1.INDEX_ID_KEY);
this.validDefinition = true;
}
// --------------------------------- QUERY HELPERS ---------------------------------
/**
* Finds the IDs of all root objects that contain objects of the given type matching the given query
* by making use of the indexes applicable to the keys in the query.
* This function only looks at the keys in the query with primitive values,
* object values in the query referencing parent objects are not considered.
* Similarly, only indexes are used, keys without index are also ignored.
*
* If an array of root IDs is provided as input,
* the result will be an intersection of this array and the found identifiers.
*
* If the result is an empty array, it means that there is no valid identifier matching the query,
* while an `undefined` result means there is no index matching any of the query keys,
* so a result can't be determined.
*/
async findIndexedRoots(type, match, rootIds) {
if (type === this.rootType && match[IndexedStorage_1.INDEX_ID_KEY]) {
// If the input is the root type with a known ID in the query,
// and we have already established that it is not this ID,
// there is no result.
if (rootIds && !rootIds.includes(match[IndexedStorage_1.INDEX_ID_KEY])) {
return [];
}
rootIds = [match[IndexedStorage_1.INDEX_ID_KEY]];
}
const indexIds = [];
for (const [key, value] of Object.entries(match)) {
if (this.indexes[type]?.has(key) && typeof value !== 'undefined') {
// We know value is a string (or boolean/number) since we can't have indexes on fields referencing other objects
indexIds.push(this.getIndexKey(type, key, value));
}
}
if (indexIds.length === 0) {
return rootIds;
}
// Use all indexes found to find matching IDs
const indexResults = await Promise.all(indexIds.map(async (id) => await this.indexStorage.get(id) ?? []));
if (Array.isArray(rootIds)) {
indexResults.push(rootIds);
}
let indexedRoots = indexResults[0];
for (const ids of indexResults.slice(1)) {
indexedRoots = indexedRoots.filter((id) => ids.includes(id));
}
return indexedRoots;
}
/**
* Finds all objects of the given type matching the query.
* The `rootIds` array can be used to restrict the IDs of root objects to look at,
* which is relevant for the recursive calls the function does.
*
* Will throw an error if there is no index that can be used to solve the query.
*/
async solveQuery(type, query, rootIds) {
this.logger.debug(`Executing "${type}" query ${JSON.stringify(query)}. Already found roots ${rootIds?.join(',')}.`);
const indexedRoots = await this.findIndexedRoots(type, query, rootIds);
// All objects of this type that we find through recursive calls
let objs;
// Either find all objects of the type from the found rootIds if the query is a leaf,
// or recursively query the parent object if it is not.
const relation = this.getParentRelation(type);
if (!relation || !query[relation.child.key]) {
// This is a leaf node of the query
if (!Array.isArray(indexedRoots)) {
this.logger.error(`Attempting to execute query without index: ${JSON.stringify(query)}`);
throw new InternalServerError_1.InternalServerError(`Attempting to execute query without index: ${JSON.stringify(query)}`);
}
const pathFromRoot = this.getPathToType(type);
// All objects of this type for all root objects we have
const roots = (await Promise.all(indexedRoots.map(async (id) => {
const root = await this.valueStorage.get(id);
if (!root) {
// Not throwing an error to sort of make server still work if an index is wrong.
this.logger.error(`Data inconsistency: index contains ${this.rootType} with ID ${id}, but this object does not exist.`);
}
return root;
}))).filter((root) => typeof root !== 'undefined');
objs = roots.flatMap((root) => this.getChildObjects(root, pathFromRoot));
}
else {
const subQuery = (typeof query[relation.child.key] === 'string' ?
{ [IndexedStorage_1.INDEX_ID_KEY]: query[relation.child.key] } :
query[relation.child.key]);
// All objects by recursively calling this function on the parent object and extracting all children of this type
objs = (await this.solveQuery(relation.parent.type, subQuery, indexedRoots))
// Parent key field can be undefined if the index structure changed
.flatMap((parentObj) => Object.values(parentObj[relation.parent.key] ?? {}));
}
// For all keys that were not handled recursively: make sure that it matches the found objects
const remainingKeys = Object.keys(query).filter((key) => key !== relation?.child.key || typeof query[key] === 'string');
for (const key of remainingKeys) {
objs = objs.filter((obj) => obj[key] === query[key]);
}
return objs;
}
// --------------------------------- INDEX HELPERS ---------------------------------
/**
* Generate the key used to store the index in the index storage.
*/
getIndexKey(type, key, value) {
return `${encodeURIComponent(type)}/${key === IndexedStorage_1.INDEX_ID_KEY ? '' : `${key}/`}${encodeURIComponent(`${value}`)}`;
}
/**
* Update all indexes for an object of the given type, and all its children.
*/
async updateDeepTypeIndex(type, rootId, oldObj, newObj) {
const promises = [];
promises.push(this.updateTypeIndex(type, rootId, oldObj, newObj));
for (const { parent, child } of this.getChildRelations(type)) {
// `oldRecord` can be undefined if the index structure changed
const oldRecord = oldObj[parent.key] ?? {};
const newRecord = newObj?.[parent.key] ?? {};
const uniqueKeys = new Set([...Object.keys(oldRecord), ...Object.keys(newRecord)]);
for (const key of uniqueKeys) {
promises.push(this.updateDeepTypeIndex(child.type, rootId, oldRecord[key], newRecord[key]));
}
}
await Promise.all(promises);
}
/**
* Updates all indexes for an object of the given type.
*/
async updateTypeIndex(type, rootId, oldObj, newObj) {
const added = [];
const removed = [];
for (const key of this.indexes[type]) {
const oldValue = oldObj?.[key];
const newValue = newObj?.[key];
if (oldValue !== newValue) {
if (typeof oldValue !== 'undefined') {
removed.push({ key, value: oldValue });
}
if (typeof newValue !== 'undefined') {
added.push({ key, value: newValue });
}
}
}
await Promise.all([
...added.map(async ({ key, value }) => this.updateKeyIndex(type, key, value, rootId, true)),
...removed.map(async ({ key, value }) => this.updateKeyIndex(type, key, value, rootId, false)),
]);
}
/**
* Updates the index for a specific key of an object of the given type.
*/
async updateKeyIndex(type, key, value, rootId, add) {
const indexKey = this.getIndexKey(type, key, value);
const indexValues = await this.indexStorage.get(indexKey) ?? [];
this.logger.debug(`Updating index ${indexKey} by ${add ? 'adding' : 'removing'} ${rootId} from ${indexValues.join(',')}`);
if (add) {
if (!indexValues.includes(rootId)) {
indexValues.push(rootId);
}
await this.indexStorage.set(indexKey, indexValues);
}
else {
const updatedValues = indexValues.filter((val) => val !== rootId);
await (updatedValues.length === 0 ?
this.indexStorage.delete(indexKey) :
this.indexStorage.set(indexKey, updatedValues));
}
}
}
exports.WrappedIndexedStorage = WrappedIndexedStorage;
//# sourceMappingURL=WrappedIndexedStorage.js.map