UNPKG

flipper-plugin

Version:

Flipper Desktop plugin SDK and components

913 lines 34.1 kB
"use strict"; /** * Copyright (c) Meta Platforms, Inc. and affiliates. * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. * * @format */ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); exports.DataSourceView = exports.DataSource = exports.createDataSource = void 0; const sortedIndexBy_1 = __importDefault(require("lodash/sortedIndexBy")); const sortedLastIndexBy_1 = __importDefault(require("lodash/sortedLastIndexBy")); const property_1 = __importDefault(require("lodash/property")); const sortBy_1 = __importDefault(require("lodash/sortBy")); const eventemitter3_1 = __importDefault(require("eventemitter3")); // If the dataSource becomes to large, after how many records will we start to drop items? const dropFactor = 0.1; // what is the default maximum amount of records before we start shifting the data set? const defaultLimit = 100 * 1000; // if a shift on a sorted dataset exceeds this tresholds, we assume it is faster to re-sort the entire set, // rather than search and remove the affected individual items const shiftRebuildTreshold = 0.05; const DEFAULT_VIEW_ID = '0'; function createDataSource(initialSet = [], options) { const ds = new DataSource(options?.key, options?.indices); if (options?.limit !== undefined) { ds.limit = options.limit; } initialSet.forEach((value) => ds.append(value)); return ds; } exports.createDataSource = createDataSource; class DataSource { constructor(keyAttribute, secondaryIndices = []) { this.nextId = 0; this._records = []; this._recordsById = new Map(); this._recordsBySecondaryIndex = new Map(); this.idToIndex = new Map(); // if we shift the window, we increase shiftOffset to correct idToIndex results, rather than remapping all values this.shiftOffset = 0; /** * The maximum amount of records this DataSource can have */ this.limit = defaultLimit; this.outputEventEmitter = new eventemitter3_1.default(); this.keyAttribute = keyAttribute; this._secondaryIndices = new Map(secondaryIndices.map((index) => { const sortedKeys = index.slice().sort(); const key = sortedKeys.join(':'); // immediately reserve a map per index this._recordsBySecondaryIndex.set(key, new Map()); return [key, sortedKeys]; })); if (this._secondaryIndices.size !== secondaryIndices.length) { throw new Error(`Duplicate index definition in ${JSON.stringify(secondaryIndices)}`); } this.view = new DataSourceView(this, DEFAULT_VIEW_ID); this.additionalViews = {}; } get size() { return this._records.length; } /** * Returns a defensive copy of the stored records. * This is a O(n) operation! Prefer using .size and .get instead if only a subset is needed. */ records() { return this._records.map(unwrap); } get(index) { return unwrap(this._records[index]); } has(key) { this.assertKeySet(); return this._recordsById.has(key); } getById(key) { this.assertKeySet(); return this._recordsById.get(key); } keys() { this.assertKeySet(); return this._recordsById.keys(); } entries() { this.assertKeySet(); return this._recordsById.entries(); } [Symbol.iterator]() { const self = this; let offset = 0; return { next() { offset++; if (offset > self.size) { return { done: true, value: undefined }; } else { return { value: self._records[offset - 1].value, }; } }, [Symbol.iterator]() { return this; }, }; } secondaryIndicesKeys() { return [...this._secondaryIndices.keys()]; } /** * Returns the index of a specific key in the *records* set. * Returns -1 if the record wansn't found */ getIndexOfKey(key) { this.assertKeySet(); const stored = this.idToIndex.get(key); return stored === undefined ? -1 : stored + this.shiftOffset; } append(value) { if (this._records.length >= this.limit) { // we're full! let's free up some space this.shift(Math.ceil(this.limit * dropFactor)); } if (this.keyAttribute) { const key = this.getKey(value); if (this._recordsById.has(key)) { const existingValue = this._recordsById.get(key); console.warn(`Tried to append value with duplicate key: ${key} (key attribute is ${this.keyAttribute.toString()}). Old/new values:`, existingValue, value); throw new Error(`Duplicate key`); } this._recordsById.set(key, value); this.storeIndexOfKey(key, this._records.length); } this.storeSecondaryIndices(value); const visibleMap = { [DEFAULT_VIEW_ID]: false }; const approxIndexMap = { [DEFAULT_VIEW_ID]: -1 }; Object.keys(this.additionalViews).forEach((viewId) => { visibleMap[viewId] = false; approxIndexMap[viewId] = -1; }); const entry = { value, id: ++this.nextId, visible: visibleMap, approxIndex: approxIndexMap, }; this._records.push(entry); this.emitDataEvent({ type: 'append', entry, }); } /** * Updates or adds a record. Returns `true` if the record already existed. * Can only be used if a key is used. */ upsert(value) { this.assertKeySet(); const key = this.getKey(value); if (this.idToIndex.has(key)) { this.update(this.getIndexOfKey(key), value); return true; } else { this.append(value); return false; } } /** * Replaces an item in the base data collection. * Note that the index is based on the insertion order, and not based on the current view */ update(index, value) { const entry = this._records[index]; const oldValue = entry.value; if (value === oldValue) { return; } const oldVisible = { ...entry.visible }; entry.value = value; if (this.keyAttribute) { const key = this.getKey(value); const currentKey = this.getKey(oldValue); if (currentKey !== key) { const existingIndex = this.getIndexOfKey(key); if (existingIndex !== -1 && existingIndex !== index) { throw new Error(`Trying to insert duplicate key '${key}', which already exist in the collection`); } this._recordsById.delete(currentKey); this.idToIndex.delete(currentKey); } this._recordsById.set(key, value); this.storeIndexOfKey(key, index); } this.removeSecondaryIndices(oldValue); this.storeSecondaryIndices(value); this.emitDataEvent({ type: 'update', entry, oldValue, oldVisible, index, }); } /** * @param index * * Warning: this operation can be O(n) if a key is set */ delete(index) { if (index < 0 || index >= this._records.length) { throw new Error(`Out of bounds: ${index}`); } const entry = this._records.splice(index, 1)[0]; if (this.keyAttribute) { const key = this.getKey(entry.value); this._recordsById.delete(key); this.idToIndex.delete(key); if (index === 0) { // lucky happy case, this is more efficient this.shiftOffset -= 1; } else { // Optimization: this is O(n)! Should be done as an async job this.idToIndex.forEach((keyIndex, key) => { if (keyIndex + this.shiftOffset > index) this.storeIndexOfKey(key, keyIndex - 1); }); } } this.removeSecondaryIndices(entry.value); this.emitDataEvent({ type: 'remove', index, entry, }); } /** * Removes the item with the given key from this dataSource. * Returns false if no record with the given key was found * * Warning: this operation can be O(n) if a key is set */ deleteByKey(keyValue) { this.assertKeySet(); const index = this.getIndexOfKey(keyValue); if (index === -1) { return false; } this.delete(index); return true; } /** * Removes the first N entries. * @param amount */ shift(amount) { amount = Math.min(amount, this._records.length); if (amount === this._records.length) { this.clear(); return; } // increase an offset variable with amount, and correct idToIndex reads / writes with that this.shiftOffset -= amount; // removes the affected records for _records, _recordsById and idToIndex const removed = this._records.splice(0, amount); if (this.keyAttribute) { removed.forEach((entry) => { const key = this.getKey(entry.value); this._recordsById.delete(key); this.idToIndex.delete(key); }); } removed.forEach((entry) => this.removeSecondaryIndices(entry.value)); if (this.view.isSorted && removed.length > 10 && removed.length > shiftRebuildTreshold * this._records.length) { // removing a large amount of items is expensive when doing it sorted, // let's fallback to the async processing of all data instead // MWE: there is a risk here that rebuilding is too blocking, as this might happen // in background when new data arrives, and not explicitly on a user interaction this.rebuild(); } else { this.emitDataEvent({ type: 'shift', entries: removed, amount, }); } } /** * The clear operation removes any records stored, but will keep the current view preferences such as sorting and filtering */ clear() { this._records.splice(0); this._recordsById.clear(); for (const m of this._recordsBySecondaryIndex.values()) { m.clear(); } this.shiftOffset = 0; this.idToIndex.clear(); this.rebuild(); this.emitDataEvent({ type: 'clear' }); } /** * The rebuild function that would support rebuilding multiple views all at once */ rebuild() { this.view.rebuild(); Object.entries(this.additionalViews).forEach(([, dataView]) => { dataView.rebuild(); }); } /** * Returns a fork of this dataSource, that shares the source data with this dataSource, * but has it's own FSRW pipeline, to allow multiple views on the same data */ fork(viewId) { this._records.forEach((entry) => { entry.visible[viewId] = entry.visible[DEFAULT_VIEW_ID]; entry.approxIndex[viewId] = entry.approxIndex[DEFAULT_VIEW_ID]; }); const newView = new DataSourceView(this, viewId); // Refresh the new view so that it has all the existing records. newView.rebuild(); return newView; } /** * Returns a new view of the `DataSource` if there doesn't exist a `DataSourceView` with the `viewId` passed in. * The view will allow different filters and sortings on the `DataSource` which can be helpful in cases * where multiple tables/views are needed. * @param viewId id for the `DataSourceView` * @returns `DataSourceView` that corresponds to the `viewId` */ getAdditionalView(viewId) { if (viewId in this.additionalViews) { return this.additionalViews[viewId]; } this.additionalViews[viewId] = this.fork(viewId); return this.additionalViews[viewId]; } deleteView(viewId) { if (viewId in this.additionalViews) { delete this.additionalViews[viewId]; // TODO: Ideally remove the viewId in the visible and approxIndex of DataView outputs this._records.forEach((entry) => { delete entry.visible[viewId]; delete entry.approxIndex[viewId]; }); } } addDataListener(event, cb) { this.outputEventEmitter.addListener(event, cb); return () => { this.outputEventEmitter.removeListener(event, cb); }; } assertKeySet() { if (!this.keyAttribute) { throw new Error('No key has been set. Records cannot be looked up by key'); } } getKey(value) { this.assertKeySet(); // TODO: Fix this the next time the file is edited. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const key = value[this.keyAttribute]; if ((typeof key === 'string' || typeof key === 'number') && key !== '') { return key; } throw new Error(`Invalid key value: '${key}'`); } storeIndexOfKey(key, index) { // de-normalize the index, so that on later look ups its corrected again this.idToIndex.set(key, index - this.shiftOffset); } emitDataEvent(event) { // Optimization: potentially we could schedule this to happen async, // using a queue, // or only if there is an active view (although that could leak memory) this.view.processEvent(event); Object.entries(this.additionalViews).forEach(([, dataView]) => { dataView.processEvent(event); }); this.outputEventEmitter.emit(event.type, event); } storeSecondaryIndices(value) { for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) { const indexValue = this.getSecondaryIndexValueFromRecord(value, sortedIndex); // maps are already set up in constructor // TODO: Fix this the next time the file is edited. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const m = this._recordsBySecondaryIndex.get(indexKey); const a = m.get(indexValue); if (!a) { // not seen this index value yet m.set(indexValue, [value]); } else { a.push(value); } this.emitDataEvent({ type: 'siNewIndexValue', indexKey: indexValue, value, firstOfKind: !a, }); } } removeSecondaryIndices(value) { for (const [indexKey, sortedIndex] of this._secondaryIndices.entries()) { const indexValue = this.getSecondaryIndexValueFromRecord(value, sortedIndex); // maps are already set up in constructor // TODO: Fix this the next time the file is edited. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const m = this._recordsBySecondaryIndex.get(indexKey); // code belows assumes that we have an entry for this secondary // TODO: Fix this the next time the file is edited. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const a = m.get(indexValue); a.splice(a.indexOf(value), 1); } } /** * Returns all items matching the specified index query. * * Note that the results are unordered, unless * records have not been updated using upsert / update, in that case * insertion order is maintained. * * Example: * `ds.getAllRecordsByIndex({title: 'subit a bug', done: false})` * * If no index has been specified for this exact keyset in the indexQuery (see options.indices), this method will throw * * @param indexQuery * @returns */ getAllRecordsByIndex(indexQuery) { // normalise indexKey, incl sorting const sortedKeys = Object.keys(indexQuery).sort(); const indexKey = sortedKeys.join(':'); const recordsByIndex = this._recordsBySecondaryIndex.get(indexKey); if (!recordsByIndex) { throw new Error(`No index has been defined for the keys ${JSON.stringify(Object.keys(indexQuery))}`); } const indexValue = JSON.stringify( // query object needs reordering and normalised to produce correct indexValue Object.fromEntries(sortedKeys.map((k) => [k, String(indexQuery[k])]))); return recordsByIndex.get(indexValue) ?? []; } /** * Like getAllRecords, but returns the first match only. * @param indexQuery * @returns */ getFirstRecordByIndex(indexQuery) { return this.getAllRecordsByIndex(indexQuery)[0]; } getAllIndexValues(index) { const sortedKeys = index.slice().sort(); const indexKey = sortedKeys.join(':'); const recordsByIndex = this._recordsBySecondaryIndex.get(indexKey); if (!recordsByIndex) { return; } return [...recordsByIndex.keys()]; } getSecondaryIndexValueFromRecord(record, // assumes keys is already ordered keys) { return JSON.stringify(Object.fromEntries(keys.map((k) => [k, String(record[k])]))); } /** * @private */ serialize() { return this.records(); } /** * @private */ deserialize(value) { this.clear(); value.forEach((record) => { this.append(record); }); } } exports.DataSource = DataSource; function unwrap(entry) { return entry?.value; } class DataSourceView { constructor(datasource, viewId) { this.sortBy = undefined; this.reverse = false; this.filter = undefined; this.filterExceptions = undefined; /** * @readonly */ this.windowStart = 0; /** * @readonly */ this.windowEnd = 0; this.outputChangeListeners = new Set(); /** * This is the base view data, that is filtered and sorted, but not reversed or windowed */ this._output = []; this.sortHelper = (a) => this.sortBy ? this.sortBy(a.value) : a.id; this.datasource = datasource; this.viewId = viewId; } get size() { return this._output.length; } get isSorted() { return !!this.sortBy; } get isFiltered() { return !!this.filter; } get isReversed() { return this.reverse; } /** * Returns a defensive copy of the current output. * Sort, filter, reverse and are applied. * Start and end behave like slice, and default to the currently active window. */ output(start = this.windowStart, end = this.windowEnd) { if (this.reverse) { return this._output .slice(this._output.length - end, this._output.length - start) .reverse() .map((e) => e.value); } else { return this._output.slice(start, end).map((e) => e.value); } } getViewIndex(entry) { return this._output.findIndex((x) => x.value === entry); } setWindow(start, end) { this.windowStart = start; this.windowEnd = end; this.notifyAllListeners({ type: 'windowChange', newStart: start, newEnd: end, }); } addListener(listener) { this.outputChangeListeners.add(listener); return () => { this.outputChangeListeners.delete(listener); }; } setSortBy(sortBy) { if (this.sortBy === sortBy) { return; } if (typeof sortBy === 'string' && (!this.sortBy || this.sortBy.sortByKey !== sortBy)) { sortBy = (0, property_1.default)(sortBy); Object.assign(sortBy, { sortByKey: sortBy, }); } this.sortBy = sortBy; this.rebuild(); } setFilter(filter) { if (this.filter !== filter) { this.filter = filter; // Filter exceptions are relevant for one filter only this.filterExceptions = undefined; this.rebuild(); } } /** * Granular control over filters to add one-off exceptions to them. * They allow us to add singular items to table views. * Extremely useful for Bloks Debugger where we have to jump between multiple types of rows that could be filtered out */ setFilterExpections(ids) { this.filterExceptions = ids ? new Set(ids) : undefined; this.rebuild(); } toggleReversed() { this.setReversed(!this.reverse); } setReversed(reverse) { if (this.reverse !== reverse) { this.reverse = reverse; this.notifyReset(this._output.length); } } /** * The reset operation resets any view preferences such as sorting and filtering, but keeps the current set of records. */ reset() { this.sortBy = undefined; this.reverse = false; this.filter = undefined; this.filterExceptions = undefined; this.windowStart = 0; this.windowEnd = 0; this.rebuild(); } normalizeIndex(viewIndex) { return this.reverse ? this._output.length - 1 - viewIndex : viewIndex; } get(viewIndex) { return this._output[this.normalizeIndex(viewIndex)]?.value; } getEntry(viewIndex) { return this._output[this.normalizeIndex(viewIndex)]; } getViewIndexOfEntry(entry) { // Note: this function leverages the fact that entry is an internal structure that is mutable, // so any changes in the entry being moved around etc will be reflected in the original `entry` object, // and we just want to verify that this entry is indeed still the same element, visible, and still present in // the output data set. if (entry.visible[this.viewId] && entry.id === this._output[entry.approxIndex[this.viewId]]?.id) { return this.normalizeIndex(entry.approxIndex[this.viewId]); } return -1; } [Symbol.iterator]() { const self = this; let offset = this.windowStart; return { next() { offset++; if (offset > self.windowEnd || offset > self.size) { return { done: true, value: undefined }; } else { return { value: self.get(offset - 1), }; } }, [Symbol.iterator]() { return this; }, }; } notifyAllListeners(change) { this.outputChangeListeners.forEach((listener) => listener(change)); } notifyItemUpdated(viewIndex) { viewIndex = this.normalizeIndex(viewIndex); if (!this.outputChangeListeners.size || viewIndex < this.windowStart || viewIndex >= this.windowEnd) { return; } this.notifyAllListeners({ type: 'update', index: viewIndex, }); } notifyItemShift(index, delta) { if (!this.outputChangeListeners.size) { return; } let viewIndex = this.normalizeIndex(index); if (this.reverse && delta < 0) { viewIndex -= delta; // we need to correct for normalize already using the new length after applying this change } // Idea: we could add an option to automatically shift the window for before events. this.notifyAllListeners({ type: 'shift', delta, index: viewIndex, newCount: this._output.length, location: viewIndex < this.windowStart ? 'before' : viewIndex >= this.windowEnd ? 'after' : 'in', }); } notifyReset(count) { this.notifyAllListeners({ type: 'reset', newCount: count, }); } /** * @private */ processEvent(event) { const { _output: output, sortBy, filter } = this; switch (event.type) { case 'append': { const { entry } = event; entry.visible[this.viewId] = filter ? filter(entry.value) : true; this.applyFilterExceptions(entry); if (!entry.visible[this.viewId]) { // not in filter? skip this entry return; } if (!sortBy) { // no sorting? insert at the end, or beginning entry.approxIndex[this.viewId] = output.length; output.push(entry); this.notifyItemShift(entry.approxIndex[this.viewId], 1); } else { this.insertSorted(entry); } break; } case 'update': { const { entry } = event; entry.visible[this.viewId] = filter ? filter(entry.value) : true; this.applyFilterExceptions(entry); // short circuit; no view active so update straight away if (!filter && !sortBy) { output[event.index].approxIndex[this.viewId] = event.index; this.notifyItemUpdated(event.index); } else if (!event.oldVisible[this.viewId]) { if (!entry.visible[this.viewId]) { // Done! } else { // insertion, not visible before this.insertSorted(entry); } } else { // Entry was visible previously const existingIndex = this.getSortedIndex(entry, event.oldValue); if (!entry.visible[this.viewId]) { // Remove from output output.splice(existingIndex, 1); this.notifyItemShift(existingIndex, -1); } else { // Entry was and still is visible if (!this.sortBy || this.sortBy(event.oldValue) === this.sortBy(entry.value)) { // Still at same position, so done! this.notifyItemUpdated(existingIndex); } else { // item needs to be moved cause of sorting // possible optimization: if we discover that old and new index would be the same, // despite different sort values, we could still emit only an update instead of two shifts output.splice(existingIndex, 1); this.notifyItemShift(existingIndex, -1); // find new sort index this.insertSorted(entry); } } } break; } case 'remove': { this.processRemoveEvent(event.index, event.entry); break; } case 'shift': { // no sorting? then all items are removed from the start so optimize for that if (!sortBy) { let amount = 0; if (!filter) { amount = event.amount; } else { // if there is a filter, count the visibles and shift those for (let i = 0; i < event.entries.length; i++) if (event.entries[i].visible[this.viewId]) amount++; } output.splice(0, amount); this.notifyItemShift(0, -amount); } else { // we have sorting, so we need to remove item by item // we do this backward, so that approxIndex is more likely to be correct for (let i = event.entries.length - 1; i >= 0; i--) { this.processRemoveEvent(i, event.entries[i]); } } break; } case 'clear': case 'siNewIndexValue': { break; } default: throw new Error('unknown event type'); } } processRemoveEvent(index, entry) { const { _output: output, sortBy, filter } = this; // filter active, and not visible? short circuilt if (!entry.visible[this.viewId]) { return; } // no sorting, no filter? if (!sortBy && !filter) { output.splice(index, 1); this.notifyItemShift(index, -1); } else { // sorting or filter is active, find the actual location const existingIndex = this.getSortedIndex(entry, entry.value); output.splice(existingIndex, 1); this.notifyItemShift(existingIndex, -1); } } /** * Rebuilds the entire view. Typically there should be no need to call this manually * @private */ rebuild() { // Pending on the size, should we batch this in smaller non-blocking steps, // which we update in a double-buffering mechanism, report progress, and swap out when done? // // MWE: 9-3-2020 postponed for now, one massive sort seems fine. It might shortly block, // but that happens only (exception: limit caused shifts) on user interaction at very roughly 1ms per 1000 records. // See also comment below const { sortBy, filter, sortHelper } = this; // copy base array or run filter (with side effecty update of visible) // @ts-ignore prevent making _record public const records = this.datasource._records; let output = filter ? records.filter((entry) => { entry.visible[this.viewId] = filter(entry.value); this.applyFilterExceptions(entry); return entry.visible[this.viewId]; }) : records.slice(); if (sortBy) { // Pending on the size, should we batch this in smaller steps? // The following sorthing method can be taskified, however, // the implementation is 20x slower than a native sort. So for now we stick to a // blocking sort, until we have some more numbers that this is hanging for anyone // const filtered = output; // output = []; // filtered.forEach((entry) => { // const insertionIndex = sortedLastIndexBy(output, entry, sortHelper); // output.splice(insertionIndex, 0, entry); // }); output = (0, sortBy_1.default)(output, sortHelper); // uses array.sort under the hood } // write approx indexes for faster lookup of entries in visible output for (let i = 0; i < output.length; i++) { output[i].approxIndex[this.viewId] = i; } this._output = output; this.notifyReset(output.length); } getSortedIndex(entry, oldValue) { const { _output: output } = this; if (output[entry.approxIndex[this.viewId]] === entry) { // yay! return entry.approxIndex[this.viewId]; } let index = (0, sortedIndexBy_1.default)(output, { value: oldValue, id: -99999, visible: entry.visible, approxIndex: entry.approxIndex, }, this.sortHelper); index--; // the item we are looking for is not necessarily the first one at the insertion index while (output[index] !== entry) { index++; if (index >= output.length) { throw new Error('illegal state: sortedIndex not found'); // sanity check to avoid browser freeze if people mess up with internals } } return index; } insertSorted(entry) { // apply sorting const insertionIndex = (0, sortedLastIndexBy_1.default)(this._output, entry, this.sortHelper); entry.approxIndex[this.viewId] = insertionIndex; this._output.splice(insertionIndex, 0, entry); this.notifyItemShift(insertionIndex, 1); } applyFilterExceptions(entry) { if (this.datasource.keyAttribute && this.filter && this.filterExceptions && !entry.visible[this.viewId]) { const keyValue = entry.value[this.datasource.keyAttribute]; entry.visible[this.viewId] = this.filterExceptions.has(keyValue); } } } exports.DataSourceView = DataSourceView; //# sourceMappingURL=DataSource.js.map