@tanstack/db
Version:
A reactive client store for building super fast apps on sync
625 lines (624 loc) • 21 kB
JavaScript
import { map, join, filter, reduce, distinct } from "@tanstack/db-ivm";
import { optimizeQuery } from "../optimizer.js";
import { DistinctRequiresSelectError, FnSelectWithGroupByError, HavingRequiresGroupByError, LimitOffsetRequireOrderByError, DuplicateAliasInSubqueryError, UnsupportedFromTypeError, CollectionInputNotFoundError } from "../../errors.js";
import { VIRTUAL_PROP_NAMES } from "../../virtual-props.js";
import { getWhereExpression, IncludesSubquery, Value } from "../ir.js";
import { compileExpression, toBooleanPredicate } from "./evaluators.js";
import { processJoins } from "./joins.js";
import { processGroupBy, containsAggregate } from "./group-by.js";
import { processOrderBy } from "./order-by.js";
import { processSelect } from "./select.js";
const INCLUDES_ROUTING = /* @__PURE__ */ Symbol(`includesRouting`);
function compileQuery(rawQuery, inputs, collections, subscriptions, callbacks, lazySources, optimizableOrderByCollections, setWindowFn, cache = /* @__PURE__ */ new WeakMap(), queryMapping = /* @__PURE__ */ new WeakMap(), parentKeyStream, childCorrelationField) {
const cachedResult = cache.get(rawQuery);
if (cachedResult) {
return cachedResult;
}
validateQueryStructure(rawQuery);
const { optimizedQuery, sourceWhereClauses } = optimizeQuery(rawQuery);
let query = optimizedQuery;
queryMapping.set(query, rawQuery);
mapNestedQueries(query, rawQuery, queryMapping);
const allInputs = { ...inputs };
const aliasToCollectionId = {};
const aliasRemapping = {};
const sources = {};
const {
alias: mainSource,
input: mainInput,
collectionId: mainCollectionId
} = processFrom(
query.from,
allInputs,
collections,
subscriptions,
callbacks,
lazySources,
optimizableOrderByCollections,
setWindowFn,
cache,
queryMapping,
aliasToCollectionId,
aliasRemapping,
sourceWhereClauses
);
sources[mainSource] = mainInput;
let filteredMainInput = mainInput;
if (parentKeyStream && childCorrelationField) {
const childFieldPath = childCorrelationField.path.slice(1);
const childRekeyed = mainInput.pipe(
map(([key, row]) => {
const correlationValue = getNestedValue(row, childFieldPath);
return [correlationValue, [key, row]];
})
);
const joined = childRekeyed.pipe(join(parentKeyStream, `inner`));
filteredMainInput = joined.pipe(
filter(([_correlationValue, [childSide]]) => {
return childSide != null;
}),
map(([correlationValue, [childSide, parentSide]]) => {
const [childKey, childRow] = childSide;
const tagged = { ...childRow, __correlationKey: correlationValue };
if (parentSide != null) {
tagged.__parentContext = parentSide;
}
const effectiveKey = parentSide != null ? `${String(childKey)}::${JSON.stringify(parentSide)}` : childKey;
return [effectiveKey, tagged];
})
);
sources[mainSource] = filteredMainInput;
}
let pipeline = filteredMainInput.pipe(
map(([key, row]) => {
const { __parentContext, ...cleanRow } = row;
const nsRow = { [mainSource]: cleanRow };
if (__parentContext) {
Object.assign(nsRow, __parentContext);
nsRow.__parentContext = __parentContext;
}
const ret = [key, nsRow];
return ret;
})
);
if (query.join && query.join.length > 0) {
pipeline = processJoins(
pipeline,
query.join,
sources,
mainCollectionId,
mainSource,
allInputs,
cache,
queryMapping,
collections,
subscriptions,
callbacks,
lazySources,
optimizableOrderByCollections,
setWindowFn,
rawQuery,
compileQuery,
aliasToCollectionId,
aliasRemapping,
sourceWhereClauses
);
}
if (query.where && query.where.length > 0) {
for (const where of query.where) {
const whereExpression = getWhereExpression(where);
const compiledWhere = compileExpression(whereExpression);
pipeline = pipeline.pipe(
filter(([_key, namespacedRow]) => {
return toBooleanPredicate(compiledWhere(namespacedRow));
})
);
}
}
if (query.fnWhere && query.fnWhere.length > 0) {
for (const fnWhere of query.fnWhere) {
pipeline = pipeline.pipe(
filter(([_key, namespacedRow]) => {
return toBooleanPredicate(fnWhere(namespacedRow));
})
);
}
}
const includesResults = [];
const includesRoutingFns = [];
if (query.select) {
const includesEntries = extractIncludesFromSelect(query.select);
if (includesEntries.length > 0) {
query = { ...query, select: { ...query.select } };
}
for (const { key, subquery } of includesEntries) {
const compiledCorrelation = compileExpression(subquery.correlationField);
let parentKeys;
if (subquery.parentProjection && subquery.parentProjection.length > 0) {
const compiledProjections = subquery.parentProjection.map((ref) => ({
alias: ref.path[0],
field: ref.path.slice(1),
compiled: compileExpression(ref)
}));
parentKeys = pipeline.pipe(
map(([_key, nsRow]) => {
const parentContext = {};
for (const proj of compiledProjections) {
if (!parentContext[proj.alias]) {
parentContext[proj.alias] = {};
}
const value = proj.compiled(nsRow);
let target = parentContext[proj.alias];
for (let i = 0; i < proj.field.length - 1; i++) {
if (!target[proj.field[i]]) {
target[proj.field[i]] = {};
}
target = target[proj.field[i]];
}
target[proj.field[proj.field.length - 1]] = value;
}
return [compiledCorrelation(nsRow), parentContext];
})
);
} else {
parentKeys = pipeline.pipe(
map(
([_key, nsRow]) => [compiledCorrelation(nsRow), null]
)
);
}
parentKeys = parentKeys.pipe(
reduce(
(values) => values.map(([v, mult]) => [v, mult > 0 ? 1 : 0])
)
);
const childQuery = subquery.parentFilters && subquery.parentFilters.length > 0 ? {
...subquery.query,
where: [
...subquery.query.where || [],
...subquery.parentFilters
]
} : subquery.query;
const childResult = compileQuery(
childQuery,
allInputs,
collections,
subscriptions,
callbacks,
lazySources,
optimizableOrderByCollections,
setWindowFn,
cache,
queryMapping,
parentKeys,
subquery.childCorrelationField
);
Object.assign(aliasToCollectionId, childResult.aliasToCollectionId);
Object.assign(aliasRemapping, childResult.aliasRemapping);
includesResults.push({
pipeline: childResult.pipeline,
fieldName: subquery.fieldName,
correlationField: subquery.correlationField,
childCorrelationField: subquery.childCorrelationField,
hasOrderBy: !!(subquery.query.orderBy && subquery.query.orderBy.length > 0),
childCompilationResult: childResult,
parentProjection: subquery.parentProjection,
materialization: subquery.materialization,
scalarField: subquery.scalarField
});
if (subquery.parentProjection && subquery.parentProjection.length > 0) {
const compiledProjs = subquery.parentProjection.map((ref) => ({
alias: ref.path[0],
field: ref.path.slice(1),
compiled: compileExpression(ref)
}));
const compiledCorr = compiledCorrelation;
includesRoutingFns.push({
fieldName: subquery.fieldName,
getRouting: (nsRow) => {
const parentContext = {};
for (const proj of compiledProjs) {
if (!parentContext[proj.alias]) {
parentContext[proj.alias] = {};
}
const value = proj.compiled(nsRow);
let target = parentContext[proj.alias];
for (let i = 0; i < proj.field.length - 1; i++) {
if (!target[proj.field[i]]) {
target[proj.field[i]] = {};
}
target = target[proj.field[i]];
}
target[proj.field[proj.field.length - 1]] = value;
}
return { correlationKey: compiledCorr(nsRow), parentContext };
}
});
} else {
includesRoutingFns.push({
fieldName: subquery.fieldName,
getRouting: (nsRow) => ({
correlationKey: compiledCorrelation(nsRow),
parentContext: null
})
});
}
replaceIncludesInSelect(query.select, key);
}
}
if (query.distinct && !query.fnSelect && !query.select) {
throw new DistinctRequiresSelectError();
}
if (query.fnSelect && query.groupBy && query.groupBy.length > 0) {
throw new FnSelectWithGroupByError();
}
if (query.fnSelect) {
pipeline = pipeline.pipe(
map(([key, namespacedRow]) => {
const selectResults = query.fnSelect(namespacedRow);
return [
key,
{
...namespacedRow,
$selected: selectResults
}
];
})
);
} else if (query.select) {
pipeline = processSelect(pipeline, query.select);
} else {
pipeline = pipeline.pipe(
map(([key, namespacedRow]) => {
const selectResults = !query.join && !query.groupBy ? namespacedRow[mainSource] : namespacedRow;
return [
key,
{
...namespacedRow,
$selected: selectResults
}
];
})
);
}
if (includesRoutingFns.length > 0) {
pipeline = pipeline.pipe(
map(([key, namespacedRow]) => {
const routing = {};
for (const { fieldName, getRouting } of includesRoutingFns) {
routing[fieldName] = getRouting(namespacedRow);
}
namespacedRow.$selected[INCLUDES_ROUTING] = routing;
return [key, namespacedRow];
})
);
}
const groupByMainSource = parentKeyStream ? mainSource : void 0;
if (query.groupBy && query.groupBy.length > 0) {
pipeline = processGroupBy(
pipeline,
query.groupBy,
query.having,
query.select,
query.fnHaving,
mainCollectionId,
groupByMainSource
);
} else if (query.select) {
const hasAggregates = Object.values(query.select).some(
(expr) => expr.type === `agg` || containsAggregate(expr)
);
if (hasAggregates) {
pipeline = processGroupBy(
pipeline,
[],
// Empty group by means single group
query.having,
query.select,
query.fnHaving,
mainCollectionId,
groupByMainSource
);
}
}
if (query.having && (!query.groupBy || query.groupBy.length === 0)) {
const hasAggregates = query.select ? Object.values(query.select).some((expr) => expr.type === `agg`) : false;
if (!hasAggregates) {
throw new HavingRequiresGroupByError();
}
}
if (query.fnHaving && query.fnHaving.length > 0 && (!query.groupBy || query.groupBy.length === 0)) {
for (const fnHaving of query.fnHaving) {
pipeline = pipeline.pipe(
filter(([_key, namespacedRow]) => {
return fnHaving(namespacedRow);
})
);
}
}
if (query.distinct) {
pipeline = pipeline.pipe(distinct(([_key, row]) => row.$selected));
}
if (query.orderBy && query.orderBy.length > 0) {
const includesGroupKeyFn = parentKeyStream && (query.limit !== void 0 || query.offset !== void 0) ? (_key, row) => {
const correlationKey = row?.[mainSource]?.__correlationKey;
const parentContext = row?.__parentContext;
if (parentContext != null) {
return JSON.stringify([correlationKey, parentContext]);
}
return correlationKey;
} : void 0;
const orderedPipeline = processOrderBy(
rawQuery,
pipeline,
query.orderBy,
query.select || {},
collections[mainCollectionId],
optimizableOrderByCollections,
setWindowFn,
query.limit,
query.offset,
includesGroupKeyFn
);
const resultPipeline2 = orderedPipeline.pipe(
map(([key, [row, orderByIndex]]) => {
const raw = row.$selected;
const finalResults = attachVirtualPropsToSelected(
unwrapValue(raw),
row
);
if (parentKeyStream) {
const correlationKey = row[mainSource]?.__correlationKey;
const parentContext = row.__parentContext ?? null;
delete finalResults.__correlationKey;
delete finalResults.__parentContext;
return [
key,
[finalResults, orderByIndex, correlationKey, parentContext]
];
}
return [key, [finalResults, orderByIndex]];
})
);
const result2 = resultPipeline2;
const compilationResult2 = {
collectionId: mainCollectionId,
pipeline: result2,
sourceWhereClauses,
aliasToCollectionId,
aliasRemapping,
includes: includesResults.length > 0 ? includesResults : void 0
};
cache.set(rawQuery, compilationResult2);
return compilationResult2;
} else if (query.limit !== void 0 || query.offset !== void 0) {
throw new LimitOffsetRequireOrderByError();
}
const resultPipeline = pipeline.pipe(
map(([key, row]) => {
const raw = row.$selected;
const finalResults = attachVirtualPropsToSelected(
unwrapValue(raw),
row
);
if (parentKeyStream) {
const correlationKey = row[mainSource]?.__correlationKey;
const parentContext = row.__parentContext ?? null;
delete finalResults.__correlationKey;
delete finalResults.__parentContext;
return [
key,
[finalResults, void 0, correlationKey, parentContext]
];
}
return [key, [finalResults, void 0]];
})
);
const result = resultPipeline;
const compilationResult = {
collectionId: mainCollectionId,
pipeline: result,
sourceWhereClauses,
aliasToCollectionId,
aliasRemapping,
includes: includesResults.length > 0 ? includesResults : void 0
};
cache.set(rawQuery, compilationResult);
return compilationResult;
}
function collectDirectCollectionAliases(query) {
const aliases = /* @__PURE__ */ new Set();
if (query.from.type === `collectionRef`) {
aliases.add(query.from.alias);
}
if (query.join) {
for (const joinClause of query.join) {
if (joinClause.from.type === `collectionRef`) {
aliases.add(joinClause.from.alias);
}
}
}
return aliases;
}
function validateQueryStructure(query, parentCollectionAliases = /* @__PURE__ */ new Set()) {
const currentLevelAliases = collectDirectCollectionAliases(query);
for (const alias of currentLevelAliases) {
if (parentCollectionAliases.has(alias)) {
throw new DuplicateAliasInSubqueryError(
alias,
Array.from(parentCollectionAliases)
);
}
}
const combinedAliases = /* @__PURE__ */ new Set([
...parentCollectionAliases,
...currentLevelAliases
]);
if (query.from.type === `queryRef`) {
validateQueryStructure(query.from.query, combinedAliases);
}
if (query.join) {
for (const joinClause of query.join) {
if (joinClause.from.type === `queryRef`) {
validateQueryStructure(joinClause.from.query, combinedAliases);
}
}
}
}
function processFrom(from, allInputs, collections, subscriptions, callbacks, lazySources, optimizableOrderByCollections, setWindowFn, cache, queryMapping, aliasToCollectionId, aliasRemapping, sourceWhereClauses) {
switch (from.type) {
case `collectionRef`: {
const input = allInputs[from.alias];
if (!input) {
throw new CollectionInputNotFoundError(
from.alias,
from.collection.id,
Object.keys(allInputs)
);
}
aliasToCollectionId[from.alias] = from.collection.id;
return { alias: from.alias, input, collectionId: from.collection.id };
}
case `queryRef`: {
const originalQuery = queryMapping.get(from.query) || from.query;
const subQueryResult = compileQuery(
originalQuery,
allInputs,
collections,
subscriptions,
callbacks,
lazySources,
optimizableOrderByCollections,
setWindowFn,
cache,
queryMapping
);
Object.assign(aliasToCollectionId, subQueryResult.aliasToCollectionId);
Object.assign(aliasRemapping, subQueryResult.aliasRemapping);
const isUserDefinedSubquery = queryMapping.has(from.query);
const subqueryFromAlias = from.query.from.alias;
const isOptimizerCreated = !isUserDefinedSubquery && from.alias === subqueryFromAlias;
if (!isOptimizerCreated) {
for (const [alias, whereClause] of subQueryResult.sourceWhereClauses) {
sourceWhereClauses.set(alias, whereClause);
}
}
const innerAlias = Object.keys(subQueryResult.aliasToCollectionId).find(
(alias) => subQueryResult.aliasToCollectionId[alias] === subQueryResult.collectionId
);
if (innerAlias && innerAlias !== from.alias) {
aliasRemapping[from.alias] = innerAlias;
}
const subQueryInput = subQueryResult.pipeline;
const extractedInput = subQueryInput.pipe(
map((data) => {
const [key, [value, _orderByIndex]] = data;
const unwrapped = unwrapValue(value);
return [key, unwrapped];
})
);
return {
alias: from.alias,
input: extractedInput,
collectionId: subQueryResult.collectionId
};
}
default:
throw new UnsupportedFromTypeError(from.type);
}
}
function isValue(raw) {
return raw instanceof Value || raw && typeof raw === `object` && `type` in raw && raw.type === `val`;
}
function unwrapValue(value) {
return isValue(value) ? value.value : value;
}
function attachVirtualPropsToSelected(selected, row) {
if (!selected || typeof selected !== `object`) {
return selected;
}
let needsMerge = false;
for (const prop of VIRTUAL_PROP_NAMES) {
if (selected[prop] == null && prop in row) {
needsMerge = true;
break;
}
}
if (!needsMerge) {
return selected;
}
for (const prop of VIRTUAL_PROP_NAMES) {
if (selected[prop] == null && prop in row) {
selected[prop] = row[prop];
}
}
return selected;
}
function mapNestedQueries(optimizedQuery, originalQuery, queryMapping) {
if (optimizedQuery.from.type === `queryRef` && originalQuery.from.type === `queryRef`) {
queryMapping.set(optimizedQuery.from.query, originalQuery.from.query);
mapNestedQueries(
optimizedQuery.from.query,
originalQuery.from.query,
queryMapping
);
}
if (optimizedQuery.join && originalQuery.join) {
for (let i = 0; i < optimizedQuery.join.length && i < originalQuery.join.length; i++) {
const optimizedJoin = optimizedQuery.join[i];
const originalJoin = originalQuery.join[i];
if (optimizedJoin.from.type === `queryRef` && originalJoin.from.type === `queryRef`) {
queryMapping.set(optimizedJoin.from.query, originalJoin.from.query);
mapNestedQueries(
optimizedJoin.from.query,
originalJoin.from.query,
queryMapping
);
}
}
}
}
function extractIncludesFromSelect(select) {
const results = [];
for (const [key, value] of Object.entries(select)) {
if (key.startsWith(`__SPREAD_SENTINEL__`)) continue;
if (value instanceof IncludesSubquery) {
results.push({ key, subquery: value });
} else if (isNestedSelectObject(value)) {
assertNoNestedIncludes(value, key);
}
}
return results;
}
function isNestedSelectObject(value) {
return value != null && typeof value === `object` && !Array.isArray(value) && typeof value.type !== `string`;
}
function assertNoNestedIncludes(obj, parentPath) {
for (const [key, value] of Object.entries(obj)) {
if (key.startsWith(`__SPREAD_SENTINEL__`)) continue;
if (value instanceof IncludesSubquery) {
throw new Error(
`Includes subqueries must be at the top level of select(). Found nested includes at "${parentPath}.${key}".`
);
}
if (isNestedSelectObject(value)) {
assertNoNestedIncludes(value, `${parentPath}.${key}`);
}
}
}
function replaceIncludesInSelect(select, key) {
select[key] = new Value(null);
}
function getNestedValue(obj, path) {
let value = obj;
for (const segment of path) {
if (value == null) return value;
value = value[segment];
}
return value;
}
export {
INCLUDES_ROUTING,
compileQuery
};
//# sourceMappingURL=index.js.map