UNPKG

@netlify/content-engine

Version:
235 lines 9.62 kB
"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.doRunQuery = doRunQuery; exports.compareByKeySuffix = compareByKeySuffix; const iterable_1 = require("../../common/iterable"); const query_1 = require("../../common/query"); const create_index_1 = require("./create-index"); const filter_using_index_1 = require("./filter-using-index"); const redux_1 = require("../../../redux"); const common_1 = require("./common"); const suggest_index_1 = require("./suggest-index"); async function doRunQuery(args) { // Note: Keeping doRunQuery method the only async method in chain for perf const context = createQueryContext(args); const totalCount = async () => runCountOnce({ ...context, limit: undefined, skip: 0 }); if (canUseIndex(context)) { await Promise.all(context.nodeTypeNames.map((typeName) => (0, create_index_1.createIndex)(context, typeName, context.suggestedIndexFields))); return { entries: performIndexScan(context), totalCount }; } return { entries: performFullTableScan(context), totalCount }; } function performIndexScan(context) { const { suggestedIndexFields, sortFields } = context; const filterContext = context.nodeTypeNames.length === 1 ? context : { ...context, skip: 0, limit: typeof context.limit === `undefined` ? undefined : context.skip + context.limit, }; let result = new iterable_1.GatsbyIterable([]); let resultOffset = filterContext.skip; for (const typeName of context.nodeTypeNames) { const indexMetadata = (0, create_index_1.getIndexMetadata)(context, typeName, suggestedIndexFields); if (!needsSorting(context)) { const { nodes, usedSkip } = filterNodes(filterContext, indexMetadata); result = result.concat(nodes); resultOffset = usedSkip; continue; } if (canUseIndexForSorting(indexMetadata, sortFields)) { const { nodes, usedSkip } = filterNodes(filterContext, indexMetadata); // Interleave nodes of different types (not expensive for already sorted chunks) result = result.mergeSorted(nodes, createNodeSortComparator(sortFields)); resultOffset = usedSkip; continue; } // The sad part - unlimited filter + in-memory sort const unlimited = { ...context, skip: 0, limit: undefined }; const { nodes, usedSkip } = filterNodes(unlimited, indexMetadata); const sortedNodes = sortNodesInMemory(context, nodes); resultOffset = usedSkip; result = result.mergeSorted(sortedNodes, createNodeSortComparator(sortFields)); } const { limit, skip = 0 } = context; const actualSkip = skip - resultOffset; if (limit || actualSkip) { result = result.slice(actualSkip, limit ? actualSkip + limit : undefined); } return result; } function runCountOnce(context) { if (typeof context.totalCount === `undefined`) { context.totalCount = runCount(context); } return context.totalCount; } function runCount(context) { let count = 0; if (!needsFiltering(context)) { for (const typeName of context.nodeTypeNames) { count += context.datastore.countNodes(typeName); } return count; } if (!canUseIndex(context)) { for (const typeName of context.nodeTypeNames) { const nodes = completeFiltering(context, new iterable_1.GatsbyIterable(context.datastore.iterateNodesByType(typeName))); for (const _ of nodes) count++; } return count; } for (const typeName of context.nodeTypeNames) { const indexMetadata = (0, create_index_1.getIndexMetadata)(context, typeName, context.suggestedIndexFields); try { count += (0, filter_using_index_1.countUsingIndexOnly)({ ...context, indexMetadata }); } catch (e) { // We cannot reliably count using index - fallback to full iteration :/ for (const _ of filterNodes(context, indexMetadata).nodes) count++; } } return count; } function performFullTableScan(context) { // console.warn(`Fallback to full table scan :/`) const { datastore, nodeTypeNames } = context; let result = new iterable_1.GatsbyIterable([]); for (const typeName of nodeTypeNames) { let nodes = new iterable_1.GatsbyIterable(datastore.iterateNodesByType(typeName)); nodes = completeFiltering(context, nodes); if (needsSorting(context)) { nodes = sortNodesInMemory(context, nodes); result = result.mergeSorted(nodes, createNodeSortComparator(context.sortFields)); } else { result = result.concat(nodes); } } const { limit, skip = 0 } = context; if (limit || skip) { result = result.slice(skip, limit ? skip + limit : undefined); } return result; } function filterNodes(context, indexMetadata) { const { entries, usedQueries, usedSkip } = (0, filter_using_index_1.filterUsingIndex)({ ...context, indexMetadata, reverse: Array.from(context.sortFields.values())[0] === -1, }); const nodes = entries .map(({ value }) => context.datastore.getNode(value)) .filter(Boolean); return { nodes: completeFiltering(context, nodes, usedQueries), usedSkip, }; } /** * Takes intermediate result and applies any remaining filterQueries. * * If result is already fully filtered - simply returns. */ function completeFiltering(context, intermediateResult, usedQueries = new Set()) { const { dbQueries } = context; if (isFullyFiltered(dbQueries, usedQueries)) { return intermediateResult; } // Apply remaining filter operations directly (last resort: slow) const resolvedNodes = redux_1.store.getState().resolvedNodesCache; const filtersToApply = dbQueries .filter((q) => !usedQueries.has(q)) .map((q) => [(0, query_1.dbQueryToDottedField)(q), (0, query_1.getFilterStatement)(q)]); return intermediateResult.filter((node) => { const resolvedFields = resolvedNodes?.get(node.internal.type)?.get(node.id); for (const [dottedField, filter] of filtersToApply) { const tmp = (0, common_1.resolveFieldValue)(dottedField, node, resolvedFields); const value = Array.isArray(tmp) ? tmp : [tmp]; if (value.some((v) => !(0, common_1.matchesFilter)(filter, v))) { // Mimic AND semantics return false; } } return true; }); } function sortNodesInMemory(context, nodes) { // TODO: Sort using index data whenever possible (maybe store data needed for sorting in index values) // TODO: Nodes can be partially sorted by index prefix - we can (and should) exploit this return new iterable_1.GatsbyIterable(() => { const arr = Array.from(nodes); arr.sort(createNodeSortComparator(context.sortFields)); return arr; }); } function createQueryContext(args) { const { queryArgs: { filter, sort, limit, skip = 0 } = {}, firstOnly } = args; return { datastore: args.datastore, databases: args.databases, nodeTypeNames: args.nodeTypeNames, dbQueries: (0, query_1.createDbQueriesFromObject)((0, query_1.prepareQueryArgs)(filter)), sortFields: new Map(sort?.fields.map((field, i) => [field, (0, common_1.isDesc)(sort?.order[i]) ? -1 : 1])), suggestedIndexFields: new Map((0, suggest_index_1.suggestIndex)({ filter, sort })), limit: firstOnly ? 1 : limit, skip, }; } function canUseIndex(context) { return context.suggestedIndexFields.size > 0; } function needsFiltering(context) { return context.dbQueries.length > 0; } function needsSorting(context) { return context.sortFields.size > 0; } /** * Based on assumption that if all sort fields exist in index * then any result received from this index is fully sorted */ function canUseIndexForSorting(index, sortFields) { const indexKeyFields = new Map(index.keyFields); for (const [field, sortOrder] of sortFields) { if (indexKeyFields.get(field) !== sortOrder) { return false; } } return true; } function isFullyFiltered(dbQueries, usedQueries) { return dbQueries.length === usedQueries.size; } function createNodeSortComparator(sortFields) { const resolvedNodesCache = redux_1.store.getState().resolvedNodesCache; return function nodeComparator(a, b) { const resolvedAFields = resolvedNodesCache?.get(a.internal.type)?.get(a.id); const resolvedBFields = resolvedNodesCache?.get(b.internal.type)?.get(b.id); for (const [field, direction] of sortFields) { const valueA = (0, common_1.resolveFieldValue)(field, a, resolvedAFields); const valueB = (0, common_1.resolveFieldValue)(field, b, resolvedBFields); if (valueA > valueB) { return direction === 1 ? 1 : -1; } else if (valueA < valueB) { return direction === 1 ? -1 : 1; } } return 0; }; } function compareByKeySuffix(prefixLength) { return function (a, b) { const aSuffix = a.key.slice(prefixLength); const bSuffix = b.key.slice(prefixLength); // @ts-ignore return (0, common_1.compareKey)(aSuffix, bSuffix); }; } //# sourceMappingURL=run-query.js.map