@sterblue/sterblue-sdk/lib/utils/stream-query.js

Sterblue Graph SDK for graphile.sterblue.com

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.streamQuery = exports.getResultFromResultFunction = exports.getResultFromCountFunction = exports.getResultFromListFunction = void 0;

var _streamingIterables = require("streaming-iterables");

var _fp = require("lodash/fp");

function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }

function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }

function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }

function _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }

function _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }

const {
  ceil,
  min
} = Math;
/**
 * All count queries should have the count at this particular path.
 */

const countPathInQuery = "connection.totalCount";
/**
 * All list queries should have the list at this particular path.
 */

const listPathInQuery = "list";
/**
 * All result queries should have the result at this particular path.
 */

const resultPathInQuery = "result";
/**
 * If no count function is provided, we make one that says there is just enough
 * total items to fulfill the offset + first that are demanded.
 * In this case hopefully the graphql server should react nicely
 * because we might actually try to list items out of bound.
 * Prisma and Graphile react nicely so it's ok
 * @param variables graphql query object. Only `offset` and `first` are used in that function
 */

const defaultCountFunction = async ({
  offset,
  first
}) => (0, _fp.set)(countPathInQuery, offset + first, {});
/**
 * From a SDK List function, make a function that directly outputs the list
 * By taking the resulting list out of the graphql response payload
 * @param listFunction a function that return the graphql response payload with the list stored in the key `list`
 */


const getResultFromListFunction = listFunction => async variables => {
  return (0, _fp.get)(listPathInQuery, await listFunction(variables));
};
/**
 * From a SDK Count function, make a function that directly outputs the count
 * By taking the resulting count out of the graphql response payload
 * @param countFunction a function that return the graphql response payload with the count stored in `connection.aggregate.count`
 */


exports.getResultFromListFunction = getResultFromListFunction;

const getResultFromCountFunction = countFunction => async variables => {
  return (0, _fp.get)(countPathInQuery, await countFunction(variables));
};
/**
 * From a SDK Result function, make a function that directly outputs the result
 * By taking the result out of the graphql response payload
 * @param resultFunction a function that return the graphql response payload with the result stored in the key `result`
 */


exports.getResultFromCountFunction = getResultFromCountFunction;

const getResultFromResultFunction = resultFunction => async variables => {
  return (0, _fp.get)(resultPathInQuery, await resultFunction(variables));
};
/**
 * Stream a list query as an async iterable using batches.
 * Allow to perform huge queries, replace queryHugeData
 */


exports.getResultFromResultFunction = getResultFromResultFunction;

const streamQuery = (listFunction, countFunction = defaultCountFunction) => async (_ref) => {
  let {
    first = 320,
    offset = 0,
    batchSize = 20,
    batchConcurrency = 4
  } = _ref,
      variables = _objectWithoutProperties(_ref, ["first", "offset", "batchSize", "batchConcurrency"]);

  // Count the number of objects to fetch based on a count query if available
  // Defaulting in the same way as in defaultCountFunction
  const count = Math.min(first, (0, _fp.defaultTo)(first + offset, await getResultFromCountFunction(countFunction)(_objectSpread({
    first,
    offset
  }, variables))) - offset); // We get the number of batches simply by dividing

  const numberOfBatches = ceil(count / batchSize); // We prepare the batches by creating their variables in a simple list

  const batchesVariables = (0, _streamingIterables.map)(batchIndex => {
    // How much to offset in this batch
    const batchOffset = offset + batchIndex * batchSize; // How many object to fetch in this batch, can be smaller than batch size, if we overshoot

    const batchFirst = min(batchSize, count + offset - batchOffset); // Return the variables for this batch

    return _objectSpread({
      first: batchFirst,
      offset: batchOffset
    }, variables);
  }, (0, _fp.range)(0, numberOfBatches)); // We use flatTransform to apply the list function to the variables making nice batches

  return (0, _streamingIterables.flatTransform)(batchConcurrency, getResultFromListFunction(listFunction), batchesVariables);
};

exports.streamQuery = streamQuery;