@getanthill/datastore/scripts/bench.js

Event-Sourced Datastore

process.env.TELEMETRY_LOGGER_LEVEL = 'critical';

const fs = require('node:fs');
const { default: services } = require('../dist/services');
const { default: App } = require('../dist/App');
const { Datastore, Aggregator } = require('../dist/sdk');

const memwatch = require('@airbnb/node-memwatch');

const DEFAULT_TIMES_PATH = '/tmp/times.csv';

const ModelConfig = {
  is_enabled: true,
  db: 'datastore',
  name: 'things',
  correlation_field: 'thing_id',
  retry_duration: 30000,
  encrypted_fields: ['firstname'],
  schema: {
    model: {
      properties: {
        firstname: {
          type: 'string',
        },
        count: {
          type: 'number',
        },
      },
    },
    events: {
      CREATED: {
        '0_0_0': {
          properties: {
            firstname: {
              type: 'string',
            },
            count: {
              type: 'number',
            },
          },
        },
      },
      UPDATED: {
        '0_0_0': {
          properties: {
            firstname: {
              type: 'string',
            },
            count: {
              type: 'number',
            },
          },
        },
      },
      RESTORED: {
        '0_0_0': {
          additionalProperties: true,
        },
      },
      ROLLBACKED: {
        '0_0_0': {
          additionalProperties: true,
        },
      },
      PATCHED: {
        '0_0_0': {
          additionalProperties: true,
        },
      },
    },
  },
};

function uuid() {
  return 'uuid' + (Math.random() * 1e16).toFixed(0);
}

function time(from) {
  const hrTime = process.hrtime(from);

  return (hrTime[0] * 1000000000 + hrTime[1]) / 1000000;
}

function average(arr) {
  return Math.round((arr.reduce((a, b) => a + b, 0) / arr.length) * 100) / 100;
}

function median(values) {
  if (values.length === 0) throw new Error('No inputs');

  values.sort(function (a, b) {
    return a - b;
  });

  var half = Math.floor(values.length / 2);

  if (values.length % 2) return values[half];

  return (values[half - 1] + values[half]) / 2.0;
}

async function flow(app, sdk, i, steps) {
  const _uuid = uuid();

  let entity = _entity;
  if (steps.withCreation === true) {
    i === -1 && console.debug('[bench#flow] With entity creation');
    const { data: __entity } = await sdk.create('projections', {
      firstname: `alice:${_uuid}`,
    });

    entity = __entity;

    steps.op += 1;
  }

  if (steps.withDecrypt === true) {
    i === -1 && console.debug('[bench#flow] With entity decrypt');
    const {
      data: [decryptedEntity],
    } = await sdk.decrypt('projections', [entity]);

    steps.op += 1;
  }

  if (steps.withUpdate === true) {
    i === -1 && console.debug('[bench#flow] With entity update');
    const { data: updated } = await sdk.update(
      'projections',
      entity.projection_id,
      {
        count: Math.random(),
      },
    );

    steps.op += 1;
  }

  if (steps.withGet === true) {
    i === -1 && console.debug('[bench#flow] With entity get');
    const { data: got } = await sdk.get('projections', entity.projection_id);

    steps.op += 1;
  }

  if (steps.withFind === true) {
    i === -1 && console.debug('[bench#flow] With entity find');
    const { data: found } = await sdk.find('projections', {
      firstname: `alice:${_uuid}`,
      _must_hash: true, // <-- Required for index
    });

    steps.op += 1;
  }

  if (steps.withAggregation === true) {
    i === -1 && console.debug('[bench#flow] With entity aggregation');
    // const aggregator = new Aggregator(new Map([['datastore', sdk]]));

    const { data } = await sdk.aggregate(
      [
        {
          type: 'map',
        },
        {
          type: 'fetch',
          datastore: 'datastore',
          model: 'projections',
          source: 'events',
          destination: 'events',
          query: {
            projection_id: entity.projection_id,
          },
        },
      ],
      {
        entity,
      },
    );

    steps.op += 1;
  }
}

function printProgress(progress) {
  process.stdout.clearLine(0);
  process.stdout.cursorTo(0);
  process.stdout.write(progress + '%');
}

let _uuid;
let _entity;

async function main(
  filePath = DEFAULT_TIMES_PATH,
  threshold = 5,
  iterations = 100,
  parallel = 10,
) {
  threshold = parseFloat(threshold);
  iterations = Number.parseInt(iterations, 10);
  parallel = Number.parseInt(parallel, 10);

  const steps = {
    withCreation: process.argv.includes('--create'),
    withDecrypt: process.argv.includes('--decrypt'),
    withUpdate: process.argv.includes('--update'),
    withGet: process.argv.includes('--get'),
    withFind: process.argv.includes('--find'),
    withAggregation: process.argv.includes('--aggregate'),
  };

  const nbOperations = Object.values(steps).reduce(
    (s, c) => s + (c === true ? 1 : 0),
    0,
  );

  console.log('[bench] Configuration', {
    threshold,
    iterations,
    parallel,
    steps,
  });

  services.config.mode = 'development';
  services.config.security.tokens = [
    {
      id: 'admin',
      level: 'admin',
      token: 'token',
    },
  ];
  services.config.features.api.admin = true;
  services.config.features.api.aggregate = steps.withAggregation;
  services.config.features.api.openAPI.isEnabled = true;
  services.config.features.api.updateSpecOnModelsChange = true;
  services.config.features.initInternalModels = true;

  services.config.security.activeNumberEncryptionKeys = 3;
  services.config.security.encryptionKeys = {
    all: [
      '0171568cb939a6a678f80a2a5204cec8',
      '0171568cb939a6a678f80a2a5204cec9',
      '0171568cb939a6a678f80a2a5204ceca',
    ],
  };

  services.config.features.amqp.isEnabled = false;

  const dbName = `datastore_${(Math.random() * 1e10).toFixed(0)}`;
  services.config.mongodb.databases[0].url = `mongodb://localhost:27017/${dbName}`;
  services.config.mongodb.databases[1].url = `mongodb://localhost:27017/${dbName}`;

  services.config.datastores = [
    {
      name: 'datastore',
      baseUrl: `http://127.0.0.1:${services.config.port}`,
      timeout: 30_000,
      token: 'token',
      debug: false,
      connector: 'http',
    },
  ];

  const app = new App(services);
  const sdk = new Datastore({
    baseUrl: `http://localhost:${app.services.config.port}`,
    debug: false,
    token: 'token',
    timeout: 300000,
  });

  await app.start();

  const modelConfig = {
    ...ModelConfig,
    name: 'projections',
    correlation_field: 'projection_id',
    encrypted_fields: ['firstname'],
    indexes: [
      {
        collection: 'projections',
        fields: { 'firstname::hash': 1 },
        opts: { name: 'email_hash_1' },
      },
    ],
  };

  await sdk.createModel(modelConfig);

  await sdk.createModelIndexes(modelConfig);

  // Global entity
  _uuid = uuid();
  const res = await sdk.create('projections', {
    firstname: `alice:${_uuid}`,
  });
  _entity = res.data;

  await flow(app, sdk, -1, steps);

  await app.restart();

  const fd = fs.createWriteStream(filePath);
  const hd = new memwatch.HeapDiff();
  const start = process.hrtime();

  let total = 0;
  let totalRatio = 0;
  let lastRatio = 1;
  let min = Infinity;

  for (let i = 0; i < iterations; i++) {
    const tic = process.hrtime();

    await Promise.all(
      new Array(parallel).fill(1).map(() => flow(app, sdk, i, steps)),
    );

    const tac = time(tic);

    total += tac;
    min = Math.min(min, tac);

    totalRatio += tac / min;
    const newRatio = totalRatio / (i + 1);

    fd.write(
      `${i}, ${tac}, ${totalRatio / (i + 1)}, ${newRatio - lastRatio}\n`,
    );
    lastRatio = newRatio;

    const timePerStep = total / i;

    printProgress((100 * i) / iterations);

    process.stdout.clearLine(0);
    process.stdout.cursorTo(0);
    process.stdout.write(
      `[bench] progress=${((100 * i) / iterations).toFixed(
        2,
      )}% op=${nbOperations} eta=${(
        ((iterations - i) * timePerStep) /
        1000
      ).toFixed(0)}s ${tac.toFixed(3)}ms/it ${(tac / parallel).toFixed(
        3,
      )}ms/req (x${(tac / min).toFixed(2)}) ${(
        tac /
        parallel /
        nbOperations
      ).toFixed(3)}ms/op`,
    );
  }

  const stop = time(start);

  const rows = fs
    .readFileSync(filePath)
    .toString()
    .split('\n')
    .filter((str) => !!str);
  const times = rows.map((str) => parseFloat(str.split(', ')[1]));
  const ratio = rows.map((str) => parseFloat(str.split(', ')[2]));

  process.stdout.clearLine(0);
  process.stdout.cursorTo(0);

  minIt = Math.min(...times);
  medianIt = median(times);
  averageIt = average(times);
  maxIt = Math.max(...times);

  console.table([
    {
      i: iterations,
      it: iterations,
      op: nbOperations,
      ratio: Math.floor(ratio[[iterations - 1]] * 100) / 100,
      min_it: Math.floor(minIt * 100) / 100,
      med_it: Math.floor(medianIt * 100) / 100,
      avg_it: Math.floor(averageIt * 100) / 100,
      max_it: Math.floor(maxIt * 100) / 100,
      min_req: Math.floor((minIt / parallel) * 100) / 100,
      med_req: Math.floor((medianIt / parallel) * 100) / 100,
      avg_req: Math.floor((averageIt / parallel) * 100) / 100,
      max_req: Math.floor((maxIt / parallel) * 100) / 100,
      min_op: Math.floor((minIt / parallel / nbOperations) * 100) / 100,
      med_op: Math.floor((medianIt / parallel / nbOperations) * 100) / 100,
      avg_op: Math.floor((averageIt / parallel / nbOperations) * 100) / 100,
      max_op: Math.floor((maxIt / parallel / nbOperations) * 100) / 100,
    },
  ]);

  process.stdout.write('[bench] Generating memory heap diff...');

  const diff = hd.end();

  process.stdout.clearLine(0);
  process.stdout.cursorTo(0);
  console.log('[bench] memory:', diff.change.size);
  console.log('[bench] duration:', `${Math.floor(stop / 1000)}s`);

  await app.stop();

  fs.writeFileSync('/tmp/memory.json', JSON.stringify(diff));

  if (Math.abs(ratio[iterations - 1]) > 1.5) {
    throw new Error(`[bench] 💥 Time response is increasing`);
  }

  if (diff.change.size_bytes / 1024 / 1024 > threshold) {
    throw new Error(
      `[bench] 💥 Memory threshold exceeded (${diff.change.size} > ${threshold} mb)`,
    );
  }
}

main(...process.argv.slice(2))
  .then(() => {
    console.log('[bench] Ended successfully');
  })
  .catch((err) => {
    console.error(err);
    if (err?.response?.data) {
      console.error('[bench] Error', err?.response?.data);
    }

    process.exit(1);
  });