@getanthill/datastore/src/sdk/projections/index.ts

Event-Sourced Datastore

import type {
  HandlerConfig,
  ModelConfig,
  RunnerServices,
  RunnerTrigger,
} from '../../typings';
import type Datastore from '../Datastore';

import { ok } from 'node:assert';
import get from 'lodash/get';
import cloneDeep from 'lodash/cloneDeep';
import Ajv from 'ajv';

import * as telemetry from '@getanthill/telemetry';

import Aggregator from '../aggregator/Aggregator';

import thingsModelConfig from '../../templates/examples/things.json';

import PostgreSQLClient from '../../services/pg';

import { build } from '../../services';
export * from './utils';

import schemas from '../schemas.json';

const validator = new Ajv({
  schemas,
  useDefaults: false,
  coerceTypes: true,
  strict: false,
});

function validate(schemaPath: string, data: unknown) {
  const isValid = validator.validate(schemaPath, data);

  if (isValid === false) {
    const err = new Error('Validation failed');

    // @ts-ignore
    err.details = [
      {
        path: schemaPath,
      },
      {
        data,
      },
      // @ts-ignore
      ...validator.errors,
    ];

    throw err;
  }
}

export async function getProjectionConfiguration(
  url: URL,
  datastores: Map<string, Datastore>,
) {
  const names = Array.from(datastores.keys());
  const projectionSource = url.searchParams.get('source') ?? names[0];

  const projectionEntityType =
    url.searchParams.get('entity_type') ?? 'projections';

  const projectionConfigurationPath = url.searchParams.get('path') ?? null;

  const configurationPath = url.searchParams.get('configuration_path') ?? null;
  const projectionField = url.searchParams.get('projection_field');
  const projectionId = url.searchParams.get('projection_id');

  let configuration = null;
  if (configurationPath) {
    configuration = require(configurationPath);
  } else if (!projectionField && projectionId) {
    const res: any = await datastores
      .get(projectionSource)
      ?.get(projectionEntityType, projectionId);

    configuration = res.data;
  } else if (projectionField && projectionId) {
    const res: any = await datastores
      .get(projectionSource)
      ?.find(projectionEntityType, {
        [projectionField]: projectionId,
      });

    configuration = res.data[0];
  }

  if (projectionConfigurationPath) {
    configuration = get(configuration, projectionConfigurationPath, null);
  }

  ok(configuration, 'Projection configuration not found');

  validate('/schemas/datastore/projection', configuration);

  return configuration;
}

export function getDestinationDatastore(
  url: URL,
  datastores: Map<string, Datastore>,
  configuration: any,
) {
  const names = Array.from(datastores.keys());
  const projectionDestination =
    url.searchParams.get('destination') ||
    configuration?.destination ||
    names[names.length - 1];

  const destination = datastores.get(projectionDestination)!;

  return destination;
}

export async function initDestinationModel(
  url: URL,
  datastores: Map<string, Datastore>,
  configuration: any,
) {
  const destination = getDestinationDatastore(url, datastores, configuration);

  ok(!!configuration.name, 'Missing Datastores configuration name');
  const projectionModelConfig: ModelConfig = {
    ...thingsModelConfig,
    ...configuration.model,
    name: configuration.name,
  };

  ok(
    !!projectionModelConfig.correlation_field,
    'Missing projection correlation_field',
  );

  try {
    await destination.createModel(projectionModelConfig);
  } catch (err: any) {
    if (err.response && err.response.status === 409) {
      await destination.updateModel(projectionModelConfig);
    } else {
      throw err;
    }
  }

  await destination.createModelIndexes(projectionModelConfig);
}

export async function getTriggers(
  url: URL,
  datastores: Map<string, Datastore>,
  aggregator: Aggregator,
  configuration: any,
) {
  const destination = getDestinationDatastore(url, datastores, configuration);

  // Trigger configuration
  const triggers: RunnerTrigger[] = (
    configuration.triggers ?? [
      {
        ...configuration.from,
        ...configuration.trigger,
      },
    ]
  ).map((trigger: any) => ({
    ...trigger,
    query: aggregator.applyMap(trigger.query, trigger.map, {}),
  }));

  // Increment feature
  const isIncremental: boolean =
    (url.searchParams.get('is_incremental') ?? 'false') === 'true';

  if (isIncremental === true) {
    const incrementalField: string =
      url.searchParams.get('incremental_field') ?? 'updated_at';

    const incrementalSourceField: string =
      url.searchParams.get('incremental_source_field') ?? incrementalField;

    const {
      data: [lastDocument],
    }: { data: [any] } = await destination.find(
      configuration.name,
      {
        _sort: { [incrementalField]: -1 },
        _fields: {
          [incrementalField]: 1,
        },
      },
      0,
      1,
    );

    if (lastDocument) {
      triggers.forEach((trigger: { [key: string]: any }) => {
        trigger.query[incrementalSourceField] = {
          'date($gt)': lastDocument?.[incrementalField],
        };
      });
    }
  }

  triggers.forEach((trigger) =>
    validate('/schemas/datastore/runner/trigger', trigger),
  );

  return triggers;
}

export const main = async function (
  url: URL,
  /* istanbul ignore next */
  services = build(),
): Promise<HandlerConfig> {
  const datastores = services.datastores;
  const heartbeat = url.searchParams.get('heartbeat') ?? 'true';
  const progress = Number.parseInt(
    url.searchParams.get('progress') ?? '1000',
    10,
  );
  const withInit = url.searchParams.get('init') === 'true';

  // Checking the Datastore configurations.
  await Promise.all(
    Array.from(services.datastores.values()).map(
      (ds) => heartbeat !== 'false' && ds.heartbeat(),
    ),
  );

  const aggregator = new Aggregator(services.datastores);

  const configuration = await getProjectionConfiguration(url, datastores);

  if (withInit === true) {
    await initDestinationModel(url, datastores, configuration);
  }

  const triggers = await getTriggers(
    url,
    datastores,
    aggregator,
    configuration,
  );

  const counts: number[] = [];

  for (const trigger of triggers) {
    const count =
      (await datastores
        .get(trigger.datastore!)
        ?.count(trigger.model!, trigger.query!, trigger.source)) ?? 0;

    counts.push(count);
  }

  // Stats
  const stats = {
    count: 0,
    total: counts.reduce((s, c) => s + c, 0),
    processed: 0,
    failed: 0,
    skipped: 0,
  };

  telemetry.logger.debug('[projections] Initialization', {
    configuration,
  });
  telemetry.logger.info('[projections] Initialization', {
    datastores_count: datastores.size,
    name: configuration.name,
    datastores_names: Array.from(datastores.keys()),
    from: configuration.from,
    triggers,
  });

  return {
    triggers,
    start: async () => ({
      datastores: Object.fromEntries(datastores),
    }),
    stop: async () => {
      return;
    },
    handler: async (
      entity: any,
      metadata: {
        handlerId?: string;
        path?: string;
        datastore?: string;
        model?: string;
        source?: string;
        raw?: boolean;
      },
    ): Promise<any> => {
      stats.count += 1;

      telemetry.logger.debug('[projections] Projecting entity...', {
        entity,
        name: configuration.name,
        metadata,
      });

      const _aggregator = new Aggregator(datastores, configuration.aggregator);

      let data: any = null;
      try {
        data = await _aggregator.aggregate(configuration.pipeline, {
          ...configuration.state,
          metadata,
          entity,
        });

        stats.processed += 1;

        _aggregator.logs.forEach((l) =>
          /* @ts-ignore */
          telemetry.logger[l.level](l.msg, l.context),
        );

        telemetry.logger.debug('[projections] Entity successfully projected', {
          entity,
          name: configuration.name,
          data,
        });
      } catch (err: any) {
        telemetry.logger.debug('[projections] Projection error - logs', {
          logs: _aggregator.logs,
        });

        if (err.message === Aggregator.ERROR_VALIDATE_STEP_FAILED.message) {
          stats.skipped += 1;
          telemetry.logger.debug(
            '[projections] Skipped entity on validation failed step',
            {
              entity,
              name: configuration.name,
              err,
            },
          );
        } else if (err === Aggregator.ERROR_ENTITY_NOT_FOUND) {
          stats.skipped += 1;
          telemetry.logger.debug(
            '[projections] Skipped entity on fetch step as entity not found',
            {
              entity,
              name: configuration.name,
              err,
            },
          );
        } else {
          stats.failed += 1;

          telemetry.logger.error('[projections] Projection error', {
            entity,
            name: configuration.name,
            details: err?.response?.data,
            err,
          });

          throw err;
        }
      }

      stats.count % progress === 0 &&
        telemetry.logger.info('[projections] Stats', {
          name: configuration.name,
          ...stats,
          progress: stats.count / stats.total,
        });

      return { stats, data };
    },
  };
};

export const syncPostgreSQL = async function (
  url: URL,
  services = build(),
): Promise<HandlerConfig> {
  const datastoreName = url.searchParams.get('datastore') ?? 'datastore';
  const source = (url.searchParams.get('source') ?? 'events') as
    | 'events'
    | 'entities';
  const query = JSON.parse(url.searchParams.get('query') ?? '{}');
  const skip = (url.searchParams.get('skip') ?? '')
    .split(',')
    .map((v) => v.trim())
    .filter((v) => !!v);
  const only = (url.searchParams.get('only') ?? '')
    .split(',')
    .map((v) => v.trim())
    .filter((v) => !!v);
  const withEncryptedData =
    url.searchParams.get('with_encrypted_data') === 'true';

  // Overwriting the pg namespace:
  const pgConfig = cloneDeep({
    ...services.config.pg,
    namespace: url.searchParams.get('ns') ?? services.config.pg.namespace,
  });

  const pg = new PostgreSQLClient(pgConfig);

  const sdk = services.datastores.get(datastoreName);

  if (!sdk) {
    throw new Error('Unknown datastore');
  }

  // @ts-ignore
  services.pg = pg;

  const { data: models } = await sdk.getModels();

  const triggers: RunnerTrigger[] = Object.keys(models)
    .filter((m) => {
      if (only.length > 0 && !only.includes(m)) {
        return false;
      }

      if (skip.length > 0 && skip.includes(m)) {
        return false;
      }

      return true;
    })
    .map((model) => ({
      datastore: datastoreName,
      model,
      source,
      query,
    }));

  const counts: number[] = [];

  for (const trigger of triggers) {
    const count = await sdk.count(
      trigger.model!,
      trigger.query!,
      trigger.source,
    );

    counts.push(count);
  }

  services.telemetry.logger.info(
    '[projections#syncPostgreSQL] Counts',
    triggers.reduce((s, c, i) => {
      s[c.model!] = counts[i];

      return s;
    }, {} as any),
  );

  const stats = {
    count: 0,
    total: counts.reduce((s, c) => s + c, 0),
    success: 0,
    error: 0,
  };

  return {
    triggers,
    start: async () => {
      services.telemetry.logger.info('[projections#syncPostgreSQL] Starting', {
        datastore: datastoreName,
        source,
        query,
      });

      services.telemetry.logger.debug(
        '[projections#syncPostgreSQL] Connecting to PostgreSQL',
      );
      await pg.connect();

      if (url.searchParams.get('init') === '1') {
        services.telemetry.logger.debug(
          '[projections#syncPostgreSQL] Database creation if not exists',
        );
        await PostgreSQLClient.init(services.config.pg);

        services.telemetry.logger.debug(
          '[projections#syncPostgreSQL] Database schema initialization',
        );
        await pg.queryAll(
          PostgreSQLClient.getSqlSchemaForModels(
            Object.values(models),
            url.searchParams.get('clean') === '1',
            pgConfig.namespace,
          ),
        );
      }

      return services as unknown as RunnerServices;
    },
    stop: async () => {
      services.telemetry.logger.info(
        '[projections#syncPostgreSQL] Ending',
        stats,
      );

      await pg.disconnect();
    },
    handler: async (event, query) => {
      stats.count % 1000 === 0 &&
        services.telemetry.logger.info(
          '[projections#syncPostgreSQL] Syncing...',
          {
            ...stats,
            progress: stats.count / stats.total,
          },
        );

      try {
        stats.count += 1;
        services.telemetry.logger.debug(
          '[projections#syncPostgreSQL] New event',
          {
            event,
            query,
            stats,
          },
        );

        await pg.insert(models[query.model!], query.source!, event, {
          with_encrypted_data: withEncryptedData,
        });

        stats.success += 1;
      } catch (err) {
        stats.error += 1;
        services.telemetry.logger.error(
          '[projections#syncPostgreSQL] Error',
          err,
        );
      }
    },
  };
};