@opengis/bi/server/routes/data/controllers/data.js

BI data visualization module

import yaml from 'js-yaml';

import {
  config,
  autoIndex,
  pgClients,
  getSelect,
  getSelectVal,
  getFilterSQL,
  getMeta,
  logger,
} from '@opengis/fastify-table/utils.js';

import chartSQL from './util/chartSQL.js';
import normalizeData from './util/normalizeData.js';

import { getWidget } from '../../../../utils.js';

const maxLimit = 100;

export default async function dataAPI(req, reply) {
  const time = Date.now();

  const { query = {}, user = {}, unittest } = req;

  query.metric = Array.isArray(query.metric) ? query.metric.pop() : query.metric;

  const { dashboard, widget, filter, search, samples } = query;

  const widgetData = await getWidget({ pg: req.pg, dashboard, widget });

  if (widgetData.status) return widgetData;

  const { type, text, data = {}, controls, style, options } = widgetData;

  const pg = widgetData.pg || req.pg || pgClients.client;

  const { fields: cols } = await pg.query(
    `select * from ${data.table} t ${widgetData.tableSQL || data.tableSQL || ''} limit 0`
  );
  const columnTypes = cols.map((el) => ({
    name: el.name,
    type: pg.pgType?.[el.dataTypeID],
  }));

  // data param
  const { x, cls, groupbyCls, metric, table, where, tableSQL, groupby, xName, yName, xType, yType, error } =
    normalizeData(widgetData, query, columnTypes);

  if (error) { return reply.status(400).send(error); }

  // auto Index
  if (pg.pk?.[data.table]) {
    autoIndex({
      table: data.table,
      pg,
      columns: [data?.time]
        .concat([xName])
        .concat([groupby])
        .filter((el) => el),
    }).catch((err) => console.log(err));
  }

  const { pk, columns = [], view } = await getMeta({ pg, table: data.table });

  if (!view && !pk) {
    return { message: `table not found: ${data.table} (${pg.options?.database})`, status: 404 };
  }

  // const columnList = columns.map(col => col.name);
  const groupbyColumnNotExists = groupby?.split?.(',')?.filter?.(el => !columnTypes.map(el => el.name).includes(el.trim()));

  if (groupby && groupbyColumnNotExists?.length) {
    return { message: `groupby column not found: ${groupbyColumnNotExists} (${data.table}/${pg.options?.database})`, status: 404 };
  }

  // get group
  const groupData = groupby
    ? await pg
      .query(
        `select ${groupby} as name ,count(*) from ${tableSQL || table} group by ${groupby} order by count(*) desc limit 20`
      )
      .then((el) => el.rows)
    : null;

  if (query.sql === '2') return { x, metric, table, tableSQL, data, groupData };

  const order = data.order || (type === 'listbar' && cols.find(el => el.name === 'metric') ? 'metric desc' : null);

  const fData =
    filter || search
      ? await getFilterSQL({
        pg,
        table,
        filter,
        search,
        filterList: widgetData.filters,
      })
      : {};

  const optimizedSQL = widgetData?.sql
    ? `${widgetData.sql} ${fData?.q && false ? fData?.q : ''} limit ${Math.min(query.limit || widgetData.limit || maxLimit, maxLimit)}`
    : (fData?.optimizedSQL || `select * from ${tableSQL || table}`);

  if (type?.includes('bar') && !metric?.length) {
    return { message: 'empty widget params: metrics', status: 400 };
  }

  const sql = widgetData.sql ? optimizedSQL : (chartSQL[type] || chartSQL.chart)({
    where: config.local && user?.user_type === 'superadmin' ? 'true' : where, // test
    metric,
    yType, // metric type
    columns: widgetData.columns,
    table: `(${optimizedSQL})q`,
    x,
    groupData,
    groupby,
    order,
    samples,
    limit: Math.min(query.limit || maxLimit, maxLimit),
    xType,
    fx: widgetData.fx,
  });

  if (query.sql) return sql;

  if (!sql || sql?.includes('undefined')) {
    return {
      message: {
        error: 'invalid sql',
        type,
        sql,
        where,
        metric,
        table: `(${optimizedSQL})q`,
        x,
        groupData,
        groupby,
      },
      status: 500,
    };
  }

  if (config.trace) console.log(sql, user?.uid);

  const { rows = [], fields = [], errorSql } = await pg.query(sql.replace('{{uid}}', user?.uid)).catch(err => {
    logger.file('bi/data', { error: err.toString(), sql });
    return { errorSql: err.toString() };
  }); // test with limit

  if (groupbyCls) {
    const { arr = [] } = await getSelect(groupbyCls, pg) || {};
    if (arr.length) {
      const ids = arr.map(el => el.id);
      const text = arr.reduce((acc, curr) => ({ ...acc, [curr.id]: curr.text }), {});
      rows.forEach(row => {
        ids.reduce((acc, curr) => {
          Object.assign(row, { [text[curr]]: row[curr] });
          delete row[curr];
          return acc;
        }, {});
      });
    }
  }

  if (cls) {
    const values = rows
      .map((row) => row[x])
      ?.filter((el, idx, arr) => el && arr.indexOf(el) === idx);
    const vals = await getSelectVal({ pg, name: cls, values });
    rows
      .filter((row) => row[x])
      .forEach((row) => {
        Object.assign(row, { [x]: vals?.[row[x]]?.text || vals?.[row[x]] || row[x] });
      });
  }

  const metaTitles = columns.reduce((acc, curr) => Object.assign(acc, { [curr.name]: curr.title || curr.ua }), {});
  const titles = Array.isArray(widgetData?.columns)
    ? widgetData.columns.reduce((acc, curr) => Object.assign(acc, { [curr.name]: curr.title || curr.ua }), {})
    : Object.keys(widgetData?.columns || {}).reduce((acc, curr) => Object.assign(acc, { [curr]: widgetData?.columns?.[curr] }), {});

  const rows1 = type === 'table' ? rows.map(row => Object.keys(row || {}).reduce((acc, curr) => Object.assign(acc, { [titles?.[curr] || metaTitles?.[curr] || curr]: row?.[curr] }), {})) : rows;

  const yml = widgetData.yml || yaml.dump(extractYml(widgetData));
  const dimensions = fields.map((el) => el.name);

  const res = {
    time: Date.now() - time,
    error: errorSql || (!widgetData.sql ? widgetData.error : undefined),
    dimensions,
    filter: xName,
    dimensionsType: [xType, yType].filter((el) => el)?.length
      ? [xType, yType].filter((el) => el)
      : fields.map((el) => pg.pgType?.[el.dataTypeID]),
    type,

    text: text || widgetData?.title || data.text,
    // data: query.format === 'data' ? dimensions.map(el => rows.map(r => r[el])) : undefined,
    source:
      query.format === 'array'
        ? dimensions.map((el) => rows1.map((r) => r[el]))
        : rows1,
    style,
    options,
    controls,
    yml,
    data: widgetData.data,
    id: query.widget,
    columns: columnTypes.map(el => Object.assign(el, { title: titles[el.name] || metaTitles?.[el.name] || el.name })),
    params: config?.local || unittest ? {
      x,
      cls,
      metric,
      table,
      tableSQL,
      where,
      groupby,
      sql,
    } : undefined,
  };
  return res;
}

function extractYml(sourceData) {
  const { title, description, type, data, style, controls } = sourceData;
  return { title, description, type, data, style, controls };
}