@opengis/fastify-table/server/routes/table/functions/getData.js

core-plugins

import config from '../../../../config.js';
import { handlebars, handlebarsSync } from '../../../helpers/index.js';

import getFilterSQL from '../../../plugins/table/funcs/getFilterSQL/index.js';
import getAccess from '../../../plugins/crud/funcs/getAccess.js';
import setToken from '../../../plugins/crud/funcs/setToken.js';
import gisIRColumn from '../../../plugins/table/funcs/gisIRColumn.js';
import applyHook from '../../../plugins/hook/funcs/applyHook.js';
import getSelect from '../../../plugins/table/funcs/getSelect.js';
import setOpt from '../../../plugins/crud/funcs/setOpt.js';
import getOpt from '../../../plugins/crud/funcs/getOpt.js';
import getFilter from '../../../plugins/table/funcs/getFilter.js';
import logger from '../../../plugins/logger/getLogger.js';
import getTemplate from '../../../plugins/table/funcs/getTemplate.js';
import getMeta from '../../../plugins/pg/funcs/getMeta.js';
import pgClients from '../../../plugins/pg/pgClients.js';
import metaFormat from '../../../plugins/table/funcs/metaFormat/index.js';

import extraDataGet from '../../../plugins/extra/extraDataGet.js';

import locales from '../controllers/utils/locales.js';

import conditions from '../controllers/utils/conditions.js';

const components = {
  'vs-widget-file': 'select \'vs-widget-file\' as component, count(*) from crm.files where entity_id=$1 and file_status<>3',
  'vs-widget-comments': 'select \'vs-widget-comments\' as component, count(*) from crm.communications where entity_id=$1',
};

const checkInline = {};
const maxLimit = 100;
const defaultLimit = 20;

export default async function dataAPI(req, reply, called) {
  const {
    pg = pgClients.client, params, headers = {}, query = {}, user = {}, contextQuery, sufix = true,
  } = req;

  const time = Date.now();
  const timeArr = [Date.now()];
  const { uid } = user;

  const checkQuery = (item) => (user?.user_type === 'superadmin' ? !item.includes('{{uid}}') : true);

  const hookData = await applyHook('preData', {
    pg, table: params?.table, id: params?.id, user,
  });

  if (hookData?.message && hookData?.status) {
    return { message: hookData?.message, status: hookData?.status };
  }

  const tokenData = await getOpt(params.table, user?.uid);

  const loadTable = await getTemplate('table', tokenData?.table || hookData?.table || params.table);

  // check sql inline fields count
  if (!checkInline[params?.table] && loadTable?.sql?.length && loadTable.table) {
    const filterSql = loadTable.sql.filter(el => !el?.disabled && (el.inline ?? true));
    const sqlTable = filterSql.map((el, i) => ` left join lateral (${el.sql}) ${el.name || `t${i}`} on 1=1 `)?.join('') || '';
    const d = await Promise.all(filterSql.map((el, i) => pg.query(`select  ${el.name || `t${i}`}.* from(select * from ${loadTable.table})t ${sqlTable} limit 0`).then(item => item.fields)));
    d.forEach((el, i) => {
      filterSql[i].inline = el.length === 1;
      filterSql[i].fields = el.map(f => f.name);
    });
    checkInline[params?.table] = loadTable.sql;
  }
  else if (checkInline[params?.table]) {
    loadTable.sql = checkInline[params?.table];
  }

  if (query.sql === '0') return loadTable;

  if (!config.pg) { return reply.status(500).send('empty pg'); }

  const pkey = pg.pk?.[params?.table] || pg.pk?.[params?.table.replace(/"/g, '')];

  if (!loadTable && !(tokenData?.table && pg.pk?.[tokenData?.table]) && !(called && pkey)) {
    return reply.status(404).send('template not found');
  }

  const id = tokenData?.id || hookData?.id || params?.id;
  const { actions = [], query: accessQuery } = await getAccess({
    table: tokenData?.table || hookData?.table || params.table, id, user,
  }, pg) || {};

  if (!actions.includes('view') && !config?.local && !called) {
    return reply.status(403).send('access restricted');
  }

  const body = loadTable || hookData || tokenData;

  const {
    table, columns = [], sql, cardSql, form, meta, sqlColumns, public: ispublic, editable = false,
  } = loadTable || hookData || tokenData || params;

  /* const filters = ((body?.filter_list || [])
    .concat(body?.filterInline || [])
    .concat(body?.filterCustom || [])
    .concat(body?.filterState || [])
    .concat(body?.filterList || [])
    .concat(body?.filters || [])
  ).filter(el => el.id || el.name); */

  if (body?.filter_list || body?.filterList) {
    console.warn('invalid filters in template: filter_list / filterList');
    logger.file('crud/warning', { msg: 'invalid filters', template: tokenData?.table || hookData?.table || params.table });
  }

  const { list: filters = [] } = await getFilter({
    pg, table: tokenData?.table || hookData?.table || params.table, user,
  }) || {};

  const tableMeta = await getMeta({ pg, table });
  timeArr.push(Date.now());
  if (tableMeta?.view) {
    if (!loadTable?.key && !tokenData?.key) return { message: `key not found: ${table}`, status: 404 };
    Object.assign(tableMeta, { pk: loadTable?.key || tokenData?.key });
  }

  const { pk, columns: dbColumns = [] } = tableMeta || {};

  const columns1 = columns || dbColumns.map(({ name, title, dataTypeID }) => ({ name, title, type: pg.pgType[dataTypeID] }));
  columns1.forEach(col => {
    Object.assign(col, locales[`${table || params.table}.${col.name}`] || {});
  });

  if (!pk) {
    return reply.status(404).send(`table not found: ${table}`);
  }

  const columnList = dbColumns.map((el) => el.name || el).join(',');
  const sqlTable = sql?.filter?.((el) => !el?.disabled && !el.inline && el?.sql?.replace && (!el.sql.includes('{{uid}}') || uid)).map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', uid)}) ${el.name || `t${i}`} on 1=1 `)?.join('') || '';
  const cardSqlFiltered = hookData?.id || params.id ? (cardSql?.filter?.((el) => !el?.disabled && el?.name && el?.sql?.replace) || []) : [];
  const cardSqlTable = cardSqlFiltered.length ? cardSqlFiltered.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', uid)}) ct${i}  on 1=1 `).join('\n') || '' : '';

  const sqlInline = loadTable?.sql?.filter?.(el => el.inline)?.map(el => `,(${el.sql})`)?.join('') || '';
  const { fields = [] } = pg.queryCache ? await pg.queryCache(`select * ${sqlInline} from ${table} t ${sqlTable}  ${cardSqlTable} limit 0`) : {};
  const dbColumnsTable = fields.map(el => el.name);
  const cols = columns.filter((el) => el.name !== 'geom' && dbColumnsTable.includes(el.name)).map((el) => `"${el.name || el}"`).join(',');
  const metaCols = Object.keys(loadTable?.meta?.cls || {}).filter((el) => !cols.includes(el)).length
    ? `,${Object.keys(loadTable?.meta?.cls || {})?.filter((el) => !cols.includes(el)).join(',')}`
    : '';

  if (params.id && columnList.includes(params.id)) {
    return gisIRColumn({
      pg,
      layer: params.table,
      column: params.id,
      sql: query.sql,
      filter: query.filter,
      search: query.search,
      state: query.state,
      custom: query.custom,
    });
  }

  const objectId = tokenData?.id || hookData?.id || params.id;
  const isdefault = !objectId ? filters.find(el => el.default) : null;

  const checkFilter = [query.filter, query.search, query.state, query.custom, isdefault].filter((el) => el).length;

  const fData = checkFilter ? await getFilterSQL({
    pg,
    table: loadTable ? params.table : table,
    filter: query.filter,
    search: query.search,
    state: query.state,
    custom: query.custom,
    uid,
    json: 1,
    objectId,
  }) : {};

  timeArr.push(Date.now());

  const keyQuery = query.key && (loadTable?.key || tokenData?.key) && !objectId ? `${loadTable?.key || tokenData?.key}=$1` : null;

  const limit = (called ? (query.limit || defaultLimit) : Math.min(maxLimit, +(query.limit || defaultLimit))) || defaultLimit;

  const offset = query.page && query.page > 0 && !objectId ? ` offset ${(query.page - 1) * limit}` : '';
  // id, query, filter
  const [orderColumn, orderDir] = (query.order || loadTable?.order || '').split(/[- ]/);

  const order = query.order && columnList.includes(orderColumn) && orderColumn?.length
    ? `order by ${orderColumn} ${query.desc || orderDir === 'desc' ? 'desc' : ''} nulls last`
    : `order by ${(loadTable?.order || 'true::boolean')} nulls last`;
  const search = loadTable?.meta?.search && query.search
    ? `(${loadTable?.meta?.search?.split(',')?.map(el => `${el} ilike '%${query.search.replace(/%/g, '\\%').replace(/'/g, "''")}%'`).join(' or ')})`
    : null;
  const queryBbox = query?.bbox ? query.bbox.replace(/ /g, ',').split(',')?.map((el) => el - 0) : [];
  const queryPolyline = meta?.bbox && query?.polyline ? `ST_Contains(ST_MakePolygon(ST_LineFromEncodedPolyline('${query?.polyline}')),${meta.bbox})` : undefined;
  const bbox = meta?.bbox && queryBbox.filter((el) => !Number.isNaN(el))?.length === 4 ? `${meta.bbox} && 'box(${queryBbox[0]} ${queryBbox[1]},${queryBbox[2]} ${queryBbox[3]})'::box2d ` : undefined;

  const interfaceQuery = params?.query ? await handlebars.compile(params?.query)({ user, uid }) : undefined;
  const where = [(tokenData?.id || hookData?.id || params.id ? ` "${pk}" = $1` : null), keyQuery, loadTable?.query, tokenData?.query, fData.q, search, accessQuery || '1=1', contextQuery, bbox, queryPolyline, interfaceQuery].filter((el) => el).filter((el) => checkQuery(el));

  // const cardColumns = cardSqlFiltered.length ? `,${cardSqlFiltered.map((el) => el.name)}` : '';
  const q = `select  ${pk ? `"${pk}" as id,` : ''} 
                     ${params.id || query.key ? '*' : sqlColumns || cols || '*'} 
                     ${metaCols} 
                     
                     ${dbColumns.filter((el) => pg.pgType?.[el.dataTypeID] === 'geometry').length ? `,${dbColumns.filter((el) => pg.pgType?.[el.dataTypeID] === 'geometry').map(el => `st_asgeojson("${el.name.replace(/'/g, "''")}")::json as "${el.name.replace(/'/g, "''")}"`).join(',')}` : ''} 
                     from (select * ${sql?.filter(el => el.inline).map(el => `,(${el.sql})`).join('') || ''} from ${table} t  ${sqlTable} ) t 
                      
                     ${params.id ? cardSqlTable : ''} 
                     where ${where.join(' and ') || 'true'} 
                     ${order} ${offset} limit ${limit}`
    .replace(/{{uid}}/g, uid);

  // if (user?.user_type === 'superadmin') console.log(q);
  if (config.trace) console.log(q);
  if (query.sql === '1') { return q; }

  const { rows = [] } = await pg.query(q, (tokenData?.id || hookData?.id || params.id ? [tokenData?.id || hookData?.id || params.id] : null) || (query.key && loadTable.key ? [query.key] : [])).catch(err => {
    console.error(err.toString());
    throw new Error(err.toString());
  });

  if (!rows.length && headers?.referer?.includes?.('/card/') && headers?.referer?.includes?.(tokenData?.table || hookData?.table || params.table)) {
    return reply.status(403).send('access restricted: empty rows');
  }

  timeArr.push(Date.now());

  if (uid && rows.length && !config.security?.disableToken && (editable || actions.includes('edit') || actions.includes('del'))) {
    rows.forEach(row => {
      row.token = setToken({
        ids: [JSON.stringify({ id: row.id, table: tokenData?.table || hookData?.table || params.table, form: loadTable?.form })],
        uid,
        array: 1,
      })?.[0];
    });
  }

  const filterWhere = [fData.q, search, bbox, queryPolyline, interfaceQuery, loadTable?.query, tokenData?.query].filter((el) => el).filter((el) => checkQuery(el));

  const aggColumns = columns.filter((el) => el.agg).reduce((acc, curr) => Object.assign(acc, { [curr.name]: curr.agg }), {});
  const aggregates = dbColumns.map((el) => ({ name: el.name, type: pg.pgType?.[el.dataTypeID] })).filter((el) => ['numeric', 'double precision'].includes(el.type) && aggColumns[el.name]);
  const qCount = `select 
            count(*)::int as total,  
            count(*) FILTER(WHERE ${filterWhere.join(' and ') || 'true'})::int as filtered
            ${aggregates.length ? `,${aggregates.map((el) => `${aggColumns[el.name]}(${el.name}) FILTER(WHERE ${filterWhere.join(' and ') || 'true'}) as ${el.name}`).join(',')}` : ''}
            from (select * ${sql?.filter(el => el.inline).map(el => `,(${el.sql})`).join('') || ''} from ${table} t ${sqlTable})q 
            where ${[loadTable?.query, tokenData?.query, accessQuery, contextQuery].filter(el => el).filter((el) => checkQuery(el)).join(' and ') || 'true'} `
    .replace(/{{uid}}/g, uid);

  if (query.sql === '2') { return qCount; }

  const counts = keyQuery || tokenData?.id || hookData?.id || params.id
    ? { total: rows.length, filtered: rows.length }
    : await pg.queryCache?.(qCount, { table: loadTable?.table || tokenData?.table, time: 5 }).then(el => el?.rows[0] || {});

  timeArr.push(Date.now());
  const { total, filtered } = counts || {};
  const agg = Object.keys(counts).filter(el => !['total', 'filtered'].includes(el)).reduce((acc, el) => ({ ...acc, [el]: counts[el] }), {});

  await extraDataGet({ rows, table: loadTable?.table, form }, pg);

  await metaFormat({ rows, table: tokenData?.table || hookData?.table || params.table, sufix }, pg);
  timeArr.push(Date.now());
  const status = [];
  if (loadTable?.meta?.status) {
    const statusColumn = loadTable.meta?.cls?.[loadTable.meta?.status]
      ? { name: loadTable.meta?.status, data: loadTable.meta?.cls?.[loadTable.meta?.status] }
      : loadTable.columns.find(col => col.name === loadTable.meta?.status) || {};

    const statusCls = statusColumn.data || statusColumn.option;
    const statusClsData = statusCls ? await getSelect(statusCls, pg) : {};
    statusClsData?.arr
      ?.forEach(el => status.push({ ...el, count: rows.filter(row => el.id === row[statusColumn.name]).length }));
  }

  const template = await getTemplate('card', tokenData?.table || hookData?.table || params.table);
  const index = template?.find(el => el[0] === 'index.yml')?.[1] || {};

  const html = {};
  const { panels = [] } = index;

  const tokens = {};

  if (template && objectId) {
    // tokens result
    if (!config.security?.disableToken && index?.tokens && typeof index?.tokens === 'object' && !Array.isArray(index?.tokens)) {
      Object.keys(index.tokens || {})
        .filter(key => index?.tokens[key]?.public
          || actions?.includes?.('edit')
          || actions?.includes?.('add')
          || !index?.tokens[key]?.table)
        .forEach(key => {
          const item = index?.tokens[key];
          Object.keys(item).filter(el => item[el]?.includes?.('{{')).forEach(el => {
            item[el] = handlebarsSync.compile(item[el])({
              user, uid: user?.uid, id, data: rows[0],
            });
          });

          const token = item.form && item.table ? setToken({
            ids: [JSON.stringify(item)],
            uid,
            array: 1,
          })[0] : setOpt(item, user.uid);
          tokens[key] = token;
        });
    }

    // conditions
    panels?.filter(el => el.items).forEach(el => {
      el.items = el.items?.filter(item => conditions(item.conditions, rows[0]));
    });

    // title, count
    await Promise.all(panels.filter(el => el.items).map(async el => {
      const filtered1 = el.items.filter(item => item.count?.toLowerCase?.().includes('select'));
      const data = await Promise.all(filtered1.map(async item => pg.query(item.count.replace(/{{id}}/g, params.id)).then(item1 => item1.rows[0] || {})));
      filtered1.forEach((el1, i) => {
        // el1.title = data[i].title;
        Object.assign(el1, data[i] || {}, data[i].count ? {} : { count: undefined });
      });
      const q1 = el.items.map((item) => (item.component ? components[item.component] : null)).filter(item => item).join(' union all ');
      const counts1 = q1 && id
        ? await pg.query(q1, [id])
          .then(e => e.rows.reduce((acc, curr) => Object.assign(acc, { [curr.component]: curr.count }), {}))
        : {};
      el.items?.filter?.(item => item.component)?.forEach(item => Object.assign(item, { count: counts1?.[item.component] }));
    }));

    // data result
    const data = {};
    const route = pg.pk?.['admin.routes'] ? await pg.query('select route_id as path, title from admin.routes where enabled and alias=$1 limit 1', [table])
      .then(el => el.rows?.[0] || {}) : {};
    Object.assign(route, { tableTitle: loadTable?.title });
    if (index?.data && index?.data?.[0]?.name) {
      await Promise.all(index.data.filter((el) => el?.name && el?.sql).map(async (el) => {
        const q2 = handlebarsSync.compile(el.sql)({
          data: rows[0], user, uid: user?.uid, id,
        });
        const { rows: sqlData } = await pg.query(q2);
        data[el.name] = sqlData;
      }));
    }

    // html
    await Promise.all(template.filter(el => typeof el[1] === 'string' && el[0].includes('.hbs')).map(async (el) => {
      try {
        const htmlContent = await handlebars.compile(el[1])({
          ...rows[0], user, data, tokens,
        });
        const name = el[0].substring(0, el[0].lastIndexOf('.'));
        html[name] = htmlContent;
      }
      catch (err) {
        const name = el[0].substring(0, el[0].lastIndexOf('.'));
        logger.file('handlebars/error', {
          table, id: params.id, error: err.toString(), stack: err.stack,
        });
        html[name] = 'handlebars compile error';
      }
    }));
  }

  const res = {
    time: {
      total: Date.now() - time,
      init: timeArr[1] - timeArr[0],
      filter: timeArr[2] - timeArr[1],
      data: timeArr[3] - timeArr[2],
      count: timeArr[4] - timeArr[3],
      format: timeArr[5] - timeArr[4],
    },

    public: ispublic,
    tokens,
    card: loadTable?.card,
    actions,
    total,
    filtered,
    count: rows.length,
    pk,
    form,
    agg,
    status,
    panels,
    html,
    rows,
    meta,
    columns: columns1,
    filters: filters?.map?.(el => ({ ...el, sql: undefined })),
  };

  if (!res.columns?.length && dbColumns?.length) {
    Object.assign(res, { columns: dbColumns.map(({ name, title, dataTypeID }) => ({ name, title, type: pg.pgType[dataTypeID] })) });
  }

  // console.log({ add: loadTable.table, form: loadTable.form });
  if (uid && !config.security?.disableToken && actions.includes('add')) {
    const addTokens = setToken({
      ids: [
        JSON.stringify({
          table: tokenData?.table || hookData?.table || params.table,
          form: loadTable?.form,
        })],
      uid,
      array: 1,
    });
    Object.assign(res, { addToken: addTokens?.[0] });
  }

  const result = await applyHook('afterData', {
    pg,
    table: loadTable?.table || tokenData?.table,
    id: tokenData?.id || hookData?.id || params.id,
    template: tokenData?.table || hookData?.table || params.table,
    payload: res,
    user,
  });

  return result || res;
}