@opengis/bi/server/routes/dataset/controllers/export.js

BI data visualization module

/* eslint-disable no-await-in-loop */
import path from 'node:path';
import { createHash } from 'node:crypto';
import { existsSync, rmSync } from 'node:fs';

import { readFile, writeFile, mkdir } from 'node:fs/promises';

import {
  config, pgClients, getMeta, getFolder, getFilterSQL, eventStream, logger,
} from '@opengis/fastify-table/utils.js';

import convertJSONToXls from '../utils/convertJSONToXls.js';
import convertJSONToCSV from '../utils/convertJSONToCSV.js';
import metaFormat from '../utils/metaFormat.js';

export default async function datasetDataExport(req, reply) {
  const {
    pg = pgClients.client, params = {}, query = {}, user = {}, unittest,
  } = req;

  if (!user?.uid) {
    return { message: 'access restricted', status: 403 };
  }

  const { user_type: userType = 'regular' } = user;

  if (!params?.id) {
    return { message: 'not enough params: id', status: 404 };
  }

  const { format = 'json' } = query;

  const dataset = await pg.query(`select dataset_id as id, table_name as table, column_list as columns,
      export_columns as "exportColumns", sql_list as sql from bi.dataset where dataset_id=$1`, [params.id])
    .then(el => el.rows[0] || {});

  if (!dataset?.id || !dataset.table) {
    return { message: 'dataset not found', status: 404 };
  }

  const { pk = dataset.pk, view, columns: dbColumns = [] } = await getMeta(dataset.table);

  if (!dataset.table || (!pg.pk?.[dataset.table] && !pk && !view)) {
    return { message: `table not found: ${dataset.table}`, status: 404 };
  }

  const rootDir = getFolder(req, 'local');

  const date = new Date();
  const sufixName = `${query.filter || '1=1'}-${query.col || 'all'}-${query.search || '1=1'}-${query.limit || 'unlimited'}`;
  const sufixDate = [date.getFullYear(), date.getMonth(), date.getDate(), date.getHours()].join('-');

  const objInfo = createHash('md5').update([sufixName, sufixDate].join('-')).digest('hex');
  const fileName = dataset.table.concat('_').concat(objInfo).concat('.').concat(format);
  const filepath = path.join(rootDir, '/files/temp', fileName);

  if (existsSync(filepath) && query.nocache) {
    rmSync(filepath);
  }

  if (existsSync(filepath) && !query?.sql) {
    const encoding = path.extname(filepath) === '.xlsx' ? 'base64' : 'utf8';
    const data = await readFile(filepath, encoding);
    return reply.headers({ 'Content-Disposition': `attachment; filename=${path.basename(filepath)}` }).send(Buffer.from(data, encoding));
  }

  const fData = await getFilterSQL({
    filter: query.filter,
    search: query?.search,
    table: dataset.table,
    json: 1,
  });

  const where = [(params.object_id ? ` "${pk}" = $1` : null), dataset.query, fData.q].filter((el) => el);

  const cols = dataset.exportColumns
    || dataset.columns?.map((el) => el.name);

  if (!cols?.length) {
    return { message: `empty export columns: ${dataset.id}`, status: 400 };
  }

  const sqlTable = dataset.sql
    ?.filter?.((el) => !el?.disabled && el?.sql?.replace)
    ?.map((el, i) => ` left join lateral (${el.sql.replace('{{uid}}', user?.uid)}) ${el.name || `t${i}`} on 1=1 `)
    ?.join('') || '';

  const { total = '0' } = await pg.query(`select count(*) as total from ${dataset.table} 
        where ${dataset.query || 'true'} and ${params.object_id ? `${pk}='${params.object_id}'` : '1=1'}`)
    .then(el => el.rows?.[0] || {});

  if (total === '0') {
    return { message: 'Немає даних, які можна експортувати', status: 200 };
  }

  const ext = path.extname(filepath);
  const filepathJSON = ['csv', 'xlsx'].includes(format) ? filepath.replace(ext, '.json') : filepath;

  const cacheFileJSON = existsSync(filepathJSON);

  const send = +total < 10000 || unittest
    ? console.log
    : eventStream(reply);

  if (cacheFileJSON && !query.sql) {
    if (query.nocache) rmSync(filepathJSON); // delete old file, prevent append
    if (!query.nocache && ['xlsx', 'csv'].includes(format)) {
      send(`Знайдено файл формату json. Пропуск обробки даних. Початок конвертації в ${format}...`);
      if (format === 'xlsx') {
        return convertJSONToXls({
          filepathJSON,
          colmodel: dataset.columns,
          domain: req.hostname,
          source: dataset.table,
          send,
        });
      }
      if (format === 'csv') {
        return convertJSONToCSV({ send, filepathJSON });
      }
    }
  }

  const limit = Math.min(query.limit, 10000) || 10000;

  const optimizedSQL = `select  ${pk ? `"${pk}" as id,` : ''} *
    ${['geojson', 'shp'].includes(format) && dbColumns.find((el) => el.name === 'geom' && pg.pgType[el.dataTypeID] === 'geometry') ? ',st_asgeojson(geom)::json as geom' : ''} 
  from (select * from ${dataset.table} where ${sqlTable ? 'true' : (where.join(' and ') || 'true')}) t 
  ${sqlTable} where ${where.join(' and ') || 'true'} offset 0 limit ${limit}`.replace(/{{uid}}/g, user?.uid);

  if (query.sql === '1' && (config.debug || userType.includes('admin'))) return optimizedSQL;

  const res = {};
  let offset = 0;
  let seq = 0;

  send(`Всього в реєстрі: ${total}`);

  while ((+total - offset > 0) && !res?.error) {
    try {
      const q = optimizedSQL.replace(/offset 0/g, `offset ${offset}`);
      const { rows = [] } = await pg.query(q, (params.object_id ? [params.object_id] : null));

      send(`Оброблено: ${offset}/${total}`);
      send(`seq: ${++seq}`);
      send(`Обробка ${rows.length} об'єктів...`);

      if (!rows.length) {
        send('Обробка даних успішно завершена');
        break;
      }

      await metaFormat({ rows, columns: dataset.columns });

      // skip non present after metaFormat
      const allowColumnList = []; // 'id'
      if (['geojson', 'shp'].includes(format)) {
        allowColumnList.push('geom');
      }
      rows.forEach((row) => {
        Object.keys(row)
          .filter((el) => !allowColumnList.includes(el.name) && !cols.includes(el))
          .forEach((key) => delete row[key]);
      });

      // convert csv / xlsx from json
      if (!existsSync(filepathJSON)) { // if json not exists
        await mkdir(path.dirname(filepath), { recursive: true });
        await writeFile(filepathJSON, JSON.stringify(rows));
      }
      else { // if json exists
        const jsonData = JSON.parse(await readFile(filepathJSON) || '{}');
        const moreData = jsonData.concat(rows); // rewrite to appendFile?
        await writeFile(filepathJSON, JSON.stringify(moreData));
      }

      offset += rows.length;
    }
    catch (err) {
      Object.assign(res, { error: err.toString(), status: 500 });
      send(`error: ${err.toString()}`, 1);
      logger.error({
        name: 'export/table',
        filepathJSON,
        total,
        offset,
        result: res,
        error: err.toString(),
      });
      return err.toString();
    }
  }

  if (!res?.error && format === 'csv') {
    send('Сформовано файл формату json. Початок конвертації в csv...');
    return convertJSONToCSV({ send, filepathJSON });
  }

  if (!res?.error && format === 'xlsx') {
    send('Сформовано файл формату json. Початок конвертації в xlsx...');
    return convertJSONToXls({
      filepathJSON,
      colmodel: dataset.columns,
      domain: req.hostname,
      source: dataset.table,
      send,
    });
  }

  const message = res.error || 'Файл успішно сформовано. Натистіть кнопку ще раз для завантаження даних';
  send(message, 1);
  return { message, status: 200 };
}