@athenox/talos/dist/io.js

Talos: a backend you describe instead of write. Declare a schema, write the rules in plain language, map URLs to sentences; a small built-in model turns them into query plans.

/**
 * Moving rows in and out: CSV import and export, and JSON APIs read into rows.
 * (Mail stays the app's: its transport, templates and sending are not Talos's.)
 */
// --- CSV ---------------------------------------------------------------------
/** CSV text to rows keyed by header; `,`, `;` or tab (whichever the header uses most), quotes and "" escapes. */
export function parseCsv(text) {
    const src = text.replace(/^/, '');
    const first = src.split(/\r?\n/, 1)[0] ?? '';
    const sep = [',', ';', '\t'].map(s => ({ s, n: first.split(s).length })).sort((a, b) => b.n - a.n)[0].s;
    const rows = [];
    let row = [], cell = '', quoted = false;
    for (let i = 0; i < src.length; i++) {
        const ch = src[i];
        if (quoted) {
            if (ch === '"' && src[i + 1] === '"') {
                cell += '"';
                i++;
            }
            else if (ch === '"')
                quoted = false;
            else
                cell += ch;
        }
        else if (ch === '"')
            quoted = true;
        else if (ch === sep) {
            row.push(cell);
            cell = '';
        }
        else if (ch === '\n' || ch === '\r') {
            if (ch === '\r' && src[i + 1] === '\n')
                i++;
            row.push(cell);
            cell = '';
            if (row.some(c => c.trim()))
                rows.push(row);
            row = [];
        }
        else
            cell += ch;
    }
    row.push(cell);
    if (row.some(c => c.trim()))
        rows.push(row);
    const [head, ...body] = rows;
    const keys = (head ?? []).map(h => h.trim());
    return body.map(r => Object.fromEntries(keys.map((k, i) => [k, (r[i] ?? '').trim()])));
}
/** Rows to CSV, with the columns given (header labels optional). */
export function toCsv(rows, columns, sep = ',') {
    const cols = Array.isArray(columns) ? Object.fromEntries(columns.map(c => [c, c])) : columns;
    const cell = (v) => {
        const s = v === null || v === undefined ? '' : v instanceof Date ? v.toISOString() : String(v);
        return /["\n\r]/.test(s) || s.includes(sep) ? `"${s.replace(/"/g, '""')}"` : s;
    };
    return [Object.values(cols).map(cell).join(sep), ...rows.map(r => Object.keys(cols).map(k => cell(r[k])).join(sep))].join('\n');
}
/**
 * Import CSV rows into a table through a write function (a sentence route's
 * runner, or anything taking values): headers mapped to columns (several
 * spellings each), a key column deciding what is already there, and a report.
 *
 *   await importCsv(text, { map: { email: ['email', 'e-mail', 'courriel'], name: ['nom', 'name'] },
 *     key: 'email', exists: email => …, write: values => … })
 */
export async function importCsv(text, o) {
    const report = { total: 0, inserted: 0, updated: 0, skipped: 0, errors: [] };
    const norm = (s) => s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '').replace(/[^a-z0-9]/g, '');
    const seen = new Set();
    for (const [i, raw] of parseCsv(text).entries()) {
        report.total++;
        const byHead = new Map(Object.entries(raw).map(([k, v]) => [norm(k), v]));
        const values = {};
        for (const [col, heads] of Object.entries(o.map)) {
            const v = [heads].flat().map(h => byHead.get(norm(h))).find(v => v !== undefined && v !== '');
            if (v !== undefined)
                values[col] = v;
        }
        const reason = o.check?.(values);
        if (reason) {
            report.errors.push({ line: i + 2, reason });
            continue;
        }
        const key = o.key ? String(values[o.key] ?? '').toLowerCase() : '';
        if (o.key && (!key || seen.has(key))) {
            report.skipped++;
            continue;
        }
        if (o.key)
            seen.add(key);
        try {
            if (o.key && o.exists && await o.exists(key)) {
                if (o.update) {
                    await o.update(values);
                    report.updated++;
                }
                else
                    report.skipped++;
            }
            else {
                await o.write(values);
                report.inserted++;
            }
        }
        catch (e) {
            report.errors.push({ line: i + 2, reason: e?.message ?? String(e) });
        }
    }
    return report;
}
// --- JSON APIs ---------------------------------------------------------------
/**
 * A JSON API read into a row: `{placeholders}` in the URL, dotted paths or
 * functions for each field, a timeout, and a small cache. Returns null when
 * the API has nothing (404, or `found` says so).
 *
 *   const product = connector({ url: 'https://world.openfoodfacts.org/api/v2/product/{code}.json',
 *     found: j => j.status === 1, map: { name: 'product.product_name', brand: 'product.brands' } })
 *   await product({ code: '3017620422003' })
 */
export function connector(o) {
    const cache = new Map();
    const at = (j, path) => path.split('.').reduce((x, k) => x?.[/^\d+$/.test(k) ? Number(k) : k], j);
    return async (params) => {
        const url = o.url.replace(/\{(\w+)\}/g, (_, k) => encodeURIComponent(String(params[k] ?? '')));
        const hit = cache.get(url);
        if (hit && Date.now() - hit.at < (o.cacheMs ?? 10 * 60_000))
            return hit.value;
        const r = await fetch(url, { headers: { accept: 'application/json', ...o.headers }, signal: AbortSignal.timeout(o.timeoutMs ?? 8000) });
        if (r.status === 404) {
            cache.set(url, { at: Date.now(), value: null });
            return null;
        }
        if (!r.ok)
            throw new Error(`${new URL(url).host} answered ${r.status}`);
        const j = await r.json();
        const value = o.found && !o.found(j) ? null
            : Object.fromEntries(Object.entries(o.map).map(([k, p]) => [k, typeof p === 'function' ? p(j) : at(j, p)]));
        cache.set(url, { at: Date.now(), value });
        return value;
    };
}