@athenox/talos
Version:
Talos: a backend you describe instead of write. Declare a schema, write the rules in plain language, map URLs to sentences; a small built-in model turns them into query plans.
2,384 lines • 128 kB
JavaScript
import { appendFileSync, existsSync, readFileSync, statSync } from 'node:fs';
import { createHash, randomUUID } from 'node:crypto';
import { dirname, resolve as joinPath } from 'node:path';
import { fileURLToPath } from 'node:url';
import * as ort from 'onnxruntime-node';
import { rank, similar } from './recommend.js';
import { classify } from './embed.js';
import { find } from './search.js';
import { extOf, imageSize, processImage } from './images.js';
import { Telemetry, platformInfo } from './telemetry.js';
import { localStore } from './local.js';
/**
* Where the weights are.
*
* Not `import.meta.url`: a bundler rewrites this file into the app's build
* output, and the weights do not follow it there. So the path is resolved
* against the working directory — which for a server is its own root — with an
* environment variable for anyone who puts the model somewhere else.
*/
// Bundled, `import.meta.url` can be a URL of no file (`file:///_entry.js` on Windows throws).
const HERE = (() => { try {
return dirname(fileURLToPath(import.meta.url));
}
catch {
return undefined;
} })();
const MODEL_DIR = [
process.env.TALOS_MODEL_DIR,
joinPath(process.cwd(), 'node_modules', '@athenox', 'talos', 'model'),
joinPath(process.cwd(), 'node_modules', 'talosjs', 'model'),
joinPath(process.cwd(), '..', 'talos', 'model'),
joinPath(process.cwd(), 'model'),
HERE && joinPath(HERE, '..', 'model'),
].find(dir => dir && existsSync(joinPath(dir, 'config.json')));
if (!MODEL_DIR) {
throw new Error('talos: no model found. Set TALOS_MODEL_DIR to the directory holding '
+ 'config.json, encoder.onnx and decoder.onnx.');
}
const cfg = JSON.parse(readFileSync(joinPath(MODEL_DIR, 'config.json'), 'utf8'));
/**
* What people actually call each enum value. These are written into the schema the
* model reads, so resolving "urgentes" onto `urgent` is a local match next to the
* value rather than something the weights have to remember. An app can extend this
* for its own vocabulary just by adding to the map.
*/
const SURFACE = JSON.parse(readFileSync(joinPath(MODEL_DIR, 'surface.json'), 'utf8'));
/**
* Teach the runtime the words an app's own vocabulary is spoken with.
*
* Declaring a value *replaces* its forms rather than adding to them. Merging
* looked harmless and was not: editing this list during development left the
* process holding the union of every version it had seen, which shifts every
* pointer position in the window — reading broke and the cause was invisible.
* The app states the full list for a value; what it states is what is used.
*/
export function addSurfaceForms(forms) {
for (const [value, words] of Object.entries(forms))
SURFACE[value] = [...words];
}
let encoder = null;
let decoder = null;
/**
* onnxruntime takes every core by default. On a machine shared with a training
* job that is a real cost, so TALOS_THREADS caps it; unset keeps the default.
*/
const THREADS = Number(process.env.TALOS_THREADS) || undefined;
const sessionOptions = THREADS ? { intraOpNumThreads: THREADS, interOpNumThreads: 1 } : {};
const sessions = async () => {
encoder ??= await ort.InferenceSession.create(joinPath(MODEL_DIR, 'encoder.onnx'), sessionOptions);
decoder ??= await ort.InferenceSession.create(joinPath(MODEL_DIR, 'decoder.onnx'), sessionOptions);
return { encoder, decoder };
};
const VOCAB_SIZE = Object.keys(cfg.vocab).length;
/**
* A word the model has never seen still gets a stable id of its own, instead of
* every unknown word collapsing into one. Two different unseen words then look
* different, so the model can still see that the word in the schema and the word
* in the rule are the same word. It cannot know what the word *means* — only
* training vocabulary buys that — but pointing keeps working.
*
* Must match `token_id` in train/gen2.py exactly.
*/
function tokenId(tok) {
const known = cfg.vocab[tok];
if (known !== undefined)
return known;
let h = 0;
for (let i = 0; i < tok.length; i++)
h = (Math.imul(h, 31) + tok.charCodeAt(i)) >>> 0;
return VOCAB_SIZE + (h % (cfg.unkBuckets ?? 256));
}
/**
* A text as the model's encoder reads it: the mean of its token states, unit
* length. Two texts about the same thing land close together — the basis of
* `recommend`. Texts longer than the window are cut.
*/
export async function embed(text) {
const toks = tokenize(text).slice(0, cfg.maxSrc);
const ids = new BigInt64Array(cfg.maxSrc);
toks.forEach((t, i) => { ids[i] = BigInt(tokenId(t)); });
const { encoder } = await sessions();
const memory = (await encoder.run({ src: new ort.Tensor('int64', ids, [1, cfg.maxSrc]) })).memory;
const data = memory.data;
const dim = data.length / cfg.maxSrc;
const out = new Float32Array(dim);
const n = Math.max(1, toks.length);
for (let p = 0; p < n; p++)
for (let d = 0; d < dim; d++)
out[d] += data[p * dim + d] / n;
let norm = 0;
for (const v of out)
norm += v * v;
norm = Math.sqrt(norm) || 1;
for (let d = 0; d < dim; d++)
out[d] /= norm;
return out;
}
export const normalize = (s) => String(s).toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '');
const TOKEN_RE = /[a-z]+|\d+|[^\sa-z\d]/g;
const tokenize = (text) => normalize(text).match(TOKEN_RE) ?? [];
/** Exact name only — see the CALL branch of `run` for why there is no fallback. */
function matchFunction(fns, target) {
if (!target)
return undefined;
return fns?.find(f => squash(f.name) === squash(target));
}
const squash = (text) => normalize(text).replace(/\s+/g, '');
/** True when every word of `text` is one the weights have actually seen. */
const knownWords = (text) => tokenize(text).every(w => w in cfg.vocab || /^\d+$/.test(w));
/** `sendNewsletter` -> ["sendnewsletter", "send newsletter"] */
function nameForms(name) {
const spaced = normalize(name.replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/[_-]+/g, ' '));
return [...new Set([squash(name), spaced])].filter(Boolean);
}
/**
* The function whose name the sentence actually says.
*
* Verbs are where the model's vocabulary runs out: `checkout` it knows, `send`,
* `newsletter` and `reindex` it has never seen, and an unknown word does not get
* ignored — it becomes a pointer target. Asked to "call reindex" the model
* answered `SELECT articles FILTER title VAL reindex index`, and every other
* plan in the same schema picked up the stray word too.
*
* So when a request names a declared function outright, the runtime takes it at
* its word. That is not the model being overruled on a judgement call: the app
* declared the name, the sentence contains it, and there is nothing to infer.
* It also keeps function naming free — an app can call its action whatever it
* likes without waiting for the word to appear in a training set.
*
* Deliberately literal: the whole name must appear, as written or with its
* camelCase split, and a one-word name must be long enough not to collide with
* ordinary prose. "send the newsletter" does not match `sendNewsletter`.
*/
/**
* The table the request names outright, when it names exactly one.
*
* With six tables in the window the model stops being able to choose between
* them: "toutes les categories", "show me the members" and "show me the orders"
* all came back REFUSE, and "show me the entries" came back as articles. But
* each of those sentences contains the name of a declared table, which the app
* wrote down and the user then said. Honouring that is not a guess.
*
* Exactly one, and only as a whole word, so "les commentaires de l article 5" —
* which names two — is left to the model, where it belongs and where it is
* right.
*/
function namedTable(input, tables) {
const hits = tables.filter(t => [t.as, ...(t.says ?? [])].some(form => mentions(input, form)));
if (hits.length === 1)
return hits[0];
// "les ordonnances du rendez-vous 4" names two tables, and one of them is a
// child — it declares `parent`. With a single number in the sentence, the
// child is what was asked for and the number is its parent. Two children, or
// two numbers, and nothing is decided here.
const children = hits.filter(t => Object.keys(t.columns).some(c => normalize(c) === 'parent'));
const numbers = tokenize(input).filter(w => /^\d+$/.test(w));
return hits.length > 1 && children.length === 1 && numbers.length === 1 ? children[0] : undefined;
}
/**
* The one table whose declared domain holds a value the request names.
*
* "les brouillons" names no table, but `brouillon` is a form of `draft` and only
* `articles` declares `draft`. When the model aims elsewhere — at a table that
* does not own the value, or at something that is not a table at all (a model
* trained on invented table names once answered `SELECT sportif politiques`,
* two surface forms of sections) — this is the table that was meant.
*
* Exactly one table, or nothing: "pending" is an order status and a comment
* status, and a word two tables share decides nothing.
*/
function tableFromValues(input, tables) {
const owners = tables.filter(t => Object.values(t.enums ?? {}).flat().some(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f))));
return owners.length === 1 ? owners[0] : undefined;
}
function namedFunction(input, fns) {
if (!fns?.length)
return undefined;
const said = ` ${normalize(input).replace(/\s+/g, ' ')} `;
const squashed = squash(input);
for (const f of fns) {
for (const form of [...nameForms(f.name), ...(f.says ?? []).map(s => normalize(s))]) {
if (form.includes(' ') && said.includes(` ${form} `))
return f;
if (!form.includes(' ') && form.length >= 6 && squashed.includes(form))
return f;
}
}
return undefined;
}
/**
* Turn the plan's `SET name VAL value` pairs into the function's named arguments.
*
* The model names arguments by pointing at the parameter list in the schema, so
* what comes back is usually right and occasionally a near miss ("id" for
* "article id"). Arguments are matched to declared parameters and anything that
* matches nothing is dropped rather than passed through — an action should never
* receive a key its signature does not mention.
*/
function bindArguments(fn, plan, ctx) {
const args = {};
const ignored = [];
for (const s of plan.sets) {
if (!s.column)
continue;
const param = fn.params.find(p => sameWord(p, s.column))
?? fn.params.find(p => normalize(p).replace(/\s+/g, '') === normalize(s.column).replace(/\s+/g, ''));
const value = s.value.includes(SELF) ? String(ctx.userId ?? '') : join(s.value);
if (!param) {
ignored.push(s.column);
continue;
}
args[param.replace(/\s+/g, '_')] = value;
}
// Whatever the session owns is taken from the session, not from the sentence.
for (const [param, key] of Object.entries(fn.fromContext ?? {}))
args[param.replace(/\s+/g, '_')] = String(ctx[key] ?? '');
const missing = (fn.required ?? []).filter(p => !args[p.replace(/\s+/g, '_')]);
return { args, missing, ignored };
}
/**
* Read `<param> <value>` straight out of the sentence.
*
* Only for parameters the plan left empty, and only from the words the user
* actually wrote: "with the id 12" gives id = 12, "to marc@paper.test" gives
* to = marc@paper.test. It fills the gap left when the model recognises the
* function but not its arguments, which is the common shape of the failure
* because argument names are short words and values are usually unseen ones.
*/
function argumentsFromSentence(fn, input) {
const words = tokenize(input);
const found = {};
for (const param of fn.params) {
const want = tokenize(param);
for (let i = 0; i + want.length < words.length; i++) {
if (!want.every((w, k) => w === words[i + k]))
continue;
const rest = words.slice(i + want.length);
// Skip the filler a sentence puts between the name and the value.
let j = 0;
while (j < rest.length && ['the', 'a', 'an', 'of', 'is', 'le', 'la', 'les', 'de', 'du'].includes(rest[j]))
j++;
const value = join(rest.slice(j, j + 6)).split(/\s+(?:and|with|et|avec)\s+/)[0];
if (value)
found[param.replace(/\s+/g, '_')] = value;
break;
}
}
// "email the author of article 2" names no parameter, but there is exactly one
// number in it and the function wants an id. A lone integer answering a single
// id-shaped parameter is not a guess worth refusing over; two numbers, or a
// parameter that is not an id, and this stays out of it.
const idParam = fn.params.find(p => /(^|\s)id$/.test(normalize(p)));
if (idParam && !found[idParam.replace(/\s+/g, '_')]) {
const numbers = words.filter(w => /^\d+$/.test(w));
if (numbers.length === 1)
found[idParam.replace(/\s+/g, '_')] = numbers[0];
}
return found;
}
/**
* Warn about schema words the model has never seen.
*
* An out-of-vocabulary word in the input window is not ignored by a pointer
* decoder — it is a position, so it becomes something to point at, and it
* competes with the right answer. Naming a column `headline` instead of `title`
* was enough to turn every filter on this newspaper into
* `SELECT articles FILTER headline VAL <the first noun of the sentence>`.
*
* Values are a different matter: a row's title or an invented section is
* *meant* to be unknown, and the pointer mechanism exists precisely so those
* work. It is the schema's own words — tables, columns, enum values — that
* should be words the weights know.
*/
export function checkVocabulary(schema, warn = console.warn) {
const unknown = new Set();
const look = (text) => {
for (const w of tokenize(text))
if (!(w in cfg.vocab) && !/^\d+$/.test(w))
unknown.add(w);
};
for (const t of schema.tables) {
look(t.as);
for (const col of Object.keys(t.columns))
look(col);
for (const [col, values] of Object.entries(t.enums ?? {})) {
look(col);
for (const v of values)
look(v);
}
}
// Functions are not checked: they never reach the model. `buildSource` leaves
// them out of the window precisely because their verbs are the words the
// weights are least likely to know, and dispatch does not need them.
if (unknown.size) {
warn(`talos: ${unknown.size} schema word(s) are outside the model's vocabulary: `
+ `${[...unknown].join(', ')}. The model will point at them rather than read `
+ `them — prefer a word it knows (title over headline, summary over standfirst), `
+ `or add a surface form with addSurfaceForms().`);
}
return [...unknown];
}
/** The three role words the model was trained on. */
const TRAINED_ROLES = ['guest', 'customer', 'admin'];
/**
* What the model is told the caller is.
*
* An app may name its roles anything — reader, journalist, editor — but those
* words are not in the model's vocabulary, and an unknown word in the context
* does not merely get ignored: it becomes a pointer target. With `role = editor`
* in the window, "montre moi tous les articles" came back as
* `SELECT articles FILTER title VAL editor`. Every skill collapsed, not just the
* ones about permission.
*
* So the model is told one of the three words it was trained on, derived from
* what the schema actually lets this role do. It costs nothing: the model does
* not decide permission, `authorise` does, and that keeps the real role.
*/
export function modelRole(schema, role, signedIn) {
if (!role || role === 'guest' || !signedIn)
return 'guest';
if (TRAINED_ROLES.includes(role))
return role;
// Staff is whoever may change the app's own records — the first table. Being
// allowed to leave a comment does not make a reader staff, so the child tables
// deliberately do not count.
const main = schema.tables[0];
const staff = [
main?.roles?.write, main?.roles?.create, main?.roles?.remove,
...Object.values(main?.roles?.columns ?? {}),
].some(list => list?.includes(role))
|| !!schema.functions?.some(f => f.roles?.includes(role));
return staff ? 'admin' : 'customer';
}
/**
* The permission gate: the only thing that decides access, and the reason no app
* in this repo has an auth.ts. Pure and exported so it can be tested against a
* plan directly — testing it through a sentence would test the model instead,
* and the model is not what enforces this.
*
* Returns the refusal message, or null when the role may proceed.
*/
export function authorise(plan, schema, role) {
const may = (allowed) => !allowed || allowed.includes(role);
if (plan.op === 'CALL') {
const declared = schema.functions?.find(f => normalize(f.name) === normalize(plan.target ?? '') || firstWord(f.name) === firstWord(plan.target ?? ''));
return declared && !may(declared.roles) ? `${plan.target} is not yours to call` : null;
}
const target = schema.tables.find(t => plan.target && firstWord(t.as) === firstWord(plan.target));
if (!target)
return null;
const writing = plan.op !== 'SELECT';
const roles = target.roles;
// INSERT and DELETE may be narrower than write; both fall back to it.
const forOp = plan.op === 'INSERT' ? roles?.create ?? roles?.write
: plan.op === 'DELETE' ? roles?.remove ?? roles?.write
: writing ? roles?.write : roles?.read;
if (!may(forOp))
return `${role}s may not ${writing ? 'change' : 'read'} the ${target.as}`;
// Then the columns this plan actually writes, one at a time.
if (writing && roles?.columns) {
for (const s of plan.sets) {
const allowed = s.column && roles.columns[s.column];
if (allowed && !may(allowed))
return `${role}s may not set the ${s.column} of an ${firstWord(target.as)}`;
}
}
return null;
}
// --- the sequence the model reads (mirrors train/gen2.py `serialize`) --------
/**
* The exact token window the model reads for a request — for diagnosis, e.g.
* comparing a real app's window with the ones training generates.
*/
export function windowFor(schema, prompt, ctx, input) {
const context = [`role = ${modelRole(schema, ctx.role, !!ctx.userId)}`,
`signed in = ${ctx.userId ? 'yes' : 'no'}`].join(' ; ');
return buildSource(schema, prompt, context, input).toks;
}
/**
* The tables that fit in the window, when they do not all fit.
*
* The model reads 512 tokens and the request comes last, so a schema of twenty
* tables leaves no room for it: the sentence is cut off. Most requests concern
* one or two tables, and the request says which — by name, by a synonym, or by
* one of their values. Those go in first; the rest fill what room is left, in
* declaration order. A small schema is untouched.
*/
function tablesFor(schema, input, room) {
// Counted in tokens: "has public profile" is three, not one.
const cost = (t) => 2 + tokenize(t.as).length + tokenize(Object.keys(t.columns).join(' ')).length
+ Object.entries(t.enums ?? {}).reduce((n, [c, vs]) => n + 1 + tokenize(c).length
+ vs.reduce((m, v) => m + tokenize(v).length + (SURFACE[v] ?? []).slice(0, 3).reduce((k, a) => k + tokenize(a).length, 0), 0), 0);
if (schema.tables.reduce((n, t) => n + cost(t), 0) <= room)
return schema.tables;
const score = (t) => ([t.as, ...(t.says ?? [])].some(w => mentions(input, w)) ? 10 : 0)
+ Object.values(t.enums ?? {}).flat().filter(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f))).length * 3
+ [...Object.keys(t.columns), ...Object.values(t.columnSays ?? {}).flat()].filter(c => mentions(input, c)).length;
const ranked = schema.tables.map((t, i) => ({ t, i, s: score(t) })).sort((a, b) => b.s - a.s || a.i - b.i);
const kept = [];
for (const r of ranked)
if ((room -= cost(r.t)) >= 0 || !kept.length)
kept.push(r);
return kept.sort((a, b) => a.i - b.i).map(r => r.t);
}
function buildSource(schema, prompt, context, input) {
const toks = [];
const push = (text) => { for (const w of tokenize(text))
toks.push(w); };
// Room for tables: the window, less the rules, the context and the request.
const rest = 4 + prompt.split('\n').reduce((n, l) => n + tokenize(l).length + 1, 0)
+ tokenize(context).length + tokenize(input).length;
toks.push('<schema>');
for (const t of tablesFor(schema, input, cfg.maxSrc - rest)) {
toks.push('<table>');
push(t.as);
push(Object.keys(t.columns).join(' '));
for (const [col, values] of Object.entries(t.enums ?? {})) {
toks.push('<enum>');
push(col);
for (const v of values) {
push(v);
for (const alias of (SURFACE[v] ?? []).slice(0, 3))
push(alias);
}
}
}
// Functions are deliberately absent from the window.
//
// They used to be here, and every one of them cost accuracy. An unseen verb is
// not inert — `reindex` turned up as a value in unrelated plans in the same
// schema (`FILTER title VAL reindex index`) — and even in-vocabulary parameter
// names collide with columns: declaring `emailAuthor(id, subject, body)` put a
// second `body` in the window and broke publishing, which had been working.
//
// Nothing needs them here any more. `namedFunction` dispatches by matching the
// declared name against the request, and `argumentsFromSentence` reads the
// arguments from the words the user wrote. The model is left to do the one
// thing it is good at: planning queries. That is also what keeps naming free —
// an app can call its action anything without waiting for the word to reach a
// training set.
toks.push('<rules>');
// A rule that describes a function is no longer the model's business: calls
// are dispatched by name against the request, not planned. Such a line only
// puts its verbs — usually ones the weights have never seen — into the window
// where they compete as pointer targets. Dropping them took an eleven-case
// probe from 64% to 91%, and the failures it removed were in unrelated
// sentences ("les brouillons" was answering with the word "reindex" in it).
const describesAFunction = (line) => (schema.functions ?? []).some(f => [...nameForms(f.name), ...(f.says ?? []).map(s => normalize(s))]
.some(form => normalize(line).includes(form)));
for (const line of prompt.split('\n').map(l => l.trim()).filter(Boolean)) {
if (describesAFunction(line))
continue;
push(line);
toks.push(';');
}
toks.push('<ctx>');
push(context);
toks.push('<user>');
// Where the request itself starts. A pointer landing before this came from the
// schema or the rules rather than from anything the caller said, which is how
// a value like `editor` — a word only the prompt contains — ends up in a
// filter. Knowing the boundary makes that checkable instead of guessable.
const userStart = toks.length;
push(input);
return { toks: toks.slice(0, cfg.maxSrc), userStart };
}
/**
* Drop what the caller never asked for, and take the word they did say.
*
* Values are pointers into the model's input, and that input is mostly *not* the
* request — it is the schema and the rules. A pointer that lands there produces
* a filter on a word nobody typed: `FILTER title VAL editor`, where `editor`
* occurs only in the prompt, or `FILTER status VAL scheduled` on a table whose
* statuses are active and expired.
*
* So a clause survives on one of two grounds, and nothing else:
*
* - the caller said it, or
* - it is a member of the column's declared domain.
*
* And when the caller did name a domain value — "en attente" for `pending` —
* that beats whatever the model pointed at elsewhere. This is the same rule as
* everywhere else in this file: the app declares, the sentence decides, the
* weights are not asked to remember.
*/
function groundClause(clause, table, input, schema) {
// A value that is one of the app's own structural words is not a value. Asked
// for "les articles de la rubrique sport" the model filtered the title on
// "rubrique" — a word this schema declares as another table's synonym. The
// caller was naming a thing, not searching for the string.
const text0 = normalize(join(clause.value));
const structural = (schema?.tables ?? [table]).some(t => [t.as, ...(t.says ?? []), ...Object.keys(t.columns),
...Object.values(t.columnSays ?? {}).flat()].some(w => sameWord(w, text0)));
if (structural)
return null;
const domain = Object.keys(table.enums ?? {}).find(c => sameWord(c, clause.column))
? table.enums[Object.keys(table.enums).find(c => sameWord(c, clause.column))]
: undefined;
if (domain) {
const said = ` ${normalize(input).replace(/\s+/g, ' ')} `;
const spoken = domain.filter(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f)));
if (spoken.length === 1)
return { ...clause, value: [spoken[0]], said: true };
const text = normalize(join(clause.value));
const member = domain.find(v => sameWord(v, text) || (SURFACE[v] ?? []).some(a => sameWord(a, text)));
// A domain value survives only if the caller said it, or said one of its
// forms. Being a legal value is not enough: "les cours pour débutants" came
// back with `room = studio` added, a real room nobody mentioned.
const formSaid = member
&& [member, ...(SURFACE[member] ?? [])].some(f => mentions(input, f));
return formSaid || clause.said ? clause : null;
}
// A free-text column can only hold what the caller typed.
return clause.said !== false ? clause : null;
}
/**
* Send a clause to the column whose declared domain actually contains its value.
*
* The model's weakest skill is picking the column: it reaches for the first
* free-text one and drops the sentence's noun in it — `FILTER title VAL
* brouillons`, `SET content VAL published`. But `brouillons` is a declared
* surface form of `draft`, and `draft` belongs to exactly one column. The schema
* already knows where that value lives, so the runtime puts it there instead of
* asking the weights to.
*
* Deliberately narrow, because the opposite mistake is worse: only a clause on a
* column that has *no* declared domain is moved, and only when the whole value
* matches a domain member or one of its surface forms. Searching articles for
* the word "sport" in their title still works — `title` holds no domain, but
* nothing moves unless the value is an exact domain member, and a free-text
* search that happens to be exactly a section name is the rare case, while
* aiming a section at the title is the common one.
*/
function repairClause(clause, table) {
const enums = table.enums ?? {};
const named = clause.column && Object.keys(table.columns).find(c => normalize(c) === normalize(clause.column));
const text = normalize(join(clause.value));
if (!text)
return { clause };
// A clause already on a domain column is left alone — unless its value is not
// in that domain. `SET section VAL published` is the model putting the right
// word on the wrong column, and `published` is a status: the schema says so.
const ownDomain = named ? enums[named] : undefined;
if (ownDomain?.some(v => sameWord(v, text) || (SURFACE[v] ?? []).some(a => sameWord(a, text))))
return { clause };
if (named && !ownDomain && !Object.keys(enums).length)
return { clause };
for (const [col, values] of Object.entries(enums)) {
const hit = values.find(v => sameWord(v, text) || (SURFACE[v] ?? []).some(a => sameWord(a, text)));
if (hit)
return { moved: `${clause.column || '?'} -> ${col}`, clause: { column: col, value: [hit] } };
}
return { clause };
}
/**
* Equal, give or take a plural.
*
* People ask for "les brouillons" and "the drafts"; the schema declares `draft`.
* Making every app spell out both forms of every value is busywork that it will
* get wrong, and a trailing s is not a judgement call.
*/
function sameWord(a, b) {
const x = normalize(a);
const y = normalize(b);
return x === y || `${x}s` === y || x === `${y}s` || (stem(x) === stem(y) && stem(x).length >= 3);
}
/**
* The inflections a table or value name actually arrives in: English -ies
* ("entry" for `entries`) and French feminine plurals ("payées" for `payé`,
* "publiées" for `publié`). Missing those left "change the status of entry 2"
* aimed at the wrong table and "les commandes payées" unfiltered.
*/
function stem(word) {
// -ées needs both e's gone: "payées" and "payé" must meet at "pay". English
// -ed too: "close ticket 21" has to reach `closed`, "archive" `archived`.
return word.replace(/ies$/, 'y').replace(/s$/, '').replace(/ed$/, '')
.replace(/e$/, '').replace(/e$/, '');
}
/**
* Does the request contain `form`, as whole words, give or take a plural on
* each word? Every check of "did the caller say this" goes through here; doing
* it with a plain substring match missed "les brouillons" for `brouillon` and
* "the drafts" for `draft`, and threw correct filters away.
*/
function mentions(input, form) {
const words = tokenize(input);
const want = tokenize(form);
if (!want.length)
return false;
for (let i = 0; i + want.length <= words.length; i++)
if (want.every((w, k) => sameWord(w, words[i + k])))
return true;
return false;
}
/**
* Drop every clause the model scoped to "the current user".
*
* Ownership is not the model's to state: on a table declared with an
* ownerColumn the runtime pins every query to the session whatever the plan
* says, and on any other table a `user = <self>` clause is simply wrong. That
* second case is not hypothetical — trained on apps whose basket always had a
* `user` column filtered to SELF, the model learned the *shape*, and a comments
* table with a `user` column got "show me the comments on article 3" answered
* with `FILTER user VAL SELF`: the reader's own comments instead of the
* article's.
*/
function dropSelf(plan) {
const keep = (c) => !c.value.includes(SELF);
const filters = plan.filters.filter(keep);
const sets = plan.sets.filter(keep);
if (filters.length === plan.filters.length && sets.length === plan.sets.length)
return plan;
return { ...plan, filters, sets, repaired: [...(plan.repaired ?? []), 'dropped a <self> clause'] };
}
/**
* A read of a child table that names one number reads that parent's children.
*
* "les commentaires de l article 5": the table has a `parent` column, the plan
* has no filter left, and the only number the caller wrote is 5. Only for
* reads, only with exactly one number, only on a table that declares `parent`.
*/
function parentFromSentence(plan, table, input) {
if (plan.op !== 'SELECT' || plan.filters.length)
return plan;
const parent = Object.keys(table.columns).find(c => normalize(c) === 'parent');
if (!parent)
return plan;
const numbers = tokenize(input).filter(w => /^\d+$/.test(w));
if (numbers.length !== 1)
return plan;
return {
...plan,
filters: [{ column: parent, value: [numbers[0]], said: true }],
repaired: [...(plan.repaired ?? []), `${parent} = ${numbers[0]} (said)`],
};
}
function repairPlan(raw, schema, input) {
const plan = dropSelf(raw);
// A filter whose value is the *name of another table* means that table was
// what was asked for. "show me the subscribers" came back as
// `SELECT articles FILTER title VAL subscribers`: the model found the right
// word and put it in the wrong slot.
//
// Only when the request actually contains that word. A table whose name the
// model has never seen leaks into values across unrelated plans, and without
// this guard the repair promoted the leak: "montre moi tous les articles"
// became `SELECT subscribers`. The sentence has to have said it.
if (plan.op === 'SELECT' && plan.filters.length === 1 && !plan.sets.length) {
const value = join(plan.filters[0].value);
const said = ` ${normalize(input)} `;
const meant = schema.tables.find(t => sameWord(t.as, value)
&& (said.includes(` ${normalize(t.as)} `) || said.includes(` ${normalize(t.as)}s `)));
// Retarget, then repair the clauses against the table that was meant.
// Returning straight away skipped everything below: "show me the confirmed
// appointments" was retargeted and then lost its `status = confirmed`.
if (meant && !sameWord(meant.as, plan.target ?? '')) {
return repairClauses({ ...plan, target: meant.as, filters: [],
repaired: [...(plan.repaired ?? []), `table -> ${meant.as}`] }, meant, input, schema);
}
}
// Same equivalence as everywhere else: the model wrote `order` for `orders`,
// the strict first-word match found no table, and nothing got repaired. The
// target is also rewritten to the declared name, so what runs is what shows.
const table = schema.tables.find(t => plan.target && firstWord(t.as) === firstWord(plan.target))
?? schema.tables.find(t => plan.target && sameWord(t.as, plan.target));
if (!table)
return plan;
const aimed = table.as === plan.target ? plan : { ...plan, target: table.as };
return repairClauses(aimed, table, input, schema);
}
/**
* A filter the caller clearly asked for and the model did not produce.
*
* "les membres actifs", "les commandes payées", "les commentaires en attente" —
* each names exactly one value of exactly one declared domain on the table being
* read, and each came back as a bare SELECT of the whole table. The app wrote
* the value down and the caller said it; there is nothing left to infer.
*
* Only for reads, only when nothing already filters that column, and only when
* the sentence names one value and no other: two candidates is ambiguity, and
* ambiguity is the model's problem, not the runtime's.
*/
function filterFromSentence(plan, table, input) {
if (plan.op !== 'SELECT')
return plan;
const said = ` ${normalize(input).replace(/\s+/g, ' ')} `;
const hits = [];
for (const [col, values] of Object.entries(table.enums ?? {})) {
if (plan.filters.some(f => sameWord(f.column, col)))
continue;
const named = values.filter(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f)));
if (named.length === 1)
hits.push({ column: col, value: [named[0]], said: true });
}
if (hits.length !== 1)
return plan;
// A single loose word beside a value the app declared is the model padding,
// not a second condition: "je veux lire de la politique" filtered the title on
// `lire` as well as the section on `politics`, and the two together match
// nothing. So is a free-text value that *contains* the word just recognised
// as the declared value: "les problèmes majeurs" filtered the title on
// "problemes majeurs" — the same word, misfiled. A multi-word value that does
// not contain it is left alone; that shape is a real title.
const hit = hits[0];
const forms = [hit.value[0], ...(SURFACE[hit.value[0]] ?? [])].map(f => normalize(f));
const kept = plan.filters.filter((f) => {
if ((table.enums ?? {})[f.column])
return true;
const text = join(f.value);
if (normalize(text).split(/\s+/).length === 1)
return false;
return !forms.some(form => mentions(text, form));
});
const dropped = plan.filters.length - kept.length;
return {
...plan,
filters: [...kept, hits[0]],
repaired: [
...(plan.repaired ?? []),
...(dropped ? [`dropped ${dropped} loose word(s)`] : []),
`${hits[0].column} = ${hits[0].value[0]} (said)`,
],
};
}
/**
* Put every clause of a plan on the right column of `table`, then throw away the
* ones the caller never asked for. Order matters: a clause is moved to the
* column that declares its value first, and only what survives that is grounded
* against the request.
*/
function repairClauses(plan, table, input, schema) {
const moves = [];
const fix = (c) => {
// A filter on the owner column is the runtime's business, never a domain value.
if (c.value.includes(SELF))
return c;
// A clause on another table's columns is not about this one: "change the
// status of appointment 7 to completed" came back with `SET speciality city
// VAL 7`, two of the doctors' columns and the appointment's number.
const words = tokenize(c.column);
const mine = [...Object.keys(table.columns), ...Object.keys(table.columnSays ?? {}),
...Object.values(table.columnSays ?? {}).flat()];
if (schema && words.length && !words.some(w => mine.some(m => sameWord(m, w)))
&& words.every(w => schema.tables.some(t => t !== table && Object.keys(t.columns).some(m => sameWord(m, w))))) {
moves.push(`dropped ${c.column} = ${join(c.value)} (another table's column)`);
return null;
}
const { moved, clause } = repairClause(c, table);
const grounded = groundClause(clause, table, input, schema);
if (!grounded) {
moves.push(`dropped ${c.column} = ${join(c.value)}`);
return null;
}
if (moved)
moves.push(moved);
else if (grounded.value.join(' ') !== clause.value.join(' '))
moves.push(`${clause.column} = ${join(grounded.value)}`);
return grounded;
};
const filters = plan.filters.map(fix).filter(Boolean);
const sets = plan.sets.map(fix).filter(Boolean);
const repaired = moves.length
? { ...plan, filters, sets, repaired: [...(plan.repaired ?? []), ...moves] }
: plan;
return setFromSentence(parentFromSentence(filterFromSentence(repaired, table, input), table, input), table, input);
}
/**
* An update that names its new value but lost the clause that sets it.
*
* "change the status of comment 8 to approved" came back as
* `UPDATE comments FILTER id VAL 8`: the model aimed the value at a column
* grounding then rightly dropped, and nothing was left to change. The sentence
* names exactly one value of exactly one declared domain on the table, so that
* is the change. Only when the update sets nothing on any domain column.
*/
function setFromSentence(plan, table, input) {
if (plan.op !== 'UPDATE')
return plan;
const enums = table.enums ?? {};
if (plan.sets.some(s => Object.keys(enums).some(c => sameWord(c, s.column))))
return plan;
const hits = [];
for (const [col, values] of Object.entries(enums)) {
const named = values.filter(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f)));
if (named.length === 1)
hits.push({ column: col, value: [named[0]], said: true });
}
if (hits.length !== 1)
return plan;
return {
...plan,
sets: [...plan.sets, hits[0]],
repaired: [...(plan.repaired ?? []), `set ${hits[0].column} = ${hits[0].value[0]} (said)`],
};
}
/**
* The plan as it will actually run, which is not always what the model wrote:
* a clause the runtime moved to the column that declares its value shows here in
* its repaired form. `plan.tokens` keeps the raw generation, and `plan.repaired`
* says what moved, so both are visible rather than one quietly standing in for
* the other.
*/
export const planText = (plan) => [
plan.op,
plan.target ?? '',
...plan.sets.map(s => `SET ${s.column} VAL ${s.value.join(' ')}`),
...plan.filters.map(f => `FILTER ${f.column} VAL ${f.value.join(' ')}`),
].join(' ').replace(/\s+/g, ' ').trim();
const SELF = '<self>';
/** Glue a pointer run back into text: URLs and decimals arrive as separate tokens. */
const join = (words) => words.filter(w => w !== SELF).join(' ').replace(/\s+([./:@_-])\s*/g, '$1').trim();
/** `OP target [FILTER|SET col… VAL value… ]*` — VAL is what makes multi-word columns work. */
function decodePlan(ids, src, userStart = 0) {
const word = (id) => (id < cfg.nOps ? cfg.ops[id] : src[id - cfg.nOps] ?? '?');
const plan = { op: 'REFUSE', target: null, filters: [], sets: [], tokens: ids.map(word) };
if (!ids.length)
return plan;
const isOp = (k) => ids[k] !== undefined && ids[k] < cfg.nOps;
const opAt = (k) => cfg.ops[ids[k]];
let i = 0;
plan.op = isOp(0) ? opAt(0) : 'REFUSE';
i = 1;
if (['REFUSE', 'DENY', '<end>'].includes(plan.op))
return plan;
/**
* Consume pointer tokens (and a bare SELF) until the next structural op,
* remembering whether any of them landed inside the caller's own sentence.
*/
let said = false;
const run = (value = false) => {
const out = [];
said = false;
while (i < ids.length) {
if (!isOp(i)) {
const at = ids[i] - cfg.nOps;
if (at >= userStart)
said = true;
out.push({ w: word(ids[i++]), mine: at >= userStart });
continue;
}
if (opAt(i) === 'SELF') {
out.push({ w: SELF, mine: true });
i++;
continue;
}
break;
}
// A value that is partly the caller's words and partly a word of the schema
// keeps only the caller's: `FILTER name VAL width formules`, where `width`
// is another table's column, is a filter on "formules" at most.
return (value && said ? out.filter(t => t.mine) : out).map(t => t.w);
};
plan.target = run().join(' ') || null;
while (i < ids.length) {
const op = opAt(i);
i++;
if (op === 'FILTER' || op === 'SET') {
const column = run().join(' ');
let value = [];
let fromUser = false;
if (i < ids.length && opAt(i) === 'VAL') {
i++;
value = run(true);
fromUser = said;
}
;
(op === 'FILTER' ? plan.filters : plan.sets).push({ column, value, said: fromUser });
}
else if (op === '<end>')
break;
}
return plan;
}
// --- compiling a plan to SQL ------------------------------------------------
const bare = (s) => {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(s))
throw new Error(`bad identifier: ${s}`);
return s;
};
const firstWord = (s) => normalize(s).split(/\s+/)[0] ?? '';
/**
* Map what the model pointed at back onto the value the column actually holds.
*
* The surface forms are in the schema so the model can find "sports" or
* "économie" next to the value they belong to — and then it points at the
* *alias*, because that is the word it matched. `section = 'sports'` finds
* nothing; `section = 'sport'` finds the articles. The schema knows the mapping,
* so the runtime applies it rather than asking the model to do it.
*/
function canonical(value, allowed) {
if (typeof value !== 'string')
return value;
const want = normalize(value);
const exact = allowed.find(v => sameWord(v, want));
if (exact)
return exact;
const byAlias = allowed.find(v => (SURFACE[v] ?? []).some(a => sameWord(a, want)));
return byAlias ?? value;
}
async function compile(plan, schema, store, ctx, shape = {}, actions = {}) {
// Quoted, because real schemas have columns called `order` and `key`.
const ident = (s) => store.dialect === 'mysql' ? `\`${bare(s)}\`` : `"${bare(s)}"`;
const target = schema.tables.find(t => plan.target && firstWord(t.as) === firstWord(plan.target))
?? schema.tables.find(t => plan.target && normalize(t.as).includes(firstWord(plan.target)));
if (!target)
return { error: `the plan names something I do not have: ${plan.target}` };
const owner = target.ownerColumn
? (target.ownerFrom === 'sid' ? ctx.sid : ctx.userId)
: undefined;
const column = (logical) => {
const keys = Object.keys(target.columns);
const key = keys.find(k => normalize(k) === normalize(logical))
?? keys.find(k => firstWord(k) === firstWord(logical));
return key ? { logical: key, physical: target.columns[key] } : null;
};
const bindValue = async (words, logical) => {
if (words.includes(SELF))
return owner ?? ctx.userId ?? -1;
const text = join(words);
if (!text)
return null;
if (logical === 'item')
return await store.lookup(schema.tables[0].table, text);
if (/^\d+$/.test(text))
return Number(text);
return text;
};
// A request field by name: `showInNav`, `show_in_nav` or the logical `visible`.
const named = (key) => {
const snake = key.replace(/[A-Z]/g, c => `_${c.toLowerCase()}`);
const logical = Object.keys(target.columns).find(k => k === key || target.columns[k] === key || target.columns[k] === snake);
return logical ? { logical, physical: target.columns[logical] } : null;
};
const Q = '\u0001';
// Relations are joined as r0, r1… in declaration order; a condition on one
// (`category.slug`, or a filter the model put on `category`) only makes
// sense for a read.
const relNames = Object.keys(target.relations ?? {});
const relCol = (rel, col) => {
const r = target.relations[rel];
return `r${relNames.indexOf(rel)}.${ident(r.columns[col] ?? col)}`;
};
let onRelation = false;
// A per-row count of rows elsewhere: `(SELECT COUNT(*) FROM views c WHERE c.article_id = t.id)`.
// With conditions of its own (pending tasks, open loans), inlined as literals
// so the subquery can sit in a select list, a sort and a filter alike.
const lit = (v) => v === null ? 'NULL' : typeof v === 'number' ? String(v) : typeof v === 'boolean' ? String(Number(v))
: `'${String(v).replace(/'/g, "''")}'`;
const tallySql = (k) => `(SELECT COUNT(*) FROM ${ident(k.table)} c WHERE c.${ident(k.on)} = ${Q}${ident('id')}`
+ Object.entries(k.where ?? {}).map(([c, v]) => Array.isArray(v) ? ` AND c.${ident(c)} IN (${v.map(lit).join(', ')})`
: v && typeof v === 'object' && 'ne' in v ? ` AND (c.${ident(c)} IS NULL OR c.${ident(c)} <> ${lit(v.ne)})`
: ` AND c.${ident(c)} ${v === null ? 'IS NULL' : `= ${lit(v)}`}`).join('') + ')';
// A value on its way to the driver: the column's own conversion, then 0/1 for booleans.
const toDriver = (physical, v) => sqlValue(v !== null && v !== undefined && physical && target.write?.[physical] ? target.write[physical](v) : v);
const pin = (where) => Object.entries(where).flatMap(([k, v]) => {
// `any: [{ starts: { null: true } }, { starts: { lte: now } }]` — one of several.
if (k === 'any' && Array.isArray(v))
return [{ col: null, sql: '', v: null, any: v }];
const tally = target.tallies?.[k];
if (tally) {
onRelation = true;
return [{ col: null, sql: tallySql(tally), v }];
}
const [rel, sub] = k.split('.');
if (sub && relNames.includes(rel)) {
onRelation = true;
return [{ col: null, sql: relCol(rel, sub), v }];
}
const col = named(k);
return col ? [{ col, sql: `${Q}${ident(col.physical)}`, v }] : [];
});
// A pinned value is equality, or one of a few comparisons the route states:
// `{ gte: since }`, `{ between: [a, b] }`, `{ ne: 0 }`, `{ in: [...] }`, `{ null: true }`.
const compare = (pins, conditions, bind) => {
for (const { sql: c, v, any, col } of pins) {
if (any) {
const alts = any.map(w => { const cs = []; compare(pin(w), cs, bind); return cs.length ? `(${cs.join(' AND ')})` : '1 = 1'; });
if (alts.length)
conditions.push(`(${alts.join(' OR ')})`);
continue;
}
// Compared in the column's stored form (a date column may store text).
const sqlValue = (x) => toDriver(col?.physical, x);
const o = (v && typeof v === 'object' && !(v instanceof Date) ? v : { eq: v });
if ('eq' in o) {
conditions.push(o.eq === null ? `${c} IS NULL` : `${c} = ?`);
if (o.eq !== null)
bind.push(sqlValue(o.eq));
}
if ('ne' in o) {
conditions.push(`${c} <> ?`);
bind.push(sqlValue(o.ne));
}
for (const [op, sym] of [['gt', '>'], ['gte', '>='], ['lt', '<'], ['lte', '<=']])
if (op in o) {
conditions.push(`${c} ${sym} ?`);
bind.push(sqlValue(o[op]));
}
if ('between' in o) {
conditions.push(`${c} BETWEEN ? AND ?`);
bind.push(sqlValue(o.between[0]), sqlValue(o.between[1]));
}
if ('in' in o) {
conditions.push(o.in.length ? `${c} IN (${o.in.map(() => '?').join(', ')})` : '1 = 0');
bind.push(...o.in.map(sqlValue));
}
if ('null' in o)
conditions.push(`${c} IS ${o.null ? '' : 'NOT '}NULL`);
}
};
const pinned = pin(shape.where ?? {});
const conditions = [];
const bind = [];
compare(pinned, conditions, bind);
for (const f of plan.filters) {
const col = column(f.column);
if (col && pinned.some(p => p.col?.logical === col.logical))
continue;
const v = await bindValue(f.value, col?.logical ?? f.column);
// "les articles de la rubrique sport": a filter on a relation matches any
// of the related row's declared columns but its key.
const rel = !col && relNames.find(r => sameWord(r, f.column));
if (rel && v !== null) {
const cols = Object.keys(target.relations[rel].columns).filter(c => c !== 'id');
if (cols.length) {
onRelation = true;
conditions.push(`(${cols.map(c => `${relCol(rel, c)} = ?`).join(' OR ')})`);
for (const _ of cols)
bind.push(v);
}
continue;
}
if (!col || v === null)
continue;
const allowed = target.enums?.[col.logical];
// Enum columns hold one of a known set, so match exactly; a free-text column
// holds a name the sentence only partly quoted, so match loosely.
if (!allowed && typeof v === 'string') {
conditions.push(`${Q}${ident(col.physical)} LIKE ?`);
bind.push(`%${v}%`);
}
else {
conditions.push(`${Q}${ident(col.physical)} = ?`);
const value = allowed ? canonical(v, allowed) : v;
const flag = target.flags?.[col.logical];
bind.push(flag ? flag.indexOf(String(value)) : value);
}
}
if (target.ownerColumn) {
if (!owner)
return { error: 'you need to be signed in for that', denied: true };
conditions.push(`${Q}${ident(target.ownerColumn)} = ?`);
bind.push(owner);
}
// Free text over the columns a table declares searchable, when the route has some.
const searchable = (target.searchable ?? []).map(column).filter(Boolean);
if (shape.search && searchable.length) {
conditions.push(`(${searchable.map(c => `${Q}${ident(c.physical)} LIKE ?`).join(' OR ')})`);
for (const _ of searchable)
bind.push(`%${shape.search}%`);
}
// Conditions are written once and qualified per statement: a SELECT that joins
// needs `t.id`, an UPDATE on one table does not.
const whereAs = (prefix) => conditions.length ? ` WHERE ${conditions.join(' AND ').replaceAll(Q, prefix)}` : '';
const where = whereAs('');
if (onRelation && plan.op !== 'SELECT')
return { error: 'a condition on a related table only works for a read' };
const adds = new Set(), flips = new Set(), copies = new Map();
const assignments = async () => {
const cols = [];
const values = [];
let reject = null;
for (const s of plan.sets) {
const col = column(s.column);
if (!col)
continue;
if (target.writable && !target.writable.includes(col.logical))
continue;
// A column set twice in one plan (`SET name VAL economie SET name VAL 4`)
// keeps its first value; SQL would refuse the statement outright.
if (cols.includes(col.physical))
continue;
const v = await bindValue(s.value, col.logical);
if (v === null)
continue;
const allowed = target.enums?.[col.logical];
if (allowed) {
// Writing "publie" into a status column must store `published`, or the
// row stops matching every filter and every listing.
const value = canonical(v, allowed);
// And a column with a known set of values must hold one of them. When
// the model aims a value at the wrong column — `section = published` —
// the alternative to refusing is silently corrupting the row.
if (!allowed.some(a => normalize(a) === normalize(String(value)))) {
reject = `${String(value)} is not one of the ${col.logical} values`;
continue;
}
const flag = target.flags?.[col.logical];
cols.push(col.physical);
values.push(flag ? flag.indexOf(String(value)) : value);
continue;
}
cols.push(col.physical);
values.push(v);
}
// Values the request holds by name replace what the model copied.
for (const [k, raw] of Object.entries(shape.values ?? {})) {
const col = named(k);
if (!col || raw === undefined)
continue;
// A field sent empty is left alone — unless the column is required, when
// emptying it is an error the rules must see, not a silent no-op.
// (Required only on insert does not make an empty edit an error: it is skipped.)
if (raw === '' && target.rules?.[col.logical]?.required !== true)
continue;
if (target.writable && !target.writable.includes(col.logical))
continue;
// A column the route pins to one value names the row and is not rewritten;
// one it only compares (`name <> 'Admin'`) may still be written.
const isEq = (v) => !(v && typeof v === 'object' && !(v instanceof Date)) || 'eq' in v;
if (pinned.some(p => p.col?.logical === col.logical && isEq(p.v)))
continue;
const flag = target.flags?.[col.logical];
const allowed = target.enums?.[col.logical];
// `{ add: 1 }` adds to what the column holds (a click counter); `{ toggle:
// true }` flips a 0/1 column (an account switched on or off).
if (raw && typeof raw === 'object' && ('add' in raw || 'toggle' in raw)) {
const toggle = 'toggle' in raw;
(toggle ? flips : adds).add(col.physical);
const n = toggle ? 1 : Number(raw.add);
const at = cols.indexOf(col.physical);
if (at >= 0)
values[at] = n;
else {
cols.push(col.physical);
values.push(n);
}
continue;
}
let v = flag && typeof raw === 'boolean' ? Number(raw) : raw;
// null clears a nullable enum column; only a value must be one of the domain.
if (allowed && !flag && v !== null) {
v = canonical(v, allowed);
if (!allowed.some(a => normalize(a) === normalize(String(v)))) {
reject = `${String(v)} is not one of the ${col.logical} values`;
continue;
}
}
const at = cols.indexOf(col.physical);
if (at >= 0)
values[at] = v;
else {
cols.push(col.physical);
values.push(v);
}
}
// Counted before the stamps: a write the caller said nothing about is not
// made into one by the route's bookkeeping.
const said = cols.length;
// Stamps are the route's own writes, never the caller's: they may touch a
// column no body can (a view counter), and take `{ add }` / `{ toggle }` too.
for (const [c, v] of Object.entries(shape.stamp ?? {})) {
// A stamp wins over a value the body sent for the same column.
const put = (x) => { const at = cols.indexOf(c); if (at >= 0)
values[at] = x;
else {
cols.push(c);
values.push(x);
} };
if (v && typeof v === 'object' && ('add' in v || 'toggle' in v)) {
;
('toggle' in v ? flips : adds).add(c);
put('toggle' in v ? 1 : Number(v.add));
}
// `{ from: 'scheduled_at' }` copies another column of the same row (an update only).
else if (v && typeof v === 'object' && 'from' in v && !(v instanceof Date)) {
copies.set(c, bare(String(v.from)));
put(null);
}
else
put(v);
}
return { cols, values, reject, said };
};
/**
* A declared slug column, filled from its source when the write names the
* source but not the slug: unique, suffixed -2, -3… On an update the row
* itself does not count as taken.
*/
const slugs = async (cols, values, updating = false) => {
for (const [slugCol, spec] of Object.entries(target.slugs ?? {})) {
const source = typeof spec === 'string' ? spec : spec.from;
if (updating && typeof spec === 'object' && spec.keep)
continue;
const src = column(source);
const at = src ? cols.indexOf(src.physical) : -1;
if (at < 0 || cols.includes(slugCol) || values[at] == null || values[at] === '')
continue;
// An edit that leaves the source as it was keeps the slug, and its links.
if (updating) {
const now = await store.all(`SELECT ${ident(src.physical)} AS v FROM ${ident(target.table)}${where}`, ...bind);
if (now.length && now.every(r => String(r.v) === String(values[at])))
continue;
}
const base = slugify(String(values[at]));
for (let i = 1;; i++) {
const slug = i === 1 ? base : `${base}-${i}`;
const taken = await store.all(`SELECT 1 FROM ${ident(target.table)} WHERE ${ident(slugCol)} = ?`
+ (updating && conditions.length ? ` AND NOT (${conditions.join(' AND ').replaceAll(Q, '')})` : ''), slug, ...(updating && conditions.length ? bind : []));
if (!taken.length) {
cols.push(slugCol);
values.push(slug);
break;
}
}
}
};
// Declared rules, on the values about to be written. On an insert a required
// column must be present; on an update only what is written is checked.
const check = (cols, values, inserting) => {
const errors = [];
for (const [logical, rule] of Object.entries(target.rules ?? {})) {
const physical = target.columns[logical];
const at = physical ? cols.indexOf(physical) : -1;
if (rule.trim && at >= 0 && typeof values[at] === 'string')
values[at] = values[at].trim();
const v = at >= 0 ? values[at] : undefined;
const empty = v === undefined || v === null || String(v).trim() === '';
const fail = (why) => errors.push({ msg: rule.message ?? `${logical} ${why}`, column: logical });
const required = rule.required === 'insert' ? inserting : !!rule.required;
if (empty) {
if (required && (inserting || at >= 0))
fail('is required');
continue;
}
const s = String(v);
if (rule.min !== undefined && s.trim().length < rule.min)
fail(`needs at least ${rule.min} characters`);
else if (rule.max !== undefined && s.length > rule.max)
fail(`takes at most ${rule.max} characters`);
else if (rule.email && !/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(s))
fail('must be an email address');
else if (rule.pattern && !rule.pattern.test(s))
fail('is not in the expected form');
}
return errors.length ? { error: errors.map(e => e.msg).join('; '), errors } : null;
};
// The route's last word before the write, once the rules have passed; then the
// schema's own defaults for what is still missing (`created_at`, `updated_at`).
const prepare = async (cols, values, when) => {
// Unique values, checked against every other row.
const errors = [];
for (const [logical, rule] of Object.entries(target.rules ?? {})) {
const at = rule.unique ? cols.indexOf(target.columns[logical] ?? '') : -1;
if (at < 0 || values[at] === null || values[at] === undefined || values[at] === '')
continue;
const others = when === 'update' && conditions.length ? ` AND NOT (${conditions.join(' AND ').replaceAll(Q, '')})` : '';
const withCols = (rule.uniqueWith ?? []).map(l => target.columns[l] ?? l).filter(p => cols.includes(p));
const taken = await store.all(`SELECT 1 FROM ${ident(target.table)} WHERE ${[cols[at], ...withCols].map(c => `${ident(c)} = ?`).join(' AND ')}${others} LIMIT 1`, ...[cols[at], ...withCols].map(c => toDriver(c, values[cols.indexOf(c)])), ...(others ? bind : []));
if (taken.length)
errors.push({ msg: typeof rule.unique === 'string' ? rule.unique : rule.message ?? `${logical} is already used`, column: logical });
}
if (errors.length)
return { error: errors.map(e => e.msg).join('; '), errors };
await ready(cols, values);
const stored = await storeFiles(cols, values, when);
if (stored && 'error' in stored)
return stored;
for (const [c, f] of Object.entries(target.hash ?? {})) {
const at = cols.indexOf(c);
if (at >= 0 && values[at] !== null && values[at] !== undefined && values[at] !== '')
values[at] = await f(String(values[at]));
}
for (const [c, f] of Object.entries(target.defaults?.[when] ?? {}))
if (!cols.includes(c)) {
cols.push(c);
values.push(f());
}
// Columns the model fills from the row's own text, on a new row left without them.
for (const [c, inf] of when === 'insert' ? Object.entries(target.infer ?? {}) : []) {
const at = cols.indexOf(c);
if (at >= 0 && values[at] !== null && values[at] !== undefined && values[at] !== '')
continue;
const text = inf.from.map(src => { const i = cols.indexOf(target.columns[src] ?? src); return i >= 0 ? String(values[i] ?? '') : ''; }).join('. ');
const [best] = text.trim() ? classify(text, inf.labels) : [];
if (!best || best.score < (inf.min ?? 0))
continue;
if (at >= 0)
values[at] = best.label;
else {
cols.push(c);
values.push(best.label);
}
}
return null;
};
// Uploaded files for the table's file columns: checked, stored, their paths written.
const written = [];
const storeFiles = async (cols, values, when) => {
for (const [c, spec] of Object.entries(target.files ?? {})) {
const logical = Object.keys(target.columns).find(k => target.columns[k] === c);
const up = shape.uploads?.find(u => u.name === c || u.name === logical);
if (!up) {
if (spec.required && when === 'insert' && !cols.includes(c))
return { error: spec.required, errors: [{ msg: spec.required, column: logical ?? c }] };
continue;
}
const bad = (spec.accept && !spec.accept.test(up.type ?? '')) || (spec.maxBytes && up.data.length > spec.maxBytes);
if (bad)
return { error: spec.message ?? `${logical ?? c}: file refused`, errors: [{ msg: spec.message ?? `${logical ?? c}: file refused`, column: logical ?? c }] };
const { mkdir, writeFile } = await import('node:fs/promises');
const { join, extname } = await import('node:path');
const { randomBytes } = await import('node:crypto');
// A file that is not the image it claims to be is refused, not stored.
let data;
const image = shape.images?.[c] ?? (logical ? shape.images?.[logical] : undefined) ?? spec.image;
try {
data = image ? await processImage(up.data, image) : up.data;
if (spec.transform)
data = await spec.transform(data, { filename: up.filename, type: up.type });
}
catch (e) {
if (/needs sharp/.test(e?.message ?? ''))
throw e;
return { error: spec.message ?? `${logical ?? c}: file refused`, errors: [{ msg: spec.message ?? `${logical ?? c}: file refused`, column: logical ?? c }] };
}
const stem = `${Date.now()}-${randomBytes(6).toString('hex')}`;
const save = async (col, bytes, ext) => {
const name = `${stem}${col === c ? '' : `-${col}`}${ext}`;
await mkdir(spec.dir, { recursive: true });
await writeFile(join(spec.dir, name), bytes);
written.push(join(spec.dir, name));
const at = cols.indexOf(col), path = `${spec.url ?? ''}${name}`;
if (at >= 0)
values[at] = path;
else {
cols.push(col);
values.push(path);
}
};
await save(c, data, spec.ext ?? (image && extOf(image)) ?? (extname(up.filename ?? '').toLowerCase() || ''));
// What was stored, for the columns that record it.
if (spec.record) {
const meta = image || /^image\//.test(up.type ?? '') ? await imageSize(data).catch(() => ({})) : {};
const facts = { ...meta, size: data.length,
type: image?.format ? `image/${image.format}` : up.type, name: up.filename };
for (const [k, col] of Object.entries(spec.record)) {
if (!col || facts[k] === undefined)
continue;
const at = cols.indexOf(col);
if (at >= 0)
values[at] = facts[k];
else {
cols.push(col);
values.push(facts[k]);
}
}
}
// The other sizes of the same upload, each in its column.
for (const [col, v] of Object.entries(spec.variants ?? {})) {
const o = { format: spec.image?.format, quality: spec.image?.quality, ...v };
await save(col, await processImage(up.data, o), extOf(o) ?? (extname(up.filename ?? '').toLowerCase() || ''));
}
}
return null;
};
// A write that fails leaves no stored file behind.
const runOrClean = async (sql, ...p) => {
try {
return await store.run(sql, ...p);
}
catch (e) {
await unlinkAll(written);
throw e;
}
};
// A stored path back to its file on disk, if it is one of ours.
// A variant column is stored and cleaned like the file it is a size of.
const fileColumns = () => [...Object.keys(target.files ?? {}), ...Object.values(target.files ?? {}).flatMap(s => Object.keys(s.variants ?? {}))];
const fileOf = (c, v) => {
const spec = target.files?.[c] ?? Object.values(target.files ?? {}).find(s => c in (s.variants ?? {}));
if (!spec || typeof v !== 'string' || !v)
return null;
const name = spec.url && v.startsWith(spec.url) ? v.slice(spec.url.length) : !spec.url ? v : null;
return name && !/[\\/]/.test(name) ? `${spec.dir}/${name}` : null;
};
const unlinkAll = async (paths) => {
const { unlink } = await import('node:fs/promises');
for (const p of paths)
if (p)
await unlink(p).catch(() => { });
};
// The paths the rows about to change hold now, to remove once they are replaced.
const filesNow = async () => fileColumns().length && conditions.length
? await store.all(`SELECT ${fileColumns().map(ident).join(', ')} FROM ${ident(target.table)}${whereAs('')}`, ...bind)
: [];
const ready = async (cols, values) => {
const more = await shape.before?.(Object.fromEntries(cols.map((c, i) => [c, values[i]])));
for (const [c, v] of Object.entries(more ?? {})) {
const at = cols.indexOf(c);
if (at >= 0)
values[at] = v;
else {
cols.push(c);
values.push(v);
}
}
};
// SQLite upserts need the unique column: the row is updated in place, keeping
// its id and the columns this write does not name (REPLACE would drop both).
const upsertSqlite = (cols) => {
if (!shape.upsert)
return '';
const on = physicalOf(typeof shape.upsert === 'string' ? shape.upsert : 'id');
const rest = cols.filter(c => c !== on);
return ` ON CONFLICT(${ident(on)}) DO ${rest.length ? `UPDATE SET ${rest.map(c => `${ident(c)} = excluded.${ident(c)}`).join(', ')}` : 'NOTHING'}`;
};
const physicalOf = (k) => named(k)?.physical ?? column(k)?.physical ?? k;
// A many-to-many written from a list of ids in the body replaces the links.
const link = async (owners) => {
for (const [name, m] of Object.entries(target.many ?? {})) {
const list = shape.values?.[m.field ?? `${name}Ids`];
if (!Array.isArray(list))
continue;
for (const owner of owners) {
await store.run(`DELETE FROM ${ident(m.through)} WHERE ${ident(m.from)} = ?`, owner);
for (const to of new Set(list))
await store.run(`INSERT INTO ${ident(m.through)} (${ident(m.from)}, ${ident(m.to)}) VALUES (?, ?)`, owner, to);
}
}
};
// Declared follow-ups of a successful write, handed what was written.
const follow = async (when, cols, values, extra) => {
const row = Object.fromEntries(cols.map((c, i) => [c, values[i]]));
for (const name of target.after?.[when] ?? []) {
const action = actions?.[name];
if (!action)
throw new Error(`${target.as} names an after-${when} function that is not implemented: ${name}`);
await action({ args: { ...row, ...extra }, ctx, plan });
}
};
switch (plan.op) {
case 'SELECT': {
// `-date` sorts descending; several keys sort in turn.
const tallies = Object.entries(target.tallies ?? {});
// `-date` descending; `date~` puts empty values last; an enum column sorts
// by its declared order (low, medium, high), not alphabetically.
const keys = [shape.orderBy ?? []].flat().flatMap(k => {
const nullsLast = k.endsWith('~');
const name = k.replace(/^-/, '').replace(/~$/, ''), desc = k.startsWith('-') || !!shape.desc;
if (target.tallies?.[name])
return [{ desc, sql: ident(name) }];
const col = named(name) ?? column(name);
if (!col)
return [];
const c = `t.${ident(col.physical)}`;
const domain = target.enums?.[col.logical];
const sql = domain && !target.flags?.[col.logical]
? `CASE ${c} ${domain.map((v, i) => `WHEN ${lit(v)} THEN ${i}`).join(' ')} ELSE ${domain.length} END` : c;
return [...(nullsLast ? [{ desc: false, sql: `(${c} IS NULL)` }] : []), { desc, sql }];
});
const order = keys.length ? ` ORDER BY ${keys.map(k => `${k.sql}${k.desc ? ' DESC' : ''}`).join(', ')}` : '';
const limit = shape.limit ? Math.floor(shape.limit) : 0;
const offset = shape.offset ?? (shape.page && limit ? (Math.max(1, Math.floor(shape.page)) - 1) * limit : 0);
// A ranked read takes a wide candidate set and cuts after ranking.
const page = shape.recommend || shape.find || shape.similar ? ' LIMIT 2000'
: limit ? ` LIMIT ${limit}${offset ? ` OFFSET ${Math.floor(offset)}` : ''}` : '';
// Declared relations become LEFT JOINs, and come back as nested objects.
const rels = Object.entries(target.relations ?? {});
const joins = rels.map(([, r], i) => ` LEFT JOIN ${ident(r.table)} r${i} ON r${i}.${ident(r.key ?? 'id')} = t.${ident(r.on)}`).join('');
const picks = rels.flatMap(([name, r], i) => Object.entries(r.columns).map(([as, phys]) => `, r${i}.${ident(phys)} AS ${ident(`${name}__${as}`)}`)).join('')
+ tallies.map(([name, k]) => `, ${tallySql(k).replaceAll(Q, 't.')} AS ${ident(name)}`).join('');
const from = `${ident(target.view ?? target.table)} t`;
const raw = await store.all(`SELECT t.*${picks} FROM ${from}${joins}${whereAs('t.')}${order}${page}`, ...bind);
let rows = raw.map(row => shapeRow(row, target, rels.map(([n]) => n)));
if (shape.recommend)
rows = await rank(rows, { limit: limit || undefined, ...shape.recommend }, store, ident);
if (shape.find)
rows = find(rows, { limit: limit || undefined, ...shape.find });
if (shape.similar) {
const s = shape.similar;
const source = typeof s.to === 'object'
? (rows.find(r => r.id == s.to.id) ?? (await store.all(`SELECT * FROM ${ident(target.table)} WHERE ${ident('id')} = ?`, s.to.id))[0])
: null;
const to = typeof s.to === 'string' ? s.to : source ? s.text.map(c => String(source[c] ?? '')).join('. ') : '';
rows = to ? similar(rows, { to, text: s.text, exclude: typeof s.to === 'object' ? s.to.id : undefined, limit: s.limit ?? (limit || undefined), min: s.min }) : [];
}
for (const r of rows)
for (const [name] of tallies)
r[name] = Number(r[name] ?? 0);
// Lists read along with the rows, in one query per list for the whole page.
const ids = rows.map(r => r.id).filter(v => v !== undefined && v !== null);
const inIds = ids.map(() => '?').join(', ');
// A route may load only some lists (`lists: ['articles']`), or none (`lists: false`).
const wanted = (n) => shape.lists === undefined || (shape.lists !== false && shape.lists.includes(n));
for (const [name, l] of ids.length ? Object.entries(target.lists ?? {}).filter(([n]) => wanted(n)) : []) {
// Listed rows read back like their own table's rows, when it is declared.
const own = schema.tables.find(t => t.table === l.table);
const conv = (p, v) => !own || v === null || v === undefined ? v
: own.read?.[p] ? own.read[p](v) : readAs(own.json?.includes(p) ? 'json' : own.types?.[p], v);
// The listed rows, their own conditions, and what each of them points at.
const lrels = Object.entries(l.relations ?? {});
const cs = [], bs = [];
compare(Object.entries(l.where ?? {}).map(([c, v]) => ({ col: null, sql: `k.${ident(c)}`, v })), cs, bs);
const kids = await store.all(`SELECT k.${ident(l.on)} AS ${ident('__owner')}, `
+ [...Object.entries(l.columns).map(([as, p]) => `k.${ident(p)} AS ${ident(as)}`),
...lrels.flatMap(([rn, r], i) => Object.entries(r.columns).map(([as, p]) => `kr${i}.${ident(p)} AS ${ident(`${rn}__${as}`)}`))].join(', ')
+ ` FROM ${ident(l.table)} k`
+ lrels.map(([, r], i) => ` LEFT JOIN ${ident(r.table)} kr${i} ON kr${i}.${ident(r.key ?? 'id')} = k.${ident(r.on)}`).join('')
+ ` WHERE k.${ident(l.on)} IN (${inIds})${cs.length ? ` AND ${cs.join(' AND ')}` : ''}`
+ (l.orderBy ? ` ORDER BY k.${ident(l.orderBy.replace(/^-/, ''))}${l.orderBy.startsWith('-') ? ' DESC' : ''}` : ''), ...ids, ...bs);
const nest = (k) => {
const out = {};
for (const [key, v] of Object.entries(k)) {
const [rn, col] = key.split('__');
if (col && l.relations?.[rn])
(out[rn] ??= {})[col] = v;
else
out[key] = conv(l.columns[key] ?? key, v);
}
for (const [rn] of lrels)
if (out[rn] && Object.values(out[rn]).every(v => v === null))
out[rn] = null;
return out;
};
for (const r of rows) {
const mine = kids.filter(k => k.__owner === r.id).map(({ __owner, ...k }) => nest(k));
r[name] = l.limit ? mine.slice(0, l.limit) : mine;
}
}
for (const [name, m] of ids.length ? Object.entries(target.many ?? {}) : []) {
const linked = await store.all(`SELECT j.${ident(m.from)} AS ${ident('__owner')}, `
+ `${Object.entries(m.columns).map(([as, p]) => `r.${ident(p)} AS ${ident(as)}`).join(', ')}`
+ ` FROM ${ident(m.through)} j JOIN ${ident(m.table)} r ON r.${ident(m.key ?? 'id')} = j.${ident(m.to)}`
+ ` WHERE j.${ident(m.from)} IN (${inIds})`, ...ids);
for (const r of rows)
r[name] = linked.filter(k => k.__owner === r.id).map(({ __owner, ...k }) => k);
}
for (const r of rows)
finishRow(r, target);
const total = shape.total
? Number(Object.values((await store.all(`SELECT COUNT(*) AS n FROM ${from}${joins}${whereAs('t.')}`, ...bind))[0] ?? {})[0] ?? 0)
: undefined;
if (shape.one)
return rows.length ? { row: rows[0] } : { error: 'not found', missing: true };
const counts = {};
for (const [name, w] of Object.entries(shape.counts ?? {})) {
const cs = [], bs = [];
compare(pin(w), cs, bs);
const sql = `SELECT COUNT(*) AS n FROM ${from}${joins}${cs.length ? ` WHERE ${cs.join(' AND ').replaceAll(Q, 't.')}` : ''}`;
counts[name] = Number(Object.values((await store.all(sql, ...bs))[0] ?? {})[0] ?? 0);
}
const sums = {};
for (const [name, s] of Object.entries(shape.sums ?? {})) {
const col = named(s.column) ?? column(s.column);
if (!col)
continue;
const cs = [], bs = [];
compare(pin(s.where ?? {}), cs, bs);
const sql = `SELECT SUM(t.${ident(col.physical)}) AS n FROM ${from}${joins}${cs.length ? ` WHERE ${cs.join(' AND ').replaceAll(Q, 't.')}` : ''}`;
sums[name] = Number(Object.values((await store.all(sql, ...bs))[0] ?? {})[0] ?? 0);
}
if (shape.counts || shape.sums)
return { rows, total, counts, sums, page: shape.page ?? 1, limit };
return total === undefined ? { rows } : { rows, total, page: shape.page ?? 1, limit };
}
case 'INSERT': {
if (shape.rows) {
// What happened to each row: new, updated in place, or left alone (a duplicate).
let written = 0, updated = 0, skipped = 0;
for (const row of shape.rows) {
const cols = [], values = [];
for (const [k, v] of Object.entries(row)) {
const col = named(k);
if (!col || (target.writable && !target.writable.includes(col.logical)))
continue;
cols.push(col.physical);
values.push(v);
}
if (!cols.length)
continue;
const invalid = check(cols, values, true);
if (invalid)
return invalid;
const put = cols.map(ident).join(', ');
const marks = cols.map(() => '?').join(', ');
const key = typeof shape.upsert === 'string' ? physicalOf(shape.upsert) : null;
const existed = key && cols.includes(key)
? (await store.all(`SELECT 1 FROM ${ident(target.table)} WHERE ${ident(key)} = ? LIMIT 1`, toDriver(key, values[cols.indexOf(key)]))).length > 0
: false;
const ignore = shape.ignore ? (store.dialect === 'mysql' ? 'IGNORE ' : 'OR IGNORE ') : '';
const r = await store.run(store.dialect === 'mysql'
? `INSERT ${ignore}INTO ${ident(target.table)} (${put}) VALUES (${marks})`
+ (shape.upsert ? ` ON DUPLICATE KEY UPDATE ${cols.map(c => `${ident(c)} = VALUES(${ident(c)})`).join(', ')}` : '')
: `INSERT ${ignore}INTO ${ident(target.table)} (${put}) VALUES (${marks})` + (shape.ignore ? '' : upsertSqlite(cols)), ...values.map((v, i) => toDriver(cols[i], v)));
if (!Number(r.changes))
skipped++;
else if (existed)
updated++;
else
written++;
}
return { created: written, updated, skipped };
}
const { cols, values, reject, said } = await assignments();
if (reject)
return { error: reject };
// A new row takes the values the URL pins (`/articles/:articleId/attachments`).
for (const p of pinned) {
const v = p.v && typeof p.v === 'object' && !(p.v instanceof Date) ? p.v.eq : p.v;
// Never the new row's own key: `/tribunes/:id/convert` names the source, not the article.
if (p.col && p.col.physical !== 'id' && v !== undefined && !cols.includes(p.col.physical)) {
cols.push(p.col.physical);
values.push(v);
}
}
// A route that is an insert may be its own stamps and files alone (an upload).
const routeInsert = !!shape.ops?.includes('INSERT') && (cols.length > 0 || !!shape.uploads?.length);
if (!said && !routeInsert)
return { error: 'the plan did not say what to insert' };
if (target.ownerColumn && !cols.includes(target.ownerColumn)) {
cols.push(target.ownerColumn);
values.push(owner);
}
const invalid = check(cols, values, true);
if (invalid)
return invalid;
const refused = await prepare(cols, values, 'insert');
if (refused)
return refused;
await slugs(cols, values);
// For an upsert, whether the row was new — a form says "subscribed" or "already subscribed".
const upKey = typeof shape.upsert === 'string' ? physicalOf(shape.upsert) : null;
const existed = upKey && cols.includes(upKey)
? (await store.all(`SELECT * FROM ${ident(target.table)} WHERE ${ident(upKey)} = ? LIMIT 1`, toDriver(upKey, values[cols.indexOf(upKey)])))[0]
: undefined;
const r = await runOrClean(`INSERT INTO ${ident(target.table)} (${cols.map(ident).join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`
+ (store.dialect === 'mysql'
? (shape.upsert ? ` ON DUPLICATE KEY UPDATE ${cols.map(c => `${ident(c)} = VALUES(${ident(c)})`).join(', ')}` : '')
: upsertSqlite(cols)), ...values.map((v, i) => toDriver(cols[i], v)));
await link([Number(r.lastInsertRowid)]);
await follow('insert', cols, values, { id: Number(r.lastInsertRowid) });
// The row as stored (its defaults and slug too), when the route wants it back.
const back = shape.returning && r.lastInsertRowid
? (await store.all(`SELECT * FROM ${ident(target.table)} WHERE ${ident('id')} = ?`, Number(r.lastInsertRowid)))[0] : undefined;
return { created: existed ? Number(existed.id ?? r.lastInsertRowid) : Number(r.lastInsertRowid), fields: cols,
...(upKey ? { action: existed ? 'updated' : 'inserted', before: existed ? finishRow(shapeRow(existed, target, []), target) : null } : {}),
...(back ? { row: finishRow(shapeRow(back, target, []), target) } : {}) };
}
case 'UPDATE': {
const { cols, values, reject, said } = await assignments();
if (reject)
return { error: reject };
const linking = Object.entries(target.many ?? {}).some(([n, m]) => Array.isArray(shape.values?.[m.field ?? `${n}Ids`]));
if (!where)
return { error: 'refusing to update every row' };
// An update may change only a join table: the article's categories.
if (!said && linking) {
const owners = (await store.all(`SELECT ${ident('id')} FROM ${ident(target.table)}${where}`, ...bind)).map(r => r.id);
await link(owners);
return { updated: owners.length };
}
// A route that is an update may consist of its own stamps alone: a like,
// a switch flipped, an unsubscribe.
const stampsOnly = !said && cols.length > 0 && !!shape.ops?.includes('UPDATE');
if (!said && !stampsOnly)
return { error: 'the plan did not say what to change' };
const invalid = check(cols, values, false);
if (invalid)
return invalid;
const before = await filesNow();
const refused = await prepare(cols, values, 'update');
if (refused)
return refused;
await slugs(cols, values, true);
const r = await runOrClean(`UPDATE ${ident(target.table)} SET ${cols.map(c => copies.has(c) ? `${ident(c)} = ${ident(copies.get(c))}`
: adds.has(c) ? `${ident(c)} = ${ident(c)} + ?`
: flips.has(c) ? `${ident(c)} = ? - ${ident(c)}` : `${ident(c)} = ?`).join(', ')}${where}`, ...values.flatMap((v, i) => copies.has(cols[i]) ? [] : [adds.has(cols[i]) || flips.has(cols[i]) ? v : toDriver(cols[i], v)]), ...bind);
// A file replaced (or cleared) by this write is removed.
await unlinkAll(before.flatMap(row => fileColumns()
.filter(c => cols.includes(c) && row[c] !== values[cols.indexOf(c)]).map(c => fileOf(c, row[c]))));
if (linking)
await link((await store.all(`SELECT ${ident('id')} FROM ${ident(target.table)}${where}`, ...bind)).map(r => r.id));
if (Number(r.changes))
await follow('update', cols, values, { where: bind });
// The rows as they now are, for a route that answers with them.
if (shape.returning && Number(r.changes)) {
// (Conditions on a column the update just wrote may no longer hold; the row key does.)
const back = (await store.all(`SELECT * FROM ${ident(target.table)}${where}`, ...bind)).map(x => finishRow(shapeRow(x, target, []), target));
return { updated: Number(r.changes), rows: back, row: back[0] };
}
return { updated: Number(r.changes) };
}
case 'DELETE': {
// Duplicates: every row but the first of each value, within the route's conditions.
if (shape.dedupe) {
const by = physicalOf(shape.dedupe.by);
const w = whereAs('');
const r = await store.run(`DELETE FROM ${ident(target.table)}${w ? `${w} AND` : ' WHERE'} ${ident('id')} NOT IN `
+ `(SELECT id FROM (SELECT MIN(${ident('id')}) AS id FROM ${ident(target.table)}${w} GROUP BY ${ident(by)}) keep)`, ...bind, ...bind);
return { deleted: Number(r.changes), dedupe: true };
}
if (!where)
return { error: 'refusing to delete every row' };
// Rows elsewhere that point here: refuse while any exist, or remove them first.
const kids = target.children ?? [];
if (kids.length) {
const ids = (await store.all(`SELECT ${ident('id')} FROM ${ident(target.table)}${where}`, ...bind)).map(r => r.id);
const list = ids.map(() => '?').join(', ');
for (const k of ids.length ? kids.filter(k => !k.cascade) : []) {
const used = await store.all(`SELECT 1 FROM ${ident(k.table)} WHERE ${ident(k.on)} IN (${list}) LIMIT 1`, ...ids);
if (used.length)
return { error: k.message ?? `still used by ${k.table}` };
}
for (const k of ids.length ? kids.filter(k => k.cascade) : [])
await store.run(`DELETE FROM ${ident(k.table)} WHERE ${ident(k.on)} IN (${list})`, ...ids);
}
// What was deleted, for a route that must clean up after it (an uploaded file).
const gone = shape.returning || target.after?.delete
? await store.all(`SELECT * FROM ${ident(target.table)}${where}`, ...bind) : undefined;
const held = await filesNow();
const r = await store.run(`DELETE FROM ${ident(target.table)}${where}`, ...bind);
if (Number(r.changes))
await unlinkAll(held.flatMap(row => Object.keys(row).map(c => fileOf(c, row[c]))));
for (const row of Number(r.changes) ? gone ?? [] : [])
await follow('delete', Object.keys(row), Object.values(row), {});
return gone ? { deleted: Number(r.changes), rows: gone } : { deleted: Number(r.changes) };
}
default:
return { error: `I cannot run a ${plan.op}` };
}
}
// SQLite drivers take neither booleans nor dates.
const sqlValue = (v) => typeof v === 'boolean' ? Number(v) : v;
/** `slugify('Économie locale')` → `economie-locale` */
export const slugify = (s) => normalize(s).replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'item';
/** `author__name` columns folded into `author: { name }`, JSON columns parsed. */
function shapeRow(row, t, rels) {
const out = {};
for (const [k, v] of Object.entries(row)) {
const [rel, col] = k.split('__');
if (col && rels.includes(rel))
(out[rel] ??= {})[col] = v;
else {
const type = t.json?.includes(k) ? 'json' : t.types?.[k];
const own = t.read?.[k] && v !== null && v !== undefined ? t.read[k](v) : v;
// A driver may hand JSON back as text even after the ORM's own reading
// (MariaDB stores JSON as LONGTEXT).
out[k] = t.read?.[k] && !(type === 'json' && typeof own === 'string') ? own : readAs(type, own);
}
}
// A relation that matched no row is null, not an object of nulls.
for (const r of rels)
if (out[r] && Object.values(out[r]).every(v => v === null))
out[r] = null;
return out;
}
/**
* Last touch on a row read, once its tallies and lists are attached: computed
* fields from the whole row (a Gravatar from the email), then hidden ones
* removed — tallies and relations included.
*/
function finishRow(out, t) {
for (const [name, f] of Object.entries(t.computed ?? {}))
out[name] = f(out);
for (const h of t.hidden ?? []) {
const [rel, col] = h.split('.');
if (col) {
if (out[rel])
delete out[rel][col];
}
else
delete out[h];
}
return out;
}
const safeJson = (s) => { try {
return JSON.parse(s);
}
catch {
return s;
} };
function readAs(type, v) {
if (v === null || v === undefined || !type)
return v;
if (type === 'json')
return typeof v === 'string' ? safeJson(v) : v;
if (type === 'boolean')
return v === true || v === 1 || v === '1' || v === 'true';
if (type === 'number')
return Number(v);
if (type === 'date') {
if (v instanceof Date)
return v;
const d = typeof v === 'number' ? new Date(v < 1e12 ? v * 1000 : v) : new Date(String(v).replace(' ', 'T').replace(/ ([+-]\d\d:\d\d)$/, '$1'));
return Number.isNaN(d.getTime()) ? v : d;
}
return v;
}
// --- the app ----------------------------------------------------------------
export function defineApp(opts) {
const { schema, prompt, actions = {}, checkVocab = true } = opts;
// No store given: Talos's own database, opened (and its tables made) in the background;
// every query waits for it, `app.ready` too.
const own = opts.store ? undefined : localStore(schema, { seed: opts.seed });
const store = opts.store ?? {
dialect: 'sqlite',
all: async (sql, ...p) => (await own).all(sql, ...p),
run: async (sql, ...p) => (await own).run(sql, ...p),
lookup: () => null,
};
// A flag is a two-word domain everywhere the model and the repairs look.
for (const t of schema.tables)
for (const [c, words] of Object.entries(t.flags ?? {}))
t.enums = { ...t.enums, [c]: [...words] };
if (checkVocab)
checkVocabulary(schema);
const telemetry = new Telemetry(opts.telemetry);
telemetry.emit('config', { type: 'config', talos: TALOS_VERSION, model: modelInfo(),
threads: THREADS ?? null, maxSrc: cfg.maxSrc, ...platformInfo() });
// A JSON round trip keeps what the schema says and drops its functions (hooks, converters).
telemetry.emit('schema', { type: 'schema', tables: JSON.parse(JSON.stringify(schema.tables)), functions: schema.functions ?? [] });
telemetry.emit('prompts', { type: 'prompt', prompt });
async function decide(input, ctx, ops, template) {
const t0 = performance.now();
const callId = randomUUID();
const asked = input;
// A typo on a word the app declares is read as that word, and a request that
// opens on an unambiguous verb has the operation that verb says — as a
// route's HTTP method does. See `spell` and `verbOps`.
input = spell(input);
ops ??= verbOps(input);
// Exactly the shape the model was trained on. Session ids and the like are for
// the runtime, not the model — feeding them in is only noise.
const contextText = [`role = ${modelRole(schema, ctx.role, !!ctx.userId)}`,
`signed in = ${ctx.userId ? 'yes' : 'no'}`].join(' ; ');
const { toks, userStart } = buildSource(schema, prompt, contextText, input);
const ids = new BigInt64Array(cfg.maxSrc);
toks.forEach((t, i) => { ids[i] = BigInt(tokenId(t)); });
const src = new ort.Tensor('int64', ids, [1, cfg.maxSrc]);
const { encoder, decoder } = await sessions();
const memory = (await encoder.run({ src })).memory;
/**
* Greedy generation. The decoder graph is a fixed width, so the fed tokens
* are padded and we read the row for the current step; causal masking makes
* the padding that follows it irrelevant.
*
* `forbidAtStart` blocks chosen ops at the first step only, which is how a
* refusal is reconsidered without touching anything else the model does.
*/
let refuseP = 0;
const routeBlocked = ops
? ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CALL'].filter(op => !ops.includes(op)).map(op => cfg.ops.indexOf(op))
: [];
const tableOps = ['SELECT', 'INSERT', 'UPDATE', 'DELETE'].map(op => cfg.ops.indexOf(op));
const tableWords = schema.tables.flatMap(t => [t.as, ...(t.says ?? [])].map(w => tokenize(w)[0]).filter(Boolean));
const tableAt = new Set(toks.flatMap((w, p) => tableWords.some(t => sameWord(t, w)) ? [p + cfg.nOps] : []));
async function generate(forbidAtStart = []) {
const out = [];
const fed = new BigInt64Array(cfg.maxTgt);
for (let step = 0; step < cfg.maxTgt - 1; step++) {
const res = await decoder.run({
memory, src, tgt_in: new ort.Tensor('int64', fed, [1, cfg.maxTgt]),
});
const all = res.logits.data;
const width = all.length / cfg.maxTgt;
const base = step * width;
if (step === 0 && !forbidAtStart.length) {
// How much the model wanted to refuse, before anything overrode it.
let max = -Infinity;
for (let k = 0; k < width; k++)
max = Math.max(max, all[base + k]);
let sum = 0;
for (let k = 0; k < width; k++)
sum += Math.exp(all[base + k] - max);
refuseP = Math.exp(all[base + cfg.ops.indexOf('REFUSE')] - max) / sum;
}
const blocked = step === 0 ? [...forbidAtStart, ...routeBlocked] : [];
// The first word of a table op's target must be a table: "mon panier"
// came back as `SELECT list`, a word from the rules. Calls are exempt —
// their names are not in the window.
const onlyTables = step === 1 && tableOps.includes(out[0]) && tableAt.size > 0;
let best = -1;
for (let k = step === 0 ? 1 : 0; k < width; k++) {
if (blocked.includes(k))
continue;
if (onlyTables && !tableAt.has(k))
continue;
if (best < 0 || all[base + k] > all[base + best])
best = k;
}
if (best <= 0)
break;
out.push(best);
fed[step + 1] = BigInt(best);
}
return out;
}
let out = await generate();
let plan = repairPlan(decodePlan(out, toks, userStart), schema, input);
/**
* A refusal is not accepted when the request names one of the app's own
* tables.
*
* With six tables in the window the model starts giving up: "toutes les
* categories", "show me the members" and "show me the orders" all came back
* REFUSE, and every one of them says the name of a declared table. So the
* ops that mean "no" are blocked and it is asked again — the runtime does
* not pick the operation, it only declines to take no for an answer about a
* word the app itself wrote down.
*/
const named = namedTable(input, schema.tables);
// A declared value counts as much as a declared table name: "de l actualité"
// names no table but does name a section this app declared, and the model
// refused it.
const saysAValue = schema.tables.some(t => Object.values(t.enums ?? {}).flat().some(v => [v, ...(SURFACE[v] ?? [])].some(f => mentions(input, f))));
if ((named || saysAValue) && (plan.op === 'REFUSE' || plan.op === 'DENY')) {
const no = ['REFUSE', 'DENY'].map(op => cfg.ops.indexOf(op)).filter(i => i > 0);
const retry = repairPlan(decodePlan(await generate(no), toks, userStart), schema, input);
if (retry.op !== 'REFUSE' && retry.op !== 'DENY') {
out = [];
plan = { ...retry, repaired: [...(retry.repaired ?? []), 'refusal reconsidered'] };
}
}
// With no table named, a value that only one table declares says which table
// was meant — and a target that is not a declared table at all is wrong
// whatever it is.
const isTable = (name) => !!name && schema.tables.some(t => sameWord(t.as, name));
const owner = !named ? tableFromValues(input, schema.tables) : undefined;
// Run before the write checks below, which need the table settled, and again
// on whatever the model says when asked a second time.
const settle = (plan) => {
if (!named && owner && plan.op !== 'CALL' && plan.op !== 'REFUSE' && plan.op !== 'DENY'
&& !(plan.target && sameWord(plan.target, owner.as))) {
const keep = isTable(plan.target) ? [] : plan.filters.filter(f => Object.keys(owner.columns).some(c => sameWord(c, f.column)));
plan = repairClauses({ ...plan, target: owner.as, filters: keep,
repaired: [...(plan.repaired ?? []), `table -> ${owner.as} (by value)`] }, owner, input, schema);
}
// And a plan aimed at a table the request never mentions, when it mentions
// exactly one, is aimed at the wrong one.
if (named && plan.op !== 'CALL' && plan.target && !sameWord(plan.target, named.as)) {
const keep = plan.filters.filter(f => Object.keys(named.columns).some(c => sameWord(c, f.column)));
plan = repairClauses({ ...plan, target: named.as, filters: keep,
repaired: [...(plan.repaired ?? []), `table -> ${named.as}`] }, named, input, schema);
}
return plan;
};
plan = settle(plan);
/**
* Changing or deleting a row needs the row: a number the caller wrote.
*
* "des romans policiers" once came back as
* `UPDATE books SET genre VAL crime FILTER id VAL rom` — a write on a
* read, identified by a fragment of a word. With no number anywhere in the
* sentence there is no row to change, so the ops that change rows are
* blocked and the model is asked again. (An update with no condition at all
* is refused further down anyway; this catches the ones that invent one.)
*
* Likewise an update needs a change. "la fiche de l article 12" came back
* as `UPDATE articles FILTER id VAL 12`: a row, but nothing the caller said
* to put in it — once grounding is done, an update that sets nothing was a
* read. Both write ops are blocked, so the retry cannot turn it into a delete.
* Not when the sentence names a declared value: that is the change, and
* `setFromSentence` puts it back once the table is settled.
*/
const saysANumber = tokenize(input).some(w => /^\d+$/.test(w));
const changesNothing = plan.op === 'UPDATE' && !plan.sets.length && !saysAValue;
// And an insert needs something to put in the table it names. "les livres en
// francais" came back as `INSERT books SET item VAL francais livres`: `item`
// is the basket's column, not a book's — a read written as a write.
const into = plan.op === 'INSERT' ? schema.tables.find(t => plan.target && sameWord(t.as, plan.target)) : undefined;
// Only on evidence: a SET on a word the app does not declare may still be the
// caller's own name for a column ("le contenu" for `content`), and blocking
// that turned "cree un article avec le titre Essai" into a read.
const ofTable = (t, col) => Object.keys(t.columns).some(c => sameWord(c, col));
const insertsNothing = !!into && !plan.sets.some(s => ofTable(into, s.column))
&& (!plan.sets.length || plan.sets.some(s => schema.tables.some(t => t !== into && ofTable(t, s.column))));
// Unless the route itself says this is a write: then the row and the values
// come from the request, and the sentence was never where they lived.
const routeWrite = !!ops?.includes(plan.op);
if (!routeWrite && (insertsNothing
|| ((!saysANumber || changesNothing) && (plan.op === 'UPDATE' || plan.op === 'DELETE')))) {
// And, as above, no refusal of a request that names something declared:
// with the write blocked, "des romans policiers" fell back to REFUSE.
const blocked = ['UPDATE', 'DELETE', ...(insertsNothing ? ['INSERT'] : []),
...(named || saysAValue ? ['REFUSE', 'DENY'] : [])];
const writes = blocked.map(op => cfg.ops.indexOf(op)).filter(i => i > 0);
const retry = repairPlan(decodePlan(await generate(writes), toks, userStart), schema, input);
plan = settle({ ...retry, repaired: [...(retry.repaired ?? []), 'write without a row reconsidered'] });
}
// A request that says a declared function's name is a call to that one,
// whatever the model made of the unfamiliar verb — including when it planned
// a different function. See `namedFunction`.
const spoken = namedFunction(input, schema.functions);
const planned = matchFunction(schema.functions, plan.target);
if (spoken && spoken !== planned) {
plan = {
...plan, op: 'CALL', target: spoken.name, filters: [],
repaired: [...(plan.repaired ?? []), `named function -> ${spoken.name}`],
};
}
// A call's arguments are read from the sentence, so they belong in the plan
// that is shown — the model is no longer trained to write them, and a plan
// that reads `CALL createServer` while the call receives `id = 42` hides
// what actually runs.
const fn = plan.op === 'CALL' ? matchFunction(schema.functions, plan.target) : undefined;
if (fn) {
const said = argumentsFromSentence(fn, input);
const have = new Set(plan.sets.map(s => normalize(s.column)));
const extra = Object.entries(said)
.filter(([k]) => !have.has(normalize(k.replace(/_/g, ' '))))
.map(([k, v]) => ({ column: k.replace(/_/g, ' '), value: [v], said: true }));
// The sentence wins over the plan for arguments, as it does when the call runs.
const kept = plan.sets.filter(s => !(normalize(s.column.replace(/\s+/g, '_')) in said));
if (extra.length || kept.length !== plan.sets.length) {
const fromSentence = Object.entries(said).map(([k, v]) => ({ column: k.replace(/_/g, ' '), value: [v], said: true }));
plan = { ...plan, sets: [...kept, ...fromSentence] };
}
}
plan = { ...plan, callId };
const ms = +(performance.now() - t0).toFixed(2);
telemetry.answered(callId, ms);
if (telemetry.has('operations') || telemetry.has('prompts') || telemetry.has('performance')) {
telemetry.emit(undefined, { type: 'call', callId, role: modelRole(schema, ctx.role, !!ctx.userId),
...telemetry.has('prompts') ? { template, sentence: template ? undefined : anonymise(asked), read: anonymise(input) } : {},
...telemetry.has('operations') ? { pinned: ops, plan: anonymisePlan(plan), refuseP } : {},
...telemetry.has('performance') ? { ms } : {} });
}
return { plan, ms, refuseP, callId };
}
async function run(input, ctx = {}, shape = {}) {
const { plan, ms } = await decide(input, ctx, shape.ops);
return execute(plan, ms, input, ctx, shape);
}
/**
* Gate and run a plan, however it was obtained. The answer carries the
* `callId` of the model call that made the plan (for feedback, and for
* anyone helping with it), and telemetry records what was done.
*/
async function execute(plan, ms, input, ctx, shape, cached) {
const t0 = performance.now();
const callId = plan.callId ?? randomUUID();
let out;
try {
out = await executePlan(plan, ms, input, ctx, shape);
}
catch (e) {
// A function the plan called threw: the caller gets it as before, telemetry sees it.
if (!e?.statusCode || e.statusCode >= 500)
telemetry.error(e, 'execute', callId);
throw e;
}
const runMs = +(performance.now() - t0).toFixed(2);
telemetry.answered(callId, undefined, runMs);
const r = out.result;
if (out.status >= 500 || (out.status === 400 && typeof r?.error === 'string' && r.error.startsWith('that plan does not fit')))
telemetry.error({ name: 'PlanError', message: r?.error }, 'execute', callId);
if (telemetry.has('operations') || telemetry.has('performance')) {
const count = (k) => typeof r?.[k] === 'number' ? r[k] : Array.isArray(r?.[k]) ? r[k].length : undefined;
telemetry.emit(undefined, { type: 'answer', callId, cached: !!cached,
...telemetry.has('operations') ? { op: plan.op, target: plan.target, status: out.status,
rows: count('rows'), created: r?.created !== undefined ? 1 : undefined, updated: count('updated'), deleted: count('deleted') } : {},
...telemetry.has('performance') ? { ms: runMs } : {} });
}
return { ...out, callId };
}
async function executePlan(plan, ms, input, ctx, shape) {
if (plan.op === 'REFUSE')
return { plan, ms, status: 422, result: { error: 'that is not something this app does' } };
if (plan.op === 'DENY')
return { plan, ms, status: 403, result: { error: 'the rules do not allow you to do that' } };
const role = ctx.role ?? 'guest';
const may = (allowed) => !allowed || allowed.includes(role);
if (plan.op === 'CALL') {
const declared = matchFunction(schema.functions, plan.target);
// No loose matching. Taking the first word of the target is how a plan
// reading `CALL beige` once found a function: an unrecognised name is a
// refusal, not a guess at which side effect was meant.
if (!declared)
return { plan, ms, status: 422, result: { error: `no such function: ${plan.target}` } };
if (!may(declared.roles))
return { plan, ms, status: 403, result: { error: `${declared.name} is not yours to call` } };
const action = actions[declared.name]
?? actions[Object.keys(actions).find(k => normalize(k) === normalize(declared.name)) ?? ''];
if (!action)
return { plan, ms, status: 501, result: { error: `${declared.name} is declared but not implemented` } };
const bound = bindArguments(declared, plan, ctx);
// The sentence wins over the plan for arguments. What the user literally
// wrote after "with the id" is firmer ground than a pointer that may have
// wandered: the model returned `id = "reindex sendnewsletter"` for a
// request that said `id 12`. The plan still fills anything the sentence
// did not name.
const args = { ...bound.args, ...argumentsFromSentence(declared, input) };
const { ignored } = bound;
const missing = (declared.required ?? []).filter(p => !args[p.replace(/\s+/g, '_')]);
// A side effect with a missing argument is worse than a refused one: an
// email to nobody, a charge with no amount.
if (missing.length) {
return { plan, ms, status: 400, result: {
error: `${declared.name} needs ${missing.join(' and ')}`,
} };
}
const result = await action({ args, ctx, plan });
const payload = ignored.length ? { ...result, ignored } : result;
return { plan, ms, status: result?.error ? 400 : 200, result: payload };
}
const denial = authorise(plan, schema, role);
if (denial)
return { plan, ms, status: 403, result: { error: denial } };
// A generated plan can be well-formed and still not be runnable SQL — an
// INSERT missing a NOT NULL column, a filter on a column that is gone. That
// is a bad request, not a crashed server, and catching it here covers every
// op rather than guarding each one.
let result;
try {
result = await compile(plan, schema, store, ctx, shape, actions);
}
catch (e) {
// An error a route's own function raised on purpose keeps its status and message.
if (e?.statusCode)
return { plan, ms, status: e.statusCode, result: { error: e.message, errors: e.data?.errors, data: e.data } };
return { plan, ms, status: 400, result: {
error: `that plan does not fit the schema: ${e?.message ?? e}`,
} };
}
return { plan, ms, status: result.denied ? 403 : result.missing ? 404 : result.error ? 400 : 200, result };
}
/**
* The app's own words — table names and synonyms, columns and their
* synonyms, declared values and their forms, function names — and a request
* word the model does not know that is one edit from exactly one of them,
* replaced by it. "supprime la categrie 3" reached the model as an unknown
* token and it lost the table (benchmark: one typo, 40.7 % of plans right).
*/
const appWords = new Set();
for (const t of schema.tables) {
for (const w of [t.as, ...(t.says ?? []), ...Object.keys(t.columns), ...Object.values(t.columnSays ?? {}).flat(),
...Object.values(t.enums ?? {}).flat().flatMap(v => [v, ...(SURFACE[v] ?? [])])])
for (const x of tokenize(w))
if (x.length >= 4)
appWords.add(x);
}
for (const f of schema.functions ?? [])
for (const x of [f.name, ...(f.says ?? [])].flatMap(s => nameForms(s)).flatMap(tokenize))
if (x.length >= 4)
appWords.add(x);
// and their singulars: "catgory" is a typo on the table "categories".
for (const w of [...appWords])
if (w.endsWith('s'))
appWords.add(w.endsWith('ies') ? `${w.slice(0, -3)}y` : w.slice(0, -1));
const oneEdit = (a, b) => {
if (Math.abs(a.length - b.length) > 1 || a === b)
return a === b;
let i = 0, j = 0, e = 0;
while (i < a.length && j < b.length) {
if (a[i] === b[j]) {
i++;
j++;
continue;
}
if (++e > 1)
return false;
if (a.length > b.length)
i++;
else if (a.length < b.length)
j++;
else {
i++;
j++;
}
}
return e + (a.length - i) + (b.length - j) <= 1;
};
// Only misspelt words change: the rest of the request (values it carries) keeps its case and accents.
function spell(input) {
return input.replace(/\p{L}+/gu, word => {
const w = normalize(word);
if (w.length < 5 || !/^[a-z]+$/.test(w) || appWords.has(w) || cfg.vocab[w] !== undefined)
return word;
// "plugin" for the table "plugins" is a singular, not a typo: the model reads those.
const near = [...appWords].filter(a => a[0] === w[0] && a !== `${w}s` && w !== `${a}s` && oneEdit(w, a));
return near.length === 1 ? near[0] : word;
});
}
/**
* The operation an opening verb leaves no doubt about (French and English).
* The planner decided the operation from how the app's rules were phrased
* more than from the request: on a two-table app it answered SELECT to
* "supprime la categorie 3". A leading verb now pins it, as a route's method
* does; without one, the model chooses.
*/
const VERB_OPS = [
[/^(supprime|supprimer|efface|effacer|retire|retirer|enleve|enlever|delete|remove|erase|drop)\b/, 'DELETE'],
[/^(cree|creer|ajoute|ajouter|enregistre|enregistrer|insere|inserer|nouvel|nouvelle|nouveau|create|add|insert|register|new)\b/, 'INSERT'],
[/^(modifie|modifier|change|changer|mets a jour|mettre a jour|edite|editer|corrige|corriger|renomme|renommer|passe|repasse|update|edit|rename|set)\b/, 'UPDATE'],
[/^(montre|montrer|affiche|afficher|liste|lister|donne|donner|voir|cherche|chercher|trouve|trouver|show|list|display|get|find|search)\b/, 'SELECT'],
];
// A misspelt verb ("suprime") is corrected like the app's words.
for (const [re] of VERB_OPS)
for (const v of re.source.match(/[a-z]{5,}/g))
appWords.add(v);
const POLITE = /^(s il (te|vous) plait|stp|svp|please|peux tu|pouvez vous|tu peux|vous pouvez|je veux|je voudrais|j aimerais|merci de|could you|can you|i want to|i would like to)\s+/;
function verbOps(input) {
let s = normalize(input).replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
for (let i = 0; i < 2 && POLITE.test(s); i++)
s = s.replace(POLITE, '');
const hit = VERB_OPS.find(([re]) => re.test(s));
// Any verb may name a function ("ajoute un clic", "supprime mon compte"): CALL stays open.
return hit ? [hit[1], 'CALL'] : undefined;
}
/**
* A request as telemetry sees it: the model's own vocabulary and the app's
* words stay, numbers become `#`, any other word `<w>` — names, emails and
* free text do not leave the process. A plan keeps its operation, table and
* columns; a value stays only when the schema declares it (`published`).
*/
const declared = new Set(schema.tables.flatMap(t => Object.values(t.enums ?? {}).flat()));
const anonymise = (text) => tokenize(text)
.map(w => /^\d+$/.test(w) ? '#' : /^[a-z]+$/.test(w) && cfg.vocab[w] === undefined && !appWords.has(w) ? '<w>' : w).join(' ');
const anonymisePlan = (p) => {
const clause = (c) => ({ column: c.column, said: c.said,
value: c.value.length === 1 && (c.value[0] === SELF || declared.has(c.value[0])) ? c.value[0] : c.value.length ? '<v>' : '' });
return { op: p.op, target: p.target, filters: p.filters.map(clause), sets: p.sets.map(clause), repaired: p.repaired };
};
/**
* With TALOS_RECORD=<file>, every sentence a route compiles is appended as
* a JSON line — the app's schema and rules once, then (sentence, role, ops,
* plan) per case. Replayed by train/bench.ts to measure a model on real apps
* without running them.
*/
let recorded = false;
function record(sentence, ctx, ops, plan) {
const file = process.env.TALOS_RECORD;
if (!file)
return;
const app = JSON.stringify({ prompt, tables: schema.tables.map(t => ({ as: t.as, table: t.table, columns: t.columns, enums: t.enums,
flags: t.flags, says: t.says, columnSays: t.columnSays, writable: t.writable })),
functions: schema.functions?.map(f => ({ name: f.name, params: f.params, says: f.says })) });
const id = createHash('sha1').update(app).digest('hex').slice(0, 12);
const name = process.env.TALOS_APP_NAME ?? process.cwd().split(/[\\/]/).pop();
if (!recorded) {
appendFileSync(file, JSON.stringify({ kind: 'app', id, name, ...JSON.parse(app) }) + '\n');
recorded = true;
}
appendFileSync(file, JSON.stringify({ kind: 'case', app: id, sentence, role: ctx.role, userId: ctx.userId ? 1 : 0, ops,
plan: { op: plan.op, target: plan.target, filters: plan.filters.map(f => f.column), sets: plan.sets.map(s => s.column) } }) + '\n');
}
/**
* A route written as a sentence, compiled once by the model.
*
* const del = app.route('supprime la categorie {id}', { ops: ['DELETE'] })
* await del({ id: 3 }, ctx)
*
* `{k}` is a parameter; `[... {k} ...]` is dropped when `k` is missing;
* `{k:no|yes}` says a boolean with one of two words.
*
* The first call for each shape of the sentence asks the model, then keeps
* the plan with every value that came from a parameter replaced by a slot.
* Later calls bind new values into that plan and never run the model: a
* route costs what its SQL costs, and answers the same way every time. A
* plan holding a value that is neither a parameter nor a word of the template
* itself is not kept — the model found it somewhere else, so it is asked
* again next time rather than trusted forever.
*/
function route(template, base = {}) {
const cache = new Map();
const SLOT = '\u0000';
const words = new Set(tokenize(template.replace(/\{[^}]*\}/g, ' ')));
return async (p, ctx = {}, shape = {}) => {
const t0 = performance.now();
const kept = [];
const say = (s) => s.replace(/\{(\w+)(?::(\w+)\|(\w+))?\}/g, (_, k, no, yes) => no ? (p[k] ? yes : no) : String(p[k] ?? ''));
let i = 0;
const sentence = template
.replace(/\[([^\]]*)\]/g, (_, part) => {
const has = [...part.matchAll(/\{(\w+)/g)].every(m => p[m[1]] !== undefined && p[m[1]] !== '');
if (has)
kept.push(i);
i++;
return has ? say(part) : '';
})
.replace(/\{[^}]+\}/g, m => say(m))
.replace(/\s+/g, ' ').trim();
const key = kept.join(',');
const opts = { ...base, ...shape };
// Compared as the model reads them, so "Faits-divers" matches `faits divers`.
const slotOf = (value) => {
const text = tokenize(join(value)).join(' ');
return Object.keys(p).find(k => p[k] !== undefined && p[k] !== ''
&& tokenize(String(p[k])).join(' ') === text);
};
const bind = (c) => c.value[0]?.startsWith(SLOT)
? { ...c, value: tokenize(String(p[c.value[0].slice(1)] ?? '')).length
? [String(p[c.value[0].slice(1)])] : [] }
: c;
const cached = cache.get(key);
if (cached) {
const plan = { ...cached, filters: cached.filters.map(bind), sets: cached.sets.map(bind) };
return { sentence, cached: true, ...await execute(plan, +(performance.now() - t0).toFixed(2), sentence, ctx, opts, true) };
}
let { plan, ms } = await decide(sentence, ctx, opts.ops, template);
record(sentence, ctx, opts.ops, plan);
let trusted = plan.op !== 'REFUSE' && plan.op !== 'DENY';
const slot = (c) => {
if (c.value.includes(SELF))
return c;
const k = slotOf(c.value);
if (k)
return { ...c, value: [SLOT + k] };
// A declared value counts as the template's own when the template says
// it or one of its forms: "les tribunes approuvees" → `approved`.
const said = (v) => tokenize(v).every(w => words.has(w));
if (!said(join(c.value)) && !(SURFACE[join(c.value)] ?? []).some(said))
trusted = false;
return c;
};
let skeleton = { ...plan, filters: plan.filters.map(slot), sets: plan.sets.map(slot) };
// The parameter that names the row is not also a value to write: "modifie la
// recette {id}" once came back as `SET course = {id} FILTER id = {id}`.
const naming = new Set(skeleton.filters.map(f => f.value[0]).filter(v => v?.startsWith(SLOT)));
const writes = skeleton.sets.map(s => !naming.has(s.value[0]));
if (writes.includes(false)) {
plan = { ...plan, sets: plan.sets.filter((_, i) => writes[i]), repaired: [...(plan.repaired ?? []), 'dropped a write of the row\'s own parameter'] };
skeleton = { ...skeleton, sets: skeleton.sets.filter((_, i) => writes[i]) };
}
if (trusted)
cache.set(key, skeleton);
return { sentence, cached: false, ...await execute(plan, ms, sentence, ctx, opts) };
};
}
return {
run, decide, execute, route, prompt, schema,
/** The database Talos reads and writes: yours, or its own. */
store,
/** Resolves once Talos's own database is open, with its file; at once with your store. */
ready: own?.then(s => ({ file: s.file })) ?? Promise.resolve({ file: undefined }),
/** This instance's id: what someone quotes to be helped with it. */
instanceId: telemetry.instanceId,
/** Feedback on one call (its `callId`): a text, a rating 1–5, or both. */
feedback: (callId, fb) => telemetry.feedback(callId, fb),
telemetry,
};
}
/**
* The model's version: its configuration and the size of both halves, hashed.
* Two directories with the same weights give the same version.
*/
const MODEL_VERSION = createHash('sha256').update(readFileSync(joinPath(MODEL_DIR, 'config.json')))
.update(`${statSync(joinPath(MODEL_DIR, 'encoder.onnx')).size}/${statSync(joinPath(MODEL_DIR, 'decoder.onnx')).size}`)
.digest('hex').slice(0, 12);
const TALOS_VERSION = (() => {
try {
return JSON.parse(readFileSync(joinPath(HERE ?? joinPath(process.cwd(), 'node_modules', '@athenox', 'talos', 'dist'), '..', 'package.json'), 'utf8')).version;
}
catch {
return 'unknown';
}
})();
export const modelInfo = () => ({
version: MODEL_VERSION, talos: TALOS_VERSION,
params: cfg.params, dim: cfg.dim, maxSrc: cfg.maxSrc, maxTgt: cfg.maxTgt,
vocab: Object.keys(cfg.vocab).length, ops: cfg.ops,
});