@nytka/plugin-typesafe/src/inputs.mjs

TypeSafe connector for nytka projects. Asks Jev typed questions (noul, choice, score) about a file or an existing dataset and registers the judgements in datasets/ with the model, the questions hash and the usage.

// The filesystem half: where the items and the questions come from.
//
// Kept apart from typesafe.mjs so the client and classify() stay pure and testable with a
// stub, and so the one place this package reads a dataset payload is a single function a
// reviewer can find. That read is the point of the connector: an agent may not load a
// payload into context, so the connector loads it and sends it to Jev on the agent's
// behalf. The agent sees counts.

import { readFile } from 'node:fs/promises'
import { existsSync } from 'node:fs'
import { basename, extname, isAbsolute, join, relative, resolve } from 'node:path'
import { TypeSafeError } from './typesafe.mjs'

/** Where the rows of a payload live, by convention across the six collectors: gsc, ga4,
 *  dataforseo and ads write `rows`, sanity writes `documents`. Tried in this order when
 *  --items does not say. */
export const ITEMS_KEYS = Object.freeze(['rows', 'documents', 'items', 'results', 'campaigns'])

/** A path as given, resolved against the project root first and the working directory
 *  second. Returns null when neither exists. */
function locate (root, given) {
  if (isAbsolute(given)) return existsSync(given) ? given : null
  const atRoot = join(root, given)
  if (existsSync(atRoot)) return atRoot
  const atCwd = resolve(process.cwd(), given)
  return existsSync(atCwd) ? atCwd : null
}

/** A path as it should be recorded: relative to the project root when it is inside it,
 *  the given string otherwise. Stable across machines, which is what an id fragment and a
 *  registry field both need to be. */
function recordable (root, abs, given) {
  const rel = relative(root, abs)
  return rel && !rel.startsWith('..') && !isAbsolute(rel) ? rel : given
}

async function readJson (abs, what) {
  const text = await readFile(abs, 'utf8')
  try {
    return JSON.parse(text)
  } catch (err) {
    throw new TypeSafeError(`${what} at ${abs} is not valid JSON: ${err.message}`)
  }
}

/** The questions map from a file, plus the name and project-relative path the id and the
 *  registry entry carry. Validation of the map itself is typesafe.mjs's, at the moment it
 *  is used — this only gets it off disk. */
export async function loadQuestions (root, given) {
  if (!given) throw new TypeSafeError('--questions is required: a JSON file holding the API\'s `questions` map, e.g. references/questions/intent.json')
  const abs = locate(root, given)
  if (!abs) throw new TypeSafeError(`--questions ${given}: no such file (looked under the project root and the working directory).`)
  const questions = await readJson(abs, 'the questions file')
  return { questions, name: basename(abs, extname(abs)), path: recordable(root, abs, given) }
}

/** The state for `ask`, from a file. JSON if it parses, otherwise the text as it stands —
 *  a passage of prose is a legitimate state and forcing it into a JSON string first helps
 *  nobody. */
export async function loadState (root, given) {
  const abs = locate(root, given)
  if (!abs) throw new TypeSafeError(`--state ${given}: no such file (looked under the project root and the working directory).`)
  const text = await readFile(abs, 'utf8')
  try { return JSON.parse(text) } catch { return text }
}

function pickItems (payload, itemsKey, what) {
  if (Array.isArray(payload)) return { items: payload, itemsKey: null }
  if (payload === null || typeof payload !== 'object') {
    throw new TypeSafeError(`${what} is neither an array nor an object with one — nothing to iterate.`)
  }
  if (itemsKey) {
    if (!Array.isArray(payload[itemsKey])) throw new TypeSafeError(`${what} has no array at "${itemsKey}" (--items). Top-level keys: ${Object.keys(payload).join(', ') || 'none'}.`)
    return { items: payload[itemsKey], itemsKey }
  }
  const found = ITEMS_KEYS.find(k => Array.isArray(payload[k]))
  if (!found) {
    throw new TypeSafeError(`${what} has no array under any of ${ITEMS_KEYS.join(', ')}. Say which key holds the items with --items <key>. Top-level keys: ${Object.keys(payload).join(', ') || 'none'}.`)
  }
  return { items: payload[found], itemsKey: found }
}

/** Items to classify, from a dataset id or a file.
 *
 *  A reference is first tried as a dataset id in datasets/index.json — the payload at
 *  that entry's rawPath is read, and the items are its rows. Otherwise it is a path:
 *  `.jsonl` is one JSON value per non-blank line, `.json` is an array or an object
 *  holding one. Returns `{ kind, ref, rawPath, itemsKey, items }`, where `ref` is what
 *  the dataset id is keyed to: the dataset id itself, or the project-relative path. */
export async function resolveInput (root, given, { itemsKey = null } = {}) {
  if (!given) throw new TypeSafeError('--input is required: a dataset id from datasets/index.json, or a .json / .jsonl file of items.')

  const indexPath = join(root, 'datasets', 'index.json')
  if (existsSync(indexPath)) {
    let idx
    try { idx = JSON.parse(await readFile(indexPath, 'utf8')) } catch { idx = null }
    const entry = idx?.datasets?.find?.(d => d?.id === given)
    if (entry) {
      if (!entry.rawPath) throw new TypeSafeError(`dataset "${given}" is registered without a rawPath — there is no payload to read.`)
      const abs = isAbsolute(entry.rawPath) ? entry.rawPath : join(root, entry.rawPath)
      if (!existsSync(abs)) {
        throw new TypeSafeError(`dataset "${given}" points at ${entry.rawPath}, which is not on this machine. Payloads are gitignored — re-run the collector that produced it, or classify a file instead.`)
      }
      const payload = await readJson(abs, `the payload of "${given}"`)
      const picked = pickItems(payload, itemsKey, `the payload of "${given}"`)
      return { kind: 'dataset', ref: given, rawPath: entry.rawPath, itemsKey: picked.itemsKey, items: picked.items }
    }
  }

  const abs = locate(root, given)
  if (!abs) {
    throw new TypeSafeError(`--input ${given}: not a dataset id in datasets/index.json and not a file (looked under the project root and the working directory).`)
  }
  const ref = recordable(root, abs, given)
  if (extname(abs).toLowerCase() === '.jsonl') {
    const text = await readFile(abs, 'utf8')
    const items = []
    const lines = text.split('\n')
    for (let i = 0; i < lines.length; i++) {
      const line = lines[i].trim()
      if (!line) continue
      try { items.push(JSON.parse(line)) } catch (err) {
        throw new TypeSafeError(`${ref} line ${i + 1} is not valid JSON: ${err.message}`)
      }
    }
    return { kind: 'jsonl', ref, rawPath: null, itemsKey: null, items }
  }
  const payload = await readJson(abs, ref)
  const picked = pickItems(payload, itemsKey, ref)
  return { kind: 'json', ref, rawPath: null, itemsKey: picked.itemsKey, items: picked.items }
}