UNPKG

@brianlovin/notion-skills

Version:

Sync agent skills from a Notion database to Claude Code, Codex, OpenCode, Cursor, Gemini CLI.

620 lines 28.4 kB
import chalk from "chalk"; import { existsSync } from "node:fs"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { confirm } from "@inquirer/prompts"; import { classifyMissing, isNotionNotFound, } from "./missing-source.js"; import { applySourceRemoval } from "./source-state.js"; import { NotionClient, readRichText, readTitle, } from "./notion.js"; import { assertNtnInstalled } from "./ntn.js"; import { buildSkillMarkdown, convertPageToSkill, slugify, } from "./convert.js"; import { diffManifest, emptyManifest, hashContent, loadManifest, writeManifest, } from "./manifest.js"; import { computeLineDiff, hasChanges, renderUnifiedDiff } from "./diff.js"; import { withSpinner } from "./commands/_progress.js"; import { HASH_V, hashBehaviorProperties, hashSkillContent, } from "./page-hash.js"; import { materializeFiles } from "./skill-files.js"; import { collidingSlugSet, detectSlugCollisions, } from "./slug-collisions.js"; import { applyRenames, detectRenames } from "./renames.js"; import { ensureSymlink, removeSymlink, targetSkillPath, targetsForKeys, } from "./targets.js"; import { MANIFEST_FILE, SKILLS_STORE } from "./paths.js"; import { detectLocalState } from "./local-state.js"; /** * Bidirectional sync between Notion and the local central store. * * - Pull: Notion's pages → ~/.notion-skills/skills/<name>/SKILL.md * - Push: locally-edited SKILL.md → Notion page properties + body * * Local edits are detected by comparing each SKILL.md's current content * hash against `local_hash` stored in the manifest from the last sync. * Remote edits are detected by Notion's `last_edited_time` + the * `props_hash` summary (Notion does NOT bump last_edited_time for * property-only edits). * * Conflicts (both sides drifted since last sync) are resolved * last-edit-wins via `localMtime` vs `remoteEditedAt`. The loser's * content is preserved by Notion's own page history; we don't try to * merge. * * Bias against deletion: when Notion has fewer skills than the local * manifest expected, prompt before removing locals. * * Safety rule: if the on-disk manifest references a different database * than the current scope, treat as fresh — don't apply that manifest's * "missing" set as deletions, and don't run any pushes (the manifest's * `local_hash` belongs to a different DB so drift signals are bogus). */ export async function runSync(scope, options = {}) { await assertNtnInstalled(); const client = new NotionClient(); const quiet = !!options.quiet; const log = (s) => { if (!quiet) console.log(s); }; const warn = (s) => { if (!quiet) console.warn(s); }; const summary = { created: [], updated: [], pushed: [], removed: [], unchanged: [], invalid: [], conflicts: [], resolutions: [], missingSources: [], }; if (scope.sources.length === 0) { return summary; } // Iterate every configured source. Each source's pages are scoped to // its data_source_id; the manifest carries source_key per entry so // we know which entries are in scope at each step. Per-source try/ // catch lets a deleted (404'd) source be surfaced + (optionally) // disconnected without aborting sync for the healthy sources behind // it in the loop. let succeededCount = 0; const missingErrors = []; for (const source of scope.sources) { try { await runSyncForSource(client, scope, source, summary, options, { log, warn, quiet }); succeededCount++; } catch (err) { if (isNotionNotFound(err)) { missingErrors.push({ source, raw: err instanceof Error ? err.message : String(err), }); continue; } throw err; } } if (missingErrors.length > 0) { const classification = classifyMissing({ succeededCount, missingCount: missingErrors.length, }); for (const m of missingErrors) { summary.missingSources.push({ key: m.source.key, classification, raw: m.raw, }); } if (!quiet) { await handleMissingSources(scope, missingErrors, classification, { log, warn, quiet }); } } return summary; } /** * Resolve sources that 404'd during sync. In TTY mode, prompt to * disconnect each one (skills stay as drafts via `applySourceRemoval`'s * "keep" mode). In non-TTY mode, surface an actionable next step. For * the ambiguous case (no source succeeded — could be a workspace * mismatch), warn and leave the config alone. */ async function handleMissingSources(scope, missing, classification, io) { if (classification === "ambiguous") { io.warn(""); io.warn(chalk.yellow(`⚠ Couldn't reach ${missing.length} ${missing.length === 1 ? "source" : "sources"} in Notion:`)); for (const m of missing) io.warn(chalk.dim(` - ${m.source.key}`)); io.warn(chalk.dim(` Most likely \`ntn\` is signed into a different workspace. Run \`ntn whoami\` to check.`)); io.warn(chalk.dim(` If you actually deleted ${missing.length === 1 ? "it" : "them"} in Notion, run \`notion-skills source remove <key>\` for each.`)); return; } for (const m of missing) { const installedCount = await countInstalledFromSource(scope, m.source.key); io.warn(""); io.warn(chalk.yellow(`⚠ Source "${m.source.key}" was deleted in Notion.`)); if (process.stdin.isTTY) { const ok = await confirm({ message: disconnectPrompt(m.source.key, installedCount), default: true, }); if (ok) { await applySourceRemoval(scope, m.source.key, "keep"); io.log(chalk.green(disconnectSuccess(m.source.key, installedCount))); } else { io.warn(chalk.dim(` Kept "${m.source.key}" in your config. ${remindCommand(m.source.key, installedCount)}`)); } } else { io.warn(chalk.dim(` ${remindCommand(m.source.key, installedCount)}`)); } } } function disconnectPrompt(key, installedCount) { if (installedCount === 0) return `Disconnect "${key}" locally?`; const noun = installedCount === 1 ? "skill" : "skills"; const draft = installedCount === 1 ? "a local draft" : "local drafts"; return `Disconnect "${key}" locally? ${installedCount} ${noun} will stay as ${draft}.`; } function disconnectSuccess(key, installedCount) { if (installedCount === 0) return `✓ Disconnected "${key}".`; const noun = installedCount === 1 ? "skill" : "skills"; const draft = installedCount === 1 ? "a local draft" : "local drafts"; return `✓ Disconnected "${key}". ${installedCount} ${noun} kept as ${draft}.`; } function remindCommand(key, installedCount) { if (installedCount === 0) { return `Run \`notion-skills source remove ${key}\` to disconnect.`; } const noun = installedCount === 1 ? "skill" : "skills"; const draft = installedCount === 1 ? "a local draft" : "local drafts"; return `Run \`notion-skills source remove ${key} --keep-skills\` to disconnect (${installedCount} ${noun} will stay as ${draft}).`; } async function countInstalledFromSource(scope, key) { const manifest = await loadOrEmptyManifest(scope); return Object.values(manifest.skills).filter((e) => e.source_key === key).length; } /** * Sync a single source into the shared manifest. Each phase is a named * helper below; the orchestrator just sequences them. The shared * manifest mutates across phases — sync's job is to converge it toward * Notion's state for this source. */ async function runSyncForSource(client, scope, source, summary, options, io) { const contentRoot = SKILLS_STORE; const queried = await queryAndSummarise(client, source, scope, io); const manifest = await loadOrEmptyManifest(scope); await applyRenamesForSource(manifest, source, queried.pages, scope, io.log); const kept = filterKept(queried.summaries, queried.colliding, manifest, source, options); const diff = computeFetchSet(manifest, source, kept); forceMultiFileRefetch(manifest, source, kept, diff); const { drift, missingPageIds } = await detectLocalState(manifest, contentRoot); const driftReminders = await handleDriftBackups(drift, diff.toFetch, kept, contentRoot, io.log); const approvedRemovals = await confirmRemovals(diff.toRemove, io); manifest.last_synced_at = new Date().toISOString(); manifest.hash_v = HASH_V; for (const localSlug of diff.toRemove) { delete manifest.skills[localSlug]; } const toFetch = kept.filter((k) => new Set([...diff.toFetch, ...missingPageIds, ...(options.extraFetchIds ?? [])]).has(k.id)); await pullPages(client, source, queried.pages, toFetch, kept, manifest, contentRoot, summary, io.log, !!options.showDiff); await backfillLocalHash(manifest, source, contentRoot); dropStaleLocalHash(manifest, diff.toRemove, approvedRemovals); await removeApprovedFromDisk(approvedRemovals, contentRoot, summary); await reconcileTargets(scope, manifest, source, approvedRemovals, contentRoot, summary); recordUnchanged(diff.unchanged, manifest, toFetch, summary); printDriftReminders(driftReminders, io.log); await writeManifest(MANIFEST_FILE, manifest); } // ---------- phase: query the source + summarise pages ---------- async function queryAndSummarise(client, source, scope, io) { const sourceLabel = scope.sources.length === 1 ? "" : ` ${chalk.dim(`[${source.key}]`)}`; const pages = io.quiet ? await client.queryDataSource(source.data_source_id) : await withSpinner(`Querying ${source.name}${sourceLabel}`, () => client.queryDataSource(source.data_source_id), { noteFor: (p) => `${p.length} ${p.length === 1 ? "page" : "pages"}` }); const summaries = pages .filter((p) => !p.archived && !p.in_trash) .map(summarisePage) .filter((s) => s !== null); const collisions = detectSlugCollisions(pages); const colliding = collidingSlugSet(collisions); if (collisions.length > 0) { io.warn(chalk.yellow(`Skipping ${collisions.length} duplicate ${collisions.length === 1 ? "slug" : "slugs"}: ${collisions.map((c) => c.slug).join(", ")}. Rename one of the colliding pages in Notion.`)); } return { pages, summaries, colliding }; } async function loadOrEmptyManifest(scope) { return (await loadManifest(scope.sources)) ?? emptyManifest(); } // ---------- phase: rename detection (mutates manifest in place) ---------- async function applyRenamesForSource(manifest, source, pages, scope, log) { const ops = detectRenames(manifest, source.key, pages); if (ops.length === 0) return; const outcomes = await applyRenames(manifest, ops, SKILLS_STORE, scope.targets); for (const outcome of outcomes) { if (outcome.status === "renamed") { const sameLocalName = outcome.newLocalSlug === outcome.op.localSlug; const sourceChanged = outcome.op.oldSourceSlug !== outcome.op.newSourceSlug; // Three log shapes: // - Source AND local both changed: "old → new (local: X → Y)" // - Only local changed (catch-up after legacy pinned-local // state): "local: X → new" // - Only source changed (collision kept local pinned, but // that's the source-only branch below — we only land here // when at least the local renames or both) let line; if (sourceChanged && !sameLocalName) { line = `↪ ${outcome.op.oldSourceSlug}${outcome.op.newSourceSlug}` + chalk.dim(` (local: ${outcome.op.localSlug}${outcome.newLocalSlug})`); } else if (!sourceChanged && !sameLocalName) { line = `↪ local: ${outcome.op.localSlug}${outcome.newLocalSlug} ${chalk.dim("(catching up)")}`; } else { line = `↪ ${outcome.op.oldSourceSlug}${outcome.op.newSourceSlug}`; } log(chalk.cyan(line)); } else { const reason = outcome.reason.kind === "collision-manifest" ? `local slug "${outcome.reason.conflictWith}" is already in use` : `local dir "${outcome.reason.path}" already exists`; log(chalk.yellow(`⚠ ${outcome.op.oldSourceSlug}${outcome.op.newSourceSlug}: ${reason}. Updated source_slug only; local '${outcome.op.localSlug}' stays.`)); } } } // ---------- phase: narrow to skills the user has installed ---------- function filterKept(summaries, colliding, manifest, source, options) { // Sync is install-narrowed: only operate on skills the user has // installed for THIS source. extraFetchIds is the publish-side escape // hatch — a just-published page hasn't landed in the manifest yet, // but we want to round-trip its content through Notion's normaliser. const trackedSourceSlugs = new Set(); for (const entry of Object.values(manifest.skills)) { if (entry.source_key === source.key) trackedSourceSlugs.add(entry.source_slug); } const extraFetchIds = options.extraFetchIds ?? new Set(); return summaries .filter((s) => !colliding.has(s.name)) .filter((s) => trackedSourceSlugs.has(s.name) || extraFetchIds.has(s.id)); } // ---------- phase: compute the diff against the manifest ---------- function computeFetchSet(manifest, source, kept) { return diffManifest(manifest, kept.map((k) => ({ name: k.name, source_key: source.key, pageId: k.id, lastEditedTime: k.lastEditedTime, propsHash: k.propsHash, })), new Set([source.key])); } function forceMultiFileRefetch(manifest, source, kept, diff) { // Notion doesn't always bump the parent's last_edited_time when only // a child page edits — so for any tracked multi-file skill we force- // include it in the refetch set on every sync. That moves it from // unchanged → toFetch; the body hash check then either confirms no // drift or flags it as outdated. for (const [localSlug, entry] of Object.entries(manifest.skills)) { if (entry.source_key !== source.key) continue; if ((entry.files?.length ?? 0) === 0) continue; const summary = kept.find((k) => k.name === entry.source_slug); if (!summary || diff.toFetch.includes(summary.id)) continue; diff.toFetch.push(summary.id); const idx = diff.unchanged.indexOf(localSlug); if (idx >= 0) diff.unchanged.splice(idx, 1); } } // ---------- phase: backup local edits when conflict ---------- async function handleDriftBackups(localDrift, remoteFetchIds, kept, contentRoot, log) { // App-store rule: sync never pushes — that's `publish`. For each // drifted skill: if remote ALSO changed, back up the local edit // before the pull phase overwrites; if remote unchanged, surface a // one-liner reminding the user to publish. const remoteChangedNames = new Set(); for (const k of kept) if (remoteFetchIds.includes(k.id)) remoteChangedNames.add(k.name); const reminders = []; for (const [name, drift] of localDrift) { if (!remoteChangedNames.has(name)) { reminders.push(name); continue; } try { const backupDir = join(contentRoot, "..", "backup", "sync-overwrite", `${name}-${conflictBackupTimestamp()}`); await mkdir(backupDir, { recursive: true }); await writeFile(join(backupDir, "SKILL.md"), drift.mdContent, "utf8"); log(chalk.yellow(`⚠ ${name}: had local edits AND a newer version was published. Backed up your edit to ${backupDir} before pulling.`)); } catch { log(chalk.yellow(`⚠ ${name}: had local edits AND a newer version was published. Backup failed; pull will overwrite.`)); } } return reminders; } // ---------- phase: confirm removals (TTY only) ---------- async function confirmRemovals(toRemove, io) { if (toRemove.length === 0 || !process.stdin.isTTY || io.quiet) return []; console.log(""); console.log(chalk.yellow(`${toRemove.length} ${toRemove.length === 1 ? "skill is" : "skills are"} no longer in Notion:`)); for (const n of toRemove) console.log(` ${chalk.dim("·")} ${n}`); const ok = await confirm({ message: "Remove them locally to match?", default: false, }); return ok ? toRemove : []; } // ---------- phase: pull pages from Notion ---------- async function pullPages(client, source, pages, toFetch, kept, manifest, contentRoot, summary, log, showDiff) { if (toFetch.length === 0) return; // Collect outcomes first, then print. Multi-file skills are // force-refetched on every sync (Notion doesn't reliably bump the // parent's last_edited_time on child edits) — so the fetch happens // unconditionally, but the report should only mention skills whose // content actually changed. A noisy "Updated: tdd-test" on a sync // where nothing changed is a lie about what the user got. const verbose = process.env.NOTION_SKILLS_DEBUG === "1"; const outcomes = []; for (const summaryPage of toFetch) { if (verbose) console.error(`Fetching "${summaryPage.title}" (${summaryPage.id})...`); const page = pages.find((p) => p.id === summaryPage.id); const converted = await convertPageToSkill(client, page); if (!converted.ok) { summary.invalid.push({ title: summaryPage.title, reason: converted.reason }); log(` ${chalk.yellow("!")} ${summaryPage.title} ${chalk.dim(`(${converted.reason})`)}`); continue; } outcomes.push(await applySkillPullResult(converted.skill, source, manifest, kept, contentRoot, showDiff)); } const changes = outcomes.filter((o) => o.kind !== "unchanged"); if (changes.length > 0) { log(chalk.dim(`Pulling ${changes.length} ${changes.length === 1 ? "page" : "pages"}:`)); for (const c of changes) { if (c.kind === "created") { summary.created.push(c.localSlug); log(` ${chalk.green("+")} ${c.localSlug}`); } else { summary.updated.push(c.localSlug); log(` ${chalk.cyan("↓")} ${c.localSlug}`); } if (c.previous !== null) logSkillDiff(c.previous, c.next, log); } } for (const o of outcomes) { if (o.kind === "unchanged") summary.unchanged.push(o.localSlug); } } async function applySkillPullResult(skill, source, manifest, kept, contentRoot, showDiff) { const md = buildSkillMarkdown({ properties: skill.properties, body: skill.body }); const sourceSlug = skill.properties.name; // Preserve local_slug across re-fetches by matching on stable // page_id. New entries — only ever created via extraFetchIds — // adopt source_slug as their initial local_slug. const existing = Object.entries(manifest.skills).find(([, e]) => e.source_key === source.key && e.page_id === skill.pageId); const localSlug = existing ? existing[0] : sourceSlug; const skillDir = join(contentRoot, localSlug); const skillFile = join(skillDir, "SKILL.md"); const newBodyHash = hashSkillContent(skill.body, skill.files); const newLocalHash = hashSkillContent(md, skill.files); const previousEntry = existing?.[1]; const contentChanged = !previousEntry || previousEntry.body_hash !== newBodyHash || previousEntry.local_hash !== newLocalHash; // Capture the previous content for diff rendering BEFORE we // overwrite. Only diff when local_hash matches manifest's record — // that means the on-disk file IS the last-synced content, so a diff // against the new pull cleanly shows "what Notion changed". If the // user has local edits (hash mismatch), skip the diff: their changes // would conflate with Notion's, producing noise. let previousContent = null; if (showDiff && contentChanged && previousEntry?.local_hash) { try { const onDisk = await readFile(skillFile, "utf8"); if (hashContent(onDisk) === previousEntry.local_hash) { previousContent = onDisk; } } catch { // No previous file or read failure: no diff for this skill. } } await mkdir(skillDir, { recursive: true }); await writeFile(skillFile, md, "utf8"); await materializeFiles(skillDir, skill.files); const matchingSummary = kept.find((k) => k.id === skill.pageId); manifest.skills[localSlug] = { source_key: source.key, source_slug: sourceSlug, page_id: skill.pageId, last_edited_time: skill.lastEditedTime, props_hash: matchingSummary?.propsHash ?? "", body_hash: newBodyHash, local_hash: newLocalHash, files: skill.files.map((f) => f.path).sort(), }; const wasNew = !existing; const kind = wasNew ? "created" : contentChanged ? "updated" : "unchanged"; return { localSlug, kind, previous: previousContent, next: md }; } function logSkillDiff(previous, next, log) { const hunks = computeLineDiff(previous, next); if (!hasChanges(hunks)) return; const rendered = renderUnifiedDiff(hunks, { context: 1, maxLines: 20 }); for (const line of rendered) { if (line.type === "add") log(chalk.green(` + ${line.text}`)); else if (line.type === "remove") log(chalk.red(` - ${line.text}`)); else if (line.type === "elide") log(chalk.dim(` ${line.text}`)); else log(chalk.dim(` ${line.text}`)); } } // ---------- phase: backfill / drop local hashes ---------- async function backfillLocalHash(manifest, source, contentRoot) { // After the pull, every tracked entry from this source has a fresh // SKILL.md on disk that matches what we just wrote — record the hash // so the next sync's drift check can short-circuit. Scope to this // source so we don't keep retrying entries belonging to sources we // haven't synced yet. for (const [localSlug, entry] of Object.entries(manifest.skills)) { if (entry.source_key !== source.key) continue; if (entry.local_hash !== undefined) continue; const file = join(contentRoot, localSlug, "SKILL.md"); if (!existsSync(file)) continue; try { const raw = await readFile(file, "utf8"); manifest.skills[localSlug] = { ...entry, local_hash: hashContent(raw) }; } catch { // Read failure: leave local_hash unset so next sync retries. } } } function dropStaleLocalHash(manifest, proposedRemoves, approved) { // Skills declined for removal stay on disk but their local_hash now // points at content that no longer matches Notion. Drop the field so // the next sync rehashes from the live file. for (const localSlug of proposedRemoves) { if (approved.includes(localSlug)) continue; if (!manifest.skills[localSlug]) continue; const { local_hash: _drop, ...rest } = manifest.skills[localSlug]; manifest.skills[localSlug] = rest; } } // ---------- phase: filesystem reconcile ---------- async function removeApprovedFromDisk(approved, contentRoot, summary) { for (const localSlug of approved) { const skillDir = join(contentRoot, localSlug); if (existsSync(skillDir)) { await rm(skillDir, { recursive: true, force: true }); } summary.removed.push(localSlug); } } async function reconcileTargets(scope, manifest, source, approved, contentRoot, summary) { const targets = targetsForKeys(scope.targets); for (const t of targets) { for (const [localSlug, entry] of Object.entries(manifest.skills)) { if (entry.source_key !== source.key) continue; const real = join(contentRoot, localSlug); const link = targetSkillPath(t, localSlug); const result = await ensureSymlink(real, link); if (result === "skipped") summary.conflicts.push({ name: localSlug, target: link }); } for (const localSlug of approved) { await removeSymlink(targetSkillPath(t, localSlug)); } } } // ---------- phase: tail bookkeeping ---------- function recordUnchanged(unchanged, manifest, toFetch, summary) { // "Unchanged" reflects what neither side touched and we didn't // force-pull — computed AFTER the pull phase so the count is honest. const touched = new Set(toFetch.map((k) => k.id)); for (const localSlug of unchanged) { const entry = manifest.skills[localSlug]; if (entry && !touched.has(entry.page_id)) summary.unchanged.push(localSlug); } } function printDriftReminders(reminders, log) { for (const localSlug of reminders) { log(chalk.yellow(`↑ ${localSlug}: you have local edits — run \`notion-skills publish ${localSlug}\` to share them with your team.`)); } } function summarisePage(page) { const title = readTitle(page.properties); if (!title) return null; const description = readRichText(page.properties, "Description"); return { id: page.id, title, name: slugify(title), description, lastEditedTime: page.last_edited_time, propsHash: hashBehaviorProperties(page), }; } function conflictBackupTimestamp() { const d = new Date(); const pad = (n) => String(n).padStart(2, "0"); return (d.getUTCFullYear().toString() + pad(d.getUTCMonth() + 1) + pad(d.getUTCDate()) + "_" + pad(d.getUTCHours()) + pad(d.getUTCMinutes()) + pad(d.getUTCSeconds())); } export function printSummary(summary) { console.log(""); if (summary.pushed.length) { console.log(chalk.green(`↑ Pushed (${summary.pushed.length}):`)); for (const n of summary.pushed) console.log(` ${n}`); } if (summary.created.length) { console.log(chalk.green(`+ Created (${summary.created.length}):`)); for (const n of summary.created) console.log(` ${n}`); } if (summary.updated.length) { console.log(chalk.cyan(`↓ Updated (${summary.updated.length}):`)); for (const n of summary.updated) console.log(` ${n}`); } if (summary.removed.length) { console.log(chalk.red(`- Removed (${summary.removed.length}):`)); for (const n of summary.removed) console.log(` ${n}`); } if (summary.resolutions.length) { console.log(chalk.yellow(`⚠ Conflicts resolved (${summary.resolutions.length}):`)); for (const r of summary.resolutions) { const kept = r.winner === "local" ? "kept local" : "kept Notion"; console.log(` ${r.name}${kept}`); } } if (summary.unchanged.length) { console.log(chalk.dim(`= Unchanged (${summary.unchanged.length})`)); } if (summary.invalid.length) { console.log(chalk.yellow(`! Skipped invalid (${summary.invalid.length}):`)); for (const i of summary.invalid) console.log(` "${i.title}" — ${i.reason}`); } if (summary.conflicts.length) { console.log(chalk.yellow(`! Symlink conflicts (${summary.conflicts.length}):`)); for (const c of summary.conflicts) { console.log(` ${c.name} — existing non-symlink at ${c.target} (skipped)`); } } console.log(""); } //# sourceMappingURL=sync.js.map