rigjs/lib/wiki/scan.ts

A multi-repos dev tool based on yarn and git.Rigjs is intended to be the simplest way to develop,share and deliver codes between different developers or different projects.

import fs from 'fs';
import path from 'path';
import crypto from 'crypto';
import print from '../print';
import { requireVault, WikiEntry } from './config';
import { getDb, recordLastRun, upsertSourceShasBulk } from './db';
import { requireMacOS } from './platform';
import { isBinaryExtension } from './fileTypes';
import { batchGitignored } from './gitignore';

interface ScanOpts {
  json?: boolean;
  baseline?: boolean;
}

export interface ScanReport {
  wiki: string;
  new: string[];
  modified: string[];
  deleted: string[];
  rawDrift: string[];
  unchanged: number;
}

export default function wikiScan(opts: ScanOpts): void {
  requireMacOS();
  const entry = requireVault();
  const report = scanOne(entry, !!opts.baseline);
  if (opts.json) {
    // eslint-disable-next-line no-console
    console.log(JSON.stringify({ ok: true, code: 0, data: [report] }, null, 2));
  } else {
    printReport(report, !!opts.baseline);
  }

  const drift = report.rawDrift.length > 0;
  recordLastRun(report.wiki, 'scan', drift ? 10 : 0);
  if (drift) process.exit(10);
}

export function scanOne(entry: WikiEntry, baseline: boolean): ScanReport {
  const root = entry.root;
  // Always exclude the vault itself (so its own wiki pages don't loop back
  // in). Other excludes (`.git/**`, `node_modules/**`, etc.) are handled
  // generically by the hidden-segment + gitignore filters below.
  const candidates = walk(root, entry.include, [
    ...entry.exclude,
    `${path.relative(root, entry.path) || path.basename(entry.path)}/**`,
  ]);
  // Filter out hidden segments (anywhere in the relative path) and any
  // file that .gitignore covers. The wiki should only see content the
  // project considers tracked + visible.
  const ignored = batchGitignored(candidates);
  const files = candidates.filter(f => {
    const rel = path.relative(root, f);
    if (rel.split(path.sep).some(s => s.startsWith('.'))) return false;
    if (ignored.has(f)) return false;
    return true;
  });

  const db = getDb();
  const seen = new Map<string, string>(); // path -> sha
  for (const file of files) {
    const sha = sha256(file);
    seen.set(path.relative(root, file), sha);
  }

  const stored = db.prepare('SELECT path, sha FROM source_sha WHERE wiki = ?').all(entry.name) as { path: string; sha: string }[];
  const storedMap = new Map(stored.map(r => [r.path, r.sha]));

  const news: string[] = [], modified: string[] = [], deleted: string[] = [];
  let unchanged = 0;
  for (const [p, sha] of seen) {
    const old = storedMap.get(p);
    if (!old) news.push(p);
    else if (old !== sha) modified.push(p);
    else unchanged++;
  }
  for (const p of storedMap.keys()) if (!seen.has(p)) deleted.push(p);

  // RAW DRIFT: file inside <wikiPath>/raw/ whose sha changed.
  const rawDrift: string[] = [];
  const rawDir = path.join(entry.path, 'raw');
  if (fs.existsSync(rawDir)) {
    for (const file of walk(rawDir, ['**/*'], [])) {
      const rel = path.relative(root, file);
      const sha = sha256(file);
      const old = storedMap.get(rel);
      if (old && old !== sha) rawDrift.push(rel);
    }
  }

  // --baseline: commit the current shas to state.db so next scan can detect
  // drift. Does NOT touch wiki content; purely a bookkeeping write.
  if (baseline) {
    const rows: { path: string; sha: string; mtimeMs: number }[] = [];
    for (const [rel, sha] of seen) {
      const abs = path.join(root, rel);
      try {
        const stat = fs.statSync(abs);
        rows.push({ path: rel, sha, mtimeMs: stat.mtimeMs });
      } catch { /* file vanished between walk and stat — skip */ }
    }
    upsertSourceShasBulk(entry.name, rows);
  }

  return { wiki: entry.name, new: news, modified, deleted, rawDrift, unchanged };
}

function walk(root: string, include: string[], exclude: string[]): string[] {
  const out: string[] = [];
  if (!fs.existsSync(root)) return out;
  const stack = [root];
  while (stack.length) {
    const cur = stack.pop()!;
    let entries: fs.Dirent[];
    try { entries = fs.readdirSync(cur, { withFileTypes: true }); } catch { continue; }
    for (const e of entries) {
      // Cheap pre-filter: skip dot-prefixed directory/file segments outright.
      // Walk-time skip avoids descending into .git/, .vscode/, .obsidian/ etc.
      if (e.name.startsWith('.')) continue;
      const full = path.join(cur, e.name);
      const rel = path.relative(root, full);
      if (excluded(rel, exclude)) continue;
      if (e.isDirectory()) stack.push(full);
      else if (e.isFile() && included(rel, include) && !isBinaryExtension(full)) out.push(full);
    }
  }
  return out;
}

function included(rel: string, patterns: string[]): boolean {
  return patterns.some(p => globMatch(rel, p));
}
function excluded(rel: string, patterns: string[]): boolean {
  return patterns.some(p => globMatch(rel, p));
}

/** Minimal glob: supports `**`, `*`, literal. No `?` / `{}` / `[…]`. */
function globMatch(s: string, pattern: string): boolean {
  const re = new RegExp('^' + pattern
    .replace(/[.+^${}()|\\\\]/g, '\\$&')
    .replace(/\*\*/g, '')
    .replace(/\*/g, '[^/]*')
    .replace(//g, '.*') + '$');
  return re.test(s);
}

function sha256(file: string): string {
  const h = crypto.createHash('sha256');
  h.update(fs.readFileSync(file));
  return h.digest('hex');
}

function printReport(r: ScanReport, baseline: boolean) {
  print.info(`scan: ${r.wiki}${baseline ? ' (baseline)' : ''}`);
  // eslint-disable-next-line no-console
  console.log(`  NEW      (${r.new.length})${r.new.length ? '\n    ' + r.new.join('\n    ') : ''}`);
  // eslint-disable-next-line no-console
  console.log(`  MODIFIED (${r.modified.length})${r.modified.length ? '\n    ' + r.modified.join('\n    ') : ''}`);
  // eslint-disable-next-line no-console
  console.log(`  DELETED  (${r.deleted.length})${r.deleted.length ? '\n    ' + r.deleted.join('\n    ') : ''}`);
  if (r.rawDrift.length) {
    print.error(`  RAW DRIFT (${r.rawDrift.length}) — raw/ files have changed; this is forbidden:`);
    // eslint-disable-next-line no-console
    console.log('    ' + r.rawDrift.join('\n    '));
  }
  // eslint-disable-next-line no-console
  console.log(`  UNCHANGED ${r.unchanged}\n`);
  if (baseline) {
    print.info(`  baselined ${r.new.length + r.modified.length + r.unchanged} file(s) into state.db.`);
  }
}