open-memex/dist/retrieve/cjk.js

Local-first memory layer and protocol for AI coding agents. Markdown source of truth, SQLite FTS5 index, zero cloud.

// CJK retrieval helpers (pure logic, no sqlite).
//
// FTS5's `porter unicode61` tokenizer treats a run of CJK ideographs as one
// token ("中文记忆" is a single token), so substring queries never match, and
// the old query builder dropped CJK characters entirely. Per D8 the default
// CJK strategy is bigram: at write time we pre-tokenize CJK runs into
// space-separated unigrams + overlapping bigrams stored in the `cjk` FTS
// column; at query time the CJK part of the query becomes an OR of bigrams
// against that column. Unigrams are included so single-character queries
// ("猫") still match. Works identically on bun:sqlite and better-sqlite3
// with no native tokenizer dependency.
const CJK_RE = /[\u3400-\u4DBF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u1100-\u11FF\u{20000}-\u{2A6DF}]/u;
const CJK_RUN_RE = /[\u3400-\u4DBF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u1100-\u11FF\u{20000}-\u{2A6DF}]+/gu;
/** True if the string contains any CJK (Han/Hiragana/Katakana/Hangul) character. */
export function hasCjk(s) {
    return CJK_RE.test(s);
}
/**
 * Build the index text for the `cjk` FTS column: for every CJK run emit each
 * character (unigram) then every overlapping bigram, space-separated.
 * "中文记忆" -> "中 文 记 忆 中文 文记 记忆". Non-CJK text yields "".
 */
export function cjkIndexText(text) {
    const out = [];
    for (const m of text.matchAll(CJK_RUN_RE)) {
        const chars = [...m[0]];
        for (const ch of chars)
            out.push(ch);
        for (let i = 0; i + 1 < chars.length; i++)
            out.push(chars[i] + chars[i + 1]);
    }
    return out.join(" ");
}
/** Escape a raw token for embedding in a double-quoted FTS5 phrase. */
function esc(t) {
    return t.replace(/"/g, '""');
}
/**
 * Convert the CJK runs of a query into an FTS5 expression for the `cjk`
 * column. Multi-char runs become an OR of bigrams (recall-oriented; bm25
 * ranks docs matching more bigrams higher). Single chars stay unigrams.
 * Returns "" when the query has no CJK.
 *
 * Options: `dropBigrams` filters question bigrams (什么/怎么/…) when other
 * terms remain; `maxTerms` caps the term count (applied after dedupe).
 */
export function cjkQueryExpr(query, opts = {}) {
    const parts = [];
    for (const m of query.matchAll(CJK_RUN_RE)) {
        const chars = [...m[0]];
        if (chars.length === 1) {
            parts.push(`"${esc(chars[0])}"`);
        }
        else {
            for (let i = 0; i + 1 < chars.length; i++) {
                parts.push(`"${esc(chars[i] + chars[i + 1])}"`);
            }
        }
    }
    let terms = [...new Set(parts)];
    if (opts.dropBigrams && terms.length > 1) {
        const kept = terms.filter((t) => !opts.dropBigrams.has(t.slice(1, -1).replace(/""/g, '"')));
        if (kept.length > 0)
            terms = kept;
    }
    if (opts.maxTerms && terms.length > opts.maxTerms) {
        terms = terms.slice(0, opts.maxTerms);
    }
    return terms.join(" OR ");
}