scai/dist/daemon/runIndexingBatch.js

> **AI-powered CLI for local code analysis, commit message suggestions, and natural-language queries.** 100% local, private, GDPR-friendly, made in Denmark/EU with ❤️.

import fs from 'fs';
import path from 'path';
import lockfile from 'proper-lockfile';
import { log } from '../utils/log.js';
import { getDbForRepo, getDbPathForRepo } from '../db/client.js';
import { sleep } from '../utils/sleep.js';
import { selectUnprocessedFiles, markFileAsSkippedByPath, countUnprocessedFiles, upsertFileTemplate, upsertFileFtsTemplate, markFileAsIndexed } from '../db/sqlTemplates.js';
// --------------------------------------------------
// DB LOCK
// --------------------------------------------------
async function lockDbWithRetry(retries = 3, delayMs = 100) {
    for (let i = 0; i < retries; i++) {
        try {
            return await lockfile.lock(getDbPathForRepo());
        }
        catch (err) {
            if (i < retries - 1) {
                log(`⏳ DB lock busy, retrying... (${i + 1})`);
                await sleep(delayMs);
            }
            else {
                log('❌ Failed to acquire DB lock after retries:', err);
                throw err;
            }
        }
    }
}
// --------------------------------------------------
// SINGLE FILE INDEXER (daemon-side)
// --------------------------------------------------
export function indexFile(filePath, summary, type) {
    const stats = fs.statSync(filePath);
    const lastModified = stats.mtime.toISOString();
    const indexedAt = new Date().toISOString();
    const normalizedPath = path.normalize(filePath).replace(/\\/g, '/');
    const filename = path.basename(normalizedPath);
    // ----------------------------------------------
    // Extract text content (guarded)
    // ----------------------------------------------
    let contentText = '';
    try {
        if (stats.size <= 2000000) {
            const buffer = fs.readFileSync(filePath);
            const nonTextRatio = buffer
                .slice(0, 1000)
                .filter(b => b < 9 || (b > 13 && b < 32))
                .length / Math.min(buffer.length, 1000);
            if (nonTextRatio <= 0.3) {
                contentText = buffer.toString('utf-8');
            }
            else {
                log(`⚠️ Binary-like content skipped: ${normalizedPath}`);
            }
        }
        else {
            log(`⚠️ Large file content skipped: ${normalizedPath}`);
        }
    }
    catch (err) {
        log(`⚠️ Failed reading content: ${normalizedPath}`, err);
    }
    const db = getDbForRepo();
    // ----------------------------------------------
    // Metadata upsert
    // ----------------------------------------------
    try {
        db.prepare(upsertFileTemplate).run({
            path: normalizedPath,
            filename,
            summary,
            type,
            lastModified,
            indexedAt,
        });
    }
    catch (err) {
        log(`⚠️ Failed metadata upsert for ${normalizedPath}:`, err);
    }
    // ----------------------------------------------
    // FTS upsert
    // ----------------------------------------------
    db.prepare(upsertFileFtsTemplate).run({
        path: normalizedPath,
        filename,
        summary,
        contentText,
    });
    // ----------------------------------------------
    // Mark as indexed
    // ----------------------------------------------
    db.prepare(markFileAsIndexed).run({ path: normalizedPath });
}
// --------------------------------------------------
// FTS REBUILD
// --------------------------------------------------
function rebuildFts() {
    const db = getDbForRepo();
    log('🔍 Rebuilding FTS index...');
    db.exec(`INSERT INTO files_fts(files_fts) VALUES('rebuild');`);
}
// --------------------------------------------------
// INDEXING BATCH
// --------------------------------------------------
export async function runIndexingBatch() {
    log('⚡ Starting indexing batch...');
    const db = getDbForRepo();
    const BATCH_SIZE = 25; // adjust as needed
    const rows = db.prepare(selectUnprocessedFiles).all(BATCH_SIZE);
    if (rows.length === 0) {
        log('✅ No files left to index.');
        return false;
    }
    const release = await lockDbWithRetry();
    let didIndexWork = false;
    log('Release: ', release);
    try {
        for (const row of rows) {
            log(`📄 Indexing: ${row.path}`);
            try {
                indexFile(row.path, null, 'auto');
                didIndexWork = true;
            }
            catch (err) {
                log(`⚠️ Failed indexing ${row.path}`, err);
                db.prepare(markFileAsSkippedByPath).run({ path: row.path });
            }
        }
        try {
            rebuildFts();
        }
        catch (err) {
            log('⚠️ Failed FTS rebuild:', err);
        }
        const remaining = db.prepare(countUnprocessedFiles).get();
        log(`📦 Remaining unindexed files: ${remaining.count}`);
        return didIndexWork;
    }
    finally {
        if (release) {
            await release();
            log('🔓 DB lock released');
        }
        else {
            log('⚠️ DB lock was not acquired, nothing to release');
        }
    }
}