@debito/hippo-lib/src/Upload.js

Double-entry accounting library for CouchDB

import config from './config.js';
import { DEBIT, CREDIT, ACCOUNT_TYPES } from './constants.js';
import JournalEntry from './JournalEntry.js';
import Account from './Account.js';
import ErrorLog from './ErrorLog.js';

const MAX_ROWS = 500;
const META_UPDATE_INTERVAL = 10;

// Upload statuses
const UPLOAD_STATUS = {
    PENDING: 'pending',
    PROCESSING: 'processing',
    COMPLETED: 'completed',
    PARTIAL: 'partial',
    FAILED: 'failed',
    ROLLED_BACK: 'rolled_back'
};

function getDb() {
    return config.db;
}

/**
 * Upload class for managing spreadsheet imports
 * Each upload creates two documents:
 *   - upload_data: immutable payload with all entries to process
 *   - upload_meta: lightweight status tracker updated periodically
 */
class Upload {

    /**
     * Create a new upload (data doc + meta doc)
     * @param {string} type - Upload type: 'ledger', 'journal', 'general'
     * @param {string} contextAccountKey - The account key for context (e.g., the ledger being imported into)
     * @param {string} contextAccountLabel - Display label for the context account
     * @param {Array} entries - Array of entry objects (already validated and resolved by client)
     * @param {string} userId - ID of the user creating the upload
     * @returns {Promise<string>} The uploadId
     */
    static async create(type, contextAccountKey, contextAccountLabel, entries, userId) {
        if (!entries || !Array.isArray(entries) || entries.length === 0) {
            throw new Error('Entries array is required and must not be empty');
        }

        if (entries.length > MAX_ROWS) {
            throw new Error(`Maximum ${MAX_ROWS} rows per upload. Got ${entries.length}`);
        }

        // Check for active imports in this book
        if (await Upload.hasActiveImport()) {
            throw new Error('Another import is currently being processed. Please wait for it to complete.');
        }

        const timestamp = Date.now();
        const rand = Math.random().toString(36).substr(2, 9);
        const uploadId = `upload_${timestamp}_${rand}`;

        // Sort entries by date chronologically for optimal balance recalculation
        const sortedEntries = entries
            .map((entry, index) => ({ ...entry, id: index + 1 }))
            .sort((a, b) => new Date(a.date) - new Date(b.date));

        const dataDoc = {
            _id: `upload-data_${timestamp}_${rand}`,
            docType: 'upload_data',
            uploadId,
            type,
            contextAccountKey,
            entries: sortedEntries,
            createdAt: new Date().toISOString(),
            createdBy: userId
        };

        const metaDoc = {
            _id: `upload-meta_${timestamp}_${rand}`,
            docType: 'upload_meta',
            uploadId,
            type,
            contextAccountKey,
            contextAccountLabel,
            status: UPLOAD_STATUS.PENDING,
            totalRows: sortedEntries.length,
            lastProcessedId: 0,
            succeeded: 0,
            failed: [],
            createdAt: new Date().toISOString(),
            createdBy: userId,
            startedAt: null,
            completedAt: null
        };

        try {
            const bulkResponse = await getDb().bulk({ docs: [dataDoc, metaDoc] });
            for (const result of bulkResponse) {
                if (result.error) {
                    throw new Error(`Failed to create upload docs: ${result.error} - ${result.reason}`);
                }
            }
            return uploadId;
        } catch (error) {
            ErrorLog.log('error', 'Upload.create', error, { uploadId });
            throw new Error(`Failed to create upload: ${error.message}`);
        }
    }

    /**
     * Process an upload - create journal entries for each row
     * @param {string} uploadId - The upload to process
     * @returns {Promise<Object>} Processing results
     */
    static async process(uploadId) {
        const meta = await Upload.getMeta(uploadId);
        if (!meta) throw new Error(`Upload not found: ${uploadId}`);

        if (meta.status !== UPLOAD_STATUS.PENDING && meta.status !== UPLOAD_STATUS.PROCESSING) {
            throw new Error(`Upload ${uploadId} cannot be processed (status: ${meta.status})`);
        }

        const dataDoc = await Upload.getDataDoc(uploadId);
        if (!dataDoc) throw new Error(`Upload data not found: ${uploadId}`);

        // Update meta to processing
        meta.status = UPLOAD_STATUS.PROCESSING;
        meta.startedAt = meta.startedAt || new Date().toISOString();
        await Upload._saveMeta(meta);

        const startFromId = meta.lastProcessedId || 0;
        let succeeded = meta.succeeded || 0;
        const failed = meta.failed || [];
        let lastProcessedId = startFromId;

        for (const entry of dataDoc.entries) {
            // Skip already-processed entries (for resume)
            if (entry.id <= startFromId) continue;

            try {
                await Upload._processEntry(entry, dataDoc.type, dataDoc.contextAccountKey, uploadId);
                succeeded++;
            } catch (err) {
                failed.push({
                    id: entry.id,
                    error: err.message
                });
            }

            lastProcessedId = entry.id;

            // Periodic meta update
            if ((lastProcessedId - startFromId) % META_UPDATE_INTERVAL === 0) {
                meta.lastProcessedId = lastProcessedId;
                meta.succeeded = succeeded;
                meta.failed = failed;
                await Upload._saveMeta(meta);
            }
        }

        // Final meta update
        meta.lastProcessedId = lastProcessedId;
        meta.succeeded = succeeded;
        meta.failed = failed;
        meta.completedAt = new Date().toISOString();

        if (failed.length === 0) {
            meta.status = UPLOAD_STATUS.COMPLETED;
        } else if (succeeded === 0) {
            meta.status = UPLOAD_STATUS.FAILED;
        } else {
            meta.status = UPLOAD_STATUS.PARTIAL;
        }

        await Upload._saveMeta(meta);

        return {
            uploadId,
            status: meta.status,
            total: dataDoc.entries.length,
            succeeded,
            failed: failed.length,
            errors: failed
        };
    }

    /**
     * Process a single entry - create a 2-line journal entry
     * @param {Object} entry - The entry to process
     * @param {string} type - Upload type
     * @param {string} contextAccountKey - Context account key
     * @param {string} uploadId - Upload ID for tagging
     * @private
     */
    static async _processEntry(entry, type, contextAccountKey, uploadId) {
        const externalRefId = `${uploadId}_row_${entry.id}`;

        // Server-side validation
        const contextAccount = await Account.findByKey(contextAccountKey);
        if (!contextAccount) {
            throw new Error(`Context account not found: ${contextAccountKey}`);
        }

        const compensatingAccount = await Account.findByKey(entry.accountKey);
        if (!compensatingAccount) {
            throw new Error(`Compensating account not found: ${entry.accountKey}`);
        }

        if (typeof entry.amount !== 'number' || entry.amount <= 0) {
            throw new Error(`Amount must be a positive number, got: ${entry.amount}`);
        }

        if (![DEBIT, CREDIT].includes(entry.debitCredit)) {
            throw new Error(`debitCredit must be 'debit' or 'credit', got: ${entry.debitCredit}`);
        }

        const description = entry.narration || `${contextAccount.label} / ${compensatingAccount.label}`;
        const oppositeType = entry.debitCredit === DEBIT ? CREDIT : DEBIT;

        // Create journal entry
        const je = await JournalEntry.new(description, [uploadId], externalRefId, entry.date);

        je.addLine(contextAccountKey, entry.debitCredit, entry.amount, description);
        je.addLine(entry.accountKey, oppositeType, entry.amount, description);

        if (!je.validate()) {
            throw new Error('Journal entry does not balance');
        }

        await je.post();
    }

    /**
     * Resume a previously interrupted upload
     * @param {string} uploadId - The upload to resume
     * @returns {Promise<Object>} Processing results
     */
    static async resume(uploadId) {
        const meta = await Upload.getMeta(uploadId);
        if (!meta) throw new Error(`Upload not found: ${uploadId}`);

        if (meta.status !== UPLOAD_STATUS.PROCESSING) {
            throw new Error(`Upload ${uploadId} is not in processing state (status: ${meta.status})`);
        }

        return await Upload.process(uploadId);
    }

    /**
     * Get upload meta document
     * @param {string} uploadId - The upload ID
     * @returns {Promise<Object|null>} Meta document or null
     */
    static async getMeta(uploadId) {
        try {
            const result = await getDb().find({
                selector: {
                    docType: 'upload_meta',
                    uploadId
                },
                limit: 1
            });
            return result.docs.length > 0 ? result.docs[0] : null;
        } catch (error) {
            throw new Error(`Failed to get upload meta: ${error.message}`);
        }
    }

    /**
     * List all upload meta documents
     * @returns {Promise<Array>} Array of meta documents
     */
    static async listAll() {
        try {
            const result = await getDb().find({
                selector: {
                    docType: 'upload_meta'
                },
                sort: [{ createdAt: 'desc' }],
                limit: 100
            });
            return result.docs;
        } catch (error) {
            throw new Error(`Failed to list uploads: ${error.message}`);
        }
    }

    /**
     * Get upload data document
     * @param {string} uploadId - The upload ID
     * @returns {Promise<Object|null>} Data document or null
     */
    static async getDataDoc(uploadId) {
        try {
            const result = await getDb().find({
                selector: {
                    docType: 'upload_data',
                    uploadId
                },
                limit: 1
            });
            return result.docs.length > 0 ? result.docs[0] : null;
        } catch (error) {
            throw new Error(`Failed to get upload data: ${error.message}`);
        }
    }

    /**
     * Rollback an upload - force delete all journal entries created by this upload
     * @param {string} uploadId - The upload to rollback
     * @returns {Promise<Object>} Rollback results
     */
    static async rollback(uploadId) {
        const meta = await Upload.getMeta(uploadId);
        if (!meta) throw new Error(`Upload not found: ${uploadId}`);

        if (meta.status === UPLOAD_STATUS.ROLLED_BACK) {
            throw new Error('Upload has already been rolled back');
        }

        if (meta.status === UPLOAD_STATUS.PROCESSING) {
            throw new Error('Cannot rollback an upload that is still processing');
        }

        try {
            // Find all journal entries tagged with this uploadId
            const result = await getDb().find({
                selector: {
                    docType: 'journal_entry',
                    tags: { $elemMatch: { $eq: uploadId } }
                },
                limit: 999999
            });

            let deleted = 0;
            let errors = [];

            for (const doc of result.docs) {
                try {
                    const je = new JournalEntry(doc);
                    await je.forceDelete(true);
                    deleted++;
                } catch (err) {
                    errors.push({ journalId: doc._id, error: err.message });
                }
            }

            // Update meta
            meta.status = UPLOAD_STATUS.ROLLED_BACK;
            meta.completedAt = new Date().toISOString();
            await Upload._saveMeta(meta);

            return { uploadId, deleted, errors };
        } catch (error) {
            ErrorLog.log('error', 'Upload.rollback', error, { uploadId });
            throw new Error(`Failed to rollback upload: ${error.message}`);
        }
    }

    /**
     * Check if there are any active (processing) imports in the current book
     * @returns {Promise<boolean>}
     */
    static async hasActiveImport() {
        try {
            const result = await getDb().find({
                selector: {
                    docType: 'upload_meta',
                    status: UPLOAD_STATUS.PROCESSING
                },
                limit: 1
            });
            return result.docs.length > 0;
        } catch (error) {
            return false;
        }
    }

    /**
     * Save/update meta document
     * @param {Object} meta - Meta document to save
     * @private
     */
    static async _saveMeta(meta) {
        try {
            const response = await getDb().insert(meta);
            meta._rev = response.rev;
        } catch (error) {
            ErrorLog.log('error', 'Upload._saveMeta', error, { uploadId: meta.uploadId });
            throw new Error(`Failed to update upload meta: ${error.message}`);
        }
    }
}

export { UPLOAD_STATUS };
export default Upload;