filecoin-pin/dist/car-blockstore.js

IPFS Pinning Service API implementation that pins to Filecoin's PDP service

import { CarWriter } from '@ipld/car';
import { EventEmitter } from 'node:events';
import { createWriteStream } from 'node:fs';
import { mkdir, open } from 'node:fs/promises';
import { dirname } from 'node:path';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { CID } from 'multiformats/cid';
import varint from 'varint';
export class CARWritingBlockstore extends EventEmitter {
    rootCID;
    outputPath;
    blockOffsets = new Map();
    stats;
    logger;
    carWriter = null;
    writeStream = null;
    currentOffset = 0;
    finalized = false;
    pipelinePromise = null;
    constructor(options) {
        super();
        this.rootCID = options.rootCID;
        this.outputPath = options.outputPath;
        this.logger = options.logger;
        this.stats = {
            blocksWritten: 0,
            missingBlocks: new Set(),
            totalSize: 0,
            startTime: Date.now(),
            finalized: false
        };
    }
    async initialize() {
        // Ensure output directory exists
        await mkdir(dirname(this.outputPath), { recursive: true });
        // Create CAR writer channel
        const { writer, out } = CarWriter.create([this.rootCID]);
        this.carWriter = writer;
        // Create write stream
        this.writeStream = createWriteStream(this.outputPath);
        // Track header size by counting bytes until first block is written
        let headerWritten = false;
        let headerSize = 0;
        const readable = Readable.from(out);
        const tracker = new Transform({
            transform: (chunk, _encoding, callback) => {
                if (!headerWritten) {
                    // The header is written before any blocks
                    headerSize += chunk.length;
                }
                callback(null, chunk);
            }
        });
        // Store the pipeline promise so we can await it on finalize
        this.pipelinePromise = pipeline(readable, tracker, this.writeStream);
        // Handle pipeline errors but don't let them crash the process
        this.pipelinePromise.catch((error) => {
            // Only emit error if not finalized (expected during cleanup)
            if (!this.finalized && error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
                this.emit('error', error);
            }
        });
        // Force header to be written by accessing the internal mutex
        // This ensures we can accurately track the header size
        await (this.carWriter)._mutex;
        // Mark header as written and set initial offset
        headerWritten = true;
        this.currentOffset = headerSize;
        this.emit('initialized', { rootCID: this.rootCID, outputPath: this.outputPath });
    }
    async put(cid, block, _options) {
        const cidStr = cid.toString();
        this.logger?.debug({ cid: cidStr, blockSize: block.length }, 'CARWritingBlockstore.put() called');
        if (this.finalized) {
            throw new Error('Cannot put blocks in finalized CAR blockstore');
        }
        if (this.carWriter == null) {
            await this.initialize();
        }
        // Calculate the varint that will be written
        const totalSectionLength = cid.bytes.length + block.length;
        const varintBytes = varint.encode(totalSectionLength);
        const varintLength = varintBytes.length;
        const currentOffset = this.currentOffset;
        // Block data starts after the varint and CID
        const blockStart = currentOffset + varintLength + cid.bytes.length;
        // Store the offset information BEFORE writing
        this.blockOffsets.set(cidStr, {
            blockStart,
            blockLength: block.length
        });
        // Update offset for next block
        this.currentOffset = blockStart + block.length;
        // Write block to CAR file
        await this.carWriter?.put({ cid, bytes: block });
        this.logger?.debug({
            cid: cidStr,
            currentOffset,
            varintLength,
            cidLength: cid.bytes.length,
            blockStart,
            blockLength: block.length
        }, 'Block offset calculated');
        // Update statistics
        this.stats.blocksWritten++;
        this.stats.totalSize += block.length;
        // Emit event for tracking
        this.emit('block:stored', { cid, size: block.length });
        this.logger?.info({ cid: cidStr, blocksWritten: this.stats.blocksWritten, totalSize: this.stats.totalSize }, 'Block written to CAR file');
        return cid;
    }
    async get(cid, _options) {
        const cidStr = cid.toString();
        this.logger?.debug({ cid: cidStr }, 'CARWritingBlockstore.get() called');
        const offset = this.blockOffsets.get(cidStr);
        if (offset == null) {
            // Track missing blocks for statistics
            this.stats.missingBlocks.add(cidStr);
            this.emit('block:missing', { cid });
            // Important: Throw a specific error that Bitswap/Helia expects
            const error = new Error(`Block not found: ${cidStr}`);
            error.code = 'ERR_NOT_FOUND';
            throw error;
        }
        // Open the file in read-only mode
        const fd = await open(this.outputPath, 'r');
        try {
            // Allocate buffer for the block data
            const buffer = Buffer.alloc(offset.blockLength);
            // Read the block from the file at the stored offset
            const { bytesRead } = await fd.read(buffer, 0, offset.blockLength, offset.blockStart);
            if (bytesRead !== offset.blockLength) {
                throw new Error(`Failed to read complete block for ${cidStr}: expected ${offset.blockLength} bytes, got ${bytesRead}`);
            }
            return new Uint8Array(buffer);
        }
        finally {
            // Always close the file descriptor
            await fd.close();
        }
    }
    async has(cid, _options) {
        const cidStr = cid.toString();
        const hasBlock = this.blockOffsets.has(cidStr);
        this.logger?.debug({ cid: cidStr, hasBlock }, 'CARWritingBlockstore.has() called');
        return hasBlock;
    }
    async delete(_cid, _options) {
        throw new Error('Delete operation not supported on CAR writing blockstore');
    }
    async *putMany(source, _options) {
        for await (const { cid, block } of source) {
            yield await this.put(cid, block);
        }
    }
    async *getMany(source, _options) {
        for await (const cid of source) {
            const block = await this.get(cid);
            yield { cid, block };
        }
    }
    async *deleteMany(_source, _options) {
        throw new Error('DeleteMany operation not supported on CAR writing blockstore');
    }
    async *getAll(_options) {
        for (const [cidStr] of this.blockOffsets.entries()) {
            const cid = CID.parse(cidStr);
            const block = await this.get(cid);
            yield { cid, block };
        }
    }
    /**
     * Finalize the CAR file and return statistics
     */
    async finalize() {
        if (this.finalized) {
            return this.stats;
        }
        // Throw error if no blocks were written
        if (this.carWriter == null) {
            throw new Error('Cannot finalize CAR blockstore without any blocks written');
        }
        // First close the CAR writer to signal no more data
        if (this.carWriter != null) {
            await this.carWriter.close();
            this.carWriter = null;
        }
        // Wait for the pipeline to complete if it exists
        if (this.pipelinePromise != null) {
            try {
                await this.pipelinePromise;
            }
            catch (error) {
                // Ignore premature close errors during finalization
                if (error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
                    throw error;
                }
            }
        }
        // Clean up the write stream
        if (this.writeStream != null) {
            this.writeStream = null;
        }
        this.finalized = true;
        this.stats.finalized = true;
        this.emit('finalized', this.stats);
        return this.stats;
    }
    /**
     * Get current statistics
     */
    getStats() {
        return {
            ...this.stats,
            missingBlocks: new Set(this.stats.missingBlocks) // Return a copy
        };
    }
    /**
     * Clean up resources (called on errors)
     */
    async cleanup() {
        try {
            if (this.carWriter != null && !this.finalized) {
                await this.carWriter.close();
            }
            if ((this.writeStream != null) && !this.writeStream.destroyed) {
                this.writeStream.destroy();
            }
        }
        catch (error) {
            // Ignore cleanup errors
        }
        this.emit('cleanup');
    }
}
//# sourceMappingURL=car-blockstore.js.map