filecoin-pin/src/car-blockstore.ts

IPFS Pinning Service API implementation that pins to Filecoin's PDP service

import { CarWriter } from '@ipld/car'
import { EventEmitter } from 'node:events'
import { createWriteStream, type WriteStream } from 'node:fs'
import { mkdir, open } from 'node:fs/promises'
import { dirname } from 'node:path'
import { Readable, Transform } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { CID } from 'multiformats/cid'
import varint from 'varint'
import type { Blockstore } from 'interface-blockstore'
import type { AbortOptions, AwaitIterable } from 'interface-store'
import type { Logger } from 'pino'

export interface CARBlockstoreStats {
  blocksWritten: number
  missingBlocks: Set<string>
  totalSize: number
  startTime: number
  finalized: boolean
}

export interface CARBlockstoreOptions {
  rootCID: CID
  outputPath: string
  logger?: Logger
}

/**
 * A blockstore that writes blocks directly to a CAR file as they arrive.
 * This eliminates the need for redundant storage during IPFS pinning operations.
 */
interface BlockOffset {
  blockStart: number // Where the actual block data starts (after varint + CID)
  blockLength: number // Length of just the block data
}

export class CARWritingBlockstore extends EventEmitter implements Blockstore {
  private readonly rootCID: CID
  private readonly outputPath: string
  private readonly blockOffsets = new Map<string, BlockOffset>()
  private readonly stats: CARBlockstoreStats
  private readonly logger: Logger | undefined
  private carWriter: any = null
  private writeStream: WriteStream | null = null
  private currentOffset = 0
  private finalized = false
  private pipelinePromise: Promise<void> | null = null

  constructor (options: CARBlockstoreOptions) {
    super()
    this.rootCID = options.rootCID
    this.outputPath = options.outputPath
    this.logger = options.logger
    this.stats = {
      blocksWritten: 0,
      missingBlocks: new Set(),
      totalSize: 0,
      startTime: Date.now(),
      finalized: false
    }
  }

  async initialize (): Promise<void> {
    // Ensure output directory exists
    await mkdir(dirname(this.outputPath), { recursive: true })

    // Create CAR writer channel
    const { writer, out } = CarWriter.create([this.rootCID])
    this.carWriter = writer

    // Create write stream
    this.writeStream = createWriteStream(this.outputPath)

    // Track header size by counting bytes until first block is written
    let headerWritten = false
    let headerSize = 0
    const readable = Readable.from(out)

    const tracker = new Transform({
      transform: (chunk, _encoding, callback) => {
        if (!headerWritten) {
          // The header is written before any blocks
          headerSize += (chunk as Buffer).length
        }
        callback(null, chunk)
      }
    })

    // Store the pipeline promise so we can await it on finalize
    this.pipelinePromise = pipeline(readable, tracker, this.writeStream)

    // Handle pipeline errors but don't let them crash the process
    this.pipelinePromise.catch((error) => {
      // Only emit error if not finalized (expected during cleanup)
      if (!this.finalized && error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
        this.emit('error', error)
      }
    })

    // Force header to be written by accessing the internal mutex
    // This ensures we can accurately track the header size
    await (this.carWriter)._mutex

    // Mark header as written and set initial offset
    headerWritten = true
    this.currentOffset = headerSize

    this.emit('initialized', { rootCID: this.rootCID, outputPath: this.outputPath })
  }

  async put (cid: CID, block: Uint8Array, _options?: AbortOptions): Promise<CID> {
    const cidStr = cid.toString()
    this.logger?.debug({ cid: cidStr, blockSize: block.length }, 'CARWritingBlockstore.put() called')

    if (this.finalized) {
      throw new Error('Cannot put blocks in finalized CAR blockstore')
    }

    if (this.carWriter == null) {
      await this.initialize()
    }

    // Calculate the varint that will be written
    const totalSectionLength = cid.bytes.length + block.length
    const varintBytes = varint.encode(totalSectionLength)
    const varintLength = varintBytes.length

    const currentOffset = this.currentOffset

    // Block data starts after the varint and CID
    const blockStart = currentOffset + varintLength + cid.bytes.length

    // Store the offset information BEFORE writing
    this.blockOffsets.set(cidStr, {
      blockStart,
      blockLength: block.length
    })

    // Update offset for next block
    this.currentOffset = blockStart + block.length

    // Write block to CAR file
    await this.carWriter?.put({ cid, bytes: block })

    this.logger?.debug({
      cid: cidStr,
      currentOffset,
      varintLength,
      cidLength: cid.bytes.length,
      blockStart,
      blockLength: block.length
    }, 'Block offset calculated')

    // Update statistics
    this.stats.blocksWritten++
    this.stats.totalSize += block.length

    // Emit event for tracking
    this.emit('block:stored', { cid, size: block.length })

    this.logger?.info({ cid: cidStr, blocksWritten: this.stats.blocksWritten, totalSize: this.stats.totalSize }, 'Block written to CAR file')

    return cid
  }

  async get (cid: CID, _options?: AbortOptions): Promise<Uint8Array> {
    const cidStr = cid.toString()
    this.logger?.debug({ cid: cidStr }, 'CARWritingBlockstore.get() called')

    const offset = this.blockOffsets.get(cidStr)
    if (offset == null) {
      // Track missing blocks for statistics
      this.stats.missingBlocks.add(cidStr)
      this.emit('block:missing', { cid })
      // Important: Throw a specific error that Bitswap/Helia expects
      const error: Error & { code?: string } = new Error(`Block not found: ${cidStr}`)
      error.code = 'ERR_NOT_FOUND'
      throw error
    }

    // Open the file in read-only mode
    const fd = await open(this.outputPath, 'r')
    try {
      // Allocate buffer for the block data
      const buffer = Buffer.alloc(offset.blockLength)

      // Read the block from the file at the stored offset
      const { bytesRead } = await fd.read(buffer, 0, offset.blockLength, offset.blockStart)

      if (bytesRead !== offset.blockLength) {
        throw new Error(`Failed to read complete block for ${cidStr}: expected ${offset.blockLength} bytes, got ${bytesRead}`)
      }

      return new Uint8Array(buffer)
    } finally {
      // Always close the file descriptor
      await fd.close()
    }
  }

  async has (cid: CID, _options?: AbortOptions): Promise<boolean> {
    const cidStr = cid.toString()
    const hasBlock = this.blockOffsets.has(cidStr)
    this.logger?.debug({ cid: cidStr, hasBlock }, 'CARWritingBlockstore.has() called')
    return hasBlock
  }

  async delete (_cid: CID, _options?: AbortOptions): Promise<void> {
    throw new Error('Delete operation not supported on CAR writing blockstore')
  }

  async * putMany (source: AwaitIterable<{ cid: CID, block: Uint8Array }>, _options?: AbortOptions): AsyncIterable<CID> {
    for await (const { cid, block } of source) {
      yield await this.put(cid, block)
    }
  }

  async * getMany (source: AwaitIterable<CID>, _options?: AbortOptions): AsyncIterable<{ cid: CID, block: Uint8Array }> {
    for await (const cid of source) {
      const block = await this.get(cid)
      yield { cid, block }
    }
  }

  async * deleteMany (_source: AwaitIterable<CID>, _options?: AbortOptions): AsyncIterable<CID> {
    throw new Error('DeleteMany operation not supported on CAR writing blockstore')
  }

  async * getAll (_options?: AbortOptions): AsyncIterable<{ cid: CID, block: Uint8Array }> {
    for (const [cidStr] of this.blockOffsets.entries()) {
      const cid = CID.parse(cidStr)
      const block = await this.get(cid)
      yield { cid, block }
    }
  }

  /**
   * Finalize the CAR file and return statistics
   */
  async finalize (): Promise<CARBlockstoreStats> {
    if (this.finalized) {
      return this.stats
    }

    // Throw error if no blocks were written
    if (this.carWriter == null) {
      throw new Error('Cannot finalize CAR blockstore without any blocks written')
    }

    // First close the CAR writer to signal no more data
    if (this.carWriter != null) {
      await this.carWriter.close()
      this.carWriter = null
    }

    // Wait for the pipeline to complete if it exists
    if (this.pipelinePromise != null) {
      try {
        await this.pipelinePromise
      } catch (error: any) {
        // Ignore premature close errors during finalization
        if (error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
          throw error
        }
      }
    }

    // Clean up the write stream
    if (this.writeStream != null) {
      this.writeStream = null
    }

    this.finalized = true
    this.stats.finalized = true

    this.emit('finalized', this.stats)

    return this.stats
  }

  /**
   * Get current statistics
   */
  getStats (): CARBlockstoreStats {
    return {
      ...this.stats,
      missingBlocks: new Set(this.stats.missingBlocks) // Return a copy
    }
  }

  /**
   * Clean up resources (called on errors)
   */
  async cleanup (): Promise<void> {
    try {
      if (this.carWriter != null && !this.finalized) {
        await this.carWriter.close()
      }
      if ((this.writeStream != null) && !this.writeStream.destroyed) {
        this.writeStream.destroy()
      }
    } catch (error) {
      // Ignore cleanup errors
    }

    this.emit('cleanup')
  }
}