filecoin-pin/src/test/unit/unixfs-car.test.ts

Bridge IPFS content to Filecoin Onchain Cloud using familiar tools

/**
 * Integration test to verify parity between add and import commands
 *
 * This test ensures that:
 * 1. Adding a file with `add` command produces a valid CAR
 * 2. Importing that CAR with `import` command works correctly
 * 3. Both paths produce the same piece CID when uploaded
 */

import { randomBytes } from 'node:crypto'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { CarReader } from '@ipld/car'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import { createCarFromPath } from '../../core/unixfs/index.js'

// Test constants
const PLACEHOLDER_CID = 'bafyaaiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'

describe('UnixFS CAR Creation', () => {
  const testDir = join(tmpdir(), 'filecoin-pin-add-import-test')
  const testFile = join(testDir, 'test-content.bin')
  // Use random data to avoid deduplication - need >1MiB for multi-block
  const testContent = randomBytes(1024 * 1024 * 1.5) // 1.5MB of random data

  const countBlocks = async (carPath: string): Promise<number> => {
    const carData = await readFile(carPath)
    const reader = await CarReader.fromBytes(carData)
    let count = 0
    for await (const _block of reader.blocks()) {
      count++
    }
    return count
  }

  beforeEach(async () => {
    // Create test directory and file
    await rm(testDir, { recursive: true, force: true })
    await mkdir(testDir, { recursive: true })
    await writeFile(testFile, testContent)
  })

  afterEach(async () => {
    // Clean up test directory
    await rm(testDir, { recursive: true, force: true })
  })

  describe('Bare mode (no directory wrapper)', () => {
    it('should create a valid CAR that can be imported', async () => {
      // Step 1: Create CAR from file using add logic
      const { carPath, rootCid } = await createCarFromPath(testFile, { bare: true })

      // Step 2: Read and validate the CAR file
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      // Verify roots
      const roots = await reader.getRoots()
      expect(roots.length).toBe(1)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Step 3: Verify we can iterate through blocks
      let blockCount = 0
      let totalSize = 0
      for await (const { cid, bytes } of reader.blocks()) {
        blockCount++
        totalSize += bytes.length

        // Root block should be present
        if (cid.toString() === rootCid.toString()) {
          expect(bytes.length).toBeGreaterThan(0)
        }
      }

      expect(blockCount).toBeGreaterThan(0)
      expect(totalSize).toBeGreaterThan(0)

      // Clean up
      await rm(carPath, { force: true })
    })

    it('should produce consistent root CIDs for same content in bare mode', async () => {
      // Create multiple CARs from same content in bare mode
      const results = await Promise.all([
        createCarFromPath(testFile, { bare: true }),
        createCarFromPath(testFile, { bare: true }),
        createCarFromPath(testFile, { bare: true }),
      ])

      // All should have the same root CID
      const rootCids = results.map((r) => r.rootCid.toString())
      expect(new Set(rootCids).size).toBe(1)

      // Clean up all temp CARs
      await Promise.all(results.map((r) => rm(r.carPath, { force: true })))
    })

    it('should handle small single-block files correctly in bare mode', async () => {
      const smallFile = join(testDir, 'small.txt')
      await writeFile(smallFile, 'tiny')

      const { carPath, rootCid } = await createCarFromPath(smallFile, { bare: true })

      // Should still produce valid CAR
      const carData = await readFile(carPath)
      expect(carData.length).toBeGreaterThan(0)

      // Should have valid UnixFS root
      // Small files use raw codec (0x55), larger files shard so will have a dag-pb (0x70) root
      expect([0x55, 0x70]).toContain(rootCid.code)

      // Verify CAR structure
      const reader = await CarReader.fromBytes(carData)
      const roots = await reader.getRoots()
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Count blocks - small file should have just 1 block
      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }
      expect(blockCount).toBe(1) // Single block for tiny file

      // Clean up
      await rm(carPath, { force: true })
    })

    it('should handle larger files with multiple blocks in bare mode', async () => {
      // Create a file large enough to require multiple UnixFS blocks
      // UnixFS chunks at 1MiB (1048576 bytes) by default in Helia
      const largeFile = join(testDir, 'large.bin')
      // Use random bytes to avoid deduplication
      const largeContent = randomBytes(1024 * 1024 * 2) // 2MB of random data
      await writeFile(largeFile, largeContent)

      const { carPath, rootCid } = await createCarFromPath(largeFile, { bare: true })

      // Read the CAR and count blocks
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      let blockCount = 0
      let hasRootBlock = false
      for await (const { cid } of reader.blocks()) {
        blockCount++
        if (cid.toString() === rootCid.toString()) {
          hasRootBlock = true
        }
      }

      // Should have multiple blocks for chunked content
      // 2 data blocks + 1 dag-pb root block linking them
      expect(blockCount).toBe(3)
      expect(hasRootBlock).toBe(true)

      // Clean up
      await rm(carPath, { force: true })
    })

    it('should validate placeholder CID is replaced in bare mode', async () => {
      const { carPath, rootCid } = await createCarFromPath(testFile, { bare: true })

      // The placeholder CID should never appear in final output
      expect(rootCid.toString()).not.toBe(PLACEHOLDER_CID)

      // Verify the CAR file has the correct root
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)
      const roots = await reader.getRoots()

      expect(roots[0]?.toString()).not.toBe(PLACEHOLDER_CID)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Clean up
      await rm(carPath, { force: true })
    })
  })

  describe('Directory wrapper mode (default)', () => {
    it('should create a valid CAR with directory wrapper', async () => {
      const { carPath, rootCid } = await createCarFromPath(testFile, { bare: false })

      // Read and validate the CAR file
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      // Verify roots
      const roots = await reader.getRoots()
      expect(roots.length).toBe(1)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Should have blocks for file data plus directory
      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }

      // Expect: 2 data blocks (1.5MB chunked at 1MiB) + 1 file metadata + 1 wrapping directory = 4
      expect(blockCount).toBe(4)

      // Clean up
      await rm(carPath, { force: true })
    })

    it('should have one extra block compared to bare mode', async () => {
      const { carPath: bareCar } = await createCarFromPath(testFile, { bare: true })
      const { carPath: dirCar } = await createCarFromPath(testFile, { bare: false })

      const bareBlocks = await countBlocks(bareCar)
      const dirBlocks = await countBlocks(dirCar)

      // Directory mode should have exactly one extra block (the directory)
      expect(dirBlocks).toBe(bareBlocks + 1)

      // Clean up
      await rm(bareCar, { force: true })
      await rm(dirCar, { force: true })
    })

    it('should produce different root CIDs between bare and directory modes', async () => {
      const { carPath: bareCar, rootCid: bareCid } = await createCarFromPath(testFile, { bare: true })
      const { carPath: dirCar, rootCid: dirCid } = await createCarFromPath(testFile, { bare: false })

      // Root CIDs should be different
      expect(bareCid.toString()).not.toBe(dirCid.toString())

      // Clean up
      await rm(bareCar, { force: true })
      await rm(dirCar, { force: true })
    })

    it('should produce consistent root CIDs for same content in directory mode', async () => {
      const results = await Promise.all([
        createCarFromPath(testFile, { bare: false }),
        createCarFromPath(testFile, { bare: false }),
        createCarFromPath(testFile, { bare: false }),
      ])

      // All should have the same root CID
      const rootCids = results.map((r) => r.rootCid.toString())
      expect(new Set(rootCids).size).toBe(1)

      // Clean up all temp CARs
      await Promise.all(results.map((r) => rm(r.carPath, { force: true })))
    })

    it('should handle small files with directory wrapper', async () => {
      const smallFile = join(testDir, 'small.txt')
      await writeFile(smallFile, 'tiny')

      const { carPath: bareCar } = await createCarFromPath(smallFile, { bare: true })
      const { carPath: dirCar } = await createCarFromPath(smallFile, { bare: false })

      const bareBlocks = await countBlocks(bareCar)
      const dirBlocks = await countBlocks(dirCar)

      // Small file: 1 data block + 1 directory block
      expect(bareBlocks).toBe(1)
      expect(dirBlocks).toBe(2)

      // Clean up
      await rm(bareCar, { force: true })
      await rm(dirCar, { force: true })
    })
  })

  describe('Directory CAR creation', () => {
    it('should create a valid CAR from a directory with multiple files', async () => {
      // Create a test directory structure
      const testDirPath = join(testDir, 'test-directory')
      await mkdir(testDirPath, { recursive: true })

      // Create multiple files with different sizes
      await writeFile(join(testDirPath, 'file1.txt'), 'content1')
      await writeFile(join(testDirPath, 'file2.bin'), randomBytes(1024))
      await writeFile(join(testDirPath, 'large.bin'), randomBytes(1024 * 1024 * 2)) // 2MB

      const { carPath, rootCid } = await createCarFromPath(testDirPath)

      // Verify the result
      expect(rootCid).toBeDefined()

      // Read and validate the CAR file
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      // Verify roots
      const roots = await reader.getRoots()
      expect(roots.length).toBe(1)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Count blocks - should have blocks for:
      // - 3 file data blocks (file1.txt, file2.bin, 2 blocks for large.bin)
      // - 1 file metadata for large.bin (since it's chunked)
      // - 1 root directory
      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }
      expect(blockCount).toBeGreaterThanOrEqual(5) // At least 5 blocks

      // Clean up
      await rm(carPath, { force: true })
      await rm(testDirPath, { recursive: true, force: true })
    })

    it('should handle nested directory structures correctly', async () => {
      // Create nested directory structure
      const rootDir = join(testDir, 'nested-test')
      const subDir1 = join(rootDir, 'subdir1')
      const subDir2 = join(rootDir, 'subdir2')
      const deepDir = join(subDir1, 'deep')

      await mkdir(deepDir, { recursive: true })
      await mkdir(subDir2, { recursive: true })

      // Add files at different levels
      await writeFile(join(rootDir, 'root.txt'), 'root content')
      await writeFile(join(subDir1, 'sub1.txt'), 'subdir1 content')
      await writeFile(join(subDir2, 'sub2.txt'), 'subdir2 content')
      await writeFile(join(deepDir, 'deep.txt'), 'deep content')

      const { carPath, rootCid } = await createCarFromPath(rootDir)

      // Verify the result
      expect(rootCid).toBeDefined()

      // Verify CAR structure
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      const roots = await reader.getRoots()
      expect(roots.length).toBe(1)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Count blocks - should have at least:
      // - 4 file blocks (one per file)
      // - Directory blocks for nested structure
      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }
      expect(blockCount).toBeGreaterThan(4) // More than just file blocks

      // Clean up
      await rm(carPath, { force: true })
      await rm(rootDir, { recursive: true, force: true })
    })

    it('should handle empty directories correctly', async () => {
      const emptyDir = join(testDir, 'empty-dir')
      await mkdir(emptyDir, { recursive: true })

      const { carPath, rootCid } = await createCarFromPath(emptyDir)

      expect(rootCid).toBeDefined()

      // Verify CAR has at least the directory block
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }
      expect(blockCount).toBe(1) // Just the empty directory block

      // Clean up
      await rm(carPath, { force: true })
      await rm(emptyDir, { recursive: true, force: true })
    })

    it('should handle directories with only subdirectories (no files)', async () => {
      const parentDir = join(testDir, 'dirs-only')
      const subDir1 = join(parentDir, 'sub1')
      const subDir2 = join(parentDir, 'sub2')
      const subSubDir = join(subDir1, 'subsub')

      await mkdir(subSubDir, { recursive: true })
      await mkdir(subDir2, { recursive: true })

      const { carPath, rootCid } = await createCarFromPath(parentDir)

      expect(rootCid).toBeDefined()

      // Verify CAR has directory blocks
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)

      let blockCount = 0
      for await (const _block of reader.blocks()) {
        blockCount++
      }
      // Empty directories deduplicate to the same CID, so we expect:
      // 1. Empty directory block (shared by all empty dirs)
      // 2. sub1 directory block (links to subsub)
      // 3. parent directory block (links to sub1 and sub2)
      expect(blockCount).toBe(3)

      // Clean up
      await rm(carPath, { force: true })
      await rm(parentDir, { recursive: true, force: true })
    })

    it('should handle mixed content (files and directories) correctly', async () => {
      const mixedDir = join(testDir, 'mixed-content')
      const subDir = join(mixedDir, 'subdir')
      await mkdir(subDir, { recursive: true })

      // Add various file types and sizes
      await writeFile(join(mixedDir, 'text.txt'), 'text content')
      await writeFile(join(mixedDir, 'binary.bin'), randomBytes(512))
      await writeFile(join(subDir, 'nested.json'), JSON.stringify({ test: true }))
      await writeFile(join(subDir, 'large.dat'), randomBytes(1024 * 1024 * 1.5)) // 1.5MB

      const { carPath, rootCid } = await createCarFromPath(mixedDir)

      expect(rootCid).toBeDefined()

      // Verify the CAR contains all expected content
      const blockCount = await countBlocks(carPath)
      // Should have blocks for: 4 files (large.dat chunked to 2) + file metadata for large.dat + 2 directories
      expect(blockCount).toBeGreaterThanOrEqual(7)

      // Clean up
      await rm(carPath, { force: true })
      await rm(mixedDir, { recursive: true, force: true })
    })

    it('should produce consistent CIDs for identical directory structures', async () => {
      // Create two identical directory structures
      const dir1 = join(testDir, 'consistent1')
      const dir2 = join(testDir, 'consistent2')

      for (const dir of [dir1, dir2]) {
        const subDir = join(dir, 'sub')
        await mkdir(subDir, { recursive: true })
        await writeFile(join(dir, 'file.txt'), 'same content')
        await writeFile(join(subDir, 'nested.txt'), 'nested content')
      }

      const result1 = await createCarFromPath(dir1)
      const result2 = await createCarFromPath(dir2)

      // Should produce identical CIDs for identical content
      expect(result1.rootCid.toString()).toBe(result2.rootCid.toString())

      // Clean up
      await rm(result1.carPath, { force: true })
      await rm(result2.carPath, { force: true })
      await rm(dir1, { recursive: true, force: true })
      await rm(dir2, { recursive: true, force: true })
    })

    it('should handle very deep directory nesting', async () => {
      // Create a deeply nested structure
      let currentPath = testDir
      const depth = 10
      for (let i = 0; i < depth; i++) {
        currentPath = join(currentPath, `level${i}`)
      }
      await mkdir(currentPath, { recursive: true })
      await writeFile(join(currentPath, 'deep-file.txt'), 'very deep')

      const basePath = join(testDir, 'level0')
      const { carPath, rootCid } = await createCarFromPath(basePath)

      expect(rootCid).toBeDefined()

      // Should have blocks for all directory levels plus the file
      const blockCount = await countBlocks(carPath)
      expect(blockCount).toBeGreaterThanOrEqual(depth + 1) // At least one block per level + file

      // Clean up
      await rm(carPath, { force: true })
      await rm(basePath, { recursive: true, force: true })
    })

    it('should create exactly 3 blocks for simple directory with 2 small files', async () => {
      // Create a simple directory with 2 small files
      const simpleDir = join(testDir, 'simple-dir')
      await mkdir(simpleDir, { recursive: true })

      // Create 2 small files (small enough to be single blocks each)
      await writeFile(join(simpleDir, 'file1.txt'), 'content of file 1')
      await writeFile(join(simpleDir, 'file2.txt'), 'content of file 2')

      const { carPath, rootCid } = await createCarFromPath(simpleDir)

      expect(rootCid).toBeDefined()

      // Expected blocks:
      // 1. file1.txt data block (raw)
      // 2. file2.txt data block (raw)
      // 3. directory block (dag-pb) linking to file1 and file2
      const blockCount = await countBlocks(carPath)
      expect(blockCount).toBe(3)

      // Clean up
      await rm(carPath, { force: true })
      await rm(simpleDir, { recursive: true, force: true })
    })

    it('should deduplicate identical content blocks', async () => {
      // Create a directory with multiple files containing identical content
      const dedupDir = join(testDir, 'dedup-test')
      await mkdir(dedupDir, { recursive: true })

      // Create 3 files with identical content - they should deduplicate to one block
      const identicalContent = 'This is the same content in all files'
      await writeFile(join(dedupDir, 'file1.txt'), identicalContent)
      await writeFile(join(dedupDir, 'file2.txt'), identicalContent)
      await writeFile(join(dedupDir, 'file3.txt'), identicalContent)

      const { carPath, rootCid } = await createCarFromPath(dedupDir)

      expect(rootCid).toBeDefined()

      // Expected blocks:
      // 1. Single data block (shared by all 3 files due to deduplication)
      // 2. directory block (dag-pb) linking to the 3 files (which all point to same data block)
      const blockCount = await countBlocks(carPath)
      expect(blockCount).toBe(2) // Only 2 blocks instead of 4 due to deduplication

      // Verify CAR still has correct structure
      const carData = await readFile(carPath)
      const reader = await CarReader.fromBytes(carData)
      const roots = await reader.getRoots()
      expect(roots.length).toBe(1)
      expect(roots[0]?.toString()).toBe(rootCid.toString())

      // Clean up
      await rm(carPath, { force: true })
      await rm(dedupDir, { recursive: true, force: true })
    })
  })
})