filecoin-pin
Version:
Bridge IPFS content to Filecoin Onchain Cloud using familiar tools
158 lines • 6.12 kB
JavaScript
/**
* Node.js filesystem-based storage backend for CAR files.
* Streams CAR data directly to disk.
*/
import { EventEmitter } from 'node:events';
import { createWriteStream } from 'node:fs';
import { mkdir, open } from 'node:fs/promises';
import { dirname } from 'node:path';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { CarWriter } from '@ipld/car';
/**
* File-based storage backend that streams to disk
*/
export class CARFileBackend extends EventEmitter {
outputPath;
logger;
carWriter = null;
writeStream = null;
pipelinePromise = null;
finalized = false;
constructor(outputPath, logger) {
super();
this.outputPath = outputPath;
this.logger = logger;
}
async initialize(rootCID) {
// Ensure output directory exists
await mkdir(dirname(this.outputPath), { recursive: true });
// Create CAR writer channel
const { writer, out } = CarWriter.create([rootCID]);
this.carWriter = writer;
// Create write stream
this.writeStream = createWriteStream(this.outputPath);
// Track header size by counting bytes until first block is written
let headerWritten = false;
let headerSize = 0;
const readable = Readable.from(out);
const tracker = new Transform({
transform: (chunk, _encoding, callback) => {
if (!headerWritten) {
// The header is written before any blocks
headerSize += chunk.length;
}
callback(null, chunk);
},
});
// Store the pipeline promise so we can await it on finalize
this.pipelinePromise = pipeline(readable, tracker, this.writeStream);
// Handle pipeline errors but don't let them crash the process
this.pipelinePromise.catch((error) => {
// Only emit error if not finalized (expected during cleanup)
if (!this.finalized && error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
this.emit('error', error);
}
});
// Force header to be written by accessing the internal mutex
// This ensures we can accurately track the header size
await this.carWriter._mutex;
// Mark header as written
headerWritten = true;
this.emit('initialized', { rootCID, outputPath: this.outputPath });
return { headerSize };
}
async writeBlock(cid, block, offset) {
const cidStr = cid.toString();
this.logger?.debug({ cid: cidStr, blockSize: block.length }, 'CARWritingBlockstore.put() called');
// Write block to CAR file
await this.carWriter?.put({ cid, bytes: block });
// Wait for the write to be queued/processed by the CAR writer
// This ensures data is in the pipeline before we return
await this.carWriter?._mutex;
const varintLength = offset.blockStart - offset.blockLength - cid.bytes.length;
this.logger?.debug({
cid: cidStr,
varintLength,
cidLength: cid.bytes.length,
blockStart: offset.blockStart,
blockLength: offset.blockLength,
}, 'Block offset calculated');
// Emit event for tracking
this.emit('block:stored', { cid, size: block.length });
this.logger?.info({ cid: cidStr }, 'Block written to CAR file');
}
async *readBlock(cid, offset) {
const cidStr = cid.toString();
this.logger?.debug({ cid: cidStr }, 'CARWritingBlockstore.get() called');
// Open the file in read-only mode
const fd = await open(this.outputPath, 'r');
try {
// Allocate buffer for the block data
const buffer = Buffer.alloc(offset.blockLength);
// Read the block from the file at the stored offset
const { bytesRead } = await fd.read(buffer, 0, offset.blockLength, offset.blockStart);
if (bytesRead !== offset.blockLength) {
throw new Error(`Failed to read complete block for ${cidStr}: expected ${offset.blockLength} bytes, got ${bytesRead}`);
}
yield new Uint8Array(buffer);
}
finally {
// Always close the file descriptor
await fd.close();
}
}
async finalize() {
if (this.finalized) {
return;
}
// Throw error if no blocks were written
if (this.carWriter == null) {
throw new Error('Cannot finalize CAR blockstore without any blocks written');
}
// First close the CAR writer to signal no more data
if (this.carWriter != null) {
await this.carWriter.close();
this.carWriter = null;
}
// Wait for the pipeline to complete if it exists
if (this.pipelinePromise != null) {
try {
await this.pipelinePromise;
}
catch (error) {
// Ignore premature close errors during finalization
if (error.code !== 'ERR_STREAM_PREMATURE_CLOSE') {
throw error;
}
}
}
// Clean up the write stream
if (this.writeStream != null) {
this.writeStream = null;
}
this.finalized = true;
this.emit('finalized');
}
async cleanup() {
// Mark as finalized to prevent further writes
this.finalized = true;
if (this.carWriter != null) {
await this.carWriter.close();
}
// Wait for pipeline to complete if it exists
if (this.pipelinePromise != null) {
try {
await this.pipelinePromise;
}
catch {
// Ignore pipeline errors during cleanup
}
}
if (this.writeStream != null && !this.writeStream.destroyed) {
this.writeStream.destroy();
}
this.emit('cleanup');
}
}
//# sourceMappingURL=car-file-backend.js.map