ipfs-unixfs-importer
Version:
JavaScript implementation of the UnixFs importer used by IPFS
258 lines • 7.76 kB
JavaScript
/**
* @packageDocumentation
*
* @example
*
* Let's create a little directory to import:
*
* ```console
* > cd /tmp
* > mkdir foo
* > echo 'hello' > foo/bar
* > echo 'world' > foo/quux
* ```
*
* And write the importing logic:
*
* ```js
* import { importer } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core/memory'
* import * as fs from 'node:fs'
*
* // Where the blocks will be stored
* const blockstore = new MemoryBlockstore()
*
* // Import path /tmp/foo/
* const source = [{
* path: '/tmp/foo/bar',
* content: fs.createReadStream('/tmp/foo/bar')
* }, {
* path: '/tmp/foo/quxx',
* content: fs.createReadStream('/tmp/foo/quux')
* }]
*
* for await (const entry of importer(source, blockstore)) {
* console.info(entry)
* }
* ```
*
* When run, metadata about DAGNodes in the created tree is printed until the root:
*
* ```js
* {
* cid: CID, // see https://github.com/multiformats/js-cid
* path: 'tmp/foo/bar',
* unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs
* }
* {
* cid: CID, // see https://github.com/multiformats/js-cid
* path: 'tmp/foo/quxx',
* unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs
* }
* {
* cid: CID, // see https://github.com/multiformats/js-cid
* path: 'tmp/foo',
* unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs
* }
* {
* cid: CID, // see https://github.com/multiformats/js-cid
* path: 'tmp',
* unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs
* }
* ```
*/
import first from 'it-first';
import parallelBatch from 'it-parallel-batch';
import { fixedSize } from './chunker/fixed-size.js';
import { defaultBufferImporter } from './dag-builder/buffer-importer.js';
import { defaultDagBuilder } from './dag-builder/index.js';
import { defaultChunkValidator } from './dag-builder/validate-chunks.js';
import { InvalidParametersError } from './errors.js';
import { balanced } from './layout/index.js';
import { defaultTreeBuilder } from './tree-builder.js';
export * from './errors.js';
/**
* The importer creates UnixFS DAGs and stores the blocks that make
* them up in the passed blockstore.
*
* @example
*
* ```typescript
* import { importer } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core'
*
* // store blocks in memory, other blockstores are available
* const blockstore = new MemoryBlockstore()
*
* const input = [{
* path: './foo.txt',
* content: Uint8Array.from([0, 1, 2, 3, 4])
* }, {
* path: './bar.txt',
* content: Uint8Array.from([0, 1, 2, 3, 4])
* }]
*
* for await (const entry of importer(input, blockstore)) {
* console.info(entry)
* // { cid: CID(), ... }
* }
* ```
*/
export async function* importer(source, blockstore, options = {}) {
let candidates;
if (Symbol.asyncIterator in source || Symbol.iterator in source) {
candidates = source;
}
else {
candidates = [source];
}
const wrapWithDirectory = options.wrapWithDirectory ?? false;
const shardSplitThresholdBytes = options.shardSplitThresholdBytes ?? 262144;
const shardFanoutBits = options.shardFanoutBits ?? 8;
const cidVersion = options.cidVersion ?? 1;
const rawLeaves = options.rawLeaves ?? true;
const leafType = options.leafType ?? 'file';
const fileImportConcurrency = options.fileImportConcurrency ?? 50;
const blockWriteConcurrency = options.blockWriteConcurrency ?? 10;
const reduceSingleLeafToSelf = options.reduceSingleLeafToSelf ?? true;
const chunker = options.chunker ?? fixedSize();
const chunkValidator = options.chunkValidator ?? defaultChunkValidator();
const buildDag = options.dagBuilder ?? defaultDagBuilder({
chunker,
chunkValidator,
wrapWithDirectory,
layout: options.layout ?? balanced(),
bufferImporter: options.bufferImporter ?? defaultBufferImporter({
cidVersion,
rawLeaves,
leafType,
onProgress: options.onProgress
}),
blockWriteConcurrency,
reduceSingleLeafToSelf,
cidVersion,
onProgress: options.onProgress
});
const buildTree = options.treeBuilder ?? defaultTreeBuilder({
wrapWithDirectory,
shardSplitThresholdBytes,
shardFanoutBits,
cidVersion,
onProgress: options.onProgress
});
for await (const entry of buildTree(parallelBatch(buildDag(candidates, blockstore), fileImportConcurrency), blockstore)) {
yield {
cid: entry.cid,
path: entry.path,
unixfs: entry.unixfs,
size: entry.size
};
}
}
/**
* `importFile` is similar to `importer` except it accepts a single
* `FileCandidate` and returns a promise of a single `ImportResult`
* instead of a stream of results.
*
* @example
*
* ```typescript
* import { importFile } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core'
*
* // store blocks in memory, other blockstores are available
* const blockstore = new MemoryBlockstore()
*
* const input: FileCandidate = {
* path: './foo.txt',
* content: Uint8Array.from([0, 1, 2, 3, 4])
* }
*
* const entry = await importFile(input, blockstore)
* ```
*/
export async function importFile(content, blockstore, options = {}) {
const result = await first(importer([content], blockstore, options));
if (result == null) {
throw new InvalidParametersError('Nothing imported');
}
return result;
}
/**
* `importDir` is similar to `importer` except it accepts a single
* `DirectoryCandidate` and returns a promise of a single `ImportResult`
* instead of a stream of results.
*
* @example
*
* ```typescript
* import { importDirectory } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core'
*
* // store blocks in memory, other blockstores are available
* const blockstore = new MemoryBlockstore()
*
* const input: DirectoryCandidate = {
* path: './foo.txt'
* }
*
* const entry = await importDirectory(input, blockstore)
* ```
*/
export async function importDirectory(content, blockstore, options = {}) {
const result = await first(importer([content], blockstore, options));
if (result == null) {
throw new InvalidParametersError('Nothing imported');
}
return result;
}
/**
* `importBytes` accepts a single Uint8Array and returns a promise
* of a single `ImportResult`.
*
* @example
*
* ```typescript
* import { importBytes } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core'
*
* // store blocks in memory, other blockstores are available
* const blockstore = new MemoryBlockstore()
*
* const input = Uint8Array.from([0, 1, 2, 3, 4])
*
* const entry = await importBytes(input, blockstore)
* ```
*/
export async function importBytes(buf, blockstore, options = {}) {
return importFile({
content: buf
}, blockstore, options);
}
/**
* `importByteStream` accepts a single stream of Uint8Arrays and
* returns a promise of a single `ImportResult`.
*
* @example
*
* ```typescript
* import { importByteStream } from 'ipfs-unixfs-importer'
* import { MemoryBlockstore } from 'blockstore-core'
*
* // store blocks in memory, other blockstores are available
* const blockstore = new MemoryBlockstore()
*
* const input = [
* Uint8Array.from([0, 1, 2, 3, 4]),
* Uint8Array.from([5, 6, 7, 8, 9])
* ]
*
* const entry = await importByteStream(input, blockstore)
* ```
*/
export async function importByteStream(bufs, blockstore, options = {}) {
return importFile({
content: bufs
}, blockstore, options);
}
//# sourceMappingURL=index.js.map