@dfinity/assets
Version:
JavaScript and TypeScript library to manage assets on the Internet Computer
381 lines • 16.2 kB
JavaScript
import { Actor, Cbor as cbor, Certificate, HttpAgent, lookup_path, lookupResultToBuffer, reconstruct, LookupPathStatus, } from '@dfinity/agent';
import { lebDecode, PipeArrayBuffer, compare } from '@dfinity/candid';
import { getAssetsCanister } from "./canisters/assets.js";
import { sha256 } from '@noble/hashes/sha2';
import { isReadable } from "./readable/readable.js";
import { ReadableFile } from "./readable/readableFile.js";
import { ReadableBlob } from "./readable/readableBlob.js";
import { ReadablePath } from "./readable/readablePath.js";
import { ReadableBytes } from "./readable/readableBytes.js";
import { limit } from "./utils/limit.js";
import { base64Decode } from "./utils/base64.js";
import fs from 'fs';
export class AssetManager {
/**
* Create assets canister manager instance
* @param config Additional configuration options, canister id is required
*/
constructor(config) {
const { concurrency, maxSingleFileSize, maxChunkSize, ...actorConfig } = config;
this._actor = getAssetsCanister(actorConfig);
this._limit = limit(concurrency ?? 16);
this._maxSingleFileSize = maxSingleFileSize ?? 1900000;
this._maxChunkSize = maxChunkSize ?? 1900000;
}
/**
* Create readable from store arguments
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
static async toReadable(...args) {
if (typeof File === 'function' && args[0] instanceof File) {
return new ReadableFile(args[0]);
}
if (typeof Blob === 'function' && args[0] instanceof Blob && args[1]?.fileName) {
return new ReadableBlob(args[1].fileName, args[0]);
}
if (typeof args[0] === 'string') {
return await ReadablePath.create(args[0]);
}
if ((Array.isArray(args[0]) || args[0] instanceof Uint8Array || args[0] instanceof ArrayBuffer) &&
args[1]?.fileName) {
return new ReadableBytes(args[1].fileName, args[0]);
}
if (isReadable(args[0])) {
return args[0];
}
throw new Error('Invalid arguments, readable could not be created');
}
/**
* Get list of all files in assets canister
* @returns All files in asset canister
*/
async list() {
return this._actor.list({});
}
/**
* Store data on assets canister
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
async store(...args) {
const readable = await AssetManager.toReadable(...args);
const [, config] = args;
const key = [config?.path ?? '', config?.fileName ?? readable.fileName].join('/');
// If asset is small enough upload in one request else upload in chunks (batch)
if (readable.length <= this._maxSingleFileSize) {
config?.onProgress?.({ current: 0, total: readable.length });
await this._limit(async () => {
await readable.open();
const bytes = await readable.slice(0, readable.length);
await readable.close();
const hash = config?.sha256 ?? sha256(new Uint8Array(bytes));
return this._actor.store({
key,
content: bytes,
content_type: readable.contentType,
sha256: [hash],
content_encoding: config?.contentEncoding ?? 'identity',
});
});
config?.onProgress?.({ current: readable.length, total: readable.length });
}
else {
// Create batch to upload asset in chunks
const batch = this.batch();
await batch.store(readable, config);
await batch.commit();
}
return key;
}
/**
* Delete file from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
*/
async delete(key) {
await this._actor.delete_asset({ key });
}
/**
* Delete all files from assets canister
*/
async clear() {
await this._actor.clear({});
}
/**
* Get asset instance from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
* @param acceptEncodings The accepted content encodings, defaults to ['identity']
*/
async get(key, acceptEncodings) {
const data = await this._actor.get({
key,
accept_encodings: acceptEncodings ?? ['identity'],
});
return new Asset(this._actor, this._limit, key, acceptEncodings ?? ['identity'], data.content, data.content_type, Number(data.total_length), data.content_encoding, data.content.length, data.sha256[0]);
}
/**
* Create a batch assets operations instance, commit multiple operations in a single request
*/
batch() {
return new AssetManagerBatch(this._actor, this._limit, this._maxChunkSize);
}
}
// Required since the sha256 type is not exported
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const hasher = sha256.create();
class AssetManagerBatch {
constructor(_actor, _limit, _maxChunkSize) {
this._actor = _actor;
this._limit = _limit;
this._maxChunkSize = _maxChunkSize;
this._scheduledOperations = [];
this._sha256 = {};
this._progress = {};
}
/**
* Insert batch operation to store data on assets canister
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
async store(...args) {
const readable = await AssetManager.toReadable(...args);
const [, config] = args;
const key = [config?.path ?? '', config?.fileName ?? readable.fileName].join('/');
if (!config?.sha256) {
this._sha256[key] = sha256.create();
}
this._progress[key] = { current: 0, total: readable.length };
config?.onProgress?.(this._progress[key]);
this._scheduledOperations.push(async (batch_id, onProgress) => {
await readable.open();
const chunkCount = Math.ceil(readable.length / this._maxChunkSize);
const chunkIds = await Promise.all(Array.from({ length: chunkCount }).map(async (_, index) => {
const content = await readable.slice(index * this._maxChunkSize, Math.min((index + 1) * this._maxChunkSize, readable.length));
if (!config?.sha256) {
this._sha256[key].update(content);
}
const { chunk_id } = await this._limit(() => this._actor.create_chunk({
content,
batch_id,
}));
this._progress[key].current += content.length;
config?.onProgress?.(this._progress[key]);
onProgress?.({
current: Object.values(this._progress).reduce((acc, val) => acc + val.current, 0),
total: Object.values(this._progress).reduce((acc, val) => acc + val.total, 0),
});
return chunk_id;
}));
await readable.close();
const headers = config?.headers ? [config.headers] : [];
return [
{
CreateAsset: { key, content_type: config?.contentType ?? readable.contentType, headers },
},
{
SetAssetContent: {
key,
sha256: [config?.sha256 ?? new Uint8Array(this._sha256[key].digest())],
chunk_ids: chunkIds,
content_encoding: config?.contentEncoding ?? 'identity',
},
},
];
});
return key;
}
/**
* Insert batch operation to delete file from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
*/
delete(key) {
this._scheduledOperations.push(async () => [{ DeleteAsset: { key } }]);
}
/**
* Commit all batch operations to assets canister
* @param args Optional arguments with optional progress callback for commit progress
*/
async commit(args) {
// Create batch
const { batch_id } = await this._limit(() => this._actor.create_batch({}));
// Progress callback
args?.onProgress?.({
current: Object.values(this._progress).reduce((acc, val) => acc + val.current, 0),
total: Object.values(this._progress).reduce((acc, val) => acc + val.total, 0),
});
// Execute scheduled operations
const operations = (await Promise.all(this._scheduledOperations.map(scheduled_operation => scheduled_operation(batch_id, args?.onProgress)))).flat();
// Commit batch
await this._limit(() => this._actor.commit_batch({ batch_id, operations }));
// Cleanup
this._scheduledOperations = [];
this._sha256 = {};
this._progress = {};
}
}
class Asset {
constructor(_actor, _limit, _key, _acceptEncodings, _content, contentType, length, contentEncoding, chunkSize, sha256) {
this._actor = _actor;
this._limit = _limit;
this._key = _key;
this._acceptEncodings = _acceptEncodings;
this._content = _content;
this.contentType = contentType;
this.length = length;
this.contentEncoding = contentEncoding;
this.chunkSize = chunkSize;
this.sha256 = sha256;
}
/**
* Get asset content as blob (web), most browsers are able to use disk storage for larger blobs
*/
async toBlob() {
const blobs = Array.from({ length: Math.ceil(this.length / this.chunkSize) });
await this.getChunks((index, chunk) => (blobs[index] = new Blob([chunk])));
return new Blob([...blobs]);
}
/**
* Get asset content as unsigned 8-bit integer array, use `toBlob` (web) or `write` (Node.js) for larger files
*/
async toUint8Array() {
const bytes = new Uint8Array(this.length);
await this.getChunks((index, chunk) => bytes.set(chunk, index * this.chunkSize));
return bytes;
}
/**
* Get asset content as number array, use `toBlob` (web) or `write` (Node.js) for larger files
*/
async toNumberArray() {
const chunks = Array.from({ length: Math.ceil(this.length / this.chunkSize) });
await this.getChunks((index, chunk) => (chunks[index] = Array.from(chunk)));
return chunks.flat();
}
/**
* Write asset content to file (Node.js)
* @param path File path to write to
*/
async write(path) {
const fd = await new Promise((resolve, reject) => fs.open(path, 'w', (err, fd) => {
if (err) {
reject(err);
return;
}
resolve(fd);
}));
await this.getChunks((index, chunk) => new Promise((resolve, reject) => fs.write(fd, chunk, 0, chunk.length, index * this.chunkSize, (err) => {
if (err) {
reject(err);
return;
}
resolve();
})));
await new Promise(resolve => fs.close(fd, () => resolve()));
}
/**
* Get All chunks of asset through `onChunk` callback, can be used for a custom storage implementation
* @param onChunk Called on each received chunk
* @param sequential Chunks are received in sequential order when true or `concurrency` is `1` in config
*/
async getChunks(onChunk, sequential) {
onChunk(0, this._content);
const chunkLimit = sequential ? limit(1) : this._limit;
await Promise.all(Array.from({ length: Math.ceil(this.length / this.chunkSize) - 1 }).map((_, index) => chunkLimit(async () => {
const { content } = await this._actor.get_chunk({
key: this._key,
content_encoding: this.contentEncoding,
index: BigInt(index + 1),
sha256: this.sha256 ? [this.sha256] : [],
});
onChunk(index + 1, content);
})));
}
/**
* Check if asset has been certified, which means that the content's hash is in the canister hash tree
*/
async isCertified() {
// Below implementation is based on Internet Computer service worker
const agent = Actor.agentOf(this._actor) ?? new HttpAgent();
const canisterId = Actor.canisterIdOf(this._actor);
if (!agent.rootKey) {
throw Error('Agent is missing root key');
}
const response = await this._limit(() => this._actor.http_request({
method: 'get',
url: this._key,
headers: [['Accept-Encoding', this._acceptEncodings.join(', ')]],
body: new Uint8Array(),
}));
let certificate;
let tree;
const certificateHeader = response.headers.find(([key]) => key.trim().toLowerCase() === 'ic-certificate');
if (!certificateHeader) {
return false;
}
const fields = certificateHeader[1].split(/,/);
for (const f of fields) {
const [, name, b64Value] = [...(f.match(/^(.*)=:(.*):$/) ?? [])].map(x => x.trim());
const value = base64Decode(b64Value);
if (name === 'certificate') {
certificate = value;
}
else if (name === 'tree') {
tree = value;
}
}
if (!certificate || !tree) {
// No certificate or tree in response header
return false;
}
const cert = await Certificate.create({
certificate,
rootKey: agent.rootKey,
canisterId,
}).catch(() => Promise.resolve());
if (!cert) {
// Certificate is not valid
return false;
}
// Check certificate time
const timeLookup = cert.lookup_path(['time']);
if (timeLookup.status !== LookupPathStatus.Found || !(timeLookup.value instanceof Uint8Array)) {
return false;
}
const decodedTime = lebDecode(new PipeArrayBuffer(timeLookup.value));
const certTime = Number(decodedTime / BigInt(1_000_000)); // Convert from nanos to millis
const now = Date.now();
const maxCertTimeOffset = 300_000; // 5 min
if (certTime - maxCertTimeOffset > now || certTime + maxCertTimeOffset < now) {
return false;
}
const hashTree = cbor.decode(tree);
const reconstructed = await reconstruct(hashTree);
const witness = cert.lookup_path(['canister', canisterId.toUint8Array(), 'certified_data']);
if (witness.status !== LookupPathStatus.Found || !(witness.value instanceof Uint8Array)) {
// Could not find certified data for this canister in the certificate
return false;
}
// First validate that the Tree is as good as the certification
if (compare(witness.value, reconstructed) !== 0) {
// Witness != Tree passed in ic-certification
return false;
}
// Lookup hash of asset in tree
const treeSha = lookupResultToBuffer(lookup_path(['http_assets', this._key], hashTree));
return !!treeSha && !!this.sha256 && compare(this.sha256, treeSha) === 0;
}
/**
* Check if the hash of the asset data is equal to the hash that has been certified
* @param bytes Optionally pass data to hash instead of waiting for asset data to be fetched and hashed
*/
async verifySha256(bytes) {
if (!this.sha256?.buffer) {
return false;
}
const hash = sha256.create();
if (bytes) {
hash.update(Array.isArray(bytes) ? new Uint8Array(bytes) : bytes);
}
else {
await this.getChunks((_, chunk) => hash.update(chunk), true);
}
return compare(this.sha256, hash.digest()) === 0;
}
}
//# sourceMappingURL=index.js.map