@dfinity/assets
Version:
JavaScript and TypeScript library to manage assets on the Internet Computer
388 lines • 16.8 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AssetManager = void 0;
const agent_1 = require("@dfinity/agent");
const candid_1 = require("@dfinity/candid");
const assets_ts_1 = require("./canisters/assets.js");
const sha2_1 = require("@noble/hashes/sha2");
const readable_ts_1 = require("./readable/readable.js");
const readableFile_ts_1 = require("./readable/readableFile.js");
const readableBlob_ts_1 = require("./readable/readableBlob.js");
const readablePath_ts_1 = require("./readable/readablePath.js");
const readableBytes_ts_1 = require("./readable/readableBytes.js");
const limit_ts_1 = require("./utils/limit.js");
const base64_ts_1 = require("./utils/base64.js");
const fs_1 = __importDefault(require("fs"));
class AssetManager {
/**
* Create assets canister manager instance
* @param config Additional configuration options, canister id is required
*/
constructor(config) {
const { concurrency, maxSingleFileSize, maxChunkSize, ...actorConfig } = config;
this._actor = (0, assets_ts_1.getAssetsCanister)(actorConfig);
this._limit = (0, limit_ts_1.limit)(concurrency ?? 16);
this._maxSingleFileSize = maxSingleFileSize ?? 1900000;
this._maxChunkSize = maxChunkSize ?? 1900000;
}
/**
* Create readable from store arguments
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
static async toReadable(...args) {
if (typeof File === 'function' && args[0] instanceof File) {
return new readableFile_ts_1.ReadableFile(args[0]);
}
if (typeof Blob === 'function' && args[0] instanceof Blob && args[1]?.fileName) {
return new readableBlob_ts_1.ReadableBlob(args[1].fileName, args[0]);
}
if (typeof args[0] === 'string') {
return await readablePath_ts_1.ReadablePath.create(args[0]);
}
if ((Array.isArray(args[0]) || args[0] instanceof Uint8Array || args[0] instanceof ArrayBuffer) &&
args[1]?.fileName) {
return new readableBytes_ts_1.ReadableBytes(args[1].fileName, args[0]);
}
if ((0, readable_ts_1.isReadable)(args[0])) {
return args[0];
}
throw new Error('Invalid arguments, readable could not be created');
}
/**
* Get list of all files in assets canister
* @returns All files in asset canister
*/
async list() {
return this._actor.list({});
}
/**
* Store data on assets canister
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
async store(...args) {
const readable = await AssetManager.toReadable(...args);
const [, config] = args;
const key = [config?.path ?? '', config?.fileName ?? readable.fileName].join('/');
// If asset is small enough upload in one request else upload in chunks (batch)
if (readable.length <= this._maxSingleFileSize) {
config?.onProgress?.({ current: 0, total: readable.length });
await this._limit(async () => {
await readable.open();
const bytes = await readable.slice(0, readable.length);
await readable.close();
const hash = config?.sha256 ?? (0, sha2_1.sha256)(new Uint8Array(bytes));
return this._actor.store({
key,
content: bytes,
content_type: readable.contentType,
sha256: [hash],
content_encoding: config?.contentEncoding ?? 'identity',
});
});
config?.onProgress?.({ current: readable.length, total: readable.length });
}
else {
// Create batch to upload asset in chunks
const batch = this.batch();
await batch.store(readable, config);
await batch.commit();
}
return key;
}
/**
* Delete file from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
*/
async delete(key) {
await this._actor.delete_asset({ key });
}
/**
* Delete all files from assets canister
*/
async clear() {
await this._actor.clear({});
}
/**
* Get asset instance from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
* @param acceptEncodings The accepted content encodings, defaults to ['identity']
*/
async get(key, acceptEncodings) {
const data = await this._actor.get({
key,
accept_encodings: acceptEncodings ?? ['identity'],
});
return new Asset(this._actor, this._limit, key, acceptEncodings ?? ['identity'], data.content, data.content_type, Number(data.total_length), data.content_encoding, data.content.length, data.sha256[0]);
}
/**
* Create a batch assets operations instance, commit multiple operations in a single request
*/
batch() {
return new AssetManagerBatch(this._actor, this._limit, this._maxChunkSize);
}
}
exports.AssetManager = AssetManager;
// Required since the sha256 type is not exported
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const hasher = sha2_1.sha256.create();
class AssetManagerBatch {
constructor(_actor, _limit, _maxChunkSize) {
this._actor = _actor;
this._limit = _limit;
this._maxChunkSize = _maxChunkSize;
this._scheduledOperations = [];
this._sha256 = {};
this._progress = {};
}
/**
* Insert batch operation to store data on assets canister
* @param args Arguments with either a file, blob, path, bytes or custom Readable implementation
*/
async store(...args) {
const readable = await AssetManager.toReadable(...args);
const [, config] = args;
const key = [config?.path ?? '', config?.fileName ?? readable.fileName].join('/');
if (!config?.sha256) {
this._sha256[key] = sha2_1.sha256.create();
}
this._progress[key] = { current: 0, total: readable.length };
config?.onProgress?.(this._progress[key]);
this._scheduledOperations.push(async (batch_id, onProgress) => {
await readable.open();
const chunkCount = Math.ceil(readable.length / this._maxChunkSize);
const chunkIds = await Promise.all(Array.from({ length: chunkCount }).map(async (_, index) => {
const content = await readable.slice(index * this._maxChunkSize, Math.min((index + 1) * this._maxChunkSize, readable.length));
if (!config?.sha256) {
this._sha256[key].update(content);
}
const { chunk_id } = await this._limit(() => this._actor.create_chunk({
content,
batch_id,
}));
this._progress[key].current += content.length;
config?.onProgress?.(this._progress[key]);
onProgress?.({
current: Object.values(this._progress).reduce((acc, val) => acc + val.current, 0),
total: Object.values(this._progress).reduce((acc, val) => acc + val.total, 0),
});
return chunk_id;
}));
await readable.close();
const headers = config?.headers ? [config.headers] : [];
return [
{
CreateAsset: { key, content_type: config?.contentType ?? readable.contentType, headers },
},
{
SetAssetContent: {
key,
sha256: [config?.sha256 ?? new Uint8Array(this._sha256[key].digest())],
chunk_ids: chunkIds,
content_encoding: config?.contentEncoding ?? 'identity',
},
},
];
});
return key;
}
/**
* Insert batch operation to delete file from assets canister
* @param key The path to the file on the assets canister e.g. /folder/to/my_file.txt
*/
delete(key) {
this._scheduledOperations.push(async () => [{ DeleteAsset: { key } }]);
}
/**
* Commit all batch operations to assets canister
* @param args Optional arguments with optional progress callback for commit progress
*/
async commit(args) {
// Create batch
const { batch_id } = await this._limit(() => this._actor.create_batch({}));
// Progress callback
args?.onProgress?.({
current: Object.values(this._progress).reduce((acc, val) => acc + val.current, 0),
total: Object.values(this._progress).reduce((acc, val) => acc + val.total, 0),
});
// Execute scheduled operations
const operations = (await Promise.all(this._scheduledOperations.map(scheduled_operation => scheduled_operation(batch_id, args?.onProgress)))).flat();
// Commit batch
await this._limit(() => this._actor.commit_batch({ batch_id, operations }));
// Cleanup
this._scheduledOperations = [];
this._sha256 = {};
this._progress = {};
}
}
class Asset {
constructor(_actor, _limit, _key, _acceptEncodings, _content, contentType, length, contentEncoding, chunkSize, sha256) {
this._actor = _actor;
this._limit = _limit;
this._key = _key;
this._acceptEncodings = _acceptEncodings;
this._content = _content;
this.contentType = contentType;
this.length = length;
this.contentEncoding = contentEncoding;
this.chunkSize = chunkSize;
this.sha256 = sha256;
}
/**
* Get asset content as blob (web), most browsers are able to use disk storage for larger blobs
*/
async toBlob() {
const blobs = Array.from({ length: Math.ceil(this.length / this.chunkSize) });
await this.getChunks((index, chunk) => (blobs[index] = new Blob([chunk])));
return new Blob([...blobs]);
}
/**
* Get asset content as unsigned 8-bit integer array, use `toBlob` (web) or `write` (Node.js) for larger files
*/
async toUint8Array() {
const bytes = new Uint8Array(this.length);
await this.getChunks((index, chunk) => bytes.set(chunk, index * this.chunkSize));
return bytes;
}
/**
* Get asset content as number array, use `toBlob` (web) or `write` (Node.js) for larger files
*/
async toNumberArray() {
const chunks = Array.from({ length: Math.ceil(this.length / this.chunkSize) });
await this.getChunks((index, chunk) => (chunks[index] = Array.from(chunk)));
return chunks.flat();
}
/**
* Write asset content to file (Node.js)
* @param path File path to write to
*/
async write(path) {
const fd = await new Promise((resolve, reject) => fs_1.default.open(path, 'w', (err, fd) => {
if (err) {
reject(err);
return;
}
resolve(fd);
}));
await this.getChunks((index, chunk) => new Promise((resolve, reject) => fs_1.default.write(fd, chunk, 0, chunk.length, index * this.chunkSize, (err) => {
if (err) {
reject(err);
return;
}
resolve();
})));
await new Promise(resolve => fs_1.default.close(fd, () => resolve()));
}
/**
* Get All chunks of asset through `onChunk` callback, can be used for a custom storage implementation
* @param onChunk Called on each received chunk
* @param sequential Chunks are received in sequential order when true or `concurrency` is `1` in config
*/
async getChunks(onChunk, sequential) {
onChunk(0, this._content);
const chunkLimit = sequential ? (0, limit_ts_1.limit)(1) : this._limit;
await Promise.all(Array.from({ length: Math.ceil(this.length / this.chunkSize) - 1 }).map((_, index) => chunkLimit(async () => {
const { content } = await this._actor.get_chunk({
key: this._key,
content_encoding: this.contentEncoding,
index: BigInt(index + 1),
sha256: this.sha256 ? [this.sha256] : [],
});
onChunk(index + 1, content);
})));
}
/**
* Check if asset has been certified, which means that the content's hash is in the canister hash tree
*/
async isCertified() {
// Below implementation is based on Internet Computer service worker
const agent = agent_1.Actor.agentOf(this._actor) ?? new agent_1.HttpAgent();
const canisterId = agent_1.Actor.canisterIdOf(this._actor);
if (!agent.rootKey) {
throw Error('Agent is missing root key');
}
const response = await this._limit(() => this._actor.http_request({
method: 'get',
url: this._key,
headers: [['Accept-Encoding', this._acceptEncodings.join(', ')]],
body: new Uint8Array(),
}));
let certificate;
let tree;
const certificateHeader = response.headers.find(([key]) => key.trim().toLowerCase() === 'ic-certificate');
if (!certificateHeader) {
return false;
}
const fields = certificateHeader[1].split(/,/);
for (const f of fields) {
const [, name, b64Value] = [...(f.match(/^(.*)=:(.*):$/) ?? [])].map(x => x.trim());
const value = (0, base64_ts_1.base64Decode)(b64Value);
if (name === 'certificate') {
certificate = value;
}
else if (name === 'tree') {
tree = value;
}
}
if (!certificate || !tree) {
// No certificate or tree in response header
return false;
}
const cert = await agent_1.Certificate.create({
certificate,
rootKey: agent.rootKey,
canisterId,
}).catch(() => Promise.resolve());
if (!cert) {
// Certificate is not valid
return false;
}
// Check certificate time
const timeLookup = cert.lookup_path(['time']);
if (timeLookup.status !== agent_1.LookupPathStatus.Found || !(timeLookup.value instanceof Uint8Array)) {
return false;
}
const decodedTime = (0, candid_1.lebDecode)(new candid_1.PipeArrayBuffer(timeLookup.value));
const certTime = Number(decodedTime / BigInt(1_000_000)); // Convert from nanos to millis
const now = Date.now();
const maxCertTimeOffset = 300_000; // 5 min
if (certTime - maxCertTimeOffset > now || certTime + maxCertTimeOffset < now) {
return false;
}
const hashTree = agent_1.Cbor.decode(tree);
const reconstructed = await (0, agent_1.reconstruct)(hashTree);
const witness = cert.lookup_path(['canister', canisterId.toUint8Array(), 'certified_data']);
if (witness.status !== agent_1.LookupPathStatus.Found || !(witness.value instanceof Uint8Array)) {
// Could not find certified data for this canister in the certificate
return false;
}
// First validate that the Tree is as good as the certification
if ((0, candid_1.compare)(witness.value, reconstructed) !== 0) {
// Witness != Tree passed in ic-certification
return false;
}
// Lookup hash of asset in tree
const treeSha = (0, agent_1.lookupResultToBuffer)((0, agent_1.lookup_path)(['http_assets', this._key], hashTree));
return !!treeSha && !!this.sha256 && (0, candid_1.compare)(this.sha256, treeSha) === 0;
}
/**
* Check if the hash of the asset data is equal to the hash that has been certified
* @param bytes Optionally pass data to hash instead of waiting for asset data to be fetched and hashed
*/
async verifySha256(bytes) {
if (!this.sha256?.buffer) {
return false;
}
const hash = sha2_1.sha256.create();
if (bytes) {
hash.update(Array.isArray(bytes) ? new Uint8Array(bytes) : bytes);
}
else {
await this.getChunks((_, chunk) => hash.update(chunk), true);
}
return (0, candid_1.compare)(this.sha256, hash.digest()) === 0;
}
}
//# sourceMappingURL=index.js.map