@wttp/site
Version:
Web3 Transfer Protocol (WTTP) - Site Contracts and deployment tools
523 lines ⢠26 kB
JavaScript
import { ethers } from "hardhat";
import fs from "fs";
import { encodeCharset, encodeMimeType, encodeEncoding, encodeLanguage, normalizePath } from "@wttp/core";
import { fetchResource } from "./fetchResource";
import { looseEqual, chunkData, getMimeType, getMimeTypeWithCharset, getChainSymbol as getChainSymbolUtil, waitForGasPriceBelowLimit as waitForGasPriceBelowLimitUtil, getDynamicGasSettings as getDynamicGasSettingsUtil, getEstimationGasPrice as getEstimationGasPriceUtil } from "../utils";
// Constants
const CHUNK_SIZE = 32 * 1024; // 32KB chunks
const DEFAULT_FILE_WARN = 100 * 1024 * 1024; // 100MB warning
const DEFAULT_FILE_LIMIT = 400 * 1024 * 1024; // 400MB limit
// const DEFAULT_GAS_LIMIT = 300; // Default gas limit in gwei
// Re-export for backward compatibility
export { looseEqual, getMimeType, getMimeTypeWithCharset, chunkData };
// Wrapper functions that use Hardhat's ethers.provider
export async function getChainSymbol() {
return getChainSymbolUtil(ethers.provider);
}
export async function waitForGasPriceBelowLimit(gasLimitGwei, checkIntervalMs = 10000) {
return waitForGasPriceBelowLimitUtil(ethers.provider, gasLimitGwei, checkIntervalMs);
}
export async function getDynamicGasSettings() {
return getDynamicGasSettingsUtil(ethers.provider);
}
export async function getEstimationGasPrice(customGasPriceGwei, rate = 2, minGasPriceGwei = 150) {
return getEstimationGasPriceUtil(ethers.provider, customGasPriceGwei, rate, minGasPriceGwei);
}
// Main upload function with enhanced error handling and validation
export async function uploadFile(wttpSite, sourcePath, destinationPath, fileLimitBytes, gasLimitGwei, customPublisher) {
console.log(`š Starting upload: ${sourcePath} ā ${destinationPath}`);
// Get currency symbol early
const currencySymbol = await getChainSymbol();
// Parameter validation
if (!wttpSite) {
throw new Error("Web3Site contract instance is required");
}
if (!sourcePath || !destinationPath) {
throw new Error("Both source and destination paths are required");
}
if (!fs.existsSync(sourcePath)) {
throw new Error(`Source file does not exist: ${sourcePath}`);
}
if (!fs.statSync(sourcePath).isFile()) {
throw new Error(`Source path is not a file: ${sourcePath}`);
}
// Normalize the destination path
try {
destinationPath = normalizePath(destinationPath);
}
catch (error) {
throw new Error(`Invalid destination path format: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
// Read file with error handling
let fileData;
try {
fileData = fs.readFileSync(sourcePath);
}
catch (error) {
throw new Error(`Failed to read file ${sourcePath}: ${error}`);
}
console.log(`š File size: ${fileData.length} bytes`);
// Use provided file limits or defaults
const fileWarn = fileLimitBytes ? fileLimitBytes * 0.25 : DEFAULT_FILE_WARN; // 25% of limit as warning
const fileLimit = fileLimitBytes || DEFAULT_FILE_LIMIT;
// Validate file size
if (fileData.length === 0) {
throw new Error("Cannot upload empty file");
}
if (fileData.length > fileWarn) {
if (fileData.length > fileLimit) {
throw new Error(`ā File size exceeds ${fileLimit / (1024 * 1024)}MB limit. Upload aborted.`);
}
console.warn(`ā ļø Large file detected (>${fileWarn / (1024 * 1024)}MB). Upload may take significant time and gas.`);
}
// let resourceExists = false;
// try{
// let existingResource = await fetchResource(await wttpSite.getAddress(), destinationPath, { datapoints: false });
// let existingData = existingResource.content;
// if (existingData && Buffer.from(existingData).equals(fileData)) {
// console.log("ā
File already exists with identical content");
// resourceExists = true;
// // return existingResource; // We need to either check properties also or remove this return since properties may need to be updated.
// }
// } catch (error) {
// console.log(`No resource found at ${destinationPath}, uploading...`);
// }
// Chunk the data
const chunks = chunkData(fileData, CHUNK_SIZE);
console.log(`Split into ${chunks.length} chunks of ${CHUNK_SIZE} bytes`);
// Prepare for upload
const signer = await ethers.provider.getSigner();
const signerAddress = await signer.getAddress();
// Use custom publisher if provided, otherwise use signer address
const publisherAddress = customPublisher || signerAddress;
// Prepare data registrations
const dataRegistrations = chunks.map((chunk, index) => ({
data: chunk,
chunkIndex: index,
publisher: publisherAddress
}));
// FIXED: Initialize royalty array with correct size to prevent index errors
let royalty = new Array(dataRegistrations.length).fill(0n);
// Get the DPS and DPR contracts once for efficiency
const dpsAddress = await wttpSite.DPS();
const dps = await ethers.getContractAt("@tw3/esp/contracts/interfaces/IDataPointStorage.sol:IDataPointStorage", dpsAddress);
const dprAddress = await wttpSite.DPR();
const dpr = await ethers.getContractAt("@tw3/esp/contracts/interfaces/IDataPointRegistry.sol:IDataPointRegistry", dprAddress);
let totalRoyalty = 0n;
let dataPointAddresses = new Array(dataRegistrations.length).fill("");
const resourceResponse = await fetchResource(await wttpSite.getAddress(), destinationPath);
const resourceDataPointAddresses = resourceResponse.response.resource.dataPoints;
const resourceExists = resourceDataPointAddresses.length > 0;
let chunksToUpload = []; // Array of chunk indices that need uploading
// Check royalties for each chunk before uploading
console.log(`š Calculating royalties for ${dataRegistrations.length} chunks...`);
for (let i = 0; i < dataRegistrations.length; i++) {
const chunk = dataRegistrations[i];
// Calculate the data point address
const dataPointAddress = await dps.calculateAddress(chunk.data);
dataPointAddresses[i] = dataPointAddress;
if (resourceDataPointAddresses.length > i) {
// Check if this chunk already exists in the resource
const existingAddress = resourceDataPointAddresses[i]?.toString() || "";
if (existingAddress === dataPointAddress) {
console.log(`ā
Chunk ${i + 1}/${dataRegistrations.length} already exists, skipping`);
royalty[i] = 0n; // No cost for existing chunks
}
else {
// Chunk needs to be uploaded
chunksToUpload.push(i);
// Get the royalty for this chunk
royalty[i] = await dpr.getDataPointRoyalty(dataPointAddress);
totalRoyalty += royalty[i];
console.log(`š¤ Chunk ${i + 1}/${dataRegistrations.length}: ${ethers.formatEther(royalty[i])} ${currencySymbol}`);
}
}
else {
// Chunk needs to be uploaded
chunksToUpload.push(i);
// Get the royalty for this chunk
royalty[i] = await dpr.getDataPointRoyalty(dataPointAddress);
totalRoyalty += royalty[i];
console.log(`š¤ Chunk ${i + 1}/${dataRegistrations.length}: ${ethers.formatEther(royalty[i])} ${currencySymbol}`);
}
}
console.log(`š° Total royalty required: ${ethers.formatEther(totalRoyalty)} ${currencySymbol}`);
console.log(`š Chunks to upload: ${chunksToUpload.length}/${dataRegistrations.length}`);
// Check if user has sufficient balance
const balance = await ethers.provider.getBalance(signerAddress);
if (balance < totalRoyalty) {
throw new Error(`Insufficient balance. Required: ${ethers.formatEther(totalRoyalty)} ${currencySymbol}, Available: ${ethers.formatEther(balance)} ${currencySymbol}`);
}
// Track gas and royalties for summary
let totalGasUsed = 0n;
let totalRoyaltiesSpent = 0n;
// Get MIME type and charset
const { mimeType, charset } = getMimeTypeWithCharset(sourcePath);
console.log(`Mime type: ${mimeType}, charset: ${charset || 'none'}`);
const mimeTypeBytes2 = encodeMimeType(mimeType);
const charsetBytes2 = charset ? encodeCharset(charset) : encodeCharset("");
// Wait for gas price to be below limit if specified
if (gasLimitGwei !== undefined) {
console.log(`ā³ Waiting for gas price to drop below ${gasLimitGwei} gwei...`);
await waitForGasPriceBelowLimit(gasLimitGwei);
}
// Gas optimization settings for faster transactions
const gasSettings = await getDynamicGasSettings();
const headRequest = {
path: destinationPath,
ifModifiedSince: 0,
ifNoneMatch: ethers.ZeroHash
};
// Use PUT to create new resource/update existing resource
console.log(`${resourceExists ? "Updating" : "Creating"} resource at ${destinationPath}`);
const putRequest = {
head: headRequest,
properties: {
mimeType: mimeTypeBytes2,
charset: charsetBytes2,
encoding: encodeEncoding("identity"),
language: encodeLanguage("en-US")
},
data: [dataRegistrations[0]]
};
// Early exit if no chunks need uploading and properties match
if (chunksToUpload.length === 0 && looseEqual(resourceResponse.response.head.metadata.properties, putRequest.properties)) {
console.log("ā
File is identical (properties and data), skipping upload");
return resourceResponse;
}
let needsPUT = false;
let patchChunks = [...chunksToUpload]; // Copy of chunks to upload
// Check if properties are different (requires PUT) - check this FIRST
if (!looseEqual(resourceResponse.response.head.metadata.properties, putRequest.properties)) {
console.log("ā File properties are different, requires PUT");
// console.log("Response properties:", resourceResponse.response.head.metadata.properties);
// console.log("Put request properties:", putRequest.properties);
needsPUT = true;
// When we do PUT, the resource is recreated. ALL chunks need to be re-attached.
// So patchChunks should include all chunks from 1 to n-1, not just those with data changes.
patchChunks = Array.from({ length: dataRegistrations.length - 1 }, (_, i) => i + 1);
// Calculate royalties for any chunks not already in chunksToUpload
for (let i = 1; i < dataRegistrations.length; i++) {
if (!chunksToUpload.includes(i)) {
// This chunk wasn't going to be uploaded (data unchanged), but now needs to be
console.log(`š Calculating royalty for chunk ${i + 1} (needed for resource rebuild)...`);
royalty[i] = await dpr.getDataPointRoyalty(dataPointAddresses[i]);
totalRoyalty += royalty[i];
console.log(`š° Additional royalty for chunk ${i + 1}: ${ethers.formatEther(royalty[i])} ${currencySymbol}`);
}
}
}
else {
console.log("ā
File properties are identical, using PATCH only");
}
// If we need PUT, handle chunk 0
if (needsPUT) {
// If chunk 0 data is unchanged but we need PUT, we still need to pay royalty for chunk 0
if (!chunksToUpload.includes(0)) {
console.log("š Calculating royalty for chunk 0 (needed for PUT with unchanged data)...");
royalty[0] = await dpr.getDataPointRoyalty(dataPointAddresses[0]);
totalRoyalty += royalty[0];
console.log(`š° Additional royalty for PUT: ${ethers.formatEther(royalty[0])} ${currencySymbol}`);
}
// Re-check balance with updated royalty
const balance = await ethers.provider.getBalance(signerAddress);
if (balance < totalRoyalty) {
throw new Error(`Insufficient balance after adding royalties for rebuild. Required: ${ethers.formatEther(totalRoyalty)} ${currencySymbol}, Available: ${ethers.formatEther(balance)} ${currencySymbol}`);
}
// Wait for gas price to be below limit if specified (before PUT)
if (gasLimitGwei !== undefined) {
await waitForGasPriceBelowLimit(gasLimitGwei);
}
console.log(`š Sending PUT transaction with optimized gas settings...`);
const tx = await wttpSite.PUT(putRequest, {
value: royalty[0],
...gasSettings // TODO: Uncomment this when gas settings are working
});
const receipt = await tx.wait();
if (receipt) {
totalGasUsed += receipt.gasUsed;
totalRoyaltiesSpent += royalty[0];
}
console.log("ā
PUT transaction completed - file properties and first chunk updated");
}
// Upload remaining chunks with progress reporting
for (let i = 0; i < patchChunks.length; i++) {
const chunkIndex = patchChunks[i];
const progress = Math.round(((i + 1) / patchChunks.length) * 100);
console.log(`š¤ Uploading chunk ${chunkIndex + 1}/${dataRegistrations.length} (${i + 1}/${patchChunks.length}, ${progress}%)`);
try {
// Get fresh gas settings for each chunk to adapt to network conditions
const currentGasSettings = await getDynamicGasSettings();
const patchRequest = {
head: headRequest,
data: [dataRegistrations[chunkIndex]]
};
console.log(`š Sending PATCH transaction ${i + 1} with updated gas settings...`);
const tx = await wttpSite.PATCH(patchRequest, {
value: royalty[chunkIndex],
...currentGasSettings
});
const receipt = await tx.wait(2);
if (receipt) {
totalGasUsed += receipt.gasUsed;
totalRoyaltiesSpent += royalty[chunkIndex];
}
console.log(`ā
Chunk ${chunkIndex + 1} uploaded successfully (${ethers.formatEther(royalty[chunkIndex])} ${currencySymbol})`);
}
catch (error) {
console.error(`ā Failed to upload chunk ${chunkIndex + 1}:`, error);
throw new Error(`Upload failed at chunk ${chunkIndex + 1}: ${error}`);
}
}
console.log(`š Upload completed! ${chunksToUpload.length} chunks uploaded successfully.`);
let fetchResult;
try {
fetchResult = await fetchResource(await wttpSite.getAddress(), destinationPath, { headRequest: true });
}
catch (error) {
throw new Error(`Upload failed: ${error}`);
}
const fetchSize = fetchResult.response.head.metadata.size;
if (fetchResult.response.head.status !== 200n &&
fetchSize === BigInt(fileData.length)) {
console.log(`Uploaded file has ${dataRegistrations.length} chunks`);
console.log(`File size: ${fetchSize} bytes`);
}
// Calculate summary statistics
const fileSizeMB = fileData.length / (1024 * 1024);
const feeData = await ethers.provider.getFeeData();
const effectiveGasPrice = feeData.gasPrice || feeData.maxFeePerGas || ethers.parseUnits("20", "gwei");
const totalGasCost = totalGasUsed * effectiveGasPrice;
const costPerMBWei = fileSizeMB > 0 ? (totalGasCost * BigInt(1e18)) / BigInt(Math.floor(fileSizeMB * 1e18)) : 0n;
// Build options string
const options = [];
if (fileLimitBytes !== undefined) {
options.push(`filelimit=${Math.floor(fileLimitBytes / (1024 * 1024))}MB`);
}
if (gasLimitGwei !== undefined) {
options.push(`gaslimit=${gasLimitGwei}gwei`);
}
console.log(`\nš Upload Summary:`);
console.log(` Total gas used: ${totalGasUsed.toString()}`);
console.log(` Total royalties spent: ${ethers.formatEther(totalRoyaltiesSpent)} ${currencySymbol}`);
console.log(` Total gas cost: ${ethers.formatEther(totalGasCost)} ${currencySymbol}`);
console.log(` Total cost: ${ethers.formatEther(totalGasCost + totalRoyaltiesSpent)} ${currencySymbol}`);
console.log(` Cost per MB: ${fileSizeMB > 0 ? ethers.formatEther(costPerMBWei) : "0"} ${currencySymbol}/MB`);
if (options.length > 0) {
console.log(` Options: ${options.join(", ")}`);
}
return fetchResult;
}
export async function estimateFile(wttpSite, sourcePath, destinationPath, gasPriceGwei, rate = 2, minGasPriceGwei = 150, customPublisher) {
console.log(`š Estimating gas for: ${sourcePath} ā ${destinationPath}`);
// Get gas price for estimation
const gasPrice = await getEstimationGasPrice(gasPriceGwei, rate, minGasPriceGwei);
console.log(`ā½ Using gas price: ${ethers.formatUnits(gasPrice, "gwei")} gwei`);
// Parameter validation
if (!wttpSite) {
throw new Error("Web3Site contract instance is required");
}
if (!sourcePath || !destinationPath) {
throw new Error("Both source and destination paths are required");
}
if (!fs.existsSync(sourcePath)) {
throw new Error(`Source file does not exist: ${sourcePath}`);
}
if (!fs.statSync(sourcePath).isFile()) {
throw new Error(`Source path is not a file: ${sourcePath}`);
}
// Normalize the destination path
try {
destinationPath = normalizePath(destinationPath);
}
catch (error) {
throw new Error(`Invalid destination path format: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
// Read file
let fileData;
try {
fileData = fs.readFileSync(sourcePath);
}
catch (error) {
throw new Error(`Failed to read file ${sourcePath}: ${error}`);
}
console.log(`š File size: ${fileData.length} bytes`);
// Validate file size
if (fileData.length === 0) {
throw new Error("Cannot estimate empty file");
}
// Chunk the data
const chunks = chunkData(fileData, CHUNK_SIZE);
console.log(`Split into ${chunks.length} chunks of ${CHUNK_SIZE} bytes`);
// Prepare data registrations
const signer = await ethers.provider.getSigner();
const signerAddress = await signer.getAddress();
// Use custom publisher if provided, otherwise use signer address
const publisherAddress = customPublisher || signerAddress;
const dataRegistrations = chunks.map((chunk, index) => ({
data: chunk,
chunkIndex: index,
publisher: publisherAddress
}));
// Get the DPS and DPR contracts
const dpsAddress = await wttpSite.DPS();
const dps = await ethers.getContractAt("@tw3/esp/contracts/interfaces/IDataPointStorage.sol:IDataPointStorage", dpsAddress);
const dprAddress = await wttpSite.DPR();
const dpr = await ethers.getContractAt("@tw3/esp/contracts/interfaces/IDataPointRegistry.sol:IDataPointRegistry", dprAddress);
let totalRoyalty = 0n;
let dataPointAddresses = new Array(dataRegistrations.length).fill("");
let chunksToUpload = [];
// Check existing resource
let resourceResponse;
let resourceDataPointAddresses = [];
try {
resourceResponse = await fetchResource(await wttpSite.getAddress(), destinationPath);
resourceDataPointAddresses = resourceResponse.response.resource.dataPoints.map(dp => dp.toString());
}
catch (error) {
// Resource doesn't exist yet
resourceResponse = null;
}
// Calculate royalties and determine which chunks need uploading
console.log(`š Calculating royalties for ${dataRegistrations.length} chunks...`);
for (let i = 0; i < dataRegistrations.length; i++) {
const chunk = dataRegistrations[i];
// Calculate the data point address
const dataPointAddress = await dps.calculateAddress(chunk.data);
dataPointAddresses[i] = dataPointAddress;
if (resourceDataPointAddresses.length > i) {
// Check if this chunk already exists in the resource
const existingAddress = resourceDataPointAddresses[i] || "";
if (existingAddress === dataPointAddress) {
console.log(`ā
Chunk ${i + 1}/${dataRegistrations.length} already exists, skipping`);
}
else {
chunksToUpload.push(i);
const royalty = await dpr.getDataPointRoyalty(dataPointAddress);
totalRoyalty += royalty;
}
}
else {
chunksToUpload.push(i);
const royalty = await dpr.getDataPointRoyalty(dataPointAddress);
totalRoyalty += royalty;
}
}
console.log(`š° Total royalty required: ${ethers.formatEther(totalRoyalty)} ETH`);
console.log(`š Chunks to upload: ${chunksToUpload.length}/${dataRegistrations.length}`);
// Get MIME type and charset
const { mimeType, charset } = getMimeTypeWithCharset(sourcePath);
const mimeTypeBytes2 = encodeMimeType(mimeType);
const charsetBytes2 = charset ? encodeCharset(charset) : encodeCharset("");
const headRequest = {
path: destinationPath,
ifModifiedSince: 0,
ifNoneMatch: ethers.ZeroHash
};
const putRequest = {
head: headRequest,
properties: {
mimeType: mimeTypeBytes2,
charset: charsetBytes2,
encoding: encodeEncoding("identity"),
language: encodeLanguage("en-US")
},
data: [dataRegistrations[0]]
};
// Use PUT for all chunks (overestimates but works without existing resources)
console.log(`š Using PUT for all chunks (conservative overestimation)`);
let totalGas = 0n;
// Estimate PUT for all chunks
// For PUT, all chunks need chunkIndex: 0 since PUT creates/replaces the resource starting from 0
for (let i = 0; i < chunksToUpload.length; i++) {
const chunkIndex = chunksToUpload[i];
const chunkRoyalty = await dpr.getDataPointRoyalty(dataPointAddresses[chunkIndex]);
// PUT always expects chunkIndex: 0 in the DataRegistration
// Create a PUT request with this chunk's data but chunkIndex set to 0
const putRequestForChunk = {
head: headRequest,
properties: {
mimeType: mimeTypeBytes2,
charset: charsetBytes2,
encoding: encodeEncoding("identity"),
language: encodeLanguage("en-US")
},
data: [{
data: dataRegistrations[chunkIndex].data,
chunkIndex: 0, // PUT always uses chunkIndex 0
publisher: dataRegistrations[chunkIndex].publisher
}]
};
try {
const putGasEstimate = await wttpSite.PUT.estimateGas(putRequestForChunk, {
value: chunkRoyalty
});
totalGas += putGasEstimate;
if ((i + 1) % 10 === 0 || i === chunksToUpload.length - 1) {
console.log(`ā½ Estimated ${i + 1}/${chunksToUpload.length} PUT transactions...`);
}
}
catch (error) {
console.warn(`ā ļø Could not estimate PUT gas for chunk ${chunkIndex}: ${error}`);
// Use a conservative estimate based on chunk 0 if available, otherwise default
if (i === 0) {
// If chunk 0 fails, use a default conservative estimate
totalGas += 200000n;
}
else {
// For subsequent chunks, use the same estimate as chunk 0 if available
// This is a conservative overestimate
totalGas += 200000n; // Conservative estimate per chunk
}
}
}
const totalCost = totalGas * gasPrice;
const transactionCount = chunksToUpload.length;
const averageGas = transactionCount > 0 ? totalGas / BigInt(transactionCount) : 0n;
// Calculate cost per MB
const fileSizeMB = fileData.length / (1024 * 1024);
const costPerMBWei = fileSizeMB > 0 ? (totalCost * BigInt(1e18)) / BigInt(Math.floor(fileSizeMB * 1e18)) : 0n;
// Get currency symbol
const currencySymbol = await getChainSymbol();
console.log(`\nš Estimation Summary:`);
console.log(` Total gas: ${totalGas.toString()}`);
console.log(` Average gas per transaction: ${averageGas.toString()}`);
console.log(` Total cost: ${ethers.formatEther(totalCost)} ${currencySymbol}`);
console.log(` Royalty cost: ${ethers.formatEther(totalRoyalty)} ${currencySymbol}`);
console.log(` Cost per MB: ${fileSizeMB > 0 ? ethers.formatEther(costPerMBWei) : "0"} ${currencySymbol}/MB`);
console.log(` Transactions: ${transactionCount} PUT (conservative overestimation)`);
console.log(` Gas price: ${ethers.formatUnits(gasPrice, "gwei")} gwei`);
console.log(` Settings: ${gasPriceGwei ? `gasprice=${gasPriceGwei}` : `rate=${rate}, min=${minGasPriceGwei}`}`);
return {
totalGas,
totalCost,
royaltyCost: totalRoyalty,
transactionCount,
chunksToUpload: chunksToUpload.length,
needsPUT: true, // Always true since we're using PUT for all
gasPrice
};
}
// Command-line interface
async function main() {
const args = process.argv.slice(2);
if (args.length < 3) {
console.error("Usage: npx hardhat upload --site <site-address> --source <source-path> --destination <destination-path> --network <network>");
process.exit(1);
}
const [siteAddress, sourcePath, destinationPath] = args;
// Connect to the WTTP site
const wttpSite = await ethers.getContractAt("Web3Site", siteAddress);
// Upload the file
await uploadFile(wttpSite, sourcePath, destinationPath);
}
// Only execute the script if it's being run directly
if (require.main === module) {
main()
.then(() => process.exit(0))
.catch((error) => {
console.error(error);
process.exit(1);
});
}
//# sourceMappingURL=uploadFile.js.map