@wttp/site
Version:
Web3 Transfer Protocol (WTTP) - Site Contracts and deployment tools
484 lines ā¢ 20.4 kB
JavaScript
import { ethers, Contract } from "ethers";
import fs from "fs";
import path from "path";
import { encodeCharset, encodeMimeType, encodeEncoding, encodeLanguage, normalizePath } from "@wttp/core";
import { getWttpSite } from "./utils";
import { getMimeTypeWithCharset, chunkData } from "../utils";
import { createWTTPIgnore } from "../scripts/wttpIgnore";
const CHUNK_SIZE = 32 * 1024; // 32KB chunks
const DPS_VERSION = 2; // Default DPS version
// Local chunk address calculation (deterministic hash)
// This matches the DPS contract's calculateAddress method
// Returns bytes32 (32 bytes), not an address (20 bytes)
function calculateChunkAddressLocal(data, version = DPS_VERSION) {
// Match Solidity: keccak256(abi.encodePacked(_data, _version))
const encoded = ethers.solidityPacked(['bytes', 'uint8'], [data, version]);
return ethers.keccak256(encoded);
}
// Helper function to check if a path is a directory
function isDirectory(sourcePath) {
return fs.statSync(sourcePath).isDirectory();
}
// Helper to get all files recursively
function getAllFiles(dirPath, wttpIgnore, arrayOfFiles = []) {
const files = fs.readdirSync(dirPath);
files.forEach((file) => {
const fullPath = path.join(dirPath, file);
if (wttpIgnore && wttpIgnore.shouldIgnore(fullPath)) {
return;
}
if (isDirectory(fullPath)) {
arrayOfFiles = getAllFiles(fullPath, wttpIgnore, arrayOfFiles);
}
else {
arrayOfFiles.push(fullPath);
}
});
return arrayOfFiles;
}
// Helper to get all directories recursively
function getAllDirectories(dirPath, basePath, wttpIgnore, arrayOfDirs = []) {
const files = fs.readdirSync(dirPath);
const relativeDirPath = path.relative(basePath, dirPath);
if (relativeDirPath) {
arrayOfDirs.push(relativeDirPath);
}
files.forEach((file) => {
const fullPath = path.join(dirPath, file);
if (wttpIgnore && wttpIgnore.shouldIgnore(fullPath)) {
return;
}
if (isDirectory(fullPath)) {
arrayOfDirs = getAllDirectories(fullPath, basePath, wttpIgnore, arrayOfDirs);
}
});
return arrayOfDirs;
}
// Find index file for directory
function findIndexFile(dirPath, wttpIgnore) {
const files = fs.readdirSync(dirPath);
const indexPriority = [
"index.html",
"index.htm",
"index.js",
"index.json",
"index.md",
"index.txt"
];
for (const indexFile of indexPriority) {
const fullPath = path.join(dirPath, indexFile);
if (files.includes(indexFile) && !wttpIgnore.shouldIgnore(fullPath)) {
return `./${indexFile}`;
}
}
return "./index.html"; // Default
}
// Check if file should use external storage
function shouldUseExternalStorage(filePath, fileSize, mimeType, rules) {
if (!rules || rules.length === 0)
return null;
const ext = path.extname(filePath).toLowerCase();
for (const rule of rules) {
let matches = true;
// Check size constraints
if (rule.minSizeBytes && fileSize < rule.minSizeBytes)
matches = false;
if (rule.maxSizeBytes && fileSize > rule.maxSizeBytes)
matches = false;
// Check mime type
if (rule.mimeTypes && rule.mimeTypes.length > 0) {
const mimeMatches = rule.mimeTypes.some(pattern => {
if (pattern.includes("*")) {
const regex = new RegExp("^" + pattern.replace("*", ".*") + "$");
return regex.test(mimeType);
}
return mimeType === pattern;
});
if (!mimeMatches)
matches = false;
}
// Check extensions
if (rule.extensions && rule.extensions.length > 0) {
if (!rule.extensions.includes(ext))
matches = false;
}
if (matches)
return rule;
}
return null;
}
// Standalone manifest generation
// Can be used with any ethers.js provider (not just Hardhat)
export async function generateManifestStandalone(sourcePath, destinationPath, config, existingManifest, options) {
const hasProvider = options?.provider !== undefined;
const hasSite = options?.wttpSiteAddress !== undefined;
const estimationMode = hasProvider && hasSite;
if (estimationMode) {
console.log(`š Generating manifest with estimates for: ${sourcePath} ā ${destinationPath}`);
}
else {
console.log(`š Generating manifest (no estimates) for: ${sourcePath} ā ${destinationPath}`);
console.log(` Chunk addresses and file structure will be calculated`);
if (!hasProvider) {
console.log(` To include cost estimates, provide a provider and site contract`);
}
}
// Validate source path
if (!fs.existsSync(sourcePath)) {
throw new Error(`Source path does not exist: ${sourcePath}`);
}
if (!isDirectory(sourcePath)) {
throw new Error(`Source path is not a directory: ${sourcePath}`);
}
destinationPath = normalizePath(destinationPath, true);
// Setup ignore patterns
let ignoreOptions = { includeDefaults: true };
if (config?.ignorePattern) {
if (config.ignorePattern === "none") {
ignoreOptions.includeDefaults = false;
}
else if (typeof config.ignorePattern === "string") {
// Load from file
const ignoreFilePath = path.resolve(sourcePath, config.ignorePattern);
if (fs.existsSync(ignoreFilePath)) {
const patterns = fs.readFileSync(ignoreFilePath, "utf-8")
.split(/\r?\n/)
.map(line => line.trim())
.filter(line => line.length > 0 && !line.startsWith("#"));
ignoreOptions.customPatterns = patterns;
}
}
else if (Array.isArray(config.ignorePattern)) {
ignoreOptions.customPatterns = config.ignorePattern;
}
}
const wttpIgnore = createWTTPIgnore(sourcePath, ignoreOptions);
console.log(`š Using ${wttpIgnore.getPatterns().length} ignore patterns`);
// Get chain info if provider is available
let currencySymbol = options?.currencySymbol || "ETH";
let siteAddress;
let dps = null;
let dpr = null;
let gasPrice = ethers.parseUnits("50", "gwei");
let chainData;
if (estimationMode && options?.provider && options?.wttpSiteAddress) {
const provider = options.provider;
// Create contract instance with proper ABI typing
const wttpSite = getWttpSite(options.wttpSiteAddress, provider);
// Get currency symbol
if (options.currencySymbol) {
currencySymbol = options.currencySymbol;
}
else {
// Try to detect from chain ID
const network = await provider.getNetwork();
currencySymbol = network.chainId === 137n ? "POL" : "ETH";
}
siteAddress = await wttpSite.getAddress();
// Get DPR contract first, then get DPS from it
const dprAddress = await wttpSite.DPR();
const dprAbi = [
"function DPS() external view returns (address)",
"function getDataPointRoyalty(bytes32) external view returns (uint256)"
];
dpr = new Contract(dprAddress, dprAbi, provider);
// Get DPS address from DPR contract (avoiding issue with wttpSite.DPS())
let dpsAddress;
try {
dpsAddress = await wttpSite.DPS();
}
catch (error) {
// Fallback: get DPS from DPR contract directly
dpsAddress = await dpr.DPS();
}
const dpsAbi = [
"function calculateAddress(bytes memory) public pure returns (bytes32)",
"function VERSION() external pure returns (uint8)"
];
dps = new Contract(dpsAddress, dpsAbi, provider);
// Get current gas price for estimates
const feeData = await provider.getFeeData();
gasPrice = feeData.maxFeePerGas || feeData.gasPrice || ethers.parseUnits("50", "gwei");
// Set chain data
const network = await provider.getNetwork();
chainData = {
contractAddress: siteAddress,
chainId: Number(network.chainId),
name: options.chainName || (network.name === "unknown" ? `chain-${network.chainId}` : network.name),
symbol: currencySymbol,
publisher: options.publisher || config?.publisher || existingManifest?.chainData?.publisher,
transactions: existingManifest?.chainData?.transactions || []
};
}
else if (options?.chainId && options?.chainName && options?.currencySymbol) {
// Allow manual chain data without provider
chainData = {
contractAddress: options.wttpSiteAddress || "0x0000000000000000000000000000000000000000",
chainId: options.chainId,
name: options.chainName,
symbol: options.currencySymbol,
publisher: options.publisher || config?.publisher || existingManifest?.chainData?.publisher,
transactions: existingManifest?.chainData?.transactions || []
};
}
// Prepare config to save in manifest
const manifestConfig = config ? {
...config,
destination: destinationPath
} : undefined;
// Initialize manifest structure
const manifest = {
name: path.basename(sourcePath),
path: `./${path.basename(sourcePath)}/`,
wttpConfig: manifestConfig,
siteData: {
directories: [],
files: []
},
chainData
};
// Process directories
const allDirs = getAllDirectories(sourcePath, sourcePath, wttpIgnore);
// Add root directory
manifest.siteData.directories.push({
path: "/",
index: findIndexFile(sourcePath, wttpIgnore)
});
// Add subdirectories
for (const dirPath of allDirs) {
const fullDirPath = path.join(sourcePath, dirPath);
const normalizedPath = "/" + dirPath.replace(/\\/g, "/") + "/";
manifest.siteData.directories.push({
path: normalizedPath,
index: findIndexFile(fullDirPath, wttpIgnore)
});
}
console.log(`š Found ${manifest.siteData.directories.length} directories`);
// Process files
const allFiles = getAllFiles(sourcePath, wttpIgnore);
console.log(`š Found ${allFiles.length} files to process`);
let signerAddress = "0x0000000000000000000000000000000000000000";
if (estimationMode && options?.signer) {
signerAddress = await options.signer.getAddress();
}
for (const filePath of allFiles) {
console.log(`\nš Processing: ${path.relative(sourcePath, filePath)}`);
const relativePath = path.relative(sourcePath, filePath);
const normalizedFilePath = "./" + relativePath.replace(/\\/g, "/");
// Read file
const fileData = fs.readFileSync(filePath);
const fileSize = fileData.length;
// Get MIME type
const { mimeType, charset } = getMimeTypeWithCharset(filePath);
// Check if file should use external storage
const externalRule = shouldUseExternalStorage(filePath, fileSize, mimeType, config?.externalStorageRules);
if (externalRule) {
console.log(` š¦ File will use external storage: ${externalRule.provider}`);
// Map provider to correct protocol
const protocolMap = {
"arweave": "ar",
"ipfs": "ipfs",
};
const protocol = protocolMap[externalRule.provider] || externalRule.provider;
const fileEntry = {
path: normalizedFilePath,
type: mimeType,
externalStorage: externalRule.provider,
size: fileSize,
status: "pending",
redirect: {
code: externalRule.redirectCode || 301,
location: `${protocol}://[pending]`
},
chunks: []
};
manifest.siteData.files.push(fileEntry);
continue;
}
// Chunk the file
const chunks = chunkData(fileData, CHUNK_SIZE);
console.log(` Split into ${chunks.length} chunks`);
// Prepare chunk data with estimates
const chunkDataArray = [];
let previousChunkAddress;
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
// Calculate data point address (works with or without site)
let dataPointAddress;
if (estimationMode && dps) {
// Use DPS contract calculation
dataPointAddress = await dps.calculateAddress(chunk);
}
else {
// Use local calculation
dataPointAddress = calculateChunkAddressLocal(chunk);
}
let royaltyEther = 0;
let gasEstimate = 0;
let gasCostEther = 0;
// Only calculate estimates if we have contracts
if (estimationMode && dpr && options?.wttpSiteAddress && options?.provider) {
// Get royalty
const royalty = await dpr.getDataPointRoyalty(dataPointAddress);
royaltyEther = parseFloat(ethers.formatEther(royalty));
// Estimate gas (conservative estimate, actual may vary)
try {
// Use PUT for estimation (conservative)
const putRequest = {
head: {
path: destinationPath + normalizedFilePath.slice(1),
ifModifiedSince: 0,
ifNoneMatch: ethers.ZeroHash
},
properties: {
mimeType: encodeMimeType(mimeType),
charset: encodeCharset(charset || ""),
encoding: encodeEncoding("identity"),
language: encodeLanguage("en-US")
},
data: [{
data: chunk,
chunkIndex: 0,
publisher: signerAddress
}]
};
// Re-create contract for estimation (with provider, not signer)
const wttpSiteForEstimate = getWttpSite(options.wttpSiteAddress, options.provider);
const estimate = await wttpSiteForEstimate.PUT.estimateGas(putRequest, {
value: royalty
});
gasEstimate = Number(estimate);
}
catch (error) {
// Use conservative default if estimation fails
gasEstimate = i === 0 ? 200000 : 150000;
console.log(` ā ļø Could not estimate chunk ${i}, using default: ${gasEstimate}`);
}
gasCostEther = parseFloat(ethers.formatEther(BigInt(gasEstimate) * gasPrice));
}
const chunkEntry = {
address: dataPointAddress
};
// Add estimate only if in estimation mode
if (estimationMode && gasEstimate > 0) {
chunkEntry.estimate = gasEstimate;
}
// Add range for multi-chunk files
if (chunks.length > 1) {
const start = i * CHUNK_SIZE;
const end = Math.min(start + chunk.length - 1, fileSize - 1);
chunkEntry.range = `${start}-${end}`;
}
// Add royalty if non-zero (only in estimation mode)
if (estimationMode && royaltyEther > 0) {
chunkEntry.royalty = royaltyEther;
}
// Add gas cost estimate (only in estimation mode)
if (estimationMode && gasCostEther > 0) {
chunkEntry.gas = gasCostEther;
}
// Add prerequisite for sequential chunks
if (i > 0 && previousChunkAddress) {
chunkEntry.prerequisite = previousChunkAddress;
}
chunkDataArray.push(chunkEntry);
previousChunkAddress = dataPointAddress;
if ((i + 1) % 10 === 0 || i === chunks.length - 1) {
console.log(` Processed ${i + 1}/${chunks.length} chunks...`);
}
}
// Create file entry
const fileEntry = {
path: normalizedFilePath,
type: mimeType,
size: fileSize,
status: "pending",
chunks: chunkDataArray
};
// Add non-default properties
if (charset && charset !== "utf-8") {
fileEntry.charset = charset;
}
manifest.siteData.files.push(fileEntry);
console.log(` ā
File processed with ${chunkDataArray.length} chunks`);
}
console.log(`\nā
Manifest generation complete!`);
console.log(` Total directories: ${manifest.siteData.directories.length}`);
console.log(` Total files: ${manifest.siteData.files.length}`);
if (estimationMode) {
// Calculate total estimates
let totalGasEstimate = 0;
let totalRoyaltyEstimate = 0;
for (const file of manifest.siteData.files) {
for (const chunk of file.chunks) {
totalGasEstimate += chunk.estimate || 0;
totalRoyaltyEstimate += chunk.royalty || 0;
}
}
const totalGasCostEther = parseFloat(ethers.formatEther(BigInt(totalGasEstimate) * gasPrice));
console.log(`\nš Estimated Costs:`);
console.log(` Gas: ${totalGasCostEther.toFixed(6)} ${currencySymbol}`);
console.log(` Royalties: ${totalRoyaltyEstimate.toFixed(6)} ${currencySymbol}`);
console.log(` Total: ${(totalGasCostEther + totalRoyaltyEstimate).toFixed(6)} ${currencySymbol}`);
}
else {
console.log(`\nš Summary:`);
let totalChunks = 0;
for (const file of manifest.siteData.files) {
totalChunks += file.chunks.length;
}
console.log(` Total chunks: ${totalChunks}`);
console.log(` (No cost estimates - provide provider and site contract for estimates)`);
}
return manifest;
}
// Save manifest to file
export function saveManifest(manifest, outputPath) {
const manifestPath = outputPath || path.join(manifest.path, "wttp.manifest.json");
const manifestJson = JSON.stringify(manifest, null, 2);
fs.writeFileSync(manifestPath, manifestJson);
console.log(`\nš¾ Manifest saved to: ${manifestPath}`);
return manifestPath;
}
// Load existing manifest
export function loadManifest(manifestPath) {
if (!fs.existsSync(manifestPath)) {
throw new Error(`Manifest file not found: ${manifestPath}`);
}
const manifestJson = fs.readFileSync(manifestPath, "utf-8");
return JSON.parse(manifestJson);
}
// Load test config from file
export function loadTestConfig(configPath, networkName) {
if (!fs.existsSync(configPath)) {
return null;
}
try {
const configJson = fs.readFileSync(configPath, "utf-8");
const config = JSON.parse(configJson);
// Support both direct config and networks-based config
if (config.networks) {
// Networks-based config
const network = networkName || config.default;
if (!network) {
console.warn(`ā ļø No network specified and no default in test config`);
return null;
}
const networkConfig = config.networks[network];
if (!networkConfig) {
console.warn(`ā ļø Network '${network}' not found in test config`);
return null;
}
return networkConfig;
}
else {
// Direct config format
return config;
}
}
catch (error) {
console.warn(`ā ļø Could not load test config: ${error}`);
return null;
}
}
//# sourceMappingURL=generateManifest.js.map