git-arweave-lfs
Version:
A Git extension for versioning large files with Arweave storage
238 lines • 10.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ArweaveManager = void 0;
const arweave_1 = __importDefault(require("arweave"));
const turbo_sdk_1 = require("@ardrive/turbo-sdk");
const fs_1 = require("fs");
const path_1 = require("path");
const crypto_1 = require("crypto");
const mime_types_1 = require("mime-types");
class ArweaveManager {
constructor(config, gitDir, gatewayUrl) {
this.config = config;
this.referencesFile = (0, path_1.join)(gitDir, 'arweave-lfs', 'references.json');
this.fileReferences = this.loadReferences();
this.gatewayUrl = gatewayUrl.replace(/\/$/, '');
const gateway = new URL(this.gatewayUrl.startsWith('http') ? this.gatewayUrl : `https://${this.gatewayUrl}`);
const protocol = gateway.protocol.replace(':', '') || 'https';
const port = gateway.port ? Number.parseInt(gateway.port, 10) : protocol === 'https' ? 443 : 80;
this.arweave = arweave_1.default.init({
host: gateway.hostname,
port,
protocol
});
}
static async create(config, gitDir) {
const turboConfig = await config.getTurboConfig();
const manager = new ArweaveManager(config, gitDir, turboConfig.gatewayUrl);
return manager;
}
async ensureTurboClient() {
if (this.turboClient) {
return;
}
const walletJson = await this.config.loadWallet();
if (!walletJson) {
throw new Error('Arweave wallet not configured. Use: git arweave-lfs config set-wallet <path>');
}
const wallet = JSON.parse(walletJson);
const signer = new turbo_sdk_1.ArweaveSigner(wallet);
this.turboClient = turbo_sdk_1.TurboFactory.authenticated({ signer });
}
loadReferences() {
const references = new Map();
try {
if ((0, fs_1.existsSync)(this.referencesFile)) {
const data = JSON.parse((0, fs_1.readFileSync)(this.referencesFile, 'utf8'));
Object.entries(data).forEach(([hash, ref]) => {
references.set(hash, {
...ref,
uploadedAt: new Date(ref.uploadedAt)
});
});
}
}
catch (error) {
console.warn('Failed to load file references, starting fresh:', error);
}
return references;
}
saveReferences() {
const dir = (0, path_1.dirname)(this.referencesFile);
(0, fs_1.mkdirSync)(dir, { recursive: true });
const data = {};
this.fileReferences.forEach((ref, hash) => {
data[hash] = ref;
});
(0, fs_1.writeFileSync)(this.referencesFile, JSON.stringify(data, null, 2));
}
computeFileHash(filePath) {
const fileBuffer = (0, fs_1.readFileSync)(filePath);
return (0, crypto_1.createHash)('sha256').update(fileBuffer).digest('hex');
}
async uploadFile(filePath) {
try {
await this.ensureTurboClient();
const fileHash = this.computeFileHash(filePath);
const fileSize = (0, fs_1.statSync)(filePath).size;
const existingRef = this.fileReferences.get(fileHash);
if (existingRef) {
console.log(`✅ File already uploaded with transaction ${existingRef.txId}`);
return existingRef.txId;
}
console.error(`📤 Uploading ${filePath} to Arweave via Turbo...`);
let lastLoggedPercent = -10;
const result = await this.turboClient.uploadFile({
file: filePath,
fileSizeFactory: () => fileSize,
dataItemOpts: {
tags: [
{ name: 'Content-Type', value: this.guessContentType(filePath) },
{ name: 'App-Name', value: 'git-arweave-lfs' },
{ name: 'file-hash', value: fileHash },
]
},
events: {
onUploadProgress: ({ totalBytes, processedBytes }) => {
if (!totalBytes) {
console.error(` ↪ Uploaded ${processedBytes} bytes...`);
return;
}
const percent = Math.floor((processedBytes / totalBytes) * 100);
if (percent - lastLoggedPercent >= 10 || percent === 100) {
const humanProcessed = this.formatBytes(processedBytes);
const humanTotal = this.formatBytes(totalBytes);
console.error(` ↪ Upload progress: ${percent}% (${humanProcessed} / ${humanTotal})`);
lastLoggedPercent = percent;
}
}
}
});
console.error(`✅ Upload complete via Turbo: ${result.id}`);
const fileRef = {
filePath,
txId: result.id,
size: fileSize,
hash: fileHash,
uploadedAt: new Date()
};
this.fileReferences.set(fileHash, fileRef);
this.saveReferences();
return result.id;
}
catch (error) {
if (error.message?.includes('insufficient funds') || error.message?.includes('balance')) {
throw new Error(`Upload failed due to insufficient funds. Please check your wallet balance at arweave.app and add more AR tokens. Error: ${error.message}`);
}
else if (error.message?.includes('authentication') || error.message?.includes('unauthorized')) {
throw new Error(`Upload failed due to authentication issues. Please check your Turbo API credentials and wallet. Error: ${error.message}`);
}
else {
throw new Error(`Failed to upload file to Arweave via Turbo: ${error.message}`);
}
}
}
async downloadFile(txId, outputPath) {
try {
console.error(`📥 Downloading ${txId} to ${outputPath}...`);
const response = await fetch(this.buildGatewayUrl(txId));
if (!response.ok) {
throw new Error(`Failed to download from Arweave: ${response.status} ${response.statusText}`);
}
const fileData = Buffer.from(await response.arrayBuffer());
const dir = (0, path_1.dirname)(outputPath);
(0, fs_1.mkdirSync)(dir, { recursive: true });
(0, fs_1.writeFileSync)(outputPath, fileData);
console.log(`✅ Download complete: ${outputPath} (${fileData.length} bytes)`);
}
catch (error) {
throw new Error(`Failed to download file from Arweave: ${error}`);
}
}
async downloadFileToMemory(txId) {
try {
console.error(`📥 Downloading ${txId}...`);
const response = await fetch(`https://arweave.net/${txId}`);
if (!response.ok) {
throw new Error(`Failed to download from Arweave: ${response.status} ${response.statusText}`);
}
const fileData = Buffer.from(await response.arrayBuffer());
console.error(`✅ Download complete: ${fileData.length} bytes`);
return fileData;
}
catch (error) {
throw new Error(`Failed to download file from Arweave: ${error}`);
}
}
guessContentType(filePath) {
const mimeType = (0, mime_types_1.lookup)(filePath);
if (!mimeType) {
const ext = filePath.toLowerCase().split('.').pop();
const fallbackMimeTypes = {
'jpg': 'image/jpeg',
'jpeg': 'image/jpeg',
'png': 'image/png',
'gif': 'image/gif',
'pdf': 'application/pdf',
'txt': 'text/plain',
'html': 'text/html',
'css': 'text/css',
'js': 'application/javascript',
'json': 'application/json',
'zip': 'application/zip',
'tar': 'application/x-tar',
'gz': 'application/gzip',
'mp4': 'video/mp4',
'avi': 'video/x-msvideo',
'mov': 'video/quicktime',
'mp3': 'audio/mpeg',
'wav': 'audio/wav'
};
return fallbackMimeTypes[ext || ''] || 'application/octet-stream';
}
return mimeType;
}
async findFileByTxId(txId) {
for (const ref of this.fileReferences.values()) {
if (ref.txId === txId) {
return ref;
}
}
return null;
}
getFileReference(filePath) {
const fileHash = this.computeFileHash(filePath);
return this.fileReferences.get(fileHash) || null;
}
getAllReferences() {
return Array.from(this.fileReferences.values());
}
getStorageStats() {
let totalSize = 0;
this.fileReferences.forEach(ref => {
totalSize += ref.size;
});
return {
totalFiles: this.fileReferences.size,
totalSize
};
}
buildGatewayUrl(txId) {
const base = this.gatewayUrl.endsWith('/') ? this.gatewayUrl : `${this.gatewayUrl}/`;
return `${base}${txId}`;
}
formatBytes(bytes) {
if (bytes === 0) {
return '0 B';
}
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
const exponent = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
const value = bytes / Math.pow(1024, exponent);
return `${value.toFixed(1)} ${units[exponent]}`;
}
}
exports.ArweaveManager = ArweaveManager;
//# sourceMappingURL=arweave.js.map