dcl-catalyst-client
Version:
A client to query and perform changes on Decentraland's catalyst servers
221 lines • 10.7 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createContentClient = exports.downloadContent = void 0;
const hashing_1 = require("@dcl/hashing");
const form_data_1 = __importDefault(require("form-data"));
const Helper_1 = require("./utils/Helper");
const retry_1 = require("./utils/retry");
function arrayBufferFrom(value) {
if (value.buffer) {
return value.buffer;
}
return value;
}
async function downloadContent(fetcher, baseUrl, contentHash, options) {
const { attempts = 3, retryDelay = 500 } = options ? options : {};
const timeout = (options === null || options === void 0 ? void 0 : options.timeout) ? { timeout: options.timeout } : {};
return (0, retry_1.retry)(`fetch file with hash ${contentHash} from ${baseUrl}`, async () => {
const content = await (await fetcher.fetch(`${baseUrl}/${contentHash}`, timeout)).buffer();
if (!(options === null || options === void 0 ? void 0 : options.avoidChecks)) {
const downloadedHash = contentHash.startsWith('Qm') ? await (0, hashing_1.hashV0)(content) : await (0, hashing_1.hashV1)(content);
// Sometimes, the downloaded file is not complete, so the hash turns out to be different.
// So we will check the hash before considering the download successful.
if (downloadedHash !== contentHash) {
throw new Error(`Failed to fetch file with hash ${contentHash} from ${baseUrl}`);
}
}
return content;
}, attempts, retryDelay);
}
exports.downloadContent = downloadContent;
function createContentClient(options) {
const { fetcher, logger } = options;
const contentUrl = (0, Helper_1.sanitizeUrl)(options.url);
const defaultParallelConfig = options === null || options === void 0 ? void 0 : options.parallelConfig;
async function fetchFromMultipleServersRace(urls, path, requestOptions) {
const controller = new AbortController();
const signal = controller.signal;
const requestOptionsWithSignal = (0, Helper_1.mergeRequestOptions)(requestOptions, { signal });
return new Promise(async (resolve) => {
let pendingRequests = urls.length;
const markRequestAsComplete = () => {
pendingRequests--;
if (pendingRequests === 0) {
resolve([]);
}
};
const handleSuccess = (entities) => {
if (entities && Array.isArray(entities) && entities.length > 0) {
controller.abort();
resolve(entities);
return true;
}
return false;
};
urls.forEach(async (url) => {
try {
const serverUrl = (0, Helper_1.sanitizeUrl)(url);
const response = await fetcher.fetch(`${serverUrl}${path}`, requestOptionsWithSignal);
if (signal.aborted) {
markRequestAsComplete();
return;
}
const entities = await response.json();
if (!handleSuccess(entities)) {
markRequestAsComplete();
}
}
catch (error) {
if (!signal.aborted) {
logger === null || logger === void 0 ? void 0 : logger.warn(`Failed to fetch from ${url}:`, error);
}
markRequestAsComplete();
}
});
});
}
async function fetchFromMultipleServersAll(urls, path, requestOptions) {
const results = await Promise.allSettled(urls.map(async (url) => {
try {
const serverUrl = (0, Helper_1.sanitizeUrl)(url);
const response = await fetcher.fetch(`${serverUrl}${path}`, requestOptions);
return await response.json();
}
catch (error) {
logger === null || logger === void 0 ? void 0 : logger.warn(`Failed to fetch from ${url}:`, error);
return [];
}
}));
const allEntities = results
.filter((result) => result.status === 'fulfilled')
.flatMap((result) => result.value);
const uniqueEntities = new Map();
allEntities.forEach((entity) => {
if (!uniqueEntities.has(entity.id) || entity.timestamp > uniqueEntities.get(entity.id).timestamp) {
uniqueEntities.set(entity.id, entity);
}
});
return Array.from(uniqueEntities.values());
}
async function buildEntityFormDataForDeployment(deployData, options) {
// Check if we are running in node or browser
const areWeRunningInNode = (0, Helper_1.isNode)();
const form = new form_data_1.default();
form.append('entityId', deployData.entityId);
(0, Helper_1.addModelToFormData)(deployData.authChain, form, 'authChain');
const alreadyUploadedHashes = await hashesAlreadyOnServer(Array.from(deployData.files.keys()), options);
for (const [fileHash, file] of deployData.files) {
if (!alreadyUploadedHashes.has(fileHash) || fileHash === deployData.entityId) {
if (areWeRunningInNode) {
// Node
form.append(fileHash, Buffer.isBuffer(file) ? file : Buffer.from(arrayBufferFrom(file)), fileHash);
}
else {
// Browser
form.append(fileHash, new Blob([arrayBufferFrom(file)]), fileHash);
}
}
}
return form;
}
async function deploy(deployData, options) {
const form = await buildEntityFormDataForDeployment(deployData, options);
const requestOptions = (0, Helper_1.mergeRequestOptions)(options ? options : {}, {
body: form,
method: 'POST'
});
return await fetcher.fetch(`${contentUrl}/entities`, requestOptions);
}
async function fetchEntitiesByPointers(pointers, options) {
if (pointers.length === 0) {
return Promise.reject(`You must set at least one pointer.`);
}
const requestOptions = (0, Helper_1.mergeRequestOptions)(options ? options : {}, {
body: JSON.stringify({ pointers }),
method: 'POST',
headers: { 'Content-Type': 'application/json' }
});
return (await fetcher.fetch(`${contentUrl}/entities/active`, requestOptions)).json();
}
async function fetchEntitiesByIds(ids, options) {
if (ids.length === 0) {
return Promise.reject(`You must set at least one id.`);
}
const requestOptions = (0, Helper_1.mergeRequestOptions)(options ? options : {}, {
body: JSON.stringify({ ids }),
method: 'POST',
headers: { 'Content-Type': 'application/json' }
});
const parallelConfig = (options === null || options === void 0 ? void 0 : options.parallel) || defaultParallelConfig;
if ((parallelConfig === null || parallelConfig === void 0 ? void 0 : parallelConfig.urls) && (parallelConfig === null || parallelConfig === void 0 ? void 0 : parallelConfig.urls.length) > 0) {
return fetchFromMultipleServersRace([contentUrl, ...parallelConfig.urls], '/entities/active', requestOptions);
}
return (await fetcher.fetch(`${contentUrl}/entities/active`, requestOptions)).json();
}
async function fetchEntityById(id, options) {
const entities = await fetchEntitiesByIds([id], options);
if (entities.length === 0) {
return Promise.reject(`Failed to find an entity with id '${id}'.`);
}
return entities[0];
}
function isContentAvailable(cids, options) {
if (cids.length === 0) {
return Promise.reject(`You must set at least one cid.`);
}
return (0, Helper_1.splitAndFetch)({
fetcher: fetcher,
options,
baseUrl: contentUrl,
path: `/available-content`,
queryParams: { name: 'cid', values: cids },
uniqueBy: 'cid'
});
}
// Given an array of file hashes, return a set with those already uploaded on the server
async function hashesAlreadyOnServer(hashes, options) {
const result = await isContentAvailable(hashes, options);
const alreadyUploaded = result.filter(($) => $.available).map(({ cid }) => cid);
return new Set(alreadyUploaded);
}
async function checkPointerConsistency(pointer, options) {
const parallelConfig = (options === null || options === void 0 ? void 0 : options.parallel) || defaultParallelConfig;
if (!(parallelConfig === null || parallelConfig === void 0 ? void 0 : parallelConfig.urls) || parallelConfig.urls.length === 0) {
throw new Error('Parallel configuration is required for checking pointer consistency');
}
const requestOptions = (0, Helper_1.mergeRequestOptions)(options ? options : {}, {
body: JSON.stringify({ pointers: [pointer] }),
method: 'POST',
headers: { 'Content-Type': 'application/json' }
});
const entities = await fetchFromMultipleServersAll([contentUrl, ...parallelConfig.urls], '/entities/active', requestOptions);
if (entities.length === 0) {
return { isConsistent: true };
}
const newestTimestamp = Math.max(...entities.map((e) => e.timestamp));
const newerEntities = entities.filter((e) => e.timestamp === newestTimestamp);
const olderEntities = entities.filter((e) => e.timestamp < newestTimestamp);
return {
isConsistent: olderEntities.length === 0,
upToDateEntities: newerEntities.length > 0 ? newerEntities : undefined,
outdatedEntities: olderEntities.length > 0 ? olderEntities : undefined
};
}
return {
buildEntityFormDataForDeployment,
fetchEntitiesByPointers,
fetchEntitiesByIds,
fetchEntityById,
downloadContent: (contentHash, options) => {
return downloadContent(fetcher, contentUrl + '/contents', contentHash, options);
},
deploy,
isContentAvailable,
checkPointerConsistency
};
}
exports.createContentClient = createContentClient;
//# sourceMappingURL=ContentClient.js.map