@strapi/data-transfer
Version:
Data transfer capabilities for Strapi
462 lines (458 loc) • 18.2 kB
JavaScript
'use strict';
var crypto = require('crypto');
var stream = require('stream');
var fp = require('lodash/fp');
var utils = require('../utils.js');
var constants = require('../../remote/constants.js');
var providers = require('../../../errors/providers.js');
var transferAssetChunk = require('../../../utils/transfer-asset-chunk.js');
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const jsonLength = (obj)=>Buffer.byteLength(JSON.stringify(obj));
/**
* Default batching for entities / links / configuration over WebSocket push.
*
* Goals: (1) enough payload per round-trip to stay efficient on large transfers,
* (2) small enough per message that the remote can process and ack without multi-minute stalls,
* (3) bounded gap between engine progress and the wire (see item cap + age).
*
* These are fixed defaults (not tuned per dataset) so behavior is predictable everywhere.
*/ const STREAM_STEP_MAX_BATCH_BYTES = 512 * 1024;
/** Caps parallel work per message and how far UI count can lead the network for tiny rows. */ const STREAM_STEP_MAX_BATCH_ITEMS = 100;
/**
* If the first row in the current batch has waited this long, flush before appending more.
* Helps mixed-size streams (e.g. occasional large rows) without relying on tiny byte caps alone.
*/ const STREAM_STEP_MAX_BATCH_AGE_MS = 450;
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _checksumsEnabled = /*#__PURE__*/ _class_private_field_loose_key("_checksumsEnabled"), _startStepOnce = /*#__PURE__*/ _class_private_field_loose_key("_startStepOnce"), _startStep = /*#__PURE__*/ _class_private_field_loose_key("_startStep"), _endStep = /*#__PURE__*/ _class_private_field_loose_key("_endStep"), _streamStep = /*#__PURE__*/ _class_private_field_loose_key("_streamStep"), _writeStream = /*#__PURE__*/ _class_private_field_loose_key("_writeStream"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _reportWarning = /*#__PURE__*/ _class_private_field_loose_key("_reportWarning");
class RemoteStrapiDestinationProvider {
resetStats() {
this.stats = {
assets: {
count: 0
},
entities: {
count: 0
},
links: {
count: 0
},
configuration: {
count: 0
}
};
}
async initTransfer() {
const { strategy, restore } = this.options;
const wantsChecksums = this.options.verifyChecksums === true;
const query = this.dispatcher?.dispatchCommand({
command: 'init',
params: {
options: {
strategy,
restore
},
transfer: 'push',
...wantsChecksums ? {
checksums: true
} : {}
}
});
const res = await query;
if (!res?.transferID) {
throw new providers.ProviderTransferError('Init failed, invalid response from the server');
}
_class_private_field_loose_base(this, _checksumsEnabled)[_checksumsEnabled] = wantsChecksums && res.checksums === true;
if (wantsChecksums && res.checksums !== true) {
_class_private_field_loose_base(this, _reportWarning)[_reportWarning]('[Data transfer][push] Checksums were requested but the remote does not support checksum negotiation; continuing without checksum validation.');
}
this.resetStats();
return res.transferID;
}
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const { url, auth } = this.options;
const validProtocols = [
'https:',
'http:'
];
let ws;
if (!validProtocols.includes(url.protocol)) {
throw new providers.ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
check: 'url',
details: {
protocol: url.protocol,
validProtocols
}
});
}
const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${wsProtocol}//${url.host}${utils.trimTrailingSlash(url.pathname)}${constants.TRANSFER_PATH}/push`;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
ws = await utils.connectToWebsocket(wsUrl, undefined, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else if (auth.type === 'token') {
const headers = {
Authorization: `Bearer ${auth.token}`
};
ws = await utils.connectToWebsocket(wsUrl, {
headers
}, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else {
throw new providers.ProviderValidationError('Auth method not available', {
check: 'auth.type',
details: {
auth: auth.type
}
});
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('established websocket connection');
this.ws = ws;
const { retryMessageOptions } = this.options;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
this.dispatcher = utils.createDispatcher(this.ws, retryMessageOptions, (message)=>_class_private_field_loose_base(this, _reportInfo)[_reportInfo](message));
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('created dispatcher');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('initialize transfer');
this.transferID = await this.initTransfer();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`initialized transfer ${this.transferID}`);
this.dispatcher.setTransferProperties({
id: this.transferID,
kind: 'push'
});
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
// Gracefully close the remote transfer process
if (this.transferID && this.dispatcher) {
await this.dispatcher.dispatchTransferAction('close');
await this.dispatcher.dispatchCommand({
command: 'end',
params: {
transferID: this.transferID
}
});
}
await new Promise((resolve)=>{
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', ()=>resolve()).close();
});
}
getMetadata() {
return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null;
}
async beforeTransfer() {
this.options.onTransferPhase?.('Remote: waiting for server to clear data and prepare destination…');
await this.dispatcher?.dispatchTransferAction('beforeTransfer');
}
async rollback() {
await this.dispatcher?.dispatchTransferAction('rollback');
}
getSchemas() {
if (!this.dispatcher) {
return Promise.resolve(null);
}
return this.dispatcher.dispatchTransferAction('getSchemas');
}
createEntitiesWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('entities');
}
createLinksWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('links');
}
createConfigurationWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('configuration');
}
createAssetsWriteStream() {
let batch = [];
let hasStarted = false;
const verifyChecksums = _class_private_field_loose_base(this, _checksumsEnabled)[_checksumsEnabled];
const batchSize = 1024 * 1024; // 1MB;
const batchLength = ()=>{
return batch.reduce((acc, chunk)=>acc + transferAssetChunk.transferAssetStreamChunkByteLength(chunk), 0);
};
const startAssetsTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce]('assets');
const flush = async ()=>{
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep]('assets', batch);
batch = [];
return streamError;
};
const safePush = async (chunk)=>{
batch.push(chunk);
if (batchLength() >= batchSize) {
const streamError = await flush();
if (streamError) {
throw streamError;
}
}
};
return new stream.Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
await flush();
}
if (hasStarted) {
const { error: endStepError } = await _class_private_field_loose_base(this, _endStep)[_endStep]('assets');
if (endStepError) {
return callback(endStepError);
}
}
return callback(null);
},
async write (asset, _encoding, callback) {
const startError = await startAssetsTransferOnce();
if (startError) {
return callback(startError);
}
hasStarted = true;
const assetID = crypto.randomUUID();
const { filename, filepath, stats, stream, metadata } = asset;
const checksumHash = verifyChecksums ? crypto.createHash('sha256') : undefined;
try {
await safePush({
action: 'start',
assetID,
data: {
filename,
filepath,
stats,
metadata
}
});
for await (const chunk of stream){
checksumHash?.update(chunk);
await safePush(transferAssetChunk.createTransferAssetStreamChunk(assetID, chunk));
}
await safePush({
action: 'end',
assetID,
...checksumHash ? {
checksum: {
algorithm: 'sha256',
value: checksumHash.digest('hex')
}
} : {}
});
callback();
} catch (error) {
callback(error instanceof Error ? error : new Error(String(error)));
}
}
});
}
constructor(options){
Object.defineProperty(this, _startStepOnce, {
value: startStepOnce
});
Object.defineProperty(this, _startStep, {
value: startStep
});
Object.defineProperty(this, _endStep, {
value: endStep
});
Object.defineProperty(this, _streamStep, {
value: streamStep
});
Object.defineProperty(this, _writeStream, {
value: writeStream
});
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _reportWarning, {
value: reportWarning
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
Object.defineProperty(this, _checksumsEnabled, {
writable: true,
value: void 0
});
this.name = 'destination::remote-strapi';
this.type = 'destination';
_class_private_field_loose_base(this, _checksumsEnabled)[_checksumsEnabled] = false;
this.options = options;
this.ws = null;
this.dispatcher = null;
this.transferID = null;
_class_private_field_loose_base(this, _checksumsEnabled)[_checksumsEnabled] = options.verifyChecksums === true;
this.resetStats();
}
}
function startStepOnce(stage) {
return fp.once(()=>_class_private_field_loose_base(this, _startStep)[_startStep](stage));
}
async function startStep(step) {
try {
await this.dispatcher?.dispatchTransferStep({
action: 'start',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
this.stats[step] = {
count: 0
};
return null;
}
async function endStep(step) {
try {
const res = await this.dispatcher?.dispatchTransferStep({
action: 'end',
step
});
return {
stats: res?.stats ?? null,
error: null
};
} catch (e) {
if (e instanceof Error) {
return {
stats: null,
error: e
};
}
if (typeof e === 'string') {
return {
stats: null,
error: new providers.ProviderTransferError(e)
};
}
return {
stats: null,
error: new providers.ProviderTransferError('Unexpected error')
};
}
}
async function streamStep(step, message) {
try {
if (step === 'assets') {
const assetMessage = message;
this.stats[step].count += assetMessage.filter((data)=>data.action === 'start').length;
} else {
this.stats[step].count += message.length;
}
await this.dispatcher?.dispatchTransferStep({
action: 'stream',
step,
data: message
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
return null;
}
function writeStream(step) {
const startTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce](step);
let batch = [];
let batchStartedAt = 0;
const batchLength = ()=>jsonLength(batch);
const flushBatch = async ()=>{
if (batch.length === 0) {
return null;
}
const payload = batch;
batch = [];
batchStartedAt = 0;
return _class_private_field_loose_base(this, _streamStep)[_streamStep](step, payload);
};
const shouldFlushBatchAfterPush = ()=>{
if (batch.length === 0) {
return false;
}
return batchLength() >= STREAM_STEP_MAX_BATCH_BYTES || batch.length >= STREAM_STEP_MAX_BATCH_ITEMS || Date.now() - batchStartedAt >= STREAM_STEP_MAX_BATCH_AGE_MS;
};
return new stream.Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
const streamError = await flushBatch();
if (streamError) {
return callback(streamError);
}
}
const { error, stats } = await _class_private_field_loose_base(this, _endStep)[_endStep](step);
const { count } = this.stats[step];
if (stats && (stats.started !== count || stats.finished !== count)) {
callback(new Error(`Data missing: sent ${this.stats[step].count} ${step}, received ${stats.started} and saved ${stats.finished} ${step}`));
}
callback(error);
},
async write (chunk, _encoding, callback) {
const startError = await startTransferOnce();
if (startError) {
return callback(startError);
}
// Flush a batch that has sat long enough before growing it further (bounded latency).
if (batch.length > 0 && Date.now() - batchStartedAt >= STREAM_STEP_MAX_BATCH_AGE_MS) {
const staleError = await flushBatch();
if (staleError) {
return callback(staleError);
}
}
batch.push(chunk);
if (batch.length === 1) {
batchStartedAt = Date.now();
}
if (shouldFlushBatchAfterPush()) {
const streamError = await flushBatch();
if (streamError) {
return callback(streamError);
}
}
callback();
}
});
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-destination-provider'
},
kind: 'info'
});
}
function reportWarning(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-destination-provider'
},
kind: 'warning'
});
}
const createRemoteStrapiDestinationProvider = (options)=>{
return new RemoteStrapiDestinationProvider(options);
};
exports.createRemoteStrapiDestinationProvider = createRemoteStrapiDestinationProvider;
//# sourceMappingURL=index.js.map