@strapi/data-transfer
Version:
Data transfer capabilities for Strapi
368 lines (365 loc) • 14.5 kB
JavaScript
import { Readable } from 'stream';
import { randomUUID, createHash } from 'crypto';
import { handlerControllerFactory, isDataTransferMessage } from './utils.mjs';
import { createTransferAssetStreamChunk, transferAssetStreamChunkByteLength } from '../../../utils/transfer-asset-chunk.mjs';
import 'path';
import 'fs-extra';
import { ProviderTransferError } from '../../../errors/providers.mjs';
import '../../queries/entity.mjs';
import 'lodash/fp';
import 'node:events';
import 'node:stream/promises';
import 'events';
import 'lodash';
import '@strapi/utils';
import '../../providers/local-destination/strategies/restore/configuration.mjs';
import { createLocalStrapiSourceProvider } from '../../providers/local-source/index.mjs';
import 'ws';
import { estimateAssetTotals } from '../../providers/local-source/estimate-asset-totals.mjs';
const TRANSFER_KIND = 'pull';
const VALID_TRANSFER_ACTIONS = [
'bootstrap',
'close',
'getMetadata',
'getSchemas'
];
const createPullController = handlerControllerFactory((proto)=>({
isTransferStarted () {
return proto.isTransferStarted.call(this) && this.provider !== undefined;
},
verifyAuth () {
return proto.verifyAuth.call(this, TRANSFER_KIND);
},
cleanup () {
proto.cleanup.call(this);
this.streams = {};
this.checksumsEnabled = false;
delete this.provider;
},
onInfo (message) {
this.diagnostics?.report({
details: {
message,
origin: 'pull-handler',
createdAt: new Date()
},
kind: 'info'
});
},
onWarning (message) {
this.diagnostics?.report({
details: {
message,
createdAt: new Date(),
origin: 'pull-handler'
},
kind: 'warning'
});
},
onError (error) {
this.diagnostics?.report({
details: {
message: error.message,
error,
createdAt: new Date(),
name: error.name,
severity: 'fatal'
},
kind: 'error'
});
},
assertValidTransferAction (action) {
// Abstract the constant to string[] to allow looser check on the given action
const validActions = VALID_TRANSFER_ACTIONS;
if (validActions.includes(action)) {
return;
}
throw new ProviderTransferError(`Invalid action provided: "${action}"`, {
action,
validActions: Object.keys(VALID_TRANSFER_ACTIONS)
});
},
async onMessage (raw) {
const msg = JSON.parse(raw.toString());
if (!isDataTransferMessage(msg)) {
return;
}
if (!msg.uuid) {
await this.respond(undefined, new Error('Missing uuid in message'));
}
if (proto.hasUUID(msg.uuid)) {
const previousResponse = proto.response;
if (previousResponse?.uuid === msg.uuid) {
await this.respond(previousResponse?.uuid, previousResponse.e, previousResponse.data);
}
return;
}
const { uuid, type } = msg;
proto.addUUID(uuid);
// Regular command message (init, end, status)
if (type === 'command') {
const { command } = msg;
this.onInfo(`received command:${command} uuid:${uuid}`);
await this.executeAndRespond(uuid, ()=>{
this.assertValidTransferCommand(command);
// The status command don't have params
if (command === 'status') {
return this.status();
}
return this[command](msg.params);
});
} else if (type === 'transfer') {
this.onInfo(`received transfer action:${msg.action} step:${msg.kind} uuid:${uuid}`);
await this.executeAndRespond(uuid, async ()=>{
await this.verifyAuth();
this.assertValidTransfer();
return this.onTransferMessage(msg);
});
} else {
await this.respond(uuid, new Error('Bad Request'));
}
},
async onTransferMessage (msg) {
const { kind } = msg;
if (kind === 'action') {
return this.onTransferAction(msg);
}
if (kind === 'step') {
return this.onTransferStep(msg);
}
},
async onTransferAction (msg) {
const { action } = msg;
this.assertValidTransferAction(action);
if (action === 'bootstrap') {
return this.provider?.[action](this.diagnostics);
}
return this.provider?.[action]();
},
async flush (stage, id) {
const batchSize = 1024 * 1024;
let batch = [];
const stream = this.streams?.[stage];
const batchLength = ()=>Buffer.byteLength(JSON.stringify(batch));
const maybeConfirm = async (data)=>{
try {
await this.confirm(data);
} catch (error) {
// Handle the error, log it, or take other appropriate actions
strapi?.log.error(`[Data transfer] Message confirmation failed: ${error?.message}`);
this.onError(error);
}
};
const sendBatch = async ()=>{
await this.confirm({
type: 'transfer',
data: batch,
ended: false,
error: null,
id
});
batch = [];
};
try {
if (!stream) {
throw new ProviderTransferError(`No available stream found for ${stage}`);
}
for await (const chunk of stream){
if (stage !== 'assets') {
batch.push(chunk);
if (batchLength() >= batchSize) {
await sendBatch();
}
} else {
await this.confirm({
type: 'transfer',
data: [
chunk
],
ended: false,
error: null,
id
});
}
}
if (batch.length > 0 && stage !== 'assets') {
await sendBatch();
}
await this.confirm({
type: 'transfer',
data: null,
ended: true,
error: null,
id
});
} catch (e) {
// TODO: if this confirm fails, can we abort the whole transfer?
await maybeConfirm({
type: 'transfer',
data: null,
ended: true,
error: e,
id
});
}
},
async onTransferStep (msg) {
const { step, action } = msg;
if (action === 'start') {
if (this.streams?.[step] instanceof Readable) {
throw new Error('Stream already created, something went wrong');
}
const flushUUID = randomUUID();
let totals;
if (step === 'assets') {
totals = await estimateAssetTotals(strapi);
}
await this.createReadableStreamForStep(step);
Promise.resolve(this.flush(step, flushUUID)).catch((err)=>{
this.onError(err instanceof Error ? err : new Error(String(err)));
});
return {
ok: true,
id: flushUUID,
...totals !== undefined ? {
totals
} : {}
};
}
if (action === 'end') {
const stream = this.streams?.[step];
if (stream?.readableEnded === false) {
await new Promise((resolve)=>{
stream?.on('close', resolve).destroy();
});
}
delete this.streams?.[step];
return {
ok: true
};
}
},
async createReadableStreamForStep (step) {
const mapper = {
entities: ()=>this.provider?.createEntitiesReadStream(),
links: ()=>this.provider?.createLinksReadStream(),
configuration: ()=>this.provider?.createConfigurationReadStream(),
assets: ()=>{
const assets = this.provider?.createAssetsReadStream();
let batch = [];
const checksumsEnabled = this.checksumsEnabled === true;
const batchLength = ()=>{
return batch.reduce((acc, chunk)=>acc + transferAssetStreamChunkByteLength(chunk), 0);
};
const BATCH_MAX_SIZE = 1024 * 1024; // 1MB
if (!assets) {
throw new Error('Assets read stream could not be created');
}
/**
* Generates batches of 1MB of data from the assets stream to avoid
* sending too many small chunks
*
* @param stream Assets stream from the local source provider
*/ async function* generator(stream) {
let hasStarted = false;
let assetID = '';
let assetChecksum;
for await (const chunk of stream){
const { stream: assetStream, ...assetData } = chunk;
if (!hasStarted) {
assetID = randomUUID();
assetChecksum = checksumsEnabled ? createHash('sha256') : undefined;
// Start the transfer of a new asset
batch.push({
action: 'start',
assetID,
data: assetData
});
hasStarted = true;
}
for await (const assetChunk of assetStream){
assetChecksum?.update(assetChunk);
batch.push(createTransferAssetStreamChunk(assetID, assetChunk));
// if the batch size is bigger than BATCH_MAX_SIZE stream the batch
if (batchLength() >= BATCH_MAX_SIZE) {
yield batch;
batch = [];
}
}
// All the asset data has been streamed and gets ready for the next one
hasStarted = false;
batch.push({
action: 'end',
assetID,
...assetChecksum ? {
checksum: {
algorithm: 'sha256',
value: assetChecksum.digest('hex')
}
} : {}
});
yield batch;
batch = [];
}
}
return Readable.from(generator(assets));
}
};
if (!(step in mapper)) {
throw new Error('Invalid transfer step, impossible to create a stream');
}
if (!this.streams) {
throw new Error('Invalid transfer state');
}
this.streams[step] = await mapper[step]();
},
// Commands
async init (params) {
if (this.transferID || this.provider) {
throw new Error('Transfer already in progress');
}
await this.verifyAuth();
this.transferID = randomUUID();
this.startedAt = Date.now();
this.checksumsEnabled = params?.checksums === true;
this.streams = {};
this.provider = createLocalStrapiSourceProvider({
autoDestroy: false,
getStrapi: ()=>strapi
});
return {
transferID: this.transferID,
checksums: true
};
},
async end (params) {
await this.verifyAuth();
if (this.transferID !== params?.transferID) {
throw new ProviderTransferError('Bad transfer ID provided');
}
this.cleanup();
return {
ok: true
};
},
async status () {
const isStarted = this.isTransferStarted();
if (!isStarted) {
const startedAt = this.startedAt;
return {
active: true,
kind: TRANSFER_KIND,
startedAt,
elapsed: Date.now() - startedAt
};
}
return {
active: false,
kind: null,
elapsed: null,
startedAt: null
};
}
}));
export { createPullController };
//# sourceMappingURL=pull.mjs.map