@strapi/data-transfer
Version:
Data transfer capabilities for Strapi
262 lines (258 loc) • 12.1 kB
JavaScript
;
var path = require('path');
var stream$1 = require('stream');
var fs = require('fs-extra');
var fp = require('lodash/fp');
var streamChain = require('stream-chain');
var Parser = require('stream-json/jsonl/Parser');
require('crypto');
var writableAsyncWrite = require('../../../utils/writable-async-write.js');
var stream = require('../../../utils/stream.js');
var schema = require('../../../utils/schema.js');
require('events');
var providers = require('../../../errors/providers.js');
var utils = require('../../../file/providers/source/utils.js');
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const METADATA_FILE_PATH = 'metadata.json';
const createLocalDirectorySourceProvider = (options)=>{
return new LocalDirectorySourceProvider(options);
};
const isPathInsideRoot = (root, candidate)=>{
const relative = path.relative(root, candidate);
return relative === '' || !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
};
var _rootResolved = /*#__PURE__*/ _class_private_field_loose_key("_rootResolved"), _metadata = /*#__PURE__*/ _class_private_field_loose_key("_metadata"), _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), /** Resolve a posix-style relative path under the export root; rejects escapes. */ _safePath = /*#__PURE__*/ _class_private_field_loose_key("_safePath"), _loadMetadata = /*#__PURE__*/ _class_private_field_loose_key("_loadMetadata"), _pipeAssetsToStream = /*#__PURE__*/ _class_private_field_loose_key("_pipeAssetsToStream"), _readAssetMetadata = /*#__PURE__*/ _class_private_field_loose_key("_readAssetMetadata"), _listJsonlFiles = /*#__PURE__*/ _class_private_field_loose_key("_listJsonlFiles"), _streamJsonlDirectory = /*#__PURE__*/ _class_private_field_loose_key("_streamJsonlDirectory"), _pipeJsonlDirectoryToStream = /*#__PURE__*/ _class_private_field_loose_key("_pipeJsonlDirectoryToStream");
class LocalDirectorySourceProvider {
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const root = _class_private_field_loose_base(this, _rootResolved)[_rootResolved];
try {
const stat = await fs.stat(root);
if (!stat.isDirectory()) {
throw new providers.ProviderInitializationError(`Path '${root}' is not a directory.`);
}
await _class_private_field_loose_base(this, _loadMetadata)[_loadMetadata]();
} catch (e) {
if (e instanceof providers.ProviderInitializationError) {
throw e;
}
throw new providers.ProviderInitializationError(`Directory '${root}' is not a valid Strapi data export.`);
}
if (!_class_private_field_loose_base(this, _metadata)[_metadata]) {
throw new providers.ProviderInitializationError('Could not load metadata from Strapi data export.');
}
}
async getMetadata() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting metadata');
if (!_class_private_field_loose_base(this, _metadata)[_metadata]) {
await _class_private_field_loose_base(this, _loadMetadata)[_loadMetadata]();
}
return _class_private_field_loose_base(this, _metadata)[_metadata] ?? null;
}
async getSchemas() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schemas');
const schemaCollection = await stream.collect(this.createSchemasReadStream());
if (fp.isEmpty(schemaCollection)) {
throw new providers.ProviderInitializationError('Could not load schemas from Strapi data export.');
}
const schemas = fp.keyBy('uid', schemaCollection);
return schema.schemasToValidJSON(schemas);
}
createEntitiesReadStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities read stream');
return _class_private_field_loose_base(this, _streamJsonlDirectory)[_streamJsonlDirectory]('entities');
}
createSchemasReadStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating schemas read stream');
return _class_private_field_loose_base(this, _streamJsonlDirectory)[_streamJsonlDirectory]('schemas');
}
createLinksReadStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links read stream');
return _class_private_field_loose_base(this, _streamJsonlDirectory)[_streamJsonlDirectory]('links');
}
createConfigurationReadStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration read stream');
return _class_private_field_loose_base(this, _streamJsonlDirectory)[_streamJsonlDirectory]('configuration');
}
createAssetsReadStream() {
const outStream = new stream$1.PassThrough({
objectMode: true
});
const uploadsDir = _class_private_field_loose_base(this, _safePath)[_safePath]('assets', 'uploads');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets read stream');
_class_private_field_loose_base(this, _pipeAssetsToStream)[_pipeAssetsToStream](outStream, uploadsDir).catch((e)=>{
outStream.destroy(e instanceof Error ? e : new providers.ProviderTransferError(String(e), {
details: {
error: e
}
}));
});
return outStream;
}
constructor(options){
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _safePath, {
value: safePath
});
Object.defineProperty(this, _loadMetadata, {
value: loadMetadata
});
Object.defineProperty(this, _pipeAssetsToStream, {
value: pipeAssetsToStream
});
Object.defineProperty(this, _readAssetMetadata, {
value: readAssetMetadata
});
Object.defineProperty(this, _listJsonlFiles, {
value: listJsonlFiles
});
Object.defineProperty(this, _streamJsonlDirectory, {
value: streamJsonlDirectory
});
Object.defineProperty(this, _pipeJsonlDirectoryToStream, {
value: pipeJsonlDirectoryToStream
});
Object.defineProperty(this, _rootResolved, {
writable: true,
value: void 0
});
Object.defineProperty(this, _metadata, {
writable: true,
value: void 0
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.type = 'source';
this.name = 'source::local-directory';
this.options = options;
_class_private_field_loose_base(this, _rootResolved)[_rootResolved] = path.resolve(options.directory.path);
}
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'directory-source-provider'
},
kind: 'info'
});
}
function safePath(...posixSegments) {
const joined = path.posix.join(...posixSegments);
const segments = joined.split(path.posix.sep).filter(Boolean);
const resolved = path.resolve(_class_private_field_loose_base(this, _rootResolved)[_rootResolved], ...segments);
if (!isPathInsideRoot(_class_private_field_loose_base(this, _rootResolved)[_rootResolved], resolved)) {
throw new providers.ProviderInitializationError(`Invalid path "${joined}" — escapes backup directory`);
}
return resolved;
}
async function loadMetadata() {
const metadataPath = _class_private_field_loose_base(this, _safePath)[_safePath](METADATA_FILE_PATH);
if (!await fs.pathExists(metadataPath)) {
throw new providers.ProviderInitializationError(`Missing ${METADATA_FILE_PATH} in export directory '${_class_private_field_loose_base(this, _rootResolved)[_rootResolved]}'.`);
}
_class_private_field_loose_base(this, _metadata)[_metadata] = await fs.readJson(metadataPath);
}
async function pipeAssetsToStream(outStream, uploadsDir) {
if (!await fs.pathExists(uploadsDir)) {
outStream.end();
return;
}
const names = (await fs.readdir(uploadsDir)).sort();
for (const name of names){
const absUpload = path.join(uploadsDir, name);
const stat = await fs.stat(absUpload);
if (stat.isFile()) {
let metadata;
try {
metadata = await _class_private_field_loose_base(this, _readAssetMetadata)[_readAssetMetadata](name);
} catch (error) {
outStream.destroy(new providers.ProviderTransferError(`Failed to read metadata for ${name}`, {
details: {
error
}
}));
return;
}
const normalizedPath = utils.unknownPathToPosix(path.posix.join('assets', 'uploads', name));
const asset = {
metadata,
filename: name,
filepath: normalizedPath,
stats: {
size: stat.size
},
stream: fs.createReadStream(absUpload)
};
await writableAsyncWrite.write(outStream, asset);
}
}
outStream.end();
}
async function readAssetMetadata(filename) {
const metadataPath = _class_private_field_loose_base(this, _safePath)[_safePath]('assets', 'metadata', `${filename}.json`);
return fs.readJson(metadataPath);
}
async function listJsonlFiles(posixSubdir) {
const dirAbs = _class_private_field_loose_base(this, _safePath)[_safePath](...posixSubdir.split('/').filter(Boolean));
if (!await fs.pathExists(dirAbs)) {
return [];
}
const names = await fs.readdir(dirAbs);
return names.filter((n)=>n.endsWith('.jsonl')).sort().map((n)=>path.join(dirAbs, n));
}
function streamJsonlDirectory(posixSubdir) {
const outStream = new stream$1.PassThrough({
objectMode: true
});
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`streaming jsonl from ${posixSubdir}`);
_class_private_field_loose_base(this, _pipeJsonlDirectoryToStream)[_pipeJsonlDirectoryToStream](outStream, posixSubdir).catch((e)=>{
outStream.destroy(e instanceof Error ? e : new providers.ProviderTransferError(String(e), {
details: {
error: e
}
}));
});
return outStream;
}
async function pipeJsonlDirectoryToStream(outStream, posixSubdir) {
const files = await _class_private_field_loose_base(this, _listJsonlFiles)[_listJsonlFiles](posixSubdir);
for (const absPath of files){
const transforms = [
Parser.parser({
checkErrors: true
}),
(line)=>line.value
];
const stream = fs.createReadStream(absPath).pipe(streamChain.chain(transforms));
try {
for await (const chunk of stream){
await writableAsyncWrite.write(outStream, chunk);
}
} catch (e) {
outStream.destroy(new providers.ProviderTransferError(`Error parsing JSONL in ${absPath}: ${e.message}`, {
details: {
error: e
}
}));
return;
}
}
outStream.end();
}
exports.createLocalDirectorySourceProvider = createLocalDirectorySourceProvider;
//# sourceMappingURL=index.js.map