@strapi/data-transfer/dist/strapi/providers/local-source/assets.mjs.map

Data transfer capabilities for Strapi

{"version":3,"file":"assets.mjs","sources":["../../../../src/strapi/providers/local-source/assets.ts"],"sourcesContent":["import { join } from 'path';\nimport { Duplex, PassThrough, Readable } from 'stream';\nimport { stat, createReadStream, ReadStream } from 'fs-extra';\nimport * as webStream from 'stream/web';\nimport type { Core } from '@strapi/types';\n\nimport type { IAsset, IFile } from '../../../../types';\n\nfunction getFileStream(\n  filepath: string,\n  strapi: Core.Strapi,\n  isLocal = false\n): PassThrough | ReadStream {\n  if (isLocal) {\n    // Todo: handle errors\n    return createReadStream(filepath);\n  }\n\n  const readableStream = new PassThrough();\n\n  // fetch the image from remote url and stream it\n  strapi\n    .fetch(filepath)\n    .then((res: Response) => {\n      if (res.status !== 200) {\n        readableStream.emit('error', new Error(`Request failed with status code ${res.status}`));\n        return;\n      }\n\n      if (res.body) {\n        // pipe the image data\n        Readable.fromWeb(res.body as webStream.ReadableStream<Uint8Array>).pipe(readableStream);\n      } else {\n        readableStream.emit('error', new Error('Empty data found for file'));\n      }\n    })\n    .catch((error: unknown) => {\n      readableStream.emit('error', error);\n    });\n\n  return readableStream;\n}\n\nexport function getFileStatsForTransfer(\n  filepath: string,\n  strapi: Core.Strapi,\n  isLocal = false\n): Promise<{ size: number }> {\n  if (isLocal) {\n    return stat(filepath);\n  }\n  return new Promise((resolve, reject) => {\n    strapi\n      .fetch(filepath)\n      .then((res: Response) => {\n        if (res.status !== 200) {\n          reject(new Error(`Request failed with status code ${res.status}`));\n          return;\n        }\n\n        const contentLength = res.headers.get('content-length');\n        const stats = {\n          size: contentLength ? parseInt(contentLength, 10) : 0,\n        };\n\n        resolve(stats);\n      })\n      .catch((error: unknown) => {\n        reject(error);\n      });\n  });\n}\n\nexport async function signUploadFileForTransfer(strapi: Core.Strapi, file: IFile) {\n  const { provider } = strapi.plugins.upload;\n  const { provider: providerName } = strapi.config.get('plugin.upload') as { provider: string };\n  const isPrivate = await provider.isPrivate();\n  if (file?.provider === providerName && isPrivate) {\n    const signUrl = async (f: IFile) => {\n      const signedUrl = await provider.getSignedUrl(f);\n      f.url = signedUrl.url;\n    };\n\n    await signUrl(file);\n    if (file.formats) {\n      for (const format of Object.keys(file.formats)) {\n        await signUrl(file.formats[format]);\n      }\n    }\n  }\n}\n\nconst missingAssetWarningMessage = (file: IFile, filepath: string, format?: string): string => {\n  const formatPart = format ? ` (format: ${format})` : '';\n  return `[Data transfer] Media item ${file.id} (hash: ${file.hash}) exists in database but no corresponding file was found to transfer${formatPart}. Path: ${filepath}`;\n};\n\n/**\n * Generate and consume assets streams in order to stream each file individually\n */\nexport const createAssetsStream = (\n  strapi: Core.Strapi,\n  options: { onWarning?: (message: string) => void } = {}\n): Duplex => {\n  const warnMissingAsset = (message: string) => {\n    strapi.log.warn(message);\n    options.onWarning?.(message);\n  };\n\n  const generator: () => AsyncGenerator<IAsset, void> = async function* () {\n    const stream: Readable = strapi.db\n      .queryBuilder('plugin::upload.file')\n      // Create a query builder instance (default type is 'select')\n      // Fetch all columns\n      .select('*')\n      // Get a readable stream\n      .stream();\n\n    for await (const file of stream) {\n      const isLocalProvider = file.provider === 'local';\n      if (!isLocalProvider) {\n        await signUploadFileForTransfer(strapi, file);\n      }\n      const filepath = isLocalProvider ? join(strapi.dirs.static.public, file.url) : file.url;\n      let stats: { size: number };\n      try {\n        stats = await getFileStatsForTransfer(filepath, strapi, isLocalProvider);\n      } catch (err: unknown) {\n        const code =\n          err && typeof err === 'object' && 'code' in err\n            ? (err as NodeJS.ErrnoException).code\n            : undefined;\n        if (code === 'ENOENT') {\n          warnMissingAsset(missingAssetWarningMessage(file, filepath));\n          continue;\n        }\n        throw err;\n      }\n      const stream = getFileStream(filepath, strapi, isLocalProvider);\n\n      yield {\n        metadata: file,\n        filepath,\n        filename: file.hash + file.ext,\n        stream,\n        stats: { size: stats.size },\n      };\n\n      if (file.formats) {\n        for (const format of Object.keys(file.formats)) {\n          const fileFormat = file.formats[format];\n          const fileFormatFilepath = isLocalProvider\n            ? join(strapi.dirs.static.public, fileFormat.url)\n            : fileFormat.url;\n          let fileFormatStats: { size: number };\n          try {\n            fileFormatStats = await getFileStatsForTransfer(\n              fileFormatFilepath,\n              strapi,\n              isLocalProvider\n            );\n          } catch (err: unknown) {\n            const code =\n              err && typeof err === 'object' && 'code' in err\n                ? (err as NodeJS.ErrnoException).code\n                : undefined;\n            if (code === 'ENOENT') {\n              warnMissingAsset(missingAssetWarningMessage(file, fileFormatFilepath, format));\n              continue;\n            }\n            throw err;\n          }\n          const fileFormatStream = getFileStream(fileFormatFilepath, strapi, isLocalProvider);\n          const metadata = { ...fileFormat, type: format, id: file.id, mainHash: file.hash };\n          yield {\n            metadata,\n            filepath: fileFormatFilepath,\n            filename: fileFormat.hash + fileFormat.ext,\n            stream: fileFormatStream,\n            stats: { size: fileFormatStats.size },\n          };\n        }\n      }\n    }\n  };\n\n  return Duplex.from(generator());\n};\n"],"names":["getFileStream","filepath","strapi","isLocal","createReadStream","readableStream","PassThrough","fetch","then","res","status","emit","Error","body","Readable","fromWeb","pipe","catch","error","getFileStatsForTransfer","stat","Promise","resolve","reject","contentLength","headers","get","stats","size","parseInt","signUploadFileForTransfer","file","provider","plugins","upload","providerName","config","isPrivate","signUrl","f","signedUrl","getSignedUrl","url","formats","format","Object","keys","missingAssetWarningMessage","formatPart","id","hash","createAssetsStream","options","warnMissingAsset","message","log","warn","onWarning","generator","stream","db","queryBuilder","select","isLocalProvider","join","dirs","static","public","err","code","undefined","metadata","filename","ext","fileFormat","fileFormatFilepath","fileFormatStats","fileFormatStream","type","mainHash","Duplex","from"],"mappings":";;;;AAQA,SAASA,cACPC,QAAgB,EAChBC,MAAmB,EACnBC,UAAU,KAAK,EAAA;AAEf,IAAA,IAAIA,OAAAA,EAAS;;AAEX,QAAA,OAAOC,gBAAAA,CAAiBH,QAAAA,CAAAA;AAC1B,IAAA;AAEA,IAAA,MAAMI,iBAAiB,IAAIC,WAAAA,EAAAA;;AAG3BJ,IAAAA,MAAAA,CACGK,KAAK,CAACN,QAAAA,CAAAA,CACNO,IAAI,CAAC,CAACC,GAAAA,GAAAA;QACL,IAAIA,GAAAA,CAAIC,MAAM,KAAK,GAAA,EAAK;YACtBL,cAAAA,CAAeM,IAAI,CAAC,OAAA,EAAS,IAAIC,KAAAA,CAAM,CAAC,gCAAgC,EAAEH,GAAAA,CAAIC,MAAM,CAAA,CAAE,CAAA,CAAA;AACtF,YAAA;AACF,QAAA;QAEA,IAAID,GAAAA,CAAII,IAAI,EAAE;;AAEZC,YAAAA,QAAAA,CAASC,OAAO,CAACN,GAAAA,CAAII,IAAI,CAAA,CAA0CG,IAAI,CAACX,cAAAA,CAAAA;QAC1E,CAAA,MAAO;AACLA,YAAAA,cAAAA,CAAeM,IAAI,CAAC,OAAA,EAAS,IAAIC,KAAAA,CAAM,2BAAA,CAAA,CAAA;AACzC,QAAA;IACF,CAAA,CAAA,CACCK,KAAK,CAAC,CAACC,KAAAA,GAAAA;QACNb,cAAAA,CAAeM,IAAI,CAAC,OAAA,EAASO,KAAAA,CAAAA;AAC/B,IAAA,CAAA,CAAA;IAEF,OAAOb,cAAAA;AACT;AAEO,SAASc,uBAAAA,CACdlB,QAAgB,EAChBC,MAAmB,EACnBC,UAAU,KAAK,EAAA;AAEf,IAAA,IAAIA,OAAAA,EAAS;AACX,QAAA,OAAOiB,IAAAA,CAAKnB,QAAAA,CAAAA;AACd,IAAA;IACA,OAAO,IAAIoB,OAAAA,CAAQ,CAACC,OAAAA,EAASC,MAAAA,GAAAA;AAC3BrB,QAAAA,MAAAA,CACGK,KAAK,CAACN,QAAAA,CAAAA,CACNO,IAAI,CAAC,CAACC,GAAAA,GAAAA;YACL,IAAIA,GAAAA,CAAIC,MAAM,KAAK,GAAA,EAAK;AACtBa,gBAAAA,MAAAA,CAAO,IAAIX,KAAAA,CAAM,CAAC,gCAAgC,EAAEH,GAAAA,CAAIC,MAAM,CAAA,CAAE,CAAA,CAAA;AAChE,gBAAA;AACF,YAAA;AAEA,YAAA,MAAMc,aAAAA,GAAgBf,GAAAA,CAAIgB,OAAO,CAACC,GAAG,CAAC,gBAAA,CAAA;AACtC,YAAA,MAAMC,KAAAA,GAAQ;gBACZC,IAAAA,EAAMJ,aAAAA,GAAgBK,QAAAA,CAASL,aAAAA,EAAe,EAAA,CAAA,GAAM;AACtD,aAAA;YAEAF,OAAAA,CAAQK,KAAAA,CAAAA;QACV,CAAA,CAAA,CACCV,KAAK,CAAC,CAACC,KAAAA,GAAAA;YACNK,MAAAA,CAAOL,KAAAA,CAAAA;AACT,QAAA,CAAA,CAAA;AACJ,IAAA,CAAA,CAAA;AACF;AAEO,eAAeY,yBAAAA,CAA0B5B,MAAmB,EAAE6B,IAAW,EAAA;AAC9E,IAAA,MAAM,EAAEC,QAAQ,EAAE,GAAG9B,MAAAA,CAAO+B,OAAO,CAACC,MAAM;IAC1C,MAAM,EAAEF,UAAUG,YAAY,EAAE,GAAGjC,MAAAA,CAAOkC,MAAM,CAACV,GAAG,CAAC,eAAA,CAAA;IACrD,MAAMW,SAAAA,GAAY,MAAML,QAAAA,CAASK,SAAS,EAAA;IAC1C,IAAIN,IAAAA,EAAMC,QAAAA,KAAaG,YAAAA,IAAgBE,SAAAA,EAAW;AAChD,QAAA,MAAMC,UAAU,OAAOC,CAAAA,GAAAA;AACrB,YAAA,MAAMC,SAAAA,GAAY,MAAMR,QAAAA,CAASS,YAAY,CAACF,CAAAA,CAAAA;YAC9CA,CAAAA,CAAEG,GAAG,GAAGF,SAAAA,CAAUE,GAAG;AACvB,QAAA,CAAA;AAEA,QAAA,MAAMJ,OAAAA,CAAQP,IAAAA,CAAAA;QACd,IAAIA,IAAAA,CAAKY,OAAO,EAAE;AAChB,YAAA,KAAK,MAAMC,MAAAA,IAAUC,MAAAA,CAAOC,IAAI,CAACf,IAAAA,CAAKY,OAAO,CAAA,CAAG;AAC9C,gBAAA,MAAML,OAAAA,CAAQP,IAAAA,CAAKY,OAAO,CAACC,MAAAA,CAAO,CAAA;AACpC,YAAA;AACF,QAAA;AACF,IAAA;AACF;AAEA,MAAMG,0BAAAA,GAA6B,CAAChB,IAAAA,EAAa9B,QAAAA,EAAkB2C,MAAAA,GAAAA;IACjE,MAAMI,UAAAA,GAAaJ,SAAS,CAAC,UAAU,EAAEA,MAAAA,CAAO,CAAC,CAAC,GAAG,EAAA;AACrD,IAAA,OAAO,CAAC,2BAA2B,EAAEb,IAAAA,CAAKkB,EAAE,CAAC,QAAQ,EAAElB,IAAAA,CAAKmB,IAAI,CAAC,oEAAoE,EAAEF,UAAAA,CAAW,QAAQ,EAAE/C,QAAAA,CAAAA,CAAU;AACxK,CAAA;AAEA;;AAEC,IACM,MAAMkD,kBAAAA,GAAqB,CAChCjD,MAAAA,EACAkD,OAAAA,GAAqD,EAAE,GAAA;AAEvD,IAAA,MAAMC,mBAAmB,CAACC,OAAAA,GAAAA;QACxBpD,MAAAA,CAAOqD,GAAG,CAACC,IAAI,CAACF,OAAAA,CAAAA;AAChBF,QAAAA,OAAAA,CAAQK,SAAS,GAAGH,OAAAA,CAAAA;AACtB,IAAA,CAAA;AAEA,IAAA,MAAMI,SAAAA,GAAgD,kBAAA;AACpD,QAAA,MAAMC,SAAmBzD,MAAAA,CAAO0D,EAAE,CAC/BC,YAAY,CAAC,sBACd;;SAECC,MAAM,CAAC,IACR;SACCH,MAAM,EAAA;QAET,WAAW,MAAM5B,QAAQ4B,MAAAA,CAAQ;YAC/B,MAAMI,eAAAA,GAAkBhC,IAAAA,CAAKC,QAAQ,KAAK,OAAA;AAC1C,YAAA,IAAI,CAAC+B,eAAAA,EAAiB;AACpB,gBAAA,MAAMjC,0BAA0B5B,MAAAA,EAAQ6B,IAAAA,CAAAA;AAC1C,YAAA;AACA,YAAA,MAAM9B,QAAAA,GAAW8D,eAAAA,GAAkBC,IAAAA,CAAK9D,MAAAA,CAAO+D,IAAI,CAACC,MAAM,CAACC,MAAM,EAAEpC,IAAAA,CAAKW,GAAG,CAAA,GAAIX,KAAKW,GAAG;YACvF,IAAIf,KAAAA;YACJ,IAAI;gBACFA,KAAAA,GAAQ,MAAMR,uBAAAA,CAAwBlB,QAAAA,EAAUC,MAAAA,EAAQ6D,eAAAA,CAAAA;AAC1D,YAAA,CAAA,CAAE,OAAOK,GAAAA,EAAc;gBACrB,MAAMC,IAAAA,GACJD,GAAAA,IAAO,OAAOA,GAAAA,KAAQ,QAAA,IAAY,UAAUA,GAAAA,GACvCA,GAAAA,CAA8BC,IAAI,GACnCC,SAAAA;AACN,gBAAA,IAAID,SAAS,QAAA,EAAU;AACrBhB,oBAAAA,gBAAAA,CAAiBN,2BAA2BhB,IAAAA,EAAM9B,QAAAA,CAAAA,CAAAA;AAClD,oBAAA;AACF,gBAAA;gBACA,MAAMmE,GAAAA;AACR,YAAA;YACA,MAAMT,MAAAA,GAAS3D,aAAAA,CAAcC,QAAAA,EAAUC,MAAAA,EAAQ6D,eAAAA,CAAAA;YAE/C,MAAM;gBACJQ,QAAAA,EAAUxC,IAAAA;AACV9B,gBAAAA,QAAAA;AACAuE,gBAAAA,QAAAA,EAAUzC,IAAAA,CAAKmB,IAAI,GAAGnB,IAAAA,CAAK0C,GAAG;AAC9Bd,gBAAAA,MAAAA;gBACAhC,KAAAA,EAAO;AAAEC,oBAAAA,IAAAA,EAAMD,MAAMC;AAAK;AAC5B,aAAA;YAEA,IAAIG,IAAAA,CAAKY,OAAO,EAAE;AAChB,gBAAA,KAAK,MAAMC,MAAAA,IAAUC,MAAAA,CAAOC,IAAI,CAACf,IAAAA,CAAKY,OAAO,CAAA,CAAG;AAC9C,oBAAA,MAAM+B,UAAAA,GAAa3C,IAAAA,CAAKY,OAAO,CAACC,MAAAA,CAAO;AACvC,oBAAA,MAAM+B,kBAAAA,GAAqBZ,eAAAA,GACvBC,IAAAA,CAAK9D,MAAAA,CAAO+D,IAAI,CAACC,MAAM,CAACC,MAAM,EAAEO,UAAAA,CAAWhC,GAAG,CAAA,GAC9CgC,WAAWhC,GAAG;oBAClB,IAAIkC,eAAAA;oBACJ,IAAI;wBACFA,eAAAA,GAAkB,MAAMzD,uBAAAA,CACtBwD,kBAAAA,EACAzE,MAAAA,EACA6D,eAAAA,CAAAA;AAEJ,oBAAA,CAAA,CAAE,OAAOK,GAAAA,EAAc;wBACrB,MAAMC,IAAAA,GACJD,GAAAA,IAAO,OAAOA,GAAAA,KAAQ,QAAA,IAAY,UAAUA,GAAAA,GACvCA,GAAAA,CAA8BC,IAAI,GACnCC,SAAAA;AACN,wBAAA,IAAID,SAAS,QAAA,EAAU;4BACrBhB,gBAAAA,CAAiBN,0BAAAA,CAA2BhB,MAAM4C,kBAAAA,EAAoB/B,MAAAA,CAAAA,CAAAA;AACtE,4BAAA;AACF,wBAAA;wBACA,MAAMwB,GAAAA;AACR,oBAAA;oBACA,MAAMS,gBAAAA,GAAmB7E,aAAAA,CAAc2E,kBAAAA,EAAoBzE,MAAAA,EAAQ6D,eAAAA,CAAAA;AACnE,oBAAA,MAAMQ,QAAAA,GAAW;AAAE,wBAAA,GAAGG,UAAU;wBAAEI,IAAAA,EAAMlC,MAAAA;AAAQK,wBAAAA,EAAAA,EAAIlB,KAAKkB,EAAE;AAAE8B,wBAAAA,QAAAA,EAAUhD,KAAKmB;AAAK,qBAAA;oBACjF,MAAM;AACJqB,wBAAAA,QAAAA;wBACAtE,QAAAA,EAAU0E,kBAAAA;AACVH,wBAAAA,QAAAA,EAAUE,UAAAA,CAAWxB,IAAI,GAAGwB,UAAAA,CAAWD,GAAG;wBAC1Cd,MAAAA,EAAQkB,gBAAAA;wBACRlD,KAAAA,EAAO;AAAEC,4BAAAA,IAAAA,EAAMgD,gBAAgBhD;AAAK;AACtC,qBAAA;AACF,gBAAA;AACF,YAAA;AACF,QAAA;AACF,IAAA,CAAA;IAEA,OAAOoD,MAAAA,CAAOC,IAAI,CAACvB,SAAAA,EAAAA,CAAAA;AACrB;;;;"}