@arcane-utils/storage/dist/storage.cjs.map

Storage helper functions.

{"version":3,"sources":["../src/storage.ts"],"sourcesContent":["import { Storage, File, Bucket, CopyOptions, GetAclResponse } from '@google-cloud/storage'\nimport { v4 as uuid } from 'uuid'\nimport promiseRetry from 'promise-retry'\n\nimport { shouldRetryGoogleRequest } from '@arcane-utils/error'\n\nconst RETRY_CONFIG = {\n  retries: 5,\n  factor: 2,\n  minTimeout: 1000\n}\nexport const HUBMETRICS_FILE_FOLDER: string = 'hubmetrics_75012'\n\nexport const getStorageClient = (gcpProject: string, serviceAccount: string): Storage => {\n  return new Storage({\n    projectId: gcpProject,\n    keyFilename: serviceAccount\n  })\n}\n\nexport const getFileObject = (bucketName: string, fileName: string, storageClient: Storage): File => {\n  return storageClient.bucket(bucketName).file(fileName)\n}\n\nexport const getFileACL = async (file: File, aclQuery: any): Promise<GetAclResponse> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      return await file.acl.get(aclQuery)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not get ACL: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\n/**\n * Set file metadata\n * @example\n * metadata = {\n *   metadata: {\n *     key1: \"value1\",\n *     key2: \"value2\",\n *   }\n * }\n */\nexport const setFileMetadata = async (file: File, metadata: any): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await file.setMetadata(metadata)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not set metadata: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const copyFile = async (\n  sourceFile: File,\n  destinationFile: string | File | Bucket,\n  options?: CopyOptions\n): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await sourceFile.copy(destinationFile, options)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not copy ${sourceFile.name} to ${destinationFile}: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const downloadFile = async (file: File, destination: string): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await file.download({\n        destination\n      })\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not download ${file} to ${destination}: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const getFileHash = async (file: File): Promise<string> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      const [metadata] = await file.getMetadata()\n      console.log(`Crc32c: ${metadata.crc32c}`)\n      const { crc32c } = metadata\n      return crc32c\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not getFileMetadata ${file} : ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const uploadFile = async (bucket: Bucket, filePath: string, destFileName: string | File) => {\n  return promiseRetry(async (retry) => {\n    try {\n      await bucket.upload(filePath, {\n        destination: destFileName\n      })\n    } catch (error) {\n      console.log(`${filePath} uploaded to ${bucket.name}`)\n      retry(error)\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const deleteFiles = async (filesList: File[]): Promise<void> => {\n  return promiseRetry(async (retry) => {\n    try {\n      await Promise.all(filesList.map((file) => file.delete()))\n    } catch (error) {\n      retry(error)\n    }\n  }, RETRY_CONFIG)\n}\n\n/**\n *\n * @param filesList list of files to combine\n * @param filePath the result file path\n * @param bucket a working bucket used for file concatenation (not datalake bucket)\n * @param metadata optional metadata to set on the result file\n * @returns\n */\nexport const concatenateSubFiles = async (filesList: File[], outputFile: File, bucket: Bucket, logger: any = console, metadata?: Object): Promise<File> => {\n  return promiseRetry(async (retry, number) => {\n    let temporaryFolder = 1\n    let tempFilesList: File[] = [...filesList]\n    let fileToDelete: File[] = []\n\n    const executionId = uuid()\n\n    while (tempFilesList.length > 32) {\n      let index = 1\n      let filesToCombine: File[] = []\n      for (let idx = 0; idx < tempFilesList.length; idx += 1) {\n        filesToCombine.push(tempFilesList[idx])\n        if ((filesToCombine.length === 32) || (idx === filesList.length - 1)) {\n          await bucket.combine(filesToCombine, `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}/file_combined_${index}`)\n          index += 1\n          filesToCombine = []\n        }\n      }\n      [tempFilesList] = await bucket.getFiles({ prefix: `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}` })\n      // We only want to delete temp files but not files passed as arg in filesList\n      fileToDelete = fileToDelete.concat([...tempFilesList])\n      temporaryFolder += 1\n    }\n    try {\n      await bucket.combine(tempFilesList, outputFile)\n      if (metadata) {\n        outputFile.setMetadata({\n          metadata \n        })\n      }\n      await deleteFiles(fileToDelete)\n      return outputFile\n\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        logger.info(`Could not concatenate sub files. Attempt is ${number}. Trying again...`)\n        logger.info(error)\n        return retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,4BAAAE,EAAA,wBAAAC,EAAA,aAAAC,EAAA,gBAAAC,EAAA,iBAAAC,EAAA,eAAAC,EAAA,gBAAAC,EAAA,kBAAAC,EAAA,qBAAAC,EAAA,oBAAAC,EAAA,eAAAC,IAAA,eAAAC,EAAAb,GAAA,IAAAc,EAAmE,iCACnEC,EAA2B,gBAC3BC,EAAyB,8BAEzBC,EAAyC,+BAEnCC,EAAe,CACnB,QAAS,EACT,OAAQ,EACR,WAAY,GACd,EACahB,EAAiC,mBAEjCQ,EAAmB,CAACS,EAAoBC,IAC5C,IAAI,UAAQ,CACjB,UAAWD,EACX,YAAaC,CACf,CAAC,EAGUX,EAAgB,CAACY,EAAoBC,EAAkBC,IAC3DA,EAAc,OAAOF,CAAU,EAAE,KAAKC,CAAQ,EAG1Cf,EAAa,MAAOiB,EAAYC,OACpC,EAAAC,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,OAAO,MAAMJ,EAAK,IAAI,IAAIC,CAAQ,CACpC,OAASI,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACxFD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGX,CAAY,EAaJP,EAAkB,MAAOa,EAAYM,OACzC,EAAAJ,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,YAAYM,CAAQ,CACjC,OAASD,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,2BAA2BA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC7FD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGX,CAAY,EAGJd,EAAW,MACtB2B,EACAC,EACAC,OAEO,EAAAP,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMG,EAAW,KAAKC,EAAiBC,CAAO,CAChD,OAASJ,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,kBAAkBE,EAAW,IAAI,OAAOC,CAAe,KAAKH,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC9HD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGX,CAAY,EAGJZ,EAAe,MAAOkB,EAAYU,OACtC,EAAAR,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,SAAS,CAClB,YAAAU,CACF,CAAC,CACH,OAASL,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBL,CAAI,OAAOU,CAAW,KAAKL,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACnHD,EAAME,CAAK,EAEf,CACF,EAAGX,CAAY,EAGJV,EAAc,MAAOgB,MACzB,EAAAE,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,GAAM,CAACE,CAAQ,EAAI,MAAMN,EAAK,YAAY,EAC1C,QAAQ,IAAI,WAAWM,EAAS,MAAM,EAAE,EACxC,GAAM,CAAE,OAAAK,CAAO,EAAIL,EACnB,OAAOK,CACT,OAASN,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,6BAA6BL,CAAI,MAAMK,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACzGD,EAAME,CAAK,EAEf,CACF,EAAGX,CAAY,EAGJN,EAAa,MAAOwB,EAAgBC,EAAkBC,OAC1D,EAAAZ,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAMS,EAAO,OAAOC,EAAU,CAC5B,YAAaC,CACf,CAAC,CACH,OAAST,EAAO,CACd,QAAQ,IAAI,GAAGQ,CAAQ,gBAAgBD,EAAO,IAAI,EAAE,EACpDT,EAAME,CAAK,CACb,CACF,EAAGX,CAAY,EAGJb,EAAc,MAAOkC,MACzB,EAAAb,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAM,QAAQ,IAAIY,EAAU,IAAKf,GAASA,EAAK,OAAO,CAAC,CAAC,CAC1D,OAASK,EAAO,CACdF,EAAME,CAAK,CACb,CACF,EAAGX,CAAY,EAWJf,EAAsB,MAAOoC,EAAmBC,EAAkBJ,EAAgBK,EAAc,QAASX,OAC7G,EAAAJ,SAAa,MAAOC,EAAOC,IAAW,CAC3C,IAAIc,EAAkB,EAClBC,EAAwB,CAAC,GAAGJ,CAAS,EACrCK,EAAuB,CAAC,EAEtBC,KAAc,EAAAC,IAAK,EAEzB,KAAOH,EAAc,OAAS,IAAI,CAChC,IAAII,EAAQ,EACRC,EAAyB,CAAC,EAC9B,QAASC,EAAM,EAAGA,EAAMN,EAAc,OAAQM,GAAO,EACnDD,EAAe,KAAKL,EAAcM,CAAG,CAAC,GACjCD,EAAe,SAAW,IAAQC,IAAQV,EAAU,OAAS,KAChE,MAAMH,EAAO,QAAQY,EAAgB,GAAGR,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,kBAAkBK,CAAK,EAAE,EAC/HA,GAAS,EACTC,EAAiB,CAAC,GAGtB,CAACL,CAAa,EAAI,MAAMP,EAAO,SAAS,CAAE,OAAQ,GAAGI,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,EAAG,CAAC,EAEvHE,EAAeA,EAAa,OAAO,CAAC,GAAGD,CAAa,CAAC,EACrDD,GAAmB,CACrB,CACA,GAAI,CACF,aAAMN,EAAO,QAAQO,EAAeH,CAAU,EAC1CV,GACFU,EAAW,YAAY,CACrB,SAAAV,CACF,CAAC,EAEH,MAAMzB,EAAYuC,CAAY,EACvBJ,CAET,OAASX,EAAY,CACnB,MAAI,4BAAyBA,EAAM,IAAI,EACrC,OAAAY,EAAO,KAAK,+CAA+Cb,CAAM,mBAAmB,EACpFa,EAAO,KAAKZ,CAAK,EACVF,EAAME,CAAK,EAEpB,MAAMA,CACR,CACF,EAAGX,CAAY","names":["storage_exports","__export","HUBMETRICS_FILE_FOLDER","concatenateSubFiles","copyFile","deleteFiles","downloadFile","getFileACL","getFileHash","getFileObject","getStorageClient","setFileMetadata","uploadFile","__toCommonJS","import_storage","import_uuid","import_promise_retry","import_error","RETRY_CONFIG","gcpProject","serviceAccount","bucketName","fileName","storageClient","file","aclQuery","promiseRetry","retry","number","error","metadata","sourceFile","destinationFile","options","destination","crc32c","bucket","filePath","destFileName","filesList","outputFile","logger","temporaryFolder","tempFilesList","fileToDelete","executionId","uuid","index","filesToCombine","idx"]}