@arcane-utils/storage/dist/index.cjs.map

Storage helper functions.

{"version":3,"sources":["../src/index.ts","../src/storage.ts"],"sourcesContent":["export * as googleStorage from '@google-cloud/storage'\nexport {\n    HUBMETRICS_FILE_FOLDER,\n    getStorageClient,\n    getFileObject,\n    getFileACL,\n    setFileMetadata,\n    copyFile,\n    downloadFile,\n    getFileHash,\n    uploadFile,\n    deleteFiles,\n    concatenateSubFiles\n} from './storage';\n","import { Storage, File, Bucket, CopyOptions, GetAclResponse } from '@google-cloud/storage'\nimport { v4 as uuid } from 'uuid'\nimport promiseRetry from 'promise-retry'\n\nimport { shouldRetryGoogleRequest } from '@arcane-utils/error'\n\nconst RETRY_CONFIG = {\n  retries: 5,\n  factor: 2,\n  minTimeout: 1000\n}\nexport const HUBMETRICS_FILE_FOLDER: string = 'hubmetrics_75012'\n\nexport const getStorageClient = (gcpProject: string, serviceAccount: string): Storage => {\n  return new Storage({\n    projectId: gcpProject,\n    keyFilename: serviceAccount\n  })\n}\n\nexport const getFileObject = (bucketName: string, fileName: string, storageClient: Storage): File => {\n  return storageClient.bucket(bucketName).file(fileName)\n}\n\nexport const getFileACL = async (file: File, aclQuery: any): Promise<GetAclResponse> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      return await file.acl.get(aclQuery)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not get ACL: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\n/**\n * Set file metadata\n * @example\n * metadata = {\n *   metadata: {\n *     key1: \"value1\",\n *     key2: \"value2\",\n *   }\n * }\n */\nexport const setFileMetadata = async (file: File, metadata: any): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await file.setMetadata(metadata)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not set metadata: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const copyFile = async (\n  sourceFile: File,\n  destinationFile: string | File | Bucket,\n  options?: CopyOptions\n): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await sourceFile.copy(destinationFile, options)\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not copy ${sourceFile.name} to ${destinationFile}: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const downloadFile = async (file: File, destination: string): Promise<void> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      await file.download({\n        destination\n      })\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not download ${file} to ${destination}: ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const getFileHash = async (file: File): Promise<string> => {\n  return promiseRetry(async (retry, number) => {\n    try {\n      const [metadata] = await file.getMetadata()\n      console.log(`Crc32c: ${metadata.crc32c}`)\n      const { crc32c } = metadata\n      return crc32c\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        console.log(`Could not getFileMetadata ${file} : ${error.message}. Attempt is ${number}. Trying again...`)\n        retry(error)\n      }\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const uploadFile = async (bucket: Bucket, filePath: string, destFileName: string | File) => {\n  return promiseRetry(async (retry) => {\n    try {\n      await bucket.upload(filePath, {\n        destination: destFileName\n      })\n    } catch (error) {\n      console.log(`${filePath} uploaded to ${bucket.name}`)\n      retry(error)\n    }\n  }, RETRY_CONFIG)\n}\n\nexport const deleteFiles = async (filesList: File[]): Promise<void> => {\n  return promiseRetry(async (retry) => {\n    try {\n      await Promise.all(filesList.map((file) => file.delete()))\n    } catch (error) {\n      retry(error)\n    }\n  }, RETRY_CONFIG)\n}\n\n/**\n *\n * @param filesList list of files to combine\n * @param filePath the result file path\n * @param bucket a working bucket used for file concatenation (not datalake bucket)\n * @param metadata optional metadata to set on the result file\n * @returns\n */\nexport const concatenateSubFiles = async (filesList: File[], outputFile: File, bucket: Bucket, logger: any = console, metadata?: Object): Promise<File> => {\n  return promiseRetry(async (retry, number) => {\n    let temporaryFolder = 1\n    let tempFilesList: File[] = [...filesList]\n    let fileToDelete: File[] = []\n\n    const executionId = uuid()\n\n    while (tempFilesList.length > 32) {\n      let index = 1\n      let filesToCombine: File[] = []\n      for (let idx = 0; idx < tempFilesList.length; idx += 1) {\n        filesToCombine.push(tempFilesList[idx])\n        if ((filesToCombine.length === 32) || (idx === filesList.length - 1)) {\n          await bucket.combine(filesToCombine, `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}/file_combined_${index}`)\n          index += 1\n          filesToCombine = []\n        }\n      }\n      [tempFilesList] = await bucket.getFiles({ prefix: `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}` })\n      // We only want to delete temp files but not files passed as arg in filesList\n      fileToDelete = fileToDelete.concat([...tempFilesList])\n      temporaryFolder += 1\n    }\n    try {\n      await bucket.combine(tempFilesList, outputFile)\n      if (metadata) {\n        outputFile.setMetadata({\n          metadata \n        })\n      }\n      await deleteFiles(fileToDelete)\n      return outputFile\n\n    } catch (error: any) {\n      if (shouldRetryGoogleRequest(error.code)) {\n        logger.info(`Could not concatenate sub files. Attempt is ${number}. Trying again...`)\n        logger.info(error)\n        return retry(error)\n      }\n      throw error\n    }\n  }, RETRY_CONFIG)\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,4BAAAE,EAAA,wBAAAC,EAAA,aAAAC,EAAA,gBAAAC,EAAA,iBAAAC,EAAA,eAAAC,EAAA,gBAAAC,EAAA,kBAAAC,EAAA,qBAAAC,EAAA,kBAAAC,EAAA,oBAAAC,EAAA,eAAAC,IAAA,eAAAC,EAAAd,GAAA,IAAAW,EAA+B,sCCA/B,IAAAI,EAAmE,iCACnEC,EAA2B,gBAC3BC,EAAyB,8BAEzBC,EAAyC,+BAEnCC,EAAe,CACnB,QAAS,EACT,OAAQ,EACR,WAAY,GACd,EACaC,EAAiC,mBAEjCC,EAAmB,CAACC,EAAoBC,IAC5C,IAAI,UAAQ,CACjB,UAAWD,EACX,YAAaC,CACf,CAAC,EAGUC,EAAgB,CAACC,EAAoBC,EAAkBC,IAC3DA,EAAc,OAAOF,CAAU,EAAE,KAAKC,CAAQ,EAG1CE,EAAa,MAAOC,EAAYC,OACpC,EAAAC,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,OAAO,MAAMJ,EAAK,IAAI,IAAIC,CAAQ,CACpC,OAASI,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACxFD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAaJgB,EAAkB,MAAON,EAAYO,OACzC,EAAAL,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,YAAYO,CAAQ,CACjC,OAASF,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,2BAA2BA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC7FD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAGJkB,EAAW,MACtBC,EACAC,EACAC,OAEO,EAAAT,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMK,EAAW,KAAKC,EAAiBC,CAAO,CAChD,OAASN,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,kBAAkBI,EAAW,IAAI,OAAOC,CAAe,KAAKL,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC9HD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAGJsB,EAAe,MAAOZ,EAAYa,OACtC,EAAAX,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,SAAS,CAClB,YAAAa,CACF,CAAC,CACH,OAASR,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBL,CAAI,OAAOa,CAAW,KAAKR,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACnHD,EAAME,CAAK,EAEf,CACF,EAAGf,CAAY,EAGJwB,EAAc,MAAOd,MACzB,EAAAE,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,GAAM,CAACG,CAAQ,EAAI,MAAMP,EAAK,YAAY,EAC1C,QAAQ,IAAI,WAAWO,EAAS,MAAM,EAAE,EACxC,GAAM,CAAE,OAAAQ,CAAO,EAAIR,EACnB,OAAOQ,CACT,OAASV,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,6BAA6BL,CAAI,MAAMK,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACzGD,EAAME,CAAK,EAEf,CACF,EAAGf,CAAY,EAGJ0B,EAAa,MAAOC,EAAgBC,EAAkBC,OAC1D,EAAAjB,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAMc,EAAO,OAAOC,EAAU,CAC5B,YAAaC,CACf,CAAC,CACH,OAASd,EAAO,CACd,QAAQ,IAAI,GAAGa,CAAQ,gBAAgBD,EAAO,IAAI,EAAE,EACpDd,EAAME,CAAK,CACb,CACF,EAAGf,CAAY,EAGJ8B,EAAc,MAAOC,MACzB,EAAAnB,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAM,QAAQ,IAAIkB,EAAU,IAAKrB,GAASA,EAAK,OAAO,CAAC,CAAC,CAC1D,OAASK,EAAO,CACdF,EAAME,CAAK,CACb,CACF,EAAGf,CAAY,EAWJgC,EAAsB,MAAOD,EAAmBE,EAAkBN,EAAgBO,EAAc,QAASjB,OAC7G,EAAAL,SAAa,MAAOC,EAAOC,IAAW,CAC3C,IAAIqB,EAAkB,EAClBC,EAAwB,CAAC,GAAGL,CAAS,EACrCM,EAAuB,CAAC,EAEtBC,KAAc,EAAAC,IAAK,EAEzB,KAAOH,EAAc,OAAS,IAAI,CAChC,IAAII,EAAQ,EACRC,EAAyB,CAAC,EAC9B,QAASC,EAAM,EAAGA,EAAMN,EAAc,OAAQM,GAAO,EACnDD,EAAe,KAAKL,EAAcM,CAAG,CAAC,GACjCD,EAAe,SAAW,IAAQC,IAAQX,EAAU,OAAS,KAChE,MAAMJ,EAAO,QAAQc,EAAgB,GAAGR,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,kBAAkBK,CAAK,EAAE,EAC/HA,GAAS,EACTC,EAAiB,CAAC,GAGtB,CAACL,CAAa,EAAI,MAAMT,EAAO,SAAS,CAAE,OAAQ,GAAGM,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,EAAG,CAAC,EAEvHE,EAAeA,EAAa,OAAO,CAAC,GAAGD,CAAa,CAAC,EACrDD,GAAmB,CACrB,CACA,GAAI,CACF,aAAMR,EAAO,QAAQS,EAAeH,CAAU,EAC1ChB,GACFgB,EAAW,YAAY,CACrB,SAAAhB,CACF,CAAC,EAEH,MAAMa,EAAYO,CAAY,EACvBJ,CAET,OAASlB,EAAY,CACnB,MAAI,4BAAyBA,EAAM,IAAI,EACrC,OAAAmB,EAAO,KAAK,+CAA+CpB,CAAM,mBAAmB,EACpFoB,EAAO,KAAKnB,CAAK,EACVF,EAAME,CAAK,EAEpB,MAAMA,CACR,CACF,EAAGf,CAAY","names":["src_exports","__export","HUBMETRICS_FILE_FOLDER","concatenateSubFiles","copyFile","deleteFiles","downloadFile","getFileACL","getFileHash","getFileObject","getStorageClient","googleStorage","setFileMetadata","uploadFile","__toCommonJS","import_storage","import_uuid","import_promise_retry","import_error","RETRY_CONFIG","HUBMETRICS_FILE_FOLDER","getStorageClient","gcpProject","serviceAccount","getFileObject","bucketName","fileName","storageClient","getFileACL","file","aclQuery","promiseRetry","retry","number","error","setFileMetadata","metadata","copyFile","sourceFile","destinationFile","options","downloadFile","destination","getFileHash","crc32c","uploadFile","bucket","filePath","destFileName","deleteFiles","filesList","concatenateSubFiles","outputFile","logger","temporaryFolder","tempFilesList","fileToDelete","executionId","uuid","index","filesToCombine","idx"]}