@altostra/core/lib/bizops/ServiceClients/SyncManager/SyncManager.js

Core library for shared types and logic

"use strict";var _SyncManager_endpoints,_SyncManager_axios,__awaiter=this&&this.__awaiter||function(e,i,t,r){return new(t||(t=Promise))((function(a,o){function s(e){try{l(r.next(e))}catch(e){o(e)}}function n(e){try{l(r.throw(e))}catch(e){o(e)}}function l(e){var i;e.done?a(e.value):(i=e.value,i instanceof t?i:new t((function(e){e(i)}))).then(s,n)}l((r=r.apply(e,i||[])).next())}))},__classPrivateFieldSet=this&&this.__classPrivateFieldSet||function(e,i,t,r,a){if("m"===r)throw new TypeError("Private method is not writable");if("a"===r&&!a)throw new TypeError("Private accessor was defined without a setter");if("function"==typeof i?e!==i||!a:!i.has(e))throw new TypeError("Cannot write private member to an object whose class did not declare it");return"a"===r?a.call(e,t):a?a.value=t:i.set(e,t),t},__classPrivateFieldGet=this&&this.__classPrivateFieldGet||function(e,i,t,r){if("a"===t&&!r)throw new TypeError("Private accessor was defined without a getter");if("function"==typeof i?e!==i||!r:!i.has(e))throw new TypeError("Cannot read private member from an object whose class did not declare it");return"m"===t?r:"a"===t?r.call(e):r?r.value:i.get(e)},__importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.SyncManager=void 0;const fs_1=require("fs"),path_1=require("path"),util_1=require("util"),BlueprintHelper_1=require("../../../blueprint/Helpers/BlueprintHelper"),NonEmptyString_1=require("../../../common/CustomTypes/NonEmptyString"),Errors_1=require("../../../common/Errors"),Utils_1=require("../../../common/Utils"),Axios_1=require("../../../common/Utils/Axios"),FS_1=require("../../../common/Utils/FS"),primitives_1=require("@altostra/type-validations/lib/primitives"),from_1=require("@reactivex/ix-es2015-cjs/iterable/from"),filter_1=require("@reactivex/ix-es2015-cjs/iterable/operators/filter"),flatmap_1=require("@reactivex/ix-es2015-cjs/iterable/operators/flatmap"),axios_1=__importDefault(require("axios")),form_data_1=__importDefault(require("form-data")),FilesSources_1=require("../../FilesSources"),BuildsManager_1=require("../BuildsManager"),ConfigurableServiceClient_1=require("../ServiceClientBase/ConfigurableServiceClient"),ProjectValidatingServiceClient_1=require("../ServiceClientBase/ProjectValidatingServiceClient"),Types_1=require("./Types"),RETRIES_COUNT=5;class SyncManager extends BuildsManager_1.BuildsServiceClientBase{constructor({axios:e=axios_1.default.create(),endpoints:i,getToken:t,logger:r,config:a,errorMapping:o={}}){super({axios:e,getToken:t,logger:r,[ConfigurableServiceClient_1.configSym]:null!=a?a:BuildsManager_1.defaultConfig,[ProjectValidatingServiceClient_1.backofficeUrlSym]:i.backoffice,errorMapping:o}),_SyncManager_endpoints.set(this,void 0),_SyncManager_axios.set(this,void 0),__classPrivateFieldSet(this,_SyncManager_endpoints,i,"f"),__classPrivateFieldSet(this,_SyncManager_axios,e,"f")}sync(e,i){var t;return __awaiter(this,void 0,void 0,(function*(){if("CUSTOM"===i.type&&0===i.stores.length)throw Errors_1.AltoError.create("Stores' names required");const r="ALL"===i.type,a="CUSTOM"===i.type?i.stores:[],{project:{blueprints:o,id:s}}=yield this.validateProjectByType(e,"altostra"),n=o.map(e=>new BlueprintHelper_1.BlueprintHelper(e)),l=yield this._withLogging("Validating buckets",()=>__awaiter(this,void 0,void 0,(function*(){const i=[...(0,from_1.from)(n).pipe((0,flatmap_1.flatMap)(e=>e.resources.ofType("resource.file-store").pipe((0,filter_1.filter)(e=>!!e.resource.filesSources&&e.resource.filesSources.length>0),(0,filter_1.filter)(e=>r||a.includes(e.name)))))];return yield this._mapAndValidateFilesExistence(i,e)}))),d=yield this._withLogging("Getting repository information",()=>__awaiter(this,void 0,void 0,(function*(){const e={bucketsIds:Object.keys(l).map(NonEmptyString_1.nonEmptyString)};return yield this._request({request:t=>__awaiter(this,void 0,void 0,(function*(){return yield t.post(buildEP(__classPrivateFieldGet(this,_SyncManager_endpoints,"f").backoffice,s,i.deploymentName,"sync"),e,{headers:yield this._getHeaders()})})),dataValidation:Types_1.isSyncCommandRes,errorWrapping:{category:"server-fault",message:"Failed to get repository information",data:e}})}))),c=Object.keys(d);for(const e of c)yield this._withLogging("Syncing files to "+(null===(t=l[e])||void 0===t?void 0:t.storeDisplayName),()=>__awaiter(this,void 0,void 0,(function*(){var t;yield this._uploadFilesToStoreUsingPresignedUrl(d[e],l[e],null!==(t=i.accessLevel)&&void 0!==t?t:"private")})))}))}_mapAndValidateFilesExistence(e,i){return __awaiter(this,void 0,void 0,(function*(){const t={};for(const r of e){const e=[];for(const t of r.resource.filesSources){const r=yield(0,FilesSources_1.validateFileSource)(t,{projectDir:i});if(!r.isValid)throw Errors_1.UserFacingError.create("File validation failed. "+r.invalidFiles[0].reason,{category:"user-fault",data:{validation:r}});e.push(...r.validFiles)}t[r.id]={files:e,storeDisplayName:r.name}}return t}))}_uploadFilesToStoreUsingPresignedUrl(e,i,t){return __awaiter(this,void 0,void 0,(function*(){const r=yield this._uploadFilesToStore({uploadInfo:e,uploadFiles:null==i?void 0:i.files,storeDisplayName:i.storeDisplayName,accessLevel:t});yield(0,Utils_1.runWithLimit)(this.config.runWithLimit,r),this._logger.verbose(`Uploaded ${r.length} files to ${null==i?void 0:i.storeDisplayName}`)}))}_uploadFilesToStore({uploadInfo:e,uploadFiles:i,storeDisplayName:t,accessLevel:r,rootPath:a}){return __awaiter(this,void 0,void 0,(function*(){const o=this,s=[],n=void 0===a;for(const l of i){const i=yield fs_1.promises.stat(l);if(void 0===a&&(a=l),i.isDirectory()){const i=(yield fs_1.promises.readdir(l)).map(e=>(0,path_1.join)(l,e)),o=yield this._uploadFilesToStore({uploadInfo:e,uploadFiles:i,storeDisplayName:t,accessLevel:r,rootPath:a});s.push(...o)}else{let i=0;s.push((function s(){return __awaiter(this,void 0,void 0,(function*(){const n=(0,fs_1.createReadStream)(l),d=(0,path_1.relative)(a,l),{formData:c,length:u}=yield o._createFormData({presignedUrlResponse:e,file:l,uploadPath:d,accessLevel:r,fileData:n});o._logger.verbose("Pushing file: ",{file:l,url:e.url});try{yield __classPrivateFieldGet(o,_SyncManager_axios,"f").post(e.url,c,{headers:Object.assign(Object.assign({},c.getHeaders()),{"Content-Length":u}),maxContentLength:1/0,maxBodyLength:1/0}),o._logger.verbose("Uploaded: "+d)}catch(r){if(r=(0,Axios_1.isAxiosError)(r)?(0,Axios_1.jsonableAxiosError)(r):r,i<5)return o._logger.verbose(`Failed to upload a file for the ${i+1} times... retrying`,{error:r,file:l,url:e.url}),i++,yield s();const a=`Failed to upload '${l}' to bucket '${t}'. The bucket has denied the file upload because of a mismatch between file signatures.`;throw o._logger.verbose(a+" Make sure you're using the latest version of the Altostra CLI by running 'npm install -g @altostra/cli' and try again. If the error persists, please contact support."),Errors_1.UserFacingError.wrapError(r,{category:"server-fault",type:"server-error",message:a,data:{file:l,storeDisplayName:t}})}finally{n.close(),c.destroy()}}))}))}n&&(a=void 0)}return s}))}_createFormData({presignedUrlResponse:e,file:i,uploadPath:t,accessLevel:r,fileData:a}){return __awaiter(this,void 0,void 0,(function*(){const o=new form_data_1.default;Object.keys(null==e?void 0:e.fields).forEach(i=>{o.append(i,null==e?void 0:e.fields[i])}),o.append("acl",r),o.append("Content-Type",(0,FS_1.getMimeType)(i)),o.append("key",t),o.append("file",a);const s=(0,util_1.promisify)(o.getLength.bind(o)),n=yield s();return{formData:o,length:n}}))}invalidate(e,i){return __awaiter(this,void 0,void 0,(function*(){if("CUSTOM"===i.type&&0===i.cdns.length)throw Errors_1.AltoError.create("You must provide at least one CDN name.");const t="ALL"===i.type,r="CUSTOM"===i.type?i.cdns:[],{project:{blueprints:a,id:o}}=yield this.validateProjectByType(e,"altostra"),s=a.map(e=>new BlueprintHelper_1.BlueprintHelper(e)),n=yield this._withLogging("Validating CDNs",()=>__awaiter(this,void 0,void 0,(function*(){return[...(0,from_1.from)(s).pipe((0,flatmap_1.flatMap)(e=>e.resources.ofType("resource.cdn").pipe((0,filter_1.filter)(e=>t||r.includes((0,NonEmptyString_1.nonEmptyString)(e.name))))))].reduce((e,i)=>(e[i.id]=i,e),{})})));yield this._withLogging("Invalidating CDN caches",()=>__awaiter(this,void 0,void 0,(function*(){var e,t;const r={cdnIds:Object.keys(n).map(NonEmptyString_1.nonEmptyString),paths:null!==(t=null===(e=i.paths)||void 0===e?void 0:e.map(NonEmptyString_1.nonEmptyString))&&void 0!==t?t:[(0,NonEmptyString_1.nonEmptyString)("/*")]};yield this._request({request:e=>__awaiter(this,void 0,void 0,(function*(){return yield e.post(buildEP(__classPrivateFieldGet(this,_SyncManager_endpoints,"f").backoffice,o,i.deploymentName,"invalidate"),r,{headers:yield this._getHeaders()})})),dataValidation:primitives_1.any,errorWrapping:{category:"server-fault",message:"Failed to invalidate CDNs",data:r}})}))),this._logger.complete("Invalidation process is in progress. This may take a while.")}))}}function buildEP(e,i,t,r){return`${e}/projects/${i}/deployments/${t}/${r}`}exports.SyncManager=SyncManager,_SyncManager_endpoints=new WeakMap,_SyncManager_axios=new WeakMap;