@altostra/core/lib/common/Utils/AWS/DynamoDB/BatchOperations.js

Core library for shared types and logic

"use strict";var __awaiter=this&&this.__awaiter||function(e,t,n,r){return new(n||(n=Promise))((function(o,s){function i(e){try{c(r.next(e))}catch(e){s(e)}}function a(e){try{c(r.throw(e))}catch(e){s(e)}}function c(e){var t;e.done?o(e.value):(t=e.value,t instanceof n?t:new n((function(e){e(t)}))).then(i,a)}c((r=r.apply(e,t||[])).next())}))},__rest=this&&this.__rest||function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&t.indexOf(r)<0&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols){var o=0;for(r=Object.getOwnPropertySymbols(e);o<r.length;o++)t.indexOf(r[o])<0&&Object.prototype.propertyIsEnumerable.call(e,r[o])&&(n[r[o]]=e[r[o]])}return n};Object.defineProperty(exports,"__esModule",{value:!0}),exports.queryAndDelete=exports.batchWrite=exports.batchRead=exports.exponentialDelay=void 0;const from_1=require("@reactivex/ix-es2015-cjs/iterable/from"),flatmap_1=require("@reactivex/ix-es2015-cjs/iterable/operators/flatmap"),map_1=require("@reactivex/ix-es2015-cjs/iterable/operators/map"),dynamodb_1=require("aws-sdk/clients/dynamodb"),Numerics_1=require("../../../CustomTypes/Numerics"),Errors_1=require("../../../Errors"),Logging_1=require("../../../Logging"),Concurrency_1=require("../../Concurrency"),Object_1=require("../../Object"),common_1=require("./common"),RETRY_DELAY_FACTOR=(0,Numerics_1.naturalNumber)(10),MAX_RETRY_COUNT=(0,Numerics_1.naturalNumber)(8);function exponentialDelay(e=RETRY_DELAY_FACTOR){return t=>Math.pow(2,t)*e}function batchRead({documentClient:e=new dynamodb_1.DocumentClient,batchRead:t,nextDelay:n=exponentialDelay(),maxRetriesCount:r=MAX_RETRY_COUNT,logger:o=(0,Logging_1.nullLogger)()}){return __awaiter(this,void 0,void 0,(function*(){const s=batching(common_1.MAX_BATCH_ITEMS_READ_COUNT,t.RequestItems,e=>e.Keys,(e,t)=>Object.assign(Object.assign({},e),{Keys:t})),i={};for(const e of s){const n=yield a(Object.assign(Object.assign({},t),{RequestItems:e}),0);for(const[e,t]of Object.entries(n))e in i?i[e].push(...t):i[e]=t}return i;function a(t,s){var i,c;return __awaiter(this,void 0,void 0,(function*(){o.verbose("Batch getting items...");const l=yield e.batchGet(t).promise();if(!l.UnprocessedKeys||0===Object.keys(l.UnprocessedKeys).length)return o.verbose("All items retrieved."),null!==(i=l.Responses)&&void 0!==i?i:{};if(s>r)throw Errors_1.AltoError.create("Can't get data from the table",{data:{batchRead:t,unprocessedKeys:l.UnprocessedKeys}});yield(0,Concurrency_1.delay)(n(s));const u=null!==(c=l.Responses)&&void 0!==c?c:{};o.verbose("Getting the rest of items...");const m=yield a({RequestItems:l.UnprocessedKeys},s+1);for(const[e,t]of Object.entries(m))e in u?u[e].push(...t):u[e]=t;return u}))}}))}function batchWrite({documentClient:e=new dynamodb_1.DocumentClient,batchWrite:t,nextDelay:n=exponentialDelay(),maxRetriesCount:r=MAX_RETRY_COUNT,logger:o=(0,Logging_1.nullLogger)()}){return __awaiter(this,void 0,void 0,(function*(){const s=batching(common_1.MAX_BATCH_ITEMS_WRITE_COUNT,t.RequestItems,e=>e,(e,t)=>t);for(const e of s)yield i(Object.assign(Object.assign({},t),{RequestItems:e}),0);function i(t,s){return __awaiter(this,void 0,void 0,(function*(){o.verbose("Batch putting/deleting items...");const a=yield e.batchWrite(t).promise();if(a.UnprocessedItems&&0!==Object.keys(a.UnprocessedItems).length){if(s>r)throw Errors_1.AltoError.create("Can't save data to the table",{data:{batchWrite:t,unprocessedItems:a.UnprocessedItems}});yield(0,Concurrency_1.delay)(n(s)),o.verbose("Writing the rest of items..."),yield i({RequestItems:a.UnprocessedItems},s+1)}else o.verbose("All items have been written.")}))}}))}function queryAndDelete(e){var{tableName:t,query:n,indexName:r,pk:o,sk:s,documentClient:i=new dynamodb_1.DocumentClient,logger:a=(0,Logging_1.nullLogger)()}=e,c=__rest(e,["tableName","query","indexName","pk","sk","documentClient","logger"]);return __awaiter(this,void 0,void 0,(function*(){const e=Object.keys(n);if(1!==e.length)throw Errors_1.AltoError.create("Invalid query. Query must include a single key for the PK.",{data:{invalidQuery:n}});a.verbose("Preparing to query items for deletion");const l=e[0],u=n[l];null!=o||(o=l);const m=o!==l;if(0===u.length)return a.verbose("Nothing to query"),0;let _=0;for(let e=0;e<u.length;e+=common_1.MAX_BATCH_ITEMS_READ_COUNT){const n=u.slice(e,e+common_1.MAX_BATCH_ITEMS_READ_COUNT),d=["#key"],p=s&&s!==l&&s!==o;p&&d.push("#sk"),m&&d.push("#pk"),a.verbose(`Querying for ${n.length} items out of ${u.length} deleted IDs.`);const g=yield Promise.all(n.map(e=>i.query(Object.assign(Object.assign({TableName:t},r&&{IndexName:r}),{KeyConditionExpression:"#key = :id",ProjectionExpression:d.join(","),ExpressionAttributeNames:Object.assign(Object.assign({"#key":l},p&&{"#sk":s}),m&&{"#pk":o}),ExpressionAttributeValues:{":id":e}})).promise()));let b=(0,from_1.from)(g).pipe((0,flatmap_1.flatMap)(({Items:e})=>null!=e?e:[]));m&&(b=b.pipe((0,map_1.map)(e=>Object.assign({[o]:e[o]},s&&{[s]:e[s]}))));const y=[...b];_+=y.length;for(let e=0;e<y.length;e+=common_1.MAX_BATCH_ITEMS_WRITE_COUNT){const n=y.slice(e,e+common_1.MAX_BATCH_ITEMS_WRITE_COUNT);a.verbose(`Deleting ${n.length} items out of a total of ${y.length} queried items`),yield batchWrite(Object.assign(Object.assign({},c),{logger:a,documentClient:i,batchWrite:{RequestItems:{[t]:n.map(e=>({DeleteRequest:{Key:e}}))}}}))}}return _}))}function*batching(e,t,n,r){let o,s=Object.entries(t).map(([e,t])=>[e,{list:t,items:n(t)}]);do{o=[];const t=[];for(const[n,{list:i,items:a}]of s){const s=a.splice(Math.max(0,a.length-e));s.length>0&&t.push([n,r(i,s)]),a.length>0&&o.push([n,{list:i,items:a}])}t.length>0&&(yield(0,Object_1.fromEntries)(t)),s=o}while(o.length>0)}exports.exponentialDelay=exponentialDelay,exports.batchRead=batchRead,exports.batchWrite=batchWrite,exports.queryAndDelete=queryAndDelete;