lemon-core
Version:
Lemon Serverless Micro-Service Platform
1,076 lines • 49.8 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DummyDynamoService = exports.DynamoService = void 0;
/**
* `dynamo-service.ts`
* - common service for dynamo
*
* @author Steve Jung <steve@lemoncloud.io>
* @date 2019-08-28 initial version
* @date 2019-10-16 cleanup and optimize log
* @date 2019-11-19 optimize 404 error case, and normalize key.
* @date 2019-12-10 support `DummyDynamoService.listItems()` for mocks
*
* @copyright (C) 2019 LemonCloud Co Ltd. - All Rights Reserved.
*/
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const engine_1 = __importStar(require("../../engine/"));
const tools_1 = require("../../tools/");
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
const client_dynamodb_2 = require("@aws-sdk/client-dynamodb");
const lib_dynamodb_1 = require("@aws-sdk/lib-dynamodb");
const client_dynamodb_streams_1 = require("@aws-sdk/client-dynamodb-streams");
const NS = engine_1.$U.NS('DYNA', 'green'); // NAMESPACE TO BE PRINTED.
//* Batch operation constants
const BATCH_GET_CHUNK = 100; // DynamoDB BatchGetItem limit
const BATCH_WRITE_CHUNK = 25; // DynamoDB BatchWriteItem limit
const MAX_RETRIES = 3; // Maximum retry attempts for unprocessed items
//* create(or get) instance.
const instance = () => {
const region = 'ap-northeast-2';
return DynamoService.instance(region);
};
//* normalize dynamo properties.
const normalize = (data) => {
if (data === '')
return null;
if (!data)
return data;
if (Array.isArray(data))
return data.map(normalize);
if (typeof data == 'object') {
return Object.keys(data).reduce((O, key) => {
const val = data[key];
O[key] = normalize(val);
return O;
}, {});
}
return data;
};
/**
* class: `DynamoService`
* - basic CRUD service for AWS DynamoDB.
*/
class DynamoService {
constructor(options) {
/**
* say hello of identity.
*/
this.hello = () => `dynamo-service:${this.options.tableName}`;
// eslint-disable-next-line prettier/prettier
(0, engine_1._inf)(NS, `DynamoService(${options.tableName}/${options.idName}${options.sortName ? '/' : ''}${options.sortName || ''})...`);
if (!options.tableName)
throw new Error('.tableName is required');
if (!options.idName)
throw new Error('.idName is required');
this.options = options;
}
/**
* simple instance maker.
* @param region (default as `ap-northeast-2`)
*/
static instance(region) {
var _a, _b;
region = `${region || 'ap-northeast-2'}`;
const $cfg = (0, tools_1.awsConfig)(engine_1.default, region);
const dynamo = (_a = DynamoService._dynamo[region]) !== null && _a !== void 0 ? _a : new client_dynamodb_2.DynamoDBClient($cfg); // Low-level DynamoDB client
const dynamostr = (_b = DynamoService._dynamostr[region]) !== null && _b !== void 0 ? _b : new client_dynamodb_streams_1.DynamoDBStreamsClient($cfg); // DynamoDB Streams client
// Create memoized function for dynamodoc
if (!DynamoService._dynamodoc[region]) {
const cache = { client: null };
DynamoService._dynamodoc[region] = () => __awaiter(this, void 0, void 0, function* () {
if (!cache.client) {
(0, engine_1._log)(NS, `! creating NEW DynamoDBDocumentClient for region[${region}]`);
const credentials = $cfg.credentials;
const dynamo = new client_dynamodb_2.DynamoDBClient(Object.assign(Object.assign({}, $cfg), { credentials }));
cache.client = lib_dynamodb_1.DynamoDBDocumentClient.from(dynamo);
}
return cache.client;
});
}
const dynamodoc = DynamoService._dynamodoc[region]; // High-level Document client
return { dynamo, dynamostr, dynamodoc };
}
/**
* prepare `CreateTable` payload.
*
* @param ReadCapacityUnits
* @param WriteCapacityUnits
* @param StreamEnabled
*/
prepareCreateTable(ReadCapacityUnits = 1, WriteCapacityUnits = 1, StreamEnabled = true) {
const { tableName, idName, sortName, idType, sortType } = this.options;
(0, engine_1._log)(NS, `prepareCreateTable(${tableName}, ${idName}, ${sortName || ''}, ${sortType || ''})...`);
const keyType = (type = '') => {
type = type || 'string';
switch (type) {
case 'number':
return 'N';
case 'string':
return 'S';
default:
break;
}
throw new Error(`invalid key-type:${type}`);
};
//* prepare payload.
const payload = {
TableName: tableName,
KeySchema: [
{
AttributeName: idName,
KeyType: 'HASH',
},
],
AttributeDefinitions: [
{
AttributeName: idName,
AttributeType: keyType(idType),
},
],
ProvisionedThroughput: { ReadCapacityUnits, WriteCapacityUnits },
StreamSpecification: { StreamEnabled, StreamViewType: client_dynamodb_1.StreamViewType.NEW_AND_OLD_IMAGES },
};
//* set sort-key.
if (sortName) {
payload.KeySchema.push({
AttributeName: sortName,
KeyType: 'RANGE',
});
payload.AttributeDefinitions.push({
AttributeName: sortName,
AttributeType: keyType(sortType),
});
}
//* returns.
return payload;
}
/**
* prepare `DeleteTable` payload.
*/
prepareDeleteTable() {
const { tableName } = this.options;
(0, engine_1._log)(NS, `prepareDeleteTable(${tableName})...`);
return {
TableName: tableName,
};
}
/**
* prepare `SaveItem` payload.
*
* @param id partition-key
* @param item
*/
prepareSaveItem(id, item) {
const { tableName, idName, sortName } = this.options;
// _log(NS, `prepareSaveItem(${tableName})...`);
// item && _log(NS, '> item =', item);
if (sortName && item[sortName] === undefined)
throw new Error(`.${sortName} is required. ${idName}:${id}`);
delete item[idName]; // clear the saved id.
const node = Object.assign({ [idName]: id }, item); // copy
const data = normalize(node);
//* prepare payload.
const payload = {
TableName: tableName,
Item: data,
};
return payload;
}
/**
* prepare `Key` by id + sort key.
*
* @param id partition-key
* @param sort sort-key
*/
prepareItemKey(id, sort) {
const { tableName, idName, sortName } = this.options;
if (!id)
throw new Error(`@id is required - prepareItemKey(${tableName}/${idName})`);
// _log(NS, `prepareItemKey(${tableName}/${id}/${sort || ''})...`);
//* prepare payload.
const payload = {
TableName: tableName,
Key: {
[idName]: id,
},
};
if (sortName) {
if (sort === undefined)
throw new Error(`@sort is required. ${idName}:${id}`);
payload.Key[sortName] = sort;
}
return payload;
}
/**
* prepare `UpdateItem` payload.
*
* @param id partition-key
* @param sort sort-key
* @param $update update set
* @param $increment increment set.
*/
prepareUpdateItem(id, sort, $update, $increment) {
const debug = 0 ? true : false;
const { tableName, idName, sortName } = this.options;
debug && (0, engine_1._log)(NS, `prepareUpdateItem(${tableName}/${id}/${sort || ''})...`);
debug && $update && (0, engine_1._log)(NS, `> $update =`, engine_1.$U.json($update));
debug && $increment && (0, engine_1._log)(NS, `> $increment =`, engine_1.$U.json($increment));
const Key = this.prepareItemKey(id, sort).Key;
const norm = (_) => `${_}`.replace(/[.\\:\/$]/g, '_');
//* prepare payload.
let payload = Object.entries($update).reduce((memo, [key, value]) => {
//* ignore if key
if (key === idName || key === sortName)
return memo;
const key2 = norm(key);
value = normalize(value);
if (value && Array.isArray(value.setIndex)) {
//* support set items in list
value.setIndex.forEach(([idx, value], seq) => {
if (idx !== undefined && value !== undefined) {
memo.ExpressionAttributeNames[`#${key2}`] = key;
memo.ExpressionAttributeValues[`:${key2}_${seq}_`] = value;
memo.UpdateExpression.SET.push(`#${key2}[${idx}] = :${key2}_${seq}_`);
}
});
}
else if (value && Array.isArray(value.removeIndex)) {
//* support removing items from list
value.removeIndex.forEach((idx) => {
if (idx !== undefined) {
memo.ExpressionAttributeNames[`#${key2}`] = key2;
memo.UpdateExpression.REMOVE.push(`#${key2}[${idx}]`);
}
});
}
else {
//* prepare update-expression.
memo.ExpressionAttributeNames[`#${key2}`] = key;
memo.ExpressionAttributeValues[`:${key2}`] = value === '' ? null : value;
memo.UpdateExpression.SET.push(`#${key2} = :${key2}`);
debug && (0, engine_1._log)(NS, '>> ' + `#${key} :=`, typeof value, engine_1.$U.json(value));
}
return memo;
}, {
TableName: tableName,
Key,
UpdateExpression: { SET: [], REMOVE: [], ADD: [], DELETE: [] },
ExpressionAttributeNames: {},
ExpressionAttributeValues: {},
ConditionExpression: null,
ReturnValues: 'UPDATED_NEW',
});
//* prepare increment update.
if ($increment) {
//* increment field.
payload = Object.entries($increment).reduce((memo, [key, value]) => {
const key2 = norm(key);
if (!Array.isArray(value)) {
memo.ExpressionAttributeNames[`#${key2}`] = key;
memo.ExpressionAttributeValues[`:${key2}`] = value;
memo.UpdateExpression.ADD.push(`#${key2} :${key2}`);
debug && (0, engine_1._log)(NS, '>> ' + `#${key2} = #${key2} + :${value}`);
}
else {
memo.ExpressionAttributeNames[`#${key2}`] = key; // target attribute name
memo.ExpressionAttributeValues[`:${key2}`] = value; // list to append like `[1,2,3]`
memo.ExpressionAttributeValues[`:${key2}_0`] = []; // empty array if not exists.
memo.UpdateExpression.SET.push(`#${key2} = list_append(if_not_exists(#${key2}, :${key2}_0), :${key2})`);
debug && (0, engine_1._log)(NS, '>> ' + `#${key2} = #${key2} + ${value}`);
}
return memo;
}, payload);
}
//* build final update expression.
payload.UpdateExpression = Object.keys(payload.UpdateExpression) // ['SET', 'REMOVE', 'ADD', 'DELETE']
.map(actionName => {
const actions = payload.UpdateExpression[actionName];
return actions.length > 0 ? `${actionName} ${actions.join(', ')}` : ''; // e.g 'SET #a = :a, #b = :b'
})
.filter(exp => exp.length > 0)
.join(' ');
(0, engine_1._log)(NS, `> UpdateExpression[${id}] =`, payload.UpdateExpression);
return payload;
}
/**
* create-table
*
* @param ReadCapacityUnits
* @param WriteCapacityUnits
*/
createTable(ReadCapacityUnits = 1, WriteCapacityUnits = 1) {
return __awaiter(this, void 0, void 0, function* () {
(0, engine_1._log)(NS, `createTable(${ReadCapacityUnits}, ${WriteCapacityUnits})...`);
const payload = this.prepareCreateTable(ReadCapacityUnits, WriteCapacityUnits);
return instance()
.dynamo.send(new client_dynamodb_2.CreateTableCommand(payload))
.then(res => {
(0, engine_1._log)(NS, '> createTable.res =', res);
return res;
});
});
}
/**
* delete-table
*
*/
deleteTable() {
return __awaiter(this, void 0, void 0, function* () {
(0, engine_1._log)(NS, `deleteTable()...`);
const payload = this.prepareDeleteTable();
return instance()
.dynamo.send(new client_dynamodb_2.DeleteTableCommand(payload))
.then(res => {
(0, engine_1._log)(NS, '> deleteTable.res =', res);
return res;
});
});
}
/**
* read-item
* - read whole data of item.
*
* @param id
* @param sort
*/
readItem(id, sort) {
return __awaiter(this, void 0, void 0, function* () {
const { tableName, idName, sortName } = this.options;
// _log(NS, `readItem(${id})...`);
const itemKey = this.prepareItemKey(id, sort);
// _log(NS, `> pkey[${id}${sort ? '/' : ''}${sort || ''}] =`, $U.json(itemKey));
const dynamodoc = yield instance().dynamodoc();
return dynamodoc
.send(new lib_dynamodb_1.GetCommand(itemKey))
.then(res => {
// _log(NS, '> readItem.res =', $U.json(res));
if (!res.Item)
throw new Error(`404 NOT FOUND - ${idName}:${id}${sort ? '/' : ''}${sort || ''}`);
return res.Item;
})
.catch((e) => {
if (`${e.message}` == 'Requested resource not found')
throw new Error(`404 NOT FOUND - ${idName}:${id}`);
throw e;
});
});
}
/**
* multiple read-item
* - chunks requests into groups of 100 (DynamoDB BatchGetItem limit)
*
* @param list array of items with { id, sort? }
* @returns BatchResult with success/failed items
*/
mreadItem(list) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
const { tableName, idName, sortName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
const dynamodoc = yield DynamoService.instance().dynamodoc();
const _convertKeyToSig = (key) => {
const id = key[idName];
const sort = sortName ? key[sortName] : undefined;
return `${id}${sortName ? '/' + sort : ''}`;
};
const _convertItemToSig = (item) => {
const id = item[idName];
const sort = sortName ? item[sortName] : undefined;
return `${id}${sortName ? '/' + sort : ''}`;
};
//* Maps to preserve original order
const successMap = new Map();
const failedMap = new Map();
for (let i = 0; i < total; i += BATCH_GET_CHUNK) {
const chunk = (_b = list === null || list === void 0 ? void 0 : list.slice(i, i + BATCH_GET_CHUNK)) !== null && _b !== void 0 ? _b : [];
const chunkKeyMap = new Map();
const chunkKeys = chunk.map(item => {
const key = this.prepareItemKey(item.id, item.sort).Key;
const sig = _convertKeyToSig(key);
if (!chunkKeyMap.has(sig))
chunkKeyMap.set(sig, key);
return key;
});
const _doBatchGet = (keys, attempt, acc) => __awaiter(this, void 0, void 0, function* () {
var _c, _d, _e, _f, _g;
const response = yield dynamodoc
.send(new lib_dynamodb_1.BatchGetCommand({ RequestItems: { [tableName]: { Keys: keys } } }))
.catch((e) => {
if (`${e.message}`.includes('ResourceNotFoundException'))
throw new Error(`404 NOT FOUND - table:${tableName}`);
throw e;
});
const items = ((_d = (_c = response.Responses) === null || _c === void 0 ? void 0 : _c[tableName]) !== null && _d !== void 0 ? _d : []);
if (items.length > 0)
acc.push(...items);
const unprocessed = ((_g = (_f = (_e = response.UnprocessedKeys) === null || _e === void 0 ? void 0 : _e[tableName]) === null || _f === void 0 ? void 0 : _f.Keys) !== null && _g !== void 0 ? _g : []);
if (unprocessed.length > 0) {
if (attempt >= MAX_RETRIES)
return { items: acc, unprocessed };
//* exponential backoff
yield new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 100));
return _doBatchGet(unprocessed, attempt + 1, acc);
}
return { items: acc, unprocessed: [] };
});
try {
const { items, unprocessed } = yield _doBatchGet(chunkKeys, 1, []);
items.forEach(item => {
const sig = _convertItemToSig(item);
successMap.set(sig, item);
});
//* Handle unprocessed items (failed after retries)
if (unprocessed.length > 0) {
const error = `UNPROCESSED after ${MAX_RETRIES} retries`;
(0, engine_1._err)(NS, `! mreadItem unprocessed:`, { count: unprocessed.length, error });
unprocessed.forEach(key => {
const sig = _convertKeyToSig(key);
const failedItem = Object.assign(Object.assign({}, key), { error });
failedMap.set(sig, failedItem);
});
}
//* Identify items not found (404)
const received = new Set(items.map(item => _convertItemToSig(item)));
const unprocessedSigs = new Set(unprocessed.map(key => _convertKeyToSig(key)));
Array.from(chunkKeyMap.entries())
.filter(([sig]) => !received.has(sig) && !unprocessedSigs.has(sig))
.forEach(([sig, key]) => {
const failedItem = Object.assign(Object.assign({}, key), { error: '404 NOT FOUND' });
failedMap.set(sig, failedItem);
});
}
catch (e) {
const error = e instanceof Error ? e.message : `${e}`;
(0, engine_1._err)(NS, `! mreadItem batch failed:`, error);
Array.from(chunkKeyMap.entries()).forEach(([sig, key]) => {
const failedItem = Object.assign(Object.assign({}, key), { error });
failedMap.set(sig, failedItem);
});
}
}
//* Preserve original order
list.forEach(item => {
const key = this.prepareItemKey(item.id, item.sort).Key;
const sig = _convertKeyToSig(key);
const successItem = successMap.get(sig);
const failedItem = failedMap.get(sig);
if (successItem) {
result.success.push(successItem);
}
else if (failedItem) {
result.failed.push(failedItem);
}
});
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! mreadItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a;
return ({
id: (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id,
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> mreadItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
/**
* save-item
* - save whole data with param (use update if partial save)
*
* **WARN** overwrited if exists.
*
* @param id
* @param item
*/
saveItem(id, item) {
return __awaiter(this, void 0, void 0, function* () {
const { tableName, idName, sortName } = this.options;
// _log(NS, `saveItem(${id})...`);
const payload = this.prepareSaveItem(id, item);
// _log(NS, '> payload :=', payload);
const dynamodoc = yield DynamoService.instance().dynamodoc();
return dynamodoc
.send(new lib_dynamodb_1.PutCommand(payload))
.then(res => {
(0, engine_1._log)(NS, '> saveItem.res =', engine_1.$U.json(res));
return payload.Item;
})
.catch((e) => {
if (`${e.message}` == 'Requested resource not found')
throw new Error(`404 NOT FOUND - ${idName}:${id}`);
throw e;
});
});
}
/**
* multiple save-item
* - chunks requests into groups of 25 (DynamoDB BatchWriteItem limit)
* - NOTE: This overwrites entire items (no partial update or increment support)
*
* @param list array of items with { id, sort?, ...model }
* @returns BatchResult with success/failed items
*/
msaveItem(list) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
const { tableName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
const dynamodoc = yield DynamoService.instance().dynamodoc();
for (let i = 0; i < total; i += BATCH_WRITE_CHUNK) {
const chunk = (_b = list === null || list === void 0 ? void 0 : list.slice(i, i + BATCH_WRITE_CHUNK)) !== null && _b !== void 0 ? _b : [];
const chunkItems = [];
//* prepare batch write requests
const putRequests = chunk.map(item => {
const { id } = item, rest = __rest(item, ["id"]);
const payload = this.prepareSaveItem(id, rest);
chunkItems.push(payload.Item);
return {
PutRequest: {
Item: payload.Item,
},
};
});
//* execute batch write with retry for unprocessed items
const _doBatchWrite = (requests, attempt) => __awaiter(this, void 0, void 0, function* () {
var _c;
const response = yield dynamodoc
.send(new lib_dynamodb_1.BatchWriteCommand({ RequestItems: { [tableName]: requests } }))
.catch((e) => {
if (`${e.message}`.includes('ResourceNotFoundException'))
throw new Error(`404 NOT FOUND - table:${tableName}`);
throw e;
});
//* check for unprocessed items
const unprocessed = (_c = response.UnprocessedItems) === null || _c === void 0 ? void 0 : _c[tableName];
if (unprocessed && (unprocessed === null || unprocessed === void 0 ? void 0 : unprocessed.length) > 0) {
if (attempt >= MAX_RETRIES) {
return unprocessed;
}
//* exponential backoff
yield new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 100));
return _doBatchWrite(unprocessed, attempt + 1);
}
return [];
});
try {
const unprocessed = yield _doBatchWrite(putRequests, 1);
//* Handle unprocessed items (failed after retries)
const unprocessedIds = new Set();
if ((unprocessed === null || unprocessed === void 0 ? void 0 : unprocessed.length) > 0) {
const error = `UNPROCESSED after ${MAX_RETRIES} retries`;
(0, engine_1._err)(NS, `! msaveItem unprocessed:`, { count: unprocessed.length, error });
unprocessed.forEach(req => {
var _a;
const item = (_a = req.PutRequest) === null || _a === void 0 ? void 0 : _a.Item;
if (item) {
const itemId = String(item[this.options.idName] || item.id);
unprocessedIds.add(itemId);
const chunkItem = chunkItems.find(ci => {
const chunkItemId = String(ci[this.options.idName] || ci.id);
return chunkItemId === itemId;
});
if (chunkItem) {
const failedItem = Object.assign(Object.assign({}, chunkItem), { error });
result.failed.push(failedItem);
}
}
});
}
//* Add successfully written items
chunkItems.forEach(item => {
const itemId = String(item[this.options.idName] || item.id);
if (!unprocessedIds.has(itemId)) {
result.success.push(item);
}
});
}
catch (e) {
//* Handle complete batch failure
const error = e instanceof Error ? e.message : `${e}`;
(0, engine_1._err)(NS, `! msaveItem batch failed:`, error);
chunkItems.forEach(item => {
const failedItem = Object.assign(Object.assign({}, item), { error });
result.failed.push(failedItem);
});
}
}
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! msaveItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a;
return ({
id: (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id,
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> msaveItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
/**
* delete-item
* - destroy whole data of item.
*
* @param id
* @param sort
*/
deleteItem(id, sort) {
return __awaiter(this, void 0, void 0, function* () {
// _log(NS, `deleteItem(${id})...`);
const payload = this.prepareItemKey(id, sort);
const dynamodoc = yield DynamoService.instance().dynamodoc();
return dynamodoc
.send(new lib_dynamodb_1.DeleteCommand(payload))
.then(res => {
(0, engine_1._log)(NS, '> deleteItem.res =', engine_1.$U.json(res));
return null;
})
.catch((e) => {
if (`${e.message}` == 'Requested resource not found')
return {};
throw e;
});
});
}
/**
* update-item (or increment-item)
* - update or create if not exists.
*
* @param id
* @param sort
* @param updates
* @param increments
*/
updateItem(id, sort, updates, increments) {
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
// _log(NS, `updateItem(${id})...`);
const payload = this.prepareUpdateItem(id, sort, updates, increments);
const dynamodoc = yield DynamoService.instance().dynamodoc();
return dynamodoc
.send(new lib_dynamodb_1.UpdateCommand(payload))
.then(res => {
(0, engine_1._log)(NS, `> updateItem[${id}].res =`, engine_1.$U.json(res));
const attr = res.Attributes;
const $key = Object.assign({}, payload.Key);
return Object.assign(attr, $key);
})
.catch((e) => {
if (`${e.message}` == 'Requested resource not found')
throw new Error(`404 NOT FOUND - ${idName}:${id}`);
throw e;
});
});
}
/**
* multiple update-item
* - chunks requests into groups of 25 (DynamoDB BatchWriteItem limit)
* - NOTE: This overwrites entire items (no partial update or increment support)
*
* @param list array of items with { id, sort?, ...model }
* @returns BatchResult with success/failed items
*/
mupdateItem(list) {
var _a, _b;
return __awaiter(this, void 0, void 0, function* () {
const { tableName, idName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
const dynamodoc = yield DynamoService.instance().dynamodoc();
//* Maps to preserve original order
const successMap = new Map();
const failedMap = new Map();
for (let i = 0; i < total; i += BATCH_WRITE_CHUNK) {
const chunk = (_b = list === null || list === void 0 ? void 0 : list.slice(i, i + BATCH_WRITE_CHUNK)) !== null && _b !== void 0 ? _b : [];
const chunkItemMap = new Map();
//* prepare batch write requests
const putRequests = chunk.map(item => {
const { id } = item, rest = __rest(item, ["id"]);
//* Use partition key field if available, otherwise use id
const actualId = rest[this.options.idName] || id;
const payload = this.prepareSaveItem(actualId, rest);
chunkItemMap.set(id, payload.Item);
return {
PutRequest: {
Item: payload.Item,
},
};
});
//* execute batch write with retry for unprocessed items
const _doBatchWrite = (requests, attempt) => __awaiter(this, void 0, void 0, function* () {
var _c;
const response = yield dynamodoc
.send(new lib_dynamodb_1.BatchWriteCommand({ RequestItems: { [tableName]: requests } }))
.catch((e) => {
if (`${e.message}`.includes('ResourceNotFoundException'))
throw new Error(`404 NOT FOUND - table:${tableName}`);
throw e;
});
//* check for unprocessed items
const unprocessed = (_c = response.UnprocessedItems) === null || _c === void 0 ? void 0 : _c[tableName];
if (unprocessed && (unprocessed === null || unprocessed === void 0 ? void 0 : unprocessed.length) > 0) {
if (attempt >= MAX_RETRIES) {
return unprocessed;
}
//* exponential backoff
yield new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 100));
return _doBatchWrite(unprocessed, attempt + 1);
}
return [];
});
try {
const unprocessed = yield _doBatchWrite(putRequests, 1);
//* Handle unprocessed items (failed after retries)
const unprocessedIds = new Set();
if ((unprocessed === null || unprocessed === void 0 ? void 0 : unprocessed.length) > 0) {
const error = `UNPROCESSED after ${MAX_RETRIES} retries`;
(0, engine_1._err)(NS, `! mupdateItem unprocessed:`, { count: unprocessed.length, error });
unprocessed.forEach(req => {
var _a;
const item = (_a = req.PutRequest) === null || _a === void 0 ? void 0 : _a.Item;
if (item) {
const itemId = String(item[this.options.idName] || item.id);
for (const [id, chunkItem] of chunkItemMap.entries()) {
const chunkItemId = String(chunkItem[this.options.idName] || chunkItem.id);
if (chunkItemId === itemId) {
unprocessedIds.add(id);
const failedItem = Object.assign(Object.assign({}, chunkItem), { error });
failedMap.set(id, failedItem);
break;
}
}
}
});
}
//* Add successfully written items
chunkItemMap.forEach((item, id) => {
if (!unprocessedIds.has(id)) {
successMap.set(id, item);
}
});
}
catch (e) {
//* Handle complete batch failure
const error = e instanceof Error ? e.message : `${e}`;
(0, engine_1._err)(NS, `! mupdateItem batch failed:`, error);
chunkItemMap.forEach((item, id) => {
const failedItem = Object.assign(Object.assign({}, item), { error });
failedMap.set(id, failedItem);
});
}
}
//* Preserve original order
list.forEach(item => {
const id = item.id;
const successItem = successMap.get(id);
const failedItem = failedMap.get(id);
if (successItem) {
result.success.push(successItem);
}
else if (failedItem) {
result.failed.push(failedItem);
}
});
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! mupdateItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a;
return ({
id: (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id,
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> mupdateItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
}
exports.DynamoService = DynamoService;
DynamoService._dynamo = {};
DynamoService._dynamostr = {};
//TODO [Steve] why to use promise here to make DynamoDBDocumentClient? @251212
DynamoService._dynamodoc = {};
/**
* export to test..
*/
DynamoService.normalize = normalize;
/** ****************************************************************************************************************
* Dummy Dynamo Service
** ****************************************************************************************************************/
/**
* class: `DummyDynamoService`
* - service in-memory dummy data
*/
class DummyDynamoService extends DynamoService {
constructor(dataFile, options) {
super(options);
this.buffer = {};
/**
* say hello()
*/
this.hello = () => `dummy-dynamo-service:${this.options.tableName}`;
(0, engine_1._log)(NS, `DummyDynamoService(${dataFile || ''})...`);
if (!dataFile)
throw new Error('@dataFile(string) is required!');
const dummy = (0, tools_1.loadDataYml)(dataFile);
this.load(dummy.data);
}
load(data) {
const { idName } = this.options;
if (!data || !Array.isArray(data))
throw new Error('@data should be array!');
data.map(item => {
const id = `${item[idName] || ''}`;
this.buffer[id] = item;
});
}
/**
* ONLY FOR DUMMY
* - send list of data.
*
* @param page page number starts from 1
* @param limit limit of count.
*/
listItems(page, limit) {
return __awaiter(this, void 0, void 0, function* () {
page = engine_1.$U.N(page, 1);
limit = engine_1.$U.N(limit, 2);
const keys = Object.keys(this.buffer);
const total = keys.length;
const list = keys.slice((page - 1) * limit, page * limit).map(_ => this.buffer[_]);
return { page, limit, total, list };
});
}
readItem(id, sort) {
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
const item = this.buffer[id];
if (item === undefined)
throw new Error(`404 NOT FOUND - ${idName}:${id}`);
return Object.assign({ [idName]: id }, item);
});
}
mreadItem(list) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const { idName, sortName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
for (const item of list) {
const data = this.buffer[item.id];
if (data === undefined) {
const failedItem = (sortName
? { [idName]: item.id, [sortName]: item.sort, error: '404 NOT FOUND' }
: { [idName]: item.id, error: '404 NOT FOUND' });
result.failed.push(failedItem);
continue;
}
result.success.push(Object.assign({ [idName]: item.id }, data));
}
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! mreadItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a, _b;
return ({
id: (_b = (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id) !== null && _b !== void 0 ? _b : item === null || item === void 0 ? void 0 : item[idName],
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> mreadItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
saveItem(id, item) {
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
this.buffer[id] = normalize(item);
return Object.assign({ [idName]: id }, this.buffer[id]);
});
}
msaveItem(list) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
//* process all saves using the in-memory buffer (overwrites entire items)
for (const item of list) {
try {
const { id } = item, rest = __rest(item, ["id"]);
this.buffer[id] = normalize(rest);
result.success.push(Object.assign({ [idName]: id }, this.buffer[id]));
}
catch (e) {
const error = e instanceof Error ? e.message : `${e}`;
(0, engine_1._err)(NS, `! msaveItem failed for id=${item === null || item === void 0 ? void 0 : item.id}:`, error);
const failedItem = Object.assign(Object.assign({}, item), { error });
result.failed.push(failedItem);
}
}
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! msaveItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a, _b;
return ({
id: (_b = (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id) !== null && _b !== void 0 ? _b : item === null || item === void 0 ? void 0 : item[idName],
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> msaveItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
deleteItem(id, sort) {
return __awaiter(this, void 0, void 0, function* () {
delete this.buffer[id];
return null;
});
}
updateItem(id, sort, updates, increments) {
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
const item = this.buffer[id];
if (item === undefined)
throw new Error(`404 NOT FOUND - ${idName}:${id}`);
this.buffer[id] = Object.assign(Object.assign({}, item), normalize(updates));
return Object.assign({ [idName]: id }, this.buffer[id]);
});
}
mupdateItem(list) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
const { idName } = this.options;
const total = (_a = list === null || list === void 0 ? void 0 : list.length) !== null && _a !== void 0 ? _a : 0;
const result = { success: [], failed: [], total };
if (!list || total === 0)
return result;
//* process all updates using the in-memory buffer (overwrites entire items)
for (const item of list) {
try {
const { id } = item, rest = __rest(item, ["id"]);
//* overwrite entire item (same as PutItem behavior)
this.buffer[id] = normalize(rest);
result.success.push(Object.assign({ [idName]: id }, this.buffer[id]));
}
catch (e) {
const error = e instanceof Error ? e.message : `${e}`;
(0, engine_1._err)(NS, `! mupdateItem failed for id=${item === null || item === void 0 ? void 0 : item.id}:`, error);
const failedItem = { [idName]: item.id, error };
result.failed.push(failedItem);
}
}
if (result.failed.length > 0) {
(0, engine_1._err)(NS, `! mupdateItem.failed =`, {
failed: result.failed.length,
sample_failed: result.failed.slice(0, 3).map((item) => {
var _a, _b;
return ({
id: (_b = (_a = item === null || item === void 0 ? void 0 : item._id) !== null && _a !== void 0 ? _a : item === null || item === void 0 ? void 0 : item.id) !== null && _b !== void 0 ? _b : item === null || item === void 0 ? void 0 : item[idName],
error: item === null || item === void 0 ? void 0 : item.error,
});
}),
});
}
(0, engine_1._log)(NS, `> mupdateItem.res =`, {
success: result.success.length,
failed: result.failed.length,
total: result.total,
});
return result;
});
}
}
exports.DummyDynamoService = DummyDynamoService;
//# sourceMappingURL=dynamo-service.js.map