@mbc-cqrs-serverless/import
Version:
403 lines • 18.2 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var ImportService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ImportService = void 0;
const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
const client_s3_1 = require("@aws-sdk/client-s3");
const util_dynamodb_1 = require("@aws-sdk/util-dynamodb");
const core_1 = require("@mbc-cqrs-serverless/core");
const common_1 = require("@nestjs/common");
const config_1 = require("@nestjs/config");
const csv_parser_1 = __importDefault(require("csv-parser"));
const stream_1 = require("stream");
const ulid_1 = require("ulid");
const constant_1 = require("./constant");
const entity_1 = require("./entity");
const enum_1 = require("./enum");
const import_module_definition_1 = require("./import.module-definition");
let ImportService = ImportService_1 = class ImportService {
constructor(dynamoDbService, snsService, config, s3Service, importStrategyMap) {
this.dynamoDbService = dynamoDbService;
this.snsService = snsService;
this.config = config;
this.s3Service = s3Service;
this.importStrategyMap = importStrategyMap;
this.logger = new common_1.Logger(ImportService_1.name);
this.tableName = dynamoDbService.getTableName('import_tmp');
this.alarmTopicArn = this.config.get('SNS_ALARM_TOPIC_ARN');
}
/**
* Handles a single import request from the API.
* It uses the appropriate ImportStrategy to transform and validate the data
* before creating a record in the temporary import table.
*/
async createWithApi(dto, options) {
const { tableName, attributes } = dto;
const strategy = this.importStrategyMap.get(tableName);
if (!strategy) {
throw new common_1.BadRequestException(`No import strategy found for table: ${tableName}`);
}
const transformedData = await strategy.transform(attributes);
await strategy.validate(transformedData);
return this.createImport({ ...dto, attributes: transformedData }, options);
}
/**
* Main router for handling CSV imports. It delegates to the correct
* processing method based on the specified execution strategy.
*/
async handleCsvImport(dto, options) {
if (dto.processingMode === 'DIRECT') {
return this._processCsvDirectly(dto, options);
}
else {
return this.createCsvJob(dto, options);
}
}
/**
* Creates a master job record for a CSV import that will be orchestrated
* by a Step Function.
*/
async createCsvJob(dto, options) {
const sourceIp = options.invokeContext?.event?.requestContext?.http?.sourceIp;
const userContext = (0, core_1.getUserContext)(options.invokeContext);
const taskCode = (0, ulid_1.ulid)();
const pk = `${constant_1.CSV_IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}${dto.tenantCode}`;
const sk = `${dto.tableName}#${taskCode}`;
const item = new entity_1.ImportEntity({
id: `${pk}#${sk}`,
pk,
sk,
version: 0,
code: taskCode,
tenantCode: dto.tenantCode,
type: 'CSV_MASTER_JOB',
name: `CSV Import: ${dto.key.split('/').pop()}`,
status: enum_1.ImportStatusEnum.CREATED,
attributes: dto,
requestId: options.invokeContext?.context?.awsRequestId,
createdAt: new Date(),
updatedAt: new Date(),
createdBy: userContext.userId,
updatedBy: userContext.userId,
createdIp: sourceIp,
updatedIp: sourceIp,
});
await this.dynamoDbService.putItem(this.tableName, item);
return item;
}
async createZipJob(dto, options) {
const sourceIp = options.invokeContext?.event?.requestContext?.http?.sourceIp;
const userContext = (0, core_1.getUserContext)(options.invokeContext);
const taskCode = (0, ulid_1.ulid)();
const pk = `${constant_1.ZIP_IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}${dto.tenantCode}`;
const sk = `ZIP#${taskCode}`;
const item = new entity_1.ImportEntity({
id: `${pk}#${sk}`,
pk,
sk,
version: 0,
code: taskCode,
tenantCode: dto.tenantCode,
type: 'ZIP_MASTER_JOB',
name: `ZIP Import: ${dto.key.split('/').pop()}`,
status: enum_1.ImportStatusEnum.CREATED,
attributes: dto,
requestId: options.invokeContext?.context?.awsRequestId,
createdAt: new Date(),
updatedAt: new Date(),
createdBy: userContext.userId,
updatedBy: userContext.userId,
createdIp: sourceIp,
updatedIp: sourceIp,
});
await this.dynamoDbService.putItem(this.tableName, item);
return item;
}
/**
* Creates a import job record for a single json import
*/
async createImport(dto, options) {
const sourceIp = options.invokeContext?.event?.requestContext?.http?.sourceIp;
const userContext = (0, core_1.getUserContext)(options.invokeContext);
const taskCode = (0, ulid_1.ulid)();
const pk = `${constant_1.IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}${dto.tenantCode}`;
const sk = dto.sourceId
? `${dto.sourceId}#${taskCode}`
: `${dto.tableName}#${taskCode}`;
const item = {
id: `${pk}${core_1.KEY_SEPARATOR}${sk}`,
pk,
sk,
version: 0,
code: taskCode,
tenantCode: dto.tenantCode,
type: dto.tableName,
name: dto.name || dto.tableName,
status: enum_1.ImportStatusEnum.CREATED,
attributes: dto.attributes,
requestId: options.invokeContext?.context?.awsRequestId,
createdAt: new Date(),
updatedAt: new Date(),
createdBy: userContext.userId,
updatedBy: userContext.userId,
createdIp: sourceIp,
updatedIp: sourceIp,
};
await this.dynamoDbService.putItem(this.tableName, item);
return new entity_1.ImportEntity(item);
}
/**
* Creates a CSV master job that is part of a larger ZIP orchestration.
* It stores the SFN Task Token needed to signal completion back to the orchestrator.
* @param dto The details of the CSV file to process.
* @param taskToken The task token from the waiting Step Function.
* @param sourceId The key of the parent ZIP_MASTER_JOB.
* @returns The created ImportEntity.
*/
async createCsvJobWithTaskToken(dto, taskToken, sourceId) {
const taskCode = (0, ulid_1.ulid)();
const pk = `${constant_1.CSV_IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}${dto.tenantCode}`;
const sk = `${dto.tableName}${core_1.KEY_SEPARATOR}${taskCode}`;
const item = new entity_1.ImportEntity({
id: `${pk}${core_1.KEY_SEPARATOR}${sk}`,
pk,
sk,
version: 0,
code: taskCode,
tenantCode: dto.tenantCode,
type: 'CSV_MASTER_JOB',
name: `CSV Import (from ZIP): ${dto.key.split('/').pop()}`,
status: enum_1.ImportStatusEnum.CREATED,
// Store both the original DTO and the crucial task token
attributes: {
...dto,
taskToken,
},
source: `${sourceId.pk}${core_1.KEY_SEPARATOR}${sourceId.sk}`, // Link back to the parent ZIP job
requestId: (0, ulid_1.ulid)(), // System-generated
createdAt: new Date(),
updatedAt: new Date(),
createdBy: 'system', // This job is created by the system, not a direct user action
updatedBy: 'system',
});
await this.dynamoDbService.putItem(this.tableName, item);
return item;
}
/**
* Handles the 'DIRECT' execution strategy by fetching the CSV from S3
* and processing its stream immediately.
*/
async _processCsvDirectly(dto, options) {
this.logger.log(`Starting DIRECT CSV processing for key: ${dto.key}`);
// 1. Fetch the S3 object stream
const { Body: s3Stream } = await this.s3Service.client.send(new client_s3_1.GetObjectCommand({
Bucket: dto.bucket,
Key: dto.key,
}));
if (!(s3Stream instanceof stream_1.Readable)) {
throw new Error('Failed to get a readable stream from S3 object.');
}
// 2. Pass the stream to the centralized processor
return this._processCsvStream(s3Stream, dto, options);
}
/**
* Centralized logic to process a CSV stream. It reads each row, uses the
* appropriate ImportStrategy to transform and validate, and creates a
* temporary import record for each valid row.
*/
async _processCsvStream(stream, attributes, options) {
const strategy = this.importStrategyMap.get(attributes.tableName);
if (!strategy) {
throw new Error(`No import strategy found for table: ${attributes.tableName}`);
}
const processingPromises = [];
return new Promise((resolve, reject) => {
const parser = (0, csv_parser_1.default)({
mapHeaders: ({ header }) => header.trim(),
mapValues: ({ value }) => value.trim(),
});
stream
.pipe(parser)
.on('data', (row) => {
const processRow = (async () => {
try {
const transformedData = await strategy.transform(row);
await strategy.validate(transformedData);
const createImportDto = {
tableName: attributes.tableName,
tenantCode: attributes.tenantCode,
attributes: transformedData,
};
// Return the created entity
return await this.createImport(createImportDto, options);
}
catch (error) {
this.logger.warn(`Skipping CSV row due to error: ${error instanceof Error ? error.message : String(error)}`, { row });
return;
}
})();
processingPromises.push(processRow);
})
.on('end', async () => {
// Wait for all row processing to complete before resolving
const results = await Promise.all(processingPromises);
const successfulImports = results.filter((result) => !!result);
this.logger.log(`Finished CSV stream. Created ${successfulImports.length} import records.`);
resolve(successfulImports);
})
.on('error', (error) => {
this.logger.error('Error parsing CSV stream:', error);
reject(error);
});
});
}
async updateStatus(key, status, payload, attributes, notifyId) {
await this.dynamoDbService.updateItem(this.tableName, key, {
set: {
status,
attributes,
result: payload?.result || payload?.error,
},
});
// notification via SNS
await this.snsService.publish({
action: 'import-status',
...key,
table: this.tableName,
id: notifyId || `${key.pk}#${key.sk}`,
tenantCode: key.pk.substring(key.pk.indexOf('#') + 1),
content: {
status,
attributes,
result: payload?.result || payload?.error,
},
});
}
/**
* Atomically increments the progress counters for a parent CSV job.
* After incrementing, it checks if the job is complete and updates the
* final status if necessary.
* @param parentKey The key of the master CSV job entity.
* @param childSucceeded True if the child job was successful, false otherwise.
*/
async incrementParentJobCounters(parentKey, childSucceeded) {
this.logger.debug(`Incrementing counters for parent job (atomic workaround): ${parentKey.pk}#${parentKey.sk}`);
// 1. Define which counters to increment.
const countersToIncrement = {
processedRows: 1,
};
if (childSucceeded) {
countersToIncrement.succeededRows = 1;
}
else {
countersToIncrement.failedRows = 1;
}
// 2. Use the local helper to build the command parts.
const { UpdateExpression, ExpressionAttributeNames, ExpressionAttributeValues, } = this._buildAtomicCounterUpdateExpression(countersToIncrement);
// 3. Manually create and send the UpdateItemCommand.
const command = new client_dynamodb_1.UpdateItemCommand({
TableName: this.tableName,
Key: (0, util_dynamodb_1.marshall)(parentKey),
UpdateExpression,
ExpressionAttributeNames,
ExpressionAttributeValues: (0, util_dynamodb_1.marshall)(ExpressionAttributeValues),
ReturnValues: 'ALL_NEW',
});
const response = await this.dynamoDbService.client.send(command);
const updatedEntity = (0, util_dynamodb_1.unmarshall)(response.Attributes);
// 4. Check if the job is complete (this logic remains the same).
const { totalRows, processedRows, failedRows } = updatedEntity;
if (totalRows > 0 && processedRows >= totalRows) {
this.logger.log(`Finalizing parent CSV job ${parentKey.pk}#${parentKey.sk}`);
const finalStatus = failedRows > 0 ? enum_1.ImportStatusEnum.COMPLETED : enum_1.ImportStatusEnum.COMPLETED;
await this.updateStatus(parentKey, finalStatus, {
result: {
message: 'All child jobs have been processed.',
total: totalRows,
succeeded: updatedEntity.succeededRows || 0,
failed: failedRows || 0,
},
});
}
return updatedEntity;
}
/**
* A private helper to build a valid DynamoDB UpdateExpression for atomic counters.
* @param counters A map of attribute names to the amount they should be incremented by.
* @returns An object with the UpdateExpression and its necessary parameter maps.
*/
_buildAtomicCounterUpdateExpression(counters) {
const setExpressions = [];
const expressionAttributeNames = {};
const expressionAttributeValues = {};
for (const key in counters) {
const attrName = `#${key}`;
const startValuePlaceholder = `:${key}Start`;
const incValuePlaceholder = `:${key}Inc`;
setExpressions.push(`${attrName} = if_not_exists(${attrName}, ${startValuePlaceholder}) + ${incValuePlaceholder}`);
expressionAttributeNames[attrName] = key;
expressionAttributeValues[startValuePlaceholder] = 0; // Always start from 0 if the attribute is new.
expressionAttributeValues[incValuePlaceholder] = counters[key]; // The amount to increment by.
}
return {
UpdateExpression: `SET ${setExpressions.join(', ')}`,
ExpressionAttributeNames: expressionAttributeNames,
ExpressionAttributeValues: expressionAttributeValues,
};
}
// add a generic update method for setting totalRows
async updateImportJob(key, payload) {
return this.dynamoDbService.updateItem(this.tableName, key, payload);
}
async publishAlarm(event, errorDetails) {
this.logger.debug('event', event);
const importKey = event.importEvent.importKey;
const tenantCode = importKey.pk.substring(importKey.pk.indexOf(core_1.KEY_SEPARATOR) + 1);
const alarm = {
action: 'sfn-alarm',
id: `${importKey.pk}#${importKey.sk}`,
table: this.tableName,
pk: importKey.pk,
sk: importKey.sk,
tenantCode,
content: {
errorMessage: errorDetails,
},
};
this.logger.error('alarm:::', alarm);
await this.snsService.publish(alarm, this.alarmTopicArn);
}
async getImportByKey(key) {
const item = await this.dynamoDbService.getItem(this.tableName, key);
if (!item) {
throw new common_1.BadRequestException(`Import item not found for key: ${key.pk}#${key.sk}`);
}
return new entity_1.ImportEntity(item);
}
};
exports.ImportService = ImportService;
exports.ImportService = ImportService = ImportService_1 = __decorate([
(0, common_1.Injectable)(),
__param(4, (0, common_1.Inject)(import_module_definition_1.IMPORT_STRATEGY_MAP)),
__metadata("design:paramtypes", [core_1.DynamoDbService,
core_1.SnsService,
config_1.ConfigService,
core_1.S3Service,
Map])
], ImportService);
//# sourceMappingURL=import.service.js.map