@mbc-cqrs-serverless/import
Version:
211 lines • 10.1 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var CsvImportSfnEventHandler_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.CsvImportSfnEventHandler = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const core_1 = require("@mbc-cqrs-serverless/core");
const common_1 = require("@nestjs/common");
const config_1 = require("@nestjs/config");
const csv_parser_1 = __importDefault(require("csv-parser"));
const stream_1 = require("stream");
const enum_1 = require("../enum");
const helpers_1 = require("../helpers");
const import_module_definition_1 = require("../import.module-definition");
const import_service_1 = require("../import.service");
const csv_import_sfn_event_1 = require("./csv-import.sfn.event");
let CsvImportSfnEventHandler = CsvImportSfnEventHandler_1 = class CsvImportSfnEventHandler {
constructor(importService, importStrategyMap, configService, s3Service) {
this.importService = importService;
this.importStrategyMap = importStrategyMap;
this.configService = configService;
this.s3Service = s3Service;
this.logger = new common_1.Logger(CsvImportSfnEventHandler_1.name);
this.alarmTopicArn = this.configService.get('SNS_ALARM_TOPIC_ARN');
}
async execute(event) {
try {
return await this.handleStepState(event);
}
catch (error) {
this.logger.error('import step execution failed', error);
throw error;
}
}
async handleStepState(event) {
this.logger.debug('Processing event:::', JSON.stringify(event, null, 2));
if (event.context.State.Name === 'csv_loader') {
const input = event.input;
// 1. Get the parent job's key from the sourceId
const parentKey = (0, helpers_1.parseId)(input.sourceId);
// 2. Count the total rows in the CSV
this.logger.log(`Counting rows for file: ${input.key}`);
const totalRows = await this.countCsvRows(input);
this.logger.log(`Found ${totalRows} rows. Updating parent job.`);
// 3. Update the parent job with the total count
const updatedEntity = await this.importService.updateImportJob(parentKey, {
set: { totalRows },
});
if (updatedEntity.processedRows >= totalRows) {
this.logger.log(`Job ${input.sourceId} already finished. Setting final status.`);
const finalStatus = updatedEntity.failedRows > 0
? enum_1.ImportStatusEnum.COMPLETED
: enum_1.ImportStatusEnum.COMPLETED;
await this.importService.updateStatus(parentKey, finalStatus);
}
// 4. Proceed to load the first batch of rows as before
return this.loadCsv(input);
}
if (event.context.State.Name === 'finalize_parent_job') {
const finalizeEvent = event.input;
return this.finalizeParentJob(finalizeEvent);
}
const input = event.input;
const items = input.Items;
const attributes = input.BatchInput.Attributes;
const createImportDtos = [];
const strategy = this.importStrategyMap.get(attributes.tableName);
if (!strategy) {
throw new Error(`No import strategy found for table: ${attributes.tableName}`);
}
for (const [index, item] of items.entries()) {
try {
const transformedData = await strategy.transform(item);
await strategy.validate(transformedData);
const createImport = {
tableName: attributes.tableName,
tenantCode: attributes.tenantCode,
attributes: transformedData,
sourceId: attributes.sourceId,
};
createImportDtos.push(createImport);
}
catch (error) {
this.logger.warn(`Row ${index + 1} failed mapping.`, { item, error });
throw error;
}
}
const invokeContext = (0, core_1.extractInvokeContext)();
const options = {
invokeContext,
};
if (createImportDtos.length > 0) {
const importPromises = createImportDtos.map((dto) => this.importService.createImport(dto, options));
await Promise.all(importPromises);
}
}
async loadCsv(input, limit = 10) {
// 1. Fetch the S3 object stream
const { Body: s3Stream } = await this.s3Service.client.send(new client_s3_1.GetObjectCommand({
Bucket: input.bucket,
Key: input.key,
}));
if (!(s3Stream instanceof stream_1.Readable)) {
// This handles cases where the Body might not be a stream (e.g., error or empty object)
throw new Error('Failed to get a readable stream from S3 object.');
}
// 2. Wrap the stream processing in a Promise for async/await compatibility
return new Promise((resolve, reject) => {
const items = [];
const parser = (0, csv_parser_1.default)({
mapHeaders: ({ header }) => header.trim(),
mapValues: ({ value }) => value.trim(),
});
const streamPipeline = s3Stream.pipe(parser);
streamPipeline
.on('data', (row) => {
// Only push items until the limit is reached
if (items.length < limit) {
items.push(row);
}
else {
// 3. Efficiently destroy the stream once the limit is met.
// This stops reading the file from S3 and parsing, saving resources.
streamPipeline.destroy();
this.logger.debug(`Limit of ${limit} rows reached. Destroying stream.`);
resolve({
BatchInput: {
Attributes: input,
},
Items: items,
});
}
})
.on('end', () => {
// This 'end' event will only be reached if the file has fewer rows than the limit.
this.logger.debug(`CSV parsing finished. Found ${items.length} rows.`);
resolve({
BatchInput: {
Attributes: input,
},
Items: items,
});
})
.on('error', (error) => {
// Handle any errors during streaming or parsing
this.logger.error('Error parsing CSV stream:', error);
reject(error);
});
});
}
async countCsvRows(input) {
const { Body: s3Stream } = await this.s3Service.client.send(new client_s3_1.GetObjectCommand({ Bucket: input.bucket, Key: input.key }));
if (!(s3Stream instanceof stream_1.Readable)) {
throw new Error('Failed to get a readable stream from S3 object.');
}
return new Promise((resolve, reject) => {
let count = 0;
const parser = (0, csv_parser_1.default)();
s3Stream
.pipe(parser)
.on('data', () => count++)
.on('end', () => resolve(count))
.on('error', (error) => reject(error));
});
}
async finalizeParentJob(event) {
const parentKey = (0, helpers_1.parseId)(event.sourceId);
const totalRows = event.MapResult.length;
this.logger.log(`Setting totalRows=${totalRows} for parent job ${event.sourceId}.`);
const updatedEntity = await this.importService.updateImportJob(parentKey, {
set: { totalRows },
});
const { processedRows, failedRows } = updatedEntity;
if (totalRows > 0 && processedRows >= totalRows) {
this.logger.log(`Finalizing parent CSV job ${parentKey.pk}#${parentKey.sk}`);
const finalStatus = failedRows > 0 ? enum_1.ImportStatusEnum.COMPLETED : enum_1.ImportStatusEnum.COMPLETED;
await this.importService.updateStatus(parentKey, finalStatus, {
result: {
message: 'All child jobs have been processed.',
total: totalRows,
succeeded: updatedEntity.succeededRows || 0,
failed: failedRows || 0,
},
});
}
}
};
exports.CsvImportSfnEventHandler = CsvImportSfnEventHandler;
exports.CsvImportSfnEventHandler = CsvImportSfnEventHandler = CsvImportSfnEventHandler_1 = __decorate([
(0, core_1.EventHandler)(csv_import_sfn_event_1.CsvImportSfnEvent),
__param(1, (0, common_1.Inject)(import_module_definition_1.IMPORT_STRATEGY_MAP)),
__metadata("design:paramtypes", [import_service_1.ImportService,
Map,
config_1.ConfigService,
core_1.S3Service])
], CsvImportSfnEventHandler);
//# sourceMappingURL=csv-import.sfn.event.handler.js.map