@pujansrt/data-genie/dist/transformers/remove-duplicates-reader.js

High performant ETL engine written in TypeScript

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RemoveDuplicatesReader = void 0;
const transformers_1 = require("../transformers/transformers");
class RemoveDuplicatesReader extends transformers_1.DataTransformer {
    constructor(reader, ...fieldNames) {
        super(reader);
        this.seenKeys = new Set();
        this.fieldNames = fieldNames;
    }
    async *read() {
        for await (const record of this.reader.read()) {
            const key = this.fieldNames.map((fieldName) => record[fieldName]).join('|'); // Create a composite key
            if (!this.seenKeys.has(key)) {
                this.seenKeys.add(key);
                yield record;
            }
        }
    }
}
exports.RemoveDuplicatesReader = RemoveDuplicatesReader;