couch-elastic-stream/lib/sink/ElasticSinkTask.js

data streaming from couchdb to elastic via kafka.

"use strict";

const { SinkTask } = require("kafka-connect");

class ElasticSinkTask extends SinkTask {
    start(properties, callback, parentConfig) {

        this.parentConfig = parentConfig;
        this.properties = properties;
        const {
            client,
            index,
            type
        } = this.properties;

        this.client = client;
        this.index = index;
        this.type = type;

        callback(null);
    }

    putRecords(records) {
        return Promise.all(records.map(record => {
            if (record.value !== null && record.value !== "null") {
                this.parentConfig.emit("model-upsert", record.key.toString());
                return this.client.update({
                    id : record.key,
                    index: this.index,
                    type: this.type,
                    body: {
                      doc : record.value,
                      doc_as_upsert: true 
                    }
                }, (err, res) => {
                    if(err) {
                        if(err.meta.statusCode === 409 && err.message === "version_conflict_engine_exception") {
                            return Promise.reject(new Error("DUPLICATE ENTRY OF DATA"));
                        } else {
                            return Promise.reject(new Error(err.meta.body));
                        }
                    }
                });
            }

            //if record.value is null, we will use the key to delete the field
            this.parentConfig.emit("model-delete", record.key.toString());
            return this.client.deleteByQuery({
                index: this.index,
                type: this.type,
                body: {
                    query : {
                        match :{ id : record.key}
                    }
                }
            },(res, err) => {
                if(err) {
                    return Promise.reject(new Error(err));
                }
            });
        }));
    }

    put(records, callback) {
        this.putRecords(records).then(() => {
            callback(null);
        }).catch(error => {
            callback(error);
        });
    }

    stop() {
        //empty (con is closed by connector)
    }
}

module.exports = ElasticSinkTask;