couch-elastic-stream/lib/source/CouchSourceTask.js

data streaming from couchdb to elastic via kafka.

"use strict";

const { SourceTask, SourceRecord } = require("kafka-connect");

class CouchSourceTask extends SourceTask {

    start(properties, callback, parentConfig) {
        
			this.parentConfig = parentConfig;
			this.properties = properties;
			const {
				couch,
				maxTasks,
				database,
				partition,
				topic,
				maxPollCount,
				currentOffset
			} = this.properties;
			
			this.couch = couch;
			this.maxTasks = maxTasks;
			this.database = database;
			this.partition = partition;
			this.topic = topic;
			this.maxPollCount = maxPollCount;
			this.currentOffset = currentOffset ? currentOffset : '0';

			callback(null);    
    }

    poll(callback){
			const query = `_changes?include_docs=true&limit=${this.maxPollCount}&since=${this.currentOffset}`;
			this.couch.get(this.database, query,{})
			.then(({data, headers, status}) => {
				this.currentOffset = data.last_seq;
				const records = data.results.map(result => {
					const record = new SourceRecord();
					record.key = result.id;
					record.keySchema = null;
					if (!record.key) {
						throw new Error("db results are missing incrementing column name or default 'id' field.");
					}
					record.value = result;
					record.valueSchema = null;
					record.timestamp = new Date().toISOString();
					record.partition = this.partition;
					record.topic = this.topic;
					this.parentConfig.emit("record-read", record.key.toString());
					return record;
				});
				callback(null, records);
			})
			.catch((error) => {
				callback(error);
			});
    }

    stop() {    
    }
}

module.exports = CouchSourceTask;