node-rdkafka-imhoff
Version:
Node.js bindings for librdkafka
197 lines (167 loc) • 9.3 kB
HTML
<html lang="en">
<head>
<meta charset="utf-8">
<title>JSDoc: Tutorial: producer-cluster</title>
<script src="scripts/prettify/prettify.js"> </script>
<script src="scripts/prettify/lang-css.js"> </script>
<!--[if lt IE 9]>
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link type="text/css" rel="stylesheet" href="styles/prettify.css">
<link type="text/css" rel="stylesheet" href="toolkit/css/toolkit/bnet-responsive.min.css">
<link type="text/css" rel="stylesheet" href="styles/main.css">
</head>
<body>
<!-- HEADER -->
<div class="navbar-static">
<header class="navbar header">
<div class="grid-container">
<div class="grid-25">
<a class="brand mark" href="index.html">
Tutorial: producer-cluster
<span class="tag">3.6.11</span>
</a>
</div>
<div class="grid-75">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown"><a class="dropdown-toggle" role="button" data-toggle="dropdown" href="#" onclick="return false;">Classes</a><ul class="dropdown-menu" role="menu"><li><a href="AdminClient.html">AdminClient</a></li><li><a href="Client.html">Client</a></li><li><a href="HighLevelProducer.html">HighLevelProducer</a></li><li><a href="KafkaConsumer.html">KafkaConsumer</a></li><li><a href="KafkaConsumerStream.html">KafkaConsumerStream</a></li><li><a href="LibrdKafkaError.html">LibrdKafkaError</a></li><li><a href="Producer.html">Producer</a></li><li><a href="ProducerStream.html">ProducerStream</a></li></ul></li><li class="dropdown"><a class="dropdown-toggle" role="button" data-toggle="dropdown" href="#" onclick="return false;">Events</a><ul class="dropdown-menu" role="menu"><li><a href="Client.html#event:disconnected">disconnected</a></li><li><a href="Client.html#event:ready">ready</a></li><li><a href="HighLevelProducer.html#event:disconnected">disconnected</a></li><li><a href="HighLevelProducer.html#event:ready">ready</a></li><li><a href="KafkaConsumer.html#event:data">data</a></li><li><a href="KafkaConsumer.html#event:disconnected">disconnected</a></li><li><a href="KafkaConsumer.html#event:ready">ready</a></li><li><a href="Producer.html#event:disconnected">disconnected</a></li><li><a href="Producer.html#event:ready">ready</a></li></ul></li><li class="dropdown"><a class="dropdown-toggle" role="button" data-toggle="dropdown" href="#" onclick="return false;">Tutorials</a><ul class="dropdown-menu" role="menu"><li><a href="tutorial-consumer-flow.html">consumer-flow</a></li><li><a href="tutorial-consumer-per-partition.html">consumer-per-partition</a></li><li><a href="tutorial-consumer.html">consumer</a></li><li><a href="tutorial-docker-alpine.html">docker-alpine</a></li><li><a href="tutorial-high-level-producer.html">high-level-producer</a></li><li><a href="tutorial-metadata.html">metadata</a></li><li><a href="tutorial-producer-cluster.html">producer-cluster</a></li><li><a href="tutorial-producer_.html">producer</a></li></ul></li><h3>Global</h3><ul><li><a href="global.html#createAdminClient">createAdminClient</a></li><li><a href="global.html#createSerializer">createSerializer</a></li><li><a href="global.html#RefCounter">RefCounter</a></li><li><a href="global.html#Topic">Topic</a></li><li><a href="global.html#TopicPartition">TopicPartition</a></li></ul>
</ul>
</div>
</div>
</header>
</div>
<!-- /HEADER -->
<div class="body-content">
<div class="grid-container project-container">
<div class="grid-75 push-25">
<h1>Tutorial: producer-cluster</h1>
<section>
<header>
<h2>producer-cluster</h2>
</header>
<article>
<pre class="prettyprint source lang-js"><code>/*
* node-rdkafka - Node.js wrapper for RdKafka C/C++ library
*
* Copyright (c) 2016 Blizzard Entertainment
*
* This software may be modified and distributed under the terms
* of the MIT license. See the LICENSE.txt file for details.
*/
var cluster = require('cluster');
var numCPUs = 6;
var Kafka = require('../');
if (cluster.isMaster) {
// Fork workers.
for (var i = 0; i < numCPUs; i++) {
cluster.fork();
}
var exited_workers = 0;
cluster.on('exit', function(worker, code, signal) {
exited_workers++;
if (exited_workers === numCPUs - 1) {
process.exit();
}
});
} else {
// Configure client
var producer = new Kafka.Producer({
'client.id': 'kafka',
'metadata.broker.list': 'localhost:9092',
'compression.codec': 'none',
'retry.backoff.ms': 200,
'message.send.max.retries': 10,
'socket.keepalive.enable': true,
'queue.buffering.max.messages': 100000,
'queue.buffering.max.ms': 1000,
'batch.num.messages': 1000000,
'dr_cb': true
});
producer.setPollInterval(100);
var total = 0;
var totalSent = 0;
var max = 20000;
var errors = 0;
var started = Date.now();
var sendMessage = function() {
var ret = producer.sendMessage({
topic: 'librdtesting-01',
message: Buffer.from('message ' + total)
}, function() {
});
total++;
if (total >= max) {
} else {
setImmediate(sendMessage);
}
};
var verified_received = 0;
var exitNextTick = false;
var errorsArr = [];
var t = setInterval(function() {
producer.poll();
if (exitNextTick) {
clearInterval(t);
return setTimeout(function() {
console.log('[%d] Received: %d, Errors: %d, Total: %d', process.pid, verified_received, errors, total);
// console.log('[%d] Finished sending %d in %d seconds', process.pid, total, parseInt((Date.now() - started) / 1000));
if (errors > 0) {
console.error(errorsArr[0]);
return process.exitCode = 1;
}
process.exitCode = 0;
setTimeout(process.exit, 1000);
}, 2000);
}
if (verified_received + errors === max) {
exitNextTick = true;
}
}, 1000);
producer.connect()
.on('event.error', function(e) {
errors++;
errorsArr.push(e);
})
.on('delivery-report', function() {
verified_received++;
})
.on('ready', sendMessage);
}
</code></pre>
</article>
</section>
</div>
<div class="grid-25 pull-75">
<div class="navigation-sidebar">
<ul class="nav nav-list">
<li><a href="javascript: void(0);">Classes</a><ul class="nav nav-list" role="menu"><li><a href="AdminClient.html">AdminClient</a></li><li><a href="Client.html">Client</a></li><li><a href="HighLevelProducer.html">HighLevelProducer</a></li><li><a href="KafkaConsumer.html">KafkaConsumer</a></li><li><a href="KafkaConsumerStream.html">KafkaConsumerStream</a></li><li><a href="LibrdKafkaError.html">LibrdKafkaError</a></li><li><a href="Producer.html">Producer</a></li><li><a href="ProducerStream.html">ProducerStream</a></li></ul></li><li><a href="javascript: void(0);">Events</a><ul class="nav nav-list" role="menu"><li><a href="Client.html#event:disconnected">disconnected</a></li><li><a href="Client.html#event:ready">ready</a></li><li><a href="HighLevelProducer.html#event:disconnected">disconnected</a></li><li><a href="HighLevelProducer.html#event:ready">ready</a></li><li><a href="KafkaConsumer.html#event:data">data</a></li><li><a href="KafkaConsumer.html#event:disconnected">disconnected</a></li><li><a href="KafkaConsumer.html#event:ready">ready</a></li><li><a href="Producer.html#event:disconnected">disconnected</a></li><li><a href="Producer.html#event:ready">ready</a></li></ul></li><li><a href="javascript: void(0);">Tutorials</a><ul class="nav nav-list" role="menu"><li><a href="tutorial-consumer-flow.html">consumer-flow</a></li><li><a href="tutorial-consumer-per-partition.html">consumer-per-partition</a></li><li><a href="tutorial-consumer.html">consumer</a></li><li><a href="tutorial-docker-alpine.html">docker-alpine</a></li><li><a href="tutorial-high-level-producer.html">high-level-producer</a></li><li><a href="tutorial-metadata.html">metadata</a></li><li><a href="tutorial-producer-cluster.html">producer-cluster</a></li><li><a href="tutorial-producer_.html">producer</a></li></ul></li><h3>Global</h3><ul><li><a href="global.html#createAdminClient">createAdminClient</a></li><li><a href="global.html#createSerializer">createSerializer</a></li><li><a href="global.html#RefCounter">RefCounter</a></li><li><a href="global.html#Topic">Topic</a></li><li><a href="global.html#TopicPartition">TopicPartition</a></li></ul>
</ul>
<!-- <ul class="nav nav-list collapse"> -->
</div>
</div>
</div>
</div>
</div> <!-- /.body-content -->
<br class="clear">
<!-- FOOTER -->
<footer class="footer">
<div class="grid-container">
<div class="footer-content">
<div class="grid-75">
<div class="legal">
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc </a> on Thu Jan 11 2024 10:36:33 GMT+0100 (Central European Standard Time)
</div>
</div>
<div class="grid-25">
<span class="blizzard">Blizzard® Entertainment</span>
</div>
</div>
</div>
</footer>
<!-- /FOOTER -->
<script> prettyPrint(); </script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
<script src="toolkit/js/toolkit/toolkit.min.js"> </script>
</body>
</html>