You can use this example to setup Kafka with three clusters using Docker compose. It also comes with Zookeeper and Kafka UI. You can then use :9193, :9293, :9393 to connect to Kafka in your application. I'll show a simple one as an example in Golang.


With this setup, each topic you create will be replicated in all clusters. Also, each produced events will be replicated in all clusters.


Docker


version: "3.7"

services:

kafka-zookeeper:
container_name: kafka-zookeeper
image: "bitnami/zookeeper:3.8.4"
ports:
- "2181:2181"
environment:
- ALLOW_ANONYMOUS_LOGIN=yes

kafka-broker-1:
container_name: kafka-broker-1
image: "bitnami/kafka:3.3.2"
ports:
- "9192:9092"
- "9193:9193"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=1
- KAFKA_CFG_NODE_ID=1
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9192,EXTERNAL://:9193
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-1:9192,EXTERNAL://localhost:9193
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper

kafka-broker-2:
container_name: kafka-broker-2
image: "bitnami/kafka:3.3.2"
ports:
- "9292:9092"
- "9293:9293"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=2
- KAFKA_CFG_NODE_ID=2
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9292,EXTERNAL://:9293
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-2:9292,EXTERNAL://localhost:9293
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper

kafka-broker-3:
container_name: kafka-broker-3
image: "bitnami/kafka:3.3.2"
ports:
- "9392:9092"
- "9393:9393"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=3
- KAFKA_CFG_NODE_ID=3
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9392,EXTERNAL://:9393
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-3:9392,EXTERNAL://localhost:9393
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper

kafka-ui:
container_name: kafka-ui
image: "provectuslabs/kafka-ui:latest"
ports:
- "8080:8080"
environment:
- KAFKA_CLUSTERS_0_NAME=kafka-cluster-1
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka-broker-1:9192
- KAFKA_CLUSTERS_0_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_CLUSTERS_1_NAME=kafka-cluster-2
- KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=kafka-broker-2:9292
- KAFKA_CLUSTERS_1_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_CLUSTERS_2_NAME=kafka-cluster-3
- KAFKA_CLUSTERS_2_BOOTSTRAPSERVERS=kafka-broker-3:9392
- KAFKA_CLUSTERS_2_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_BROKERCONNECT=kafka-broker-1:9192,kafka-broker-2:9292,kafka-broker-3:9392
depends_on:
- kafka-zookeeper
- kafka-broker-1
- kafka-broker-2
- kafka-broker-3

Result


$ docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
16d2b3327e26 provectuslabs/kafka-ui:latest "/bin/sh -c 'java --…" 4 minutes ago Up 4 minutes 0.0.0.0:8080->8080/tcp kafka-ui
a4e50718d619 bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9193->9193/tcp, 0.0.0.0:9192->9092/tcp kafka-broker-1
f5ac6987882d bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9293->9293/tcp, 0.0.0.0:9292->9092/tcp kafka-broker-2
be7987b98dbb bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9393->9393/tcp, 0.0.0.0:9392->9092/tcp kafka-broker-3
ff111217cc08 bitnami/zookeeper:3.8.4 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp, 8080/tcp kafka-zookeeper

Connection


// Using Sarama client

pro, err := sarama.NewSyncProducer([]string{":9193",":9293",":9393"}, cfg)
if err != nil {
log.Fatalln(err)
}
defer pro.Close()