27/05/2024 - DOCKER, GO, KAFKA
You can use this example to setup Kafka with three clusters using Docker compose. It also comes with Zookeeper and Kafka UI. You can then use :9193, :9293, :9393
to connect to Kafka in your application. I'll show a simple one as an example in Golang.
With this setup, each topic you create will be replicated in all clusters. Also, each produced events will be replicated in all clusters.
version: "3.7"
services:
kafka-zookeeper:
container_name: kafka-zookeeper
image: "bitnami/zookeeper:3.8.4"
ports:
- "2181:2181"
environment:
- ALLOW_ANONYMOUS_LOGIN=yes
kafka-broker-1:
container_name: kafka-broker-1
image: "bitnami/kafka:3.3.2"
ports:
- "9192:9092"
- "9193:9193"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=1
- KAFKA_CFG_NODE_ID=1
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9192,EXTERNAL://:9193
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-1:9192,EXTERNAL://localhost:9193
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper
kafka-broker-2:
container_name: kafka-broker-2
image: "bitnami/kafka:3.3.2"
ports:
- "9292:9092"
- "9293:9293"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=2
- KAFKA_CFG_NODE_ID=2
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9292,EXTERNAL://:9293
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-2:9292,EXTERNAL://localhost:9293
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper
kafka-broker-3:
container_name: kafka-broker-3
image: "bitnami/kafka:3.3.2"
ports:
- "9392:9092"
- "9393:9393"
environment:
- BITNAMI_DEBUG=yes
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_BROKER_ID=3
- KAFKA_CFG_NODE_ID=3
- KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_CFG_LISTENERS=INTERNAL://:9392,EXTERNAL://:9393
- KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-3:9392,EXTERNAL://localhost:9393
- KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
- KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
depends_on:
- kafka-zookeeper
kafka-ui:
container_name: kafka-ui
image: "provectuslabs/kafka-ui:latest"
ports:
- "8080:8080"
environment:
- KAFKA_CLUSTERS_0_NAME=kafka-cluster-1
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka-broker-1:9192
- KAFKA_CLUSTERS_0_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_CLUSTERS_1_NAME=kafka-cluster-2
- KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=kafka-broker-2:9292
- KAFKA_CLUSTERS_1_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_CLUSTERS_2_NAME=kafka-cluster-3
- KAFKA_CLUSTERS_2_BOOTSTRAPSERVERS=kafka-broker-3:9392
- KAFKA_CLUSTERS_2_ZOOKEEPER=kafka-zookeeper:2181
- KAFKA_BROKERCONNECT=kafka-broker-1:9192,kafka-broker-2:9292,kafka-broker-3:9392
depends_on:
- kafka-zookeeper
- kafka-broker-1
- kafka-broker-2
- kafka-broker-3
$ docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
16d2b3327e26 provectuslabs/kafka-ui:latest "/bin/sh -c 'java --…" 4 minutes ago Up 4 minutes 0.0.0.0:8080->8080/tcp kafka-ui
a4e50718d619 bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9193->9193/tcp, 0.0.0.0:9192->9092/tcp kafka-broker-1
f5ac6987882d bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9293->9293/tcp, 0.0.0.0:9292->9092/tcp kafka-broker-2
be7987b98dbb bitnami/kafka:3.3.2 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 0.0.0.0:9393->9393/tcp, 0.0.0.0:9392->9092/tcp kafka-broker-3
ff111217cc08 bitnami/zookeeper:3.8.4 "/opt/bitnami/script…" 4 minutes ago Up 4 minutes 2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp, 8080/tcp kafka-zookeeper
// Using Sarama client
pro, err := sarama.NewSyncProducer([]string{":9193",":9293",":9393"}, cfg)
if err != nil {
log.Fatalln(err)
}
defer pro.Close()