Docker compose ile çoklu küme Kafka sunucusunu kurma

27/05/2024 - DOCKER, GO, KAFKA

Docker compose'u kullanarak Kafka'yı üç kümeyle ayarlamak için bu örneği kullanabilirsiniz. Ayrıca Zookeeper ve Kafka kullanıcı arayüzü ile birlikte gelir. Daha sonra uygulamanızda Kafka'ya bağlanmak için :9193, :9293, :9393 portlarını kullanabilirsiniz. Golang'da örnek olarak basit bir tane göstereceğim.


Bu kurulumla oluşturduğunuz her konu tüm kümelerde çoğaltılacaktır. Ayrıca üretilen her etkinlik tüm kümelerde çoğaltılacaktır.


Docker


version: "3.7"

services:

  kafka-zookeeper:
    container_name: kafka-zookeeper
    image: "bitnami/zookeeper:3.8.4"
    ports:
      - "2181:2181"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes

  kafka-broker-1:
    container_name: kafka-broker-1
    image: "bitnami/kafka:3.3.2"
    ports:
      - "9192:9092"
      - "9193:9193"
    environment:
      - BITNAMI_DEBUG=yes
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_LISTENERS=INTERNAL://:9192,EXTERNAL://:9193
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-1:9192,EXTERNAL://localhost:9193
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
    depends_on:
      - kafka-zookeeper

  kafka-broker-2:
    container_name: kafka-broker-2
    image: "bitnami/kafka:3.3.2"
    ports:
      - "9292:9092"
      - "9293:9293"
    environment:
      - BITNAMI_DEBUG=yes
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_BROKER_ID=2
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_LISTENERS=INTERNAL://:9292,EXTERNAL://:9293
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-2:9292,EXTERNAL://localhost:9293
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
    depends_on:
      - kafka-zookeeper

  kafka-broker-3:
    container_name: kafka-broker-3
    image: "bitnami/kafka:3.3.2"
    ports:
      - "9392:9092"
      - "9393:9393"
    environment:
      - BITNAMI_DEBUG=yes
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_BROKER_ID=3
      - KAFKA_CFG_NODE_ID=3
      - KAFKA_CFG_ZOOKEEPER_CONNECT=kafka-zookeeper:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_LISTENERS=INTERNAL://:9392,EXTERNAL://:9393
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-broker-3:9392,EXTERNAL://localhost:9393
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
      - KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL
    depends_on:
      - kafka-zookeeper

  kafka-ui:
    container_name: kafka-ui
    image: "provectuslabs/kafka-ui:latest"
    ports:
      - "8080:8080"
    environment:
      - KAFKA_CLUSTERS_0_NAME=kafka-cluster-1
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka-broker-1:9192
      - KAFKA_CLUSTERS_0_ZOOKEEPER=kafka-zookeeper:2181
      - KAFKA_CLUSTERS_1_NAME=kafka-cluster-2
      - KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=kafka-broker-2:9292
      - KAFKA_CLUSTERS_1_ZOOKEEPER=kafka-zookeeper:2181
      - KAFKA_CLUSTERS_2_NAME=kafka-cluster-3
      - KAFKA_CLUSTERS_2_BOOTSTRAPSERVERS=kafka-broker-3:9392
      - KAFKA_CLUSTERS_2_ZOOKEEPER=kafka-zookeeper:2181
      - KAFKA_BROKERCONNECT=kafka-broker-1:9192,kafka-broker-2:9292,kafka-broker-3:9392
    depends_on:
      - kafka-zookeeper
      - kafka-broker-1
      - kafka-broker-2
      - kafka-broker-3

Sonuç


$ docker ps -a
CONTAINER ID   IMAGE                           COMMAND                  CREATED         STATUS         PORTS                                                  NAMES
16d2b3327e26   provectuslabs/kafka-ui:latest   "/bin/sh -c 'java --…"   4 minutes ago   Up 4 minutes   0.0.0.0:8080->8080/tcp                                 kafka-ui
a4e50718d619   bitnami/kafka:3.3.2             "/opt/bitnami/script…"   4 minutes ago   Up 4 minutes   0.0.0.0:9193->9193/tcp, 0.0.0.0:9192->9092/tcp         kafka-broker-1
f5ac6987882d   bitnami/kafka:3.3.2             "/opt/bitnami/script…"   4 minutes ago   Up 4 minutes   0.0.0.0:9293->9293/tcp, 0.0.0.0:9292->9092/tcp         kafka-broker-2
be7987b98dbb   bitnami/kafka:3.3.2             "/opt/bitnami/script…"   4 minutes ago   Up 4 minutes   0.0.0.0:9393->9393/tcp, 0.0.0.0:9392->9092/tcp         kafka-broker-3
ff111217cc08   bitnami/zookeeper:3.8.4         "/opt/bitnami/script…"   4 minutes ago   Up 4 minutes   2888/tcp, 3888/tcp, 0.0.0.0:2181->2181/tcp, 8080/tcp   kafka-zookeeper

Bağlantı


// Using Sarama client

pro, err := sarama.NewSyncProducer([]string{":9193",":9293",":9393"}, cfg)
if err != nil {
	log.Fatalln(err)
}
defer pro.Close()