[kafka] 카프카 클러스터를 GUI(kafdrop)으로 구축하기

docker-compose.yml

version: '2'

services:

  # wurstmeister/zookeeper:3.4.6
  zookeeper:
    hostname: zookeeper
    container_name: zookeeper
    image: wurstmeister/zookeeper:3.4.6
    ports:
      - "2181:2181"

  # wurstmeister/kafka:2.13-2.8.1 - broker 1
  kafka1:
    hostname: kafka1
    container_name: kafka1
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test1:1:1"
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper

  # wurstmeister/kafka:2.13-2.8.1 - broker 2
  kafka2:
    hostname: kafka2
    container_name: kafka2
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test2:1:1"
    ports:
      - "9093:9092"
    depends_on:
      - zookeeper

  # wurstmeister/kafka:2.13-2.8.1 - broker 3
  kafka3:
    hostname: kafka3
    container_name: kafka3
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka3
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test3:1:1"
    ports:
      - "9094:9092"
    depends_on:
      - zookeeper

  # obsidiandynamics/kafdrop
  kafdrop:
    hostname: kafdrop
    container_name: kafdrop
    image: obsidiandynamics/kafdrop
    restart: "always"
    ports:
      - "9000:9000"
    environment:
      KAFKA_BROKERCONNECT: "kafka1:9092, kafka2:9092, kafka3:9092"
      JVM_OPTS: "-Xms32M -Xmx64M"
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3

다음 명령어 입력

docker-compose up -d

인터넷을 켜서 localhost:9000 으로 접속해보자

kafka1 브로커에 들어가보자

다시 뒤로 빠져나오자

test1 이라는 토픽에 들어가보자

view message 에 들어가보자

도커 컴포즈 종료해보자

docker-compose down

도커컴포즈 파일 수정

version: '2'

services:

  # wurstmeister/zookeeper:3.4.6
  zookeeper:
    hostname: zookeeper
    container_name: zookeeper
    image: wurstmeister/zookeeper:3.4.6
    ports:
      - "2181:2181"

  # wurstmeister/kafka:2.13-2.8.1 - broker 1
  kafka1:
    hostname: kafka1
    container_name: kafka1
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test1:3:3"
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper

  # wurstmeister/kafka:2.13-2.8.1 - broker 2
  kafka2:
    hostname: kafka2
    container_name: kafka2
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test2:3:3"
    ports:
      - "9093:9092"
    depends_on:
      - zookeeper

  # wurstmeister/kafka:2.13-2.8.1 - broker 3
  kafka3:
    hostname: kafka3
    container_name: kafka3
    image: wurstmeister/kafka:2.13-2.8.1
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_HOST_NAME: kafka3
      BOOTSTRAP_SERVERS: kafka1:9092, kafka2:9093, kafka3:9094
      KAFKA_CREATE_TOPICS: "test3:3:3"
    ports:
      - "9094:9092"
    depends_on:
      - zookeeper

  # obsidiandynamics/kafdrop
  kafdrop:
    hostname: kafdrop
    container_name: kafdrop
    image: obsidiandynamics/kafdrop
    restart: "always"
    ports:
      - "9000:9000"
    environment:
      KAFKA_BROKERCONNECT: "kafka1:9092, kafka2:9092, kafka3:9092"
      JVM_OPTS: "-Xms32M -Xmx64M"
    depends_on:
      - zookeeper
      - kafka1
      - kafka2
      - kafka3

다시 도커컴포즈 업

docker-compose up -d

다시 웹에 localhost:9000 으로 접속

메인화면

맨 아래에 있는 new 를 눌러보자

test4 라는 이름을 가진 토픽을 만드는데

파티션은 100개

레플리카는 2개로 만들어보자

메인화면으로 되돌아가자

test5 라는 이름을 가진 토픽을 만드는데

파티션은 100개

레플리카는 1개로 만들어보자

test4 토픽으로 들어가보자

test5 토픽으로 들어가보자

test1 토픽에 들어가보자

test2 토픽에 들어가보자

test3 토픽에 들어가보자

test1, test2, test3 토픽을 삭제하고

메인화면을 확인해보자



test4 토픽을 삭제하고 메인화면을 확인해보자

test5 토픽을 삭제하고 메인화면을 보자