kafka初探

version: "3.3"

services:
  openjdk8:
    # kafka自带的JDK11似乎没法进入java调试
    image: "openjdk:8u181"
    volumes:
      # 将jdk8 java home映射出来，提供给kafka使用
      - "openjdk8_java_home:/usr/lib/jvm/java-8-openjdk-amd64"
      # etc目录是jdk8链接方式引用的，也需要映射出来
      - "openjdk8_java_etc:/etc/java-8-openjdk"
  zookeeper-server:
    image: zookeeper:3.7
    volumes:
      # zookeeper快照数据
      - "zookeeper_data:/data"
      # zookeeper事务日志
      - "zookeeper_datalog:/datalog"
      # zookeeper log4j日志
      - "zookeeper_log4j_logs:/logs"
    ports:
      # 对client提供服务
      - "3181:2181"
    restart: always
  zookeeper-ui:
    image: juris/zkui:latest
    ports:
      # zookeeper-ui的http端口，访问http://主机ip:主机映射端口(如3180)
      - "3180:9090"
    environment:
      # zk-server地址和端口(配置服务名:容器端口)
      ZK_SERVER: "zookeeper-server:2181"
      # admin密码
      USER_SET: '{"users":[{"username":"admin", "password":"admin", "role":"ADMIN"}]}'
    restart: always
    depends_on:
      - zookeeper-server

  kafka-server:
    image: bitnami/kafka:3.4.0
    ports:
      # kafka客户端连接端口：无须身份认证
      - "3092:9092"
      # kafka客户端连接端口：需要身份认证
      - "3093:9093"
      # java调试端口
      - "3050:5005"
    volumes:
      # kafka数据卷
      - "kafka_data:/bitnami/kafka"
      # 启用SASL_PLAINTEXT身份认证
      - "./kafka-config/kafka_jaas.conf:/opt/bitnami/kafka/config/kafka_jaas.conf"
      # 下面是两个身份认证需要的库文件 jackson-all-1.8.4.jar & VSaslLoginModule-1.0.jar
      - "./kafka-libs/jackson-all-1.8.4.jar:/opt/bitnami/kafka/libs/jackson-all-1.8.4.jar"
      - "./kafka-libs/VSaslLoginModule-1.0.jar:/opt/bitnami/kafka/libs/VSaslLoginModule-1.0.jar"
      
      # 引入外部jdk8 ，kafka原始镜像自带的jdk11，在eclipse下一直连不上调试端口
      - "openjdk8_java_home:/usr/lib/jvm/java-8-openjdk-amd64"
      - "openjdk8_java_etc:/etc/java-8-openjdk"
    environment:
      # 详细的环境变量参考：https://github.com/bitnami/containers/tree/main/bitnami/kafka，所有server.properties中的配置可以转化成：KAFKA_CFG_，例如：KAFKA_CFG_BACKGROUND_THREADS等价于background.threads
      # zookeeper地址:端口(配置服务名:容器端口)
      KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper-server:2181"
      # 允许客户端 免密码登录、明文通讯
      ALLOW_PLAINTEXT_LISTENER: "yes"
      # kafka监听端口 9092为免登录端口，9093需要身份认证
      KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,SASL_PLAINTEXT://:9093"
      # 本服务供外部访问的地址
      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://10.1.27.89:3092,SASL_PLAINTEXT://10.1.27.89:3093"
      # 猜测：指示kafka节点之间的访问使用哪种协议-->对应前面的ADVERTISED_LISTENERS参数
      KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL: "PLAINTEXT"
      # 猜测：和SECURITY_INTER_BROKER_PROTOCOL一个含义
      KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: "PLAIN"
      # 猜测：和SECURITY_INTER_BROKER_PROTOCOL一个含义
      KAFKA_CFG_SASL_ENABLED_MECHANISMS: "PLAIN"
      # SASL_PLAINTEXT协议指定的身份认证回调
      KAFKA_CFG_LISTENER_NAME_SASL_PLAINTEXT_PLAIN_SASL_SERVER_CALLBACK_HANDLER_CLASS: "com.toone.v3.platform.kafka.server.login.VSaslServerCallbackHandler"
      # kafka集群中本节点的标识
      KAFKA_BROKER_ID: "1"
      
      #内存设置
      KAFKA_HEAP_OPTS: "-Xms1024m -Xmx2048m"
      
      #调试设置，打开调试：KAFKA_DEBUG: "on"， 关闭调试：KAFKA_DEBUG: "" (或者注释掉KAFKA_DEBUG环境变量)
      KAFKA_DEBUG: "on"
      # 引入外部的JDK，供调试用
      JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-amd64"
      
      #修正控制台logs -f中文乱码
      LC_ALL: "C.UTF-8"
    restart: always
    depends_on:
      - openjdk8
      - zookeeper-server
  kafka-ui:
    # 参见：https://www.kafkamagic.com/start/
    image: digitsy/kafka-magic:latest
    ports:
      # 访问kafka 管理界面：http://主机ip:主机映射端口(如3080)
      - "3080:80"
    volumes:
      # 持久化kafka集群配置
      - "./kafka_ui_config:/config"
    environment:
      KMAGIC_ALLOW_TOPIC_DELETE: "true"
      KMAGIC_ALLOW_SCHEMA_DELETE: "true"
      KMAGIC_CONFIG_STORE_TYPE: "file"
      KMAGIC_CONFIG_STORE_CONNECTION: "Data Source=/config/KafkaMagicConfig.db;"
      KMAGIC_CONFIG_ENCRYPTION_KEY: "ENTER_YOUR_KEY_HERE"
    restart: always
volumes:
  zookeeper_data:
  zookeeper_datalog:
  zookeeper_log4j_logs:
  kafka_data:
  openjdk8_java_home:
  openjdk8_java_etc: